Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

401 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Measured-budget memory policy for the local diffusion backend.

From the resolved device target, a free-memory snapshot and a coarse model footprint estimate, this
picks a CPU-offload policy and VAE slice/tile settings, then applies them to a built diffusers
pipeline. A model that will not fit resident is kept running by streaming weights through the GPU
one module at a time, which is lossless (offload / VAE slicing change placement and decode chunking,
not numerics).

The choice is coarse (sizes the model, not every activation), so ``auto`` is best-effort and the
explicit ``fast`` / ``balanced`` / ``low_vram`` modes are a hard override. torch / psutil imported
lazily.
"""

from __future__ import annotations

import contextlib
import contextvars
import functools
import math
import os
import sys
import threading
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Iterable, Iterator, Optional
from .host_pinning import host_pinning_allowed
from .rocm_bf16 import is_rocm_torch


MEMORY_MODE_AUTO = "auto"
MEMORY_MODE_FAST = "fast"
MEMORY_MODE_BALANCED = "balanced"
MEMORY_MODE_LOW_VRAM = "low_vram"
MEMORY_MODES = (
    MEMORY_MODE_AUTO,
    MEMORY_MODE_FAST,
    MEMORY_MODE_BALANCED,
    MEMORY_MODE_LOW_VRAM,
)

# Tiers: none, model, group, streaming, sequential (sequential is broken for GGUF
# through diffusers 0.39, kept as an escape hatch).
OFFLOAD_NONE = "none"
OFFLOAD_MODEL = "model"
OFFLOAD_GROUP = "group"
OFFLOAD_STREAMING = "streaming"
OFFLOAD_SEQUENTIAL = "sequential"

# Transformer blocks resident per group under group offloading: fewer = lower VRAM, more host-to-device traffic.
DEFAULT_GROUP_BLOCKS = 1

DEFAULT_IMAGE_WIDTH = 1024
DEFAULT_IMAGE_HEIGHT = 1024
# flat allowance for fixed pipeline costs (scheduler, embeddings, CUDA context, fragmentation)
DEFAULT_BASE_OVERHEAD_MIB = 2048
# Reserve kind for an mps reading bounded by Metal's recommended working set (see snapshot_device_memory).
METAL_WORKING_SET = "metal_working_set"


_host_memory_reclaim_warning_logged = False
_host_memory_reclaim_unsupported_logged = False

# Optional (no _ctypes); a module attribute because the reclaimer tests monkeypatch it.
try:
    import ctypes
except Exception:  # noqa: BLE001
    ctypes = None  # type: ignore[assignment]


@functools.lru_cache(maxsize = 1)
def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:
    """Resolve this process allocator's native pressure API once, if the OS exposes one."""
    if ctypes is None:
        return None
    try:
        if sys.platform.startswith("linux"):
            allocator = ctypes.CDLL(None)
            pressure = allocator.malloc_trim
            pressure.argtypes = [ctypes.c_size_t]
            pressure.restype = ctypes.c_int

            def reclaim() -> None:
                pressure(0)

        elif sys.platform == "darwin":
            allocator = ctypes.CDLL(None)
            pressure = allocator.malloc_zone_pressure_relief
            pressure.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
            pressure.restype = ctypes.c_size_t

            def reclaim() -> None:
                pressure(None, 0)

        elif sys.platform == "win32":
            pressure = None
            for library_name in ("ucrtbase.dll", "msvcrt.dll"):
                try:
                    allocator = ctypes.CDLL(library_name)
                    pressure = allocator._heapmin
                    break
                except Exception:
                    continue
            if pressure is None:
                return None
            pressure.argtypes = []
            pressure.restype = ctypes.c_int

            def reclaim() -> None:
                if pressure() == -1:
                    raise OSError("_heapmin failed")

        else:
            return None
    except Exception:
        return None
    return reclaim


OFFLOAD_KEEP_CPU_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_KEEP_CPU"
_KEEP_ATTR = "_unsloth_offload_keep_cpu"
# diffusion_vae_upcast.DTYPE_ONLY_TO_ATTR
_DTYPE_ONLY_TO_ATTR = "_unsloth_dtype_only_to"


def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
    """Offload by re-pointing unmodified weights at their host tensors; enable_model_cpu_offload is wrapped since diffusers rebuilds hooks."""
    if (os.environ.get(OFFLOAD_KEEP_CPU_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return 0
    try:
        from accelerate.hooks import CpuOffload
    except Exception:  # noqa: BLE001 - no accelerate, no offload hooks to wrap
        return 0
    enable = getattr(pipe, "enable_model_cpu_offload", None)
    if callable(enable) and not getattr(enable, _KEEP_ATTR, False):

        def _enable(*args: Any, **kwargs: Any) -> Any:
            # The re-enable after every call starts with pipe.to("cpu"), which would copy back the module run last.
            _offload_through_kept_hooks(pipe, CpuOffload, logger)
            with _keep_device_cache(pipe):
                out = enable(*args, **kwargs)
            _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
            return out

        setattr(_enable, _KEEP_ATTR, True)
        try:
            pipe.enable_model_cpu_offload = _enable
        except Exception:  # noqa: BLE001 - a pipeline refusing the attribute keeps the stock hooks
            return 0
    return _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)


def _offload_through_kept_hooks(
    pipe: Any,
    hook_cls: type,
    logger: Any = None,
) -> None:
    components = getattr(pipe, "components", None) or {}
    for module in components.values():
        hook = getattr(module, "_hf_hook", None)
        if not isinstance(hook, hook_cls) or not getattr(hook, _KEEP_ATTR, False):
            continue
        try:
            onloaded = [*module.parameters(), *(b for _, b in _named_buffers(module))]
            if any(t.device.type != "cpu" for t in onloaded):
                hook.init_hook(module)
        except Exception as exc:  # noqa: BLE001 - the stock re-enable still moves it
            if logger is not None:
                logger.debug(
                    "diffusion.memory: kept offload of %s before re-enable failed: %s",
                    type(module).__name__,
                    exc,
                )


def _gguf_parameter_class() -> Optional[type]:
    module = sys.modules.get("diffusers.quantizers.gguf.utils")
    return getattr(module, "GGUFParameter", None) if module is not None else None


def _wrapper_inner(t: Any) -> Optional[list]:
    """``[(attr, tensor), ...]`` of a traceable wrapper subclass (torchao int8 / fp8 weights) whose inner tensors are
    all plain tensors, else None."""
    import torch

    flatten = getattr(type(t), "__tensor_flatten__", None)
    if flatten is None or type(t) in (torch.Tensor, torch.nn.Parameter):
        return None
    try:
        from torch.utils._python_dispatch import is_traceable_wrapper_subclass

        if not is_traceable_wrapper_subclass(t):
            return None
        names, _ctx = t.__tensor_flatten__()
        inner = [(name, getattr(t, name)) for name in names]
    except Exception:  # noqa: BLE001
        return None
    if not inner or any(type(x) is not torch.Tensor for _, x in inner):
        return None
    return inner


def _tensor_ptr(t: Any) -> Any:
    """Storage address of ``t``; for a wrapper subclass, the tuple of its inner tensors' addresses."""
    inner = _wrapper_inner(t)
    if inner is not None:
        return tuple(int(x.data_ptr()) for _, x in inner)
    return t.data_ptr()


def _repoint(param: Any, kept: Any) -> None:
    """Point ``param`` at its kept host tensor without a copy. A wrapper subclass cannot change device through
    ``.data``; it swaps with a parameter over the kept tensors, as ``Module.to`` moves it."""
    if _wrapper_inner(kept) is None:
        param.data = kept
        return
    import torch

    torch.utils.swap_tensors(param, torch.nn.Parameter(kept, requires_grad = param.requires_grad))


def _keepable(param: Any) -> bool:
    import torch

    data = param.data
    if type(data) is torch.Tensor:
        return True
    # torchao weights (int8 / fp8): a wrapper of plain tensors; kept and re-pointed like a dense weight.
    if _wrapper_inner(data) is not None and type(data) is type(param):
        return True
    # GGUF weights are packed bytes; the quant type lives on the Parameter, which the swap never replaces.
    gguf = _gguf_parameter_class()
    return gguf is not None and type(param) is gguf and type(data) is gguf


def _wrap_cpu_offload_hooks(
    pipe: Any,
    hook_cls: type,
    logger: Any = None,
) -> int:
    wrapped = 0
    components = getattr(pipe, "components", None) or {}
    for module in components.values():
        hook = getattr(module, "_hf_hook", None)
        if not isinstance(hook, hook_cls) or getattr(hook, _KEEP_ATTR, False):
            continue
        try:
            _wrap_cpu_offload_hook(hook, module, logger)
            module.__dict__[_KEEP_ATTR]["exact_pin"] = bool(
                getattr(pipe, _FAST_PIN_REQUEST_ATTR, False)
            )
            module.__dict__[_KEEP_ATTR]["exact_pin_device"] = getattr(
                hook, "execution_device", None
            )
            wrapped += 1
        except Exception as exc:  # noqa: BLE001 - that module keeps the stock copy
            if logger is not None:
                logger.debug(
                    "diffusion.memory: keep-cpu offload not applied to %s: %s",
                    type(module).__name__,
                    exc,
                )
    return wrapped


OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_PIN"
# The host allocator rounds every block up to a power of two, so weights share chunks instead of one pin each.
_PIN_CHUNK_BYTES = 1 << 30
_PIN_ALIGN = 256
_PIN_RESERVE_MIN_BYTES = 4 << 30
_PIN_RESERVE_FRACTION = 0.15


def _pin_mode() -> str:
    raw = (os.environ.get(OFFLOAD_PIN_ENV) or "").strip().lower()
    if raw in ("0", "off", "false", "no"):
        return "off"
    if raw in ("1", "on", "true", "yes", "force"):
        return "on"
    return "auto"


def _pow2_ceil(n: int) -> int:
    return 1 << max(0, int(n) - 1).bit_length()


def release_pinned_host_memory() -> None:
    _host_empty_cache()


def _host_empty_cache() -> None:
    try:
        import torch
        empty = getattr(torch._C, "_host_emptyCache", None)
        if callable(empty) and torch.cuda.is_available():
            empty()
    except Exception:  # noqa: BLE001
        pass


def _pinnable_layout(data: Any) -> bool:
    if data.is_contiguous():
        return True
    import torch

    fmt = {4: torch.channels_last, 5: torch.channels_last_3d}.get(data.dim())
    return fmt is not None and data.is_contiguous(memory_format = fmt)


def _pin_host_weights(
    module: Any,
    host: dict,
    logger: Any = None,
    buffer_host: Optional[dict] = None,
    layout: Optional[list] = None,
    exact: bool = False,
    device: Any = None,
) -> int:
    """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Each copy keeps its source strides.
    ``layout`` (if given) receives ``(chunk, [(name, start, view), ...])`` per chunk for ``_upload_pinned_chunks``.
    ``exact`` (video loads, see ``_exact_pin_supported``) sizes each chunk to its bytes instead of a power of two."""
    import torch

    mode = _pin_mode()
    if mode == "off" or not torch.cuda.is_available() or not host_pinning_allowed(device):
        return 0
    # (name, owner tensor, inner attribute or None, the tensor to pin)
    params: list = []
    for name, p in module.named_parameters():
        if host.get(name) is None or p.device.type != "cpu" or not _keepable(p):
            continue
        inner = _wrapper_inner(p.data)
        if inner is None:
            if _pinnable_layout(p.data) and not p.data.is_pinned():
                params.append((name, p, None, p.data))
            continue
        for attr, t in inner:
            if t.device.type == "cpu" and _pinnable_layout(t) and not t.is_pinned():
                params.append((name, p, attr, t))
    buffers = [
        (name, b)
        for name, b in _named_buffers(module)
        if (buffer_host or {}).get(name) is b
        and b.device.type == "cpu"
        and _keepable_buffer(b)
        and b.is_contiguous()
        and not b.is_pinned()
    ]
    params = params + [("\0" + name, b, None, b) for name, b in buffers]
    sizes = [-(-t.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, _, _, t in params]
    if not sizes or sum(sizes) == 0:
        return 0
    # Lay out chunks first so the RAM gate counts chunk tails and last-chunk rounding, not just weight bytes.
    chunk = max(_PIN_CHUNK_BYTES, _pow2_ceil(max(sizes)))
    chunks: list = []
    slots: list = []
    off, left = 0, sum(sizes)
    for size in sizes:
        if not chunks or off + size > chunks[-1]:
            chunks.append(chunk if left >= chunk else _pow2_ceil(left))
            off = 0
        slots.append((len(chunks) - 1, off))
        off += size
        left -= size
    exact = exact and _exact_pin_supported(device)
    if exact:
        # The same slots, so the device copies are unchanged; only the host side loses the rounding.
        used = [0] * len(chunks)
        for size, (index, start) in zip(sizes, slots):
            used[index] = max(used[index], start + size)
        chunks = [-(-n // _EXACT_PIN_PAGE) * _EXACT_PIN_PAGE for n in used]
    need = sum(chunks)
    if mode == "auto":
        # Pinned pages are charged to an enforcing cgroup: size from the cgroup-capped readings.
        available_mib = _available_system_memory_mib()
        total_mib = _host_ram_capacity_mib()
        if available_mib is None or total_mib is None:
            return 0  # no way to size it, so do not lock memory
        available, total = int(available_mib) << 20, int(total_mib) << 20
        reserve = max(_PIN_RESERVE_MIN_BYTES, int(total * _PIN_RESERVE_FRACTION))
        if available - need < reserve:
            if logger is not None:
                logger.info(
                    "diffusion.memory: not pinning %s (%.1f GiB): %.1f GiB of host RAM available",
                    type(module).__name__,
                    need / 2**30,
                    available / 2**30,
                )
            return 0
    if exact:
        try:
            return _pin_host_weights_exact(
                module, host, logger, buffer_host, layout, params, slots, chunks, device
            )
        except Exception as exc:  # noqa: BLE001 - registration refused before any weight moved: the pinned allocator
            if logger is not None:
                logger.info(
                    "diffusion.memory: exact pin of %s failed (%s); using the pinned allocator",
                    type(module).__name__,
                    exc,
                )
            return _pin_host_weights(
                module, host, logger, buffer_host = buffer_host, layout = layout, device = device
            )
    bufs: list = []
    placed: list = []
    try:
        for (name, p, attr, src), (index, start) in zip(params, slots):
            if index == len(bufs):
                bufs.append(torch.empty(chunks[index], dtype = torch.uint8, pin_memory = True))
            view = bufs[index][start : start + src.nbytes].view(src.dtype)
            view = (
                view.view(src.shape)
                if src.is_contiguous()
                else view.as_strided(src.shape, src.stride())
            )
            view.copy_(src)
            placed.append((name, p, attr, view, index, start))
    except Exception as exc:  # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weights
        bufs.clear()
        placed.clear()
        view = None
        _host_empty_cache()
        if logger is not None:
            logger.info(
                "diffusion.memory: pinning %s failed (%s); weights stay pageable",
                type(module).__name__,
                exc,
            )
        return 0
    members: list = [[] for _ in bufs]
    wrapped: dict = {}
    for name, p, attr, view, index, start in placed:
        # A torchao wrapper weight gets one entry per inner tensor, under the weight's name.
        members[index].append((name, start, view))
        if name.startswith("\0"):
            name = name[1:]
            _set_buffer(module, name, view)
            buffer_host[name] = view
            continue
        if attr is not None:
            setattr(p, attr, view)
            wrapped[name] = p
            continue
        p.data = view
        host[name] = view
    for name, p in wrapped.items():
        host[name] = p.data  # the kept host weight now reads the pinned inner tensors
    if layout is not None:
        layout[:] = [(buf, entries) for buf, entries in zip(bufs, members) if entries]
    return need


# Video loads only. "0" restores the power-of-two pinned chunks of the whole-module offload tier.
EXACT_PIN_ENV = "UNSLOTH_VIDEO_EXACT_PIN"
_EXACT_PIN_PAGE = 4096


def _exact_pin_supported(device: Any = None) -> bool:
    if (os.environ.get(EXACT_PIN_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return False
    return _fast_pin_supported(device)


def _pin_host_weights_exact(
    module: Any,
    host: dict,
    logger: Any,
    buffer_host: Optional[dict],
    layout: Optional[list],
    params: list,
    slots: list,
    chunks: list,
    device: Any = None,
) -> int:
    """``_pin_host_weights`` into page-granular registered chunks (torch's allocator rounds up to a power of two).
    Weights move per chunk so the module is never held twice; a refusal after the first chunk keeps the moved
    weights pinned and leaves the rest pageable (a first-chunk refusal raises)."""
    import torch

    target = torch.device(device) if device is not None else None
    scope = (
        torch.cuda.device(target)
        if target is not None and target.type == "cuda" and target.index is not None
        else contextlib.nullcontext()
    )
    with scope:
        return _fill_exact_chunks(module, host, logger, buffer_host, layout, params, slots, chunks)


def _fill_exact_chunks(
    module: Any,
    host: dict,
    logger: Any,
    buffer_host: Optional[dict],
    layout: Optional[list],
    params: list,
    slots: list,
    chunks: list,
) -> int:
    import numpy as np
    import torch

    members: list = [[] for _ in chunks]
    for item, (index, start) in zip(range(len(params)), slots):
        members[index].append((item, start))
    done: list = []
    pinned = 0
    for index, size in enumerate(chunks):
        try:
            holder = _RegisteredHostBuffer(size)
            buf = torch.from_numpy(np.asarray(holder))
            placed = []
            for item, start in members[index]:
                name, p, attr, src = params[item]
                view = buf[start : start + src.nbytes].view(src.dtype)
                view = (
                    view.view(src.shape)
                    if src.is_contiguous()
                    else view.as_strided(src.shape, src.stride())
                )
                view.copy_(src)
                placed.append((item, start, view))
            holder.register()
        except Exception:
            if not done:
                raise
            if logger is not None:
                logger.info(
                    "diffusion.memory: pinned %d of %d chunks of %s; the rest stays pageable",
                    len(done),
                    len(chunks),
                    type(module).__name__,
                )
            break
        entries = []
        wrapped: dict = {}
        for item, start, view in placed:
            name, p, attr, _src = params[item]
            params[item] = None
            entries.append((name, start, view))
            if name.startswith("\0"):
                name = name[1:]
                _set_buffer(module, name, view)
                buffer_host[name] = view
                continue
            if attr is not None:
                setattr(p, attr, view)
                wrapped[name] = p
                continue
            p.data = view
            host[name] = view
        for name, p in wrapped.items():
            host[name] = p.data  # the kept host weight now reads the pinned inner tensors
        done.append((buf, entries))
        pinned += size
    if layout is not None:
        layout[:] = [(buf, entries) for buf, entries in done if entries]
    return pinned


OFFLOAD_FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_FAST_UPLOAD"


def _on_device(module: Any) -> bool:
    first = next(iter(module.parameters()), None)
    return first is not None and first.device.type != "cpu"


@contextlib.contextmanager
def _keep_device_cache(pipe: Any):
    """Skip the empty_cache inside diffusers' enable_model_cpu_offload (re-run after every call) once the
    components onload through pinned chunks: the chunks freed by the last offload are reused by the next onload,
    while an emptied cache makes every onload cudaMalloc again and every offload cudaFree."""
    module = sys.modules.get("diffusers.pipelines.pipeline_utils")
    original = getattr(module, "empty_device_cache", None) if module is not None else None
    components = getattr(pipe, "components", None) or {}
    engaged = _fast_upload_enabled() and any(
        bool((getattr(m, "__dict__", {}).get(_KEEP_ATTR) or {}).get("pin_layout"))
        for m in components.values()
    )
    if not engaged or not callable(original):
        yield
        return
    # Process-global patch: overlapping enables share one count, and the last one out restores the real function.
    with _cache_patch_lock:
        if not _cache_patch:
            _cache_patch.update(module = module, original = original, depth = 0)
            module.empty_device_cache = _skip_empty_device_cache
        _cache_patch["depth"] += 1
    try:
        yield
    finally:
        with _cache_patch_lock:
            _cache_patch["depth"] -= 1
            if _cache_patch["depth"] <= 0:
                if _cache_patch["module"].empty_device_cache is _skip_empty_device_cache:
                    _cache_patch["module"].empty_device_cache = _cache_patch["original"]
                _cache_patch.clear()


_cache_patch_lock = threading.Lock()
_cache_patch: dict = {}


def _skip_empty_device_cache(*args: Any, **kwargs: Any) -> None:
    return None


def _fast_upload_enabled() -> bool:
    return (os.environ.get(OFFLOAD_FAST_UPLOAD_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def _same_view(tensor: Any, view: Any) -> bool:
    return (
        tensor.device.type == "cpu"
        and tensor.data_ptr() == view.data_ptr()
        and tensor.dtype == view.dtype
        and tensor.shape == view.shape
        and tensor.stride() == view.stride()
    )


def _prune_pin_layout(layout: list, host: dict, buffer_host: dict) -> None:
    """Drop, in place, the layout entries whose kept host weight is no longer their pinned view, and the chunks left
    with none: the layout's own references would otherwise keep those pinned chunks alive."""
    kept = []
    for buf, entries in layout:
        live = []
        for entry in entries:
            name, view = entry[0], entry[2]
            tensor = buffer_host.get(name[1:]) if name.startswith("\0") else host.get(name)
            if tensor is None:
                continue
            # A torchao wrapper weight has one entry per inner tensor (as ``_upload_pinned_chunks`` reads them).
            inner = None if name.startswith("\0") else _wrapper_inner(tensor)
            if inner is not None:
                if any(_same_view(t, view) for _, t in inner):
                    live.append(entry)
            elif _same_view(tensor, view):
                live.append(entry)
        if live:
            kept.append((buf, live))
    layout[:] = kept


def _prune_state_layout(state: dict) -> None:
    layout = state.get("pin_layout")
    if layout:
        _prune_pin_layout(layout, state.get("host") or {}, state.get("buffer_host") or {})
        if not layout:
            state.pop("pin_layout", None)


def _upload_pinned_chunks(
    module: Any, layout: list, host: dict, owner: dict, buffer_host: dict, device: Any
) -> int:
    """Onload a module's pinned weights chunk by chunk: one device allocation and one non_blocking copy per pinned
    chunk, then every weight is re-pointed at its slice. The stock ``module.to`` allocates every tensor on its own,
    and with the device cache emptied between components each of those was a cudaMalloc (and a cudaFree on the way
    out): under host load that ran at 57-120 MiB/s against a 50 GiB/s link. Chunks are near-uniform, so the next
    component reuses the previous one's cached chunks. Same bytes, same strides, so outputs are bit-identical.
    Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. A torchao
    wrapper weight (int8 / fp8) is rebuilt over its inner tensors' device slices with ``__tensor_unflatten__`` and
    swapped in, as the stock ``_to_copy`` + ``Module.to`` do. Returns bytes."""
    import torch

    params = dict(module.named_parameters())
    buffers = dict(_named_buffers(module))
    copied = 0
    moved: list = []
    wrapped: dict = {}
    for name, p in params.items():
        kept = host.get(name)
        inner = _wrapper_inner(kept) if kept is not None else None
        if (
            inner is None
            or owner.get(name) is not p
            or p.device.type != "cpu"
            or type(p) is not type(kept)
            or _tensor_ptr(p.data) != _tensor_ptr(kept)
        ):
            continue
        wrapped[name] = dict(inner)
    # (name, inner attr) -> device slice once uploaded
    inner_dev: dict = {}

    def _inner_attr(name: str, view: Any) -> Optional[str]:
        for attr, t in (wrapped.get(name) or {}).items():
            if _same_view(t, view):
                return attr
        return None

    for _buf, entries in layout:
        for name, _start, view in entries:
            attr = _inner_attr(name, view)
            if attr is not None:
                inner_dev[(name, attr)] = None
    # Only a wrapper with every inner tensor in the layout is rebuilt; the rest stay for the stock module.to.
    whole = {
        name for name, inner in wrapped.items() if all((name, attr) in inner_dev for attr in inner)
    }
    try:
        for buf, entries in layout:
            live = []
            for name, start, view in entries:
                # By address, not identity: ``p.data`` and the re-captured host map hand out fresh tensor objects.
                if name.startswith("\0"):
                    leaf = name[1:]
                    b = buffers.get(leaf)
                    if b is not None and buffer_host.get(leaf) is b and _same_view(b, view):
                        live.append((name, start, view))
                    continue
                if name in wrapped:
                    if name in whole and _inner_attr(name, view) is not None:
                        live.append((name, start, view))
                    continue
                p = params.get(name)
                kept = host.get(name)
                if (
                    p is not None
                    and owner.get(name) is p
                    and kept is not None
                    and _wrapper_inner(kept) is None
                    and _same_view(kept, view)
                    and _same_view(p.data, view)
                ):
                    live.append((name, start, view))
            if not live:
                continue
            # Pack live weights so skipped ones leave no allocated holes.
            used, slots = 0, []
            for name, start, view in sorted(live, key = lambda e: e[1]):
                used = -(-used // _PIN_ALIGN) * _PIN_ALIGN
                slots.append((name, start, view, used))
                used += view.nbytes
            dense = all(start == at for _, start, _, at in slots)
            # Same stream as the forward that follows, so no sync: kernels queue behind the copy.
            dev_buf = torch.empty(used, dtype = torch.uint8, device = device)
            if not torch._has_compatible_shallow_copy_type(buf, dev_buf):
                # e.g. meta: the stock move hands back new Parameters there instead of re-pointing them.
                raise TypeError("device tensors cannot back host parameters")
            if dense:
                dev_buf.copy_(buf[:used], non_blocking = True)
            else:
                for _, start, view, at in slots:
                    dev_buf[at : at + view.nbytes].copy_(
                        buf[start : start + view.nbytes], non_blocking = True
                    )
            copied += used
            for name, _, view, at in slots:
                dev = dev_buf[at : at + view.nbytes].view(view.dtype)
                dev = (
                    dev.view(view.shape)
                    if view.is_contiguous()
                    else dev.as_strided(view.shape, view.stride())
                )
                if name.startswith("\0"):
                    _set_buffer(module, name[1:], dev)
                elif name in wrapped:
                    inner_dev[(name, _inner_attr(name, view))] = dev
                    continue
                else:
                    params[name].data = dev
                moved.append((name, view))
        for name in sorted(whole):
            p, kept = params[name], host[name]
            names, ctx = kept.__tensor_flatten__()
            rebuilt = type(kept).__tensor_unflatten__(
                {attr: inner_dev[(name, attr)] for attr in names}, ctx, kept.shape, kept.stride()
            )
            torch.utils.swap_tensors(p, torch.nn.Parameter(rebuilt, requires_grad = p.requires_grad))
            moved.append((name, kept))
    except Exception:
        # Put every weight back on its host view; the stock per-tensor copy then runs (and raises a real OOM itself).
        for name, view in moved:
            try:
                if name.startswith("\0"):
                    _set_buffer(module, name[1:], view)
                elif name in wrapped:
                    _repoint(params[name], view)
                else:
                    params[name].data = view
            except Exception:  # noqa: BLE001
                pass
        return 0
    return copied


def _named_buffers(module: Any) -> list:
    named = getattr(module, "named_buffers", None)
    return list(named()) if callable(named) else []


def _keepable_buffer(buffer: Any) -> bool:
    import torch
    return type(buffer) is torch.Tensor


def _buffer_version(buffer: Any) -> Optional[int]:
    """``_version``, or None for an inference tensor (moved under inference_mode), which has no version counter."""
    try:
        return None if buffer.is_inference() else int(buffer._version)
    except Exception:  # noqa: BLE001 - untracked: the stock copy handles it
        return None


def _set_buffer(module: Any, name: str, tensor: Any) -> None:
    prefix, _, leaf = name.rpartition(".")
    owner = module.get_submodule(prefix) if prefix else module
    owner._buffers[leaf] = tensor


def _wrap_cpu_offload_hook(
    hook: Any,
    module: Any,
    logger: Any = None,
) -> None:
    # Keyed by name: a move can hand the module new Parameter objects, and a stale key must miss, not alias.
    state = module.__dict__.get(_KEEP_ATTR)
    if state is None:
        state = {"host": {}, "owner": {}, "version": {}}
        module.__dict__[_KEEP_ATTR] = state
    state.setdefault("owner", {})
    host, owner, version = state["host"], state["owner"], state["version"]
    # Plain buffers too: the stock offload re-copied them every call.
    buffer_host = state.setdefault("buffer_host", {})
    buffer_version = state.setdefault("buffer_version", {})

    def _capture(mod: Any) -> None:
        host.clear()
        owner.clear()
        buffer_host.clear()
        for name, p in mod.named_parameters():
            if p.device.type == "cpu" and _keepable(p):
                host[name] = p.data
                owner[name] = p
        for name, b in _named_buffers(mod):
            if b.device.type == "cpu" and _keepable_buffer(b):
                buffer_host[name] = b

    _capture(module)
    version.clear()
    buffer_version.clear()
    init_hook, pre_forward = hook.init_hook, hook.pre_forward

    def _init_hook(mod: Any) -> Any:
        for name, b in _named_buffers(mod):
            kept = buffer_host.get(name)
            seen = buffer_version.get(name)
            if (
                kept is None
                or b.device.type == "cpu"
                or seen is None
                or seen[0] is not b
                or seen[1] != _buffer_version(b)
                or seen[2] != b.data_ptr()
            ):
                continue
            try:
                _set_buffer(mod, name, kept)
            except Exception:  # noqa: BLE001 - the stock copy below handles it
                pass
        for name, p in mod.named_parameters():
            kept = host.get(name)
            seen = version.get(name)
            # Identity and data_ptr too: a replacement can match the version, and `p.data = ...` does not bump it.
            if (
                kept is None
                or p.device.type == "cpu"
                or seen is None
                or seen[0] is not p
                or seen[1] != p._version
                or seen[2] != _tensor_ptr(p)
            ):
                continue
            try:
                _repoint(p, kept)
            except Exception:  # noqa: BLE001 - incompatible tensor types: the stock copy below handles it
                pass
        out = init_hook(mod)
        _capture(mod)
        # a weight cast or replaced on the device comes back as a new host tensor: release its pinned chunk now
        _prune_state_layout(state)
        version.clear()
        buffer_version.clear()
        return out

    def _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:
        # `host` gate: an all-subclass module (torchao) never fills `version`, so would rescan every forward.
        onload = not version and not buffer_version and bool(host or buffer_host)
        if onload:
            for name, p in mod.named_parameters():
                kept = host.get(name)
                if kept is not None and (
                    owner.get(name) is not p
                    or p.device.type != "cpu"
                    or _tensor_ptr(p.data) != _tensor_ptr(kept)
                ):
                    host.pop(name, None)
                    owner.pop(name, None)
            current = dict(_named_buffers(mod))
            for name in list(buffer_host):
                if current.get(name) is not buffer_host[name]:
                    buffer_host.pop(name, None)
            # changed while offloaded (a dtype cast, a replacement): its pinned chunk is not kept for nothing
            _prune_state_layout(state)
        if onload and not state.get("pin_tried"):
            # Once per module, on its first onload, so loading pays nothing.
            state["pin_tried"] = True
            layout: list = []
            try:
                pinned = _pin_host_weights(
                    mod,
                    host,
                    logger,
                    buffer_host = buffer_host,
                    layout = layout,
                    exact = bool(state.get("exact_pin")),
                    device = state.get("exact_pin_device"),
                )
            except Exception:  # noqa: BLE001 - pinning is an optimisation, never a failure
                pinned = 0
            if pinned:
                state["pin_layout"] = layout
            if pinned and logger is not None:
                logger.info(
                    "diffusion.memory: pinned %.1f GiB of %s host weights",
                    pinned / 2**30,
                    type(mod).__name__,
                )
        layout = state.get("pin_layout") if onload and _fast_upload_enabled() else None
        prior_to = mod.__dict__.get("to")
        if layout and prior_to is not None and not getattr(prior_to, _DTYPE_ONLY_TO_ATTR, False):
            # An instance-level mover is already installed (stable arena, pinned swap): defer to it. A dtype-only
            # one (diffusion_vae_upcast) passes device moves through, so the pinned chunks still go up first.
            layout = None
        if not layout:
            out = pre_forward(mod, *args, **kwargs)
        else:
            # Offload the previous component through its kept hook here, so the stock pre_forward finds it on the
            # host and skips its empty_cache: the freed chunks stay in the caching allocator for this onload.
            prev = getattr(hook, "prev_module_hook", None)
            prev_model = getattr(prev, "model", None)
            if (
                prev_model is not None
                and getattr(getattr(prev, "hook", None), _KEEP_ATTR, False)
                and callable(getattr(prev, "offload", None))
                and _on_device(prev_model)
            ):
                prev.offload()

            # The stock pre_forward offloads the previous component, then calls module.to(device) only if the module
            # is not there yet. Shadow that one call so the pinned chunks go up first; module.to then moves whatever
            # the chunks did not cover (unpinned or replaced weights) exactly as before.
            def _put_back_to() -> None:
                if prior_to is None:
                    mod.__dict__.pop("to", None)
                else:
                    mod.__dict__["to"] = prior_to

            def _to(*to_args: Any, **to_kwargs: Any) -> Any:
                _put_back_to()
                import torch

                target = to_args[0] if to_args else to_kwargs.get("device")
                try:
                    target = torch.device(target)
                except Exception:  # noqa: BLE001 - not a plain device move: stock only
                    target = None
                if (
                    target is not None
                    and target.type != "cpu"
                    and len(to_args) + len(to_kwargs) == 1
                ):
                    _upload_pinned_chunks(mod, layout, host, owner, buffer_host, target)
                return mod.to(*to_args, **to_kwargs)

            mod.__dict__["to"] = _to
            try:
                out = pre_forward(mod, *args, **kwargs)
            finally:
                _put_back_to()
        if onload:
            for name, p in mod.named_parameters():
                if name in host and owner.get(name) is p and p.device.type != "cpu":
                    version[name] = (p, p._version, _tensor_ptr(p))
            for name, b in _named_buffers(mod):
                if (
                    name in buffer_host
                    and b.device.type != "cpu"
                    and _buffer_version(b) is not None
                ):
                    buffer_version[name] = (b, _buffer_version(b), b.data_ptr())
        return out

    try:
        import torch

        # Same as accelerate's own pre_forward: a compiled forward must not trace the hook.
        _init_hook, _pre_forward = (
            torch.compiler.disable(_init_hook),
            torch.compiler.disable(_pre_forward),
        )
    except Exception:  # noqa: BLE001 - older torch: the hook runs outside any compiled region anyway
        pass
    hook.init_hook, hook.pre_forward = _init_hook, _pre_forward
    setattr(hook, _KEEP_ATTR, True)


def offload_host_copy(module: Any, name: str, tensor: Any) -> Any:
    """The keep-cpu host tensor for ``name`` while ``tensor`` is on the device unchanged since its onload (``_init_hook``'s
    re-point rule: same object, version and storage), else None."""
    state = getattr(module, "__dict__", {}).get(_KEEP_ATTR)
    if (
        not isinstance(state, dict)
        or getattr(getattr(tensor, "device", None), "type", "cpu") == "cpu"
    ):
        return None
    try:
        kept = (state.get("host") or {}).get(name)
        if kept is not None:
            seen = (state.get("version") or {}).get(name)
            if (
                (state.get("owner") or {}).get(name) is tensor
                and seen is not None
                and seen[0] is tensor
                and seen[1] == tensor._version
                and seen[2] == _tensor_ptr(tensor)
            ):
                return kept
            return None
        kept = (state.get("buffer_host") or {}).get(name)
        seen = (state.get("buffer_version") or {}).get(name)
        if (
            kept is not None
            and seen is not None
            and seen[0] is tensor
            and seen[1] == _buffer_version(tensor)
            and seen[2] == tensor.data_ptr()
        ):
            return kept
    except Exception:  # noqa: BLE001 - a state mid-update: the caller reads the device copy
        return None
    return None


def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:
    """Return unused allocator pages after whole-model CPU offload, without touching live
    tensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""
    if offload_policy != OFFLOAD_MODEL:
        return False
    return reclaim_host_memory(logger = logger)


def reclaim_host_memory(logger: Any = None) -> bool:
    """Return freed allocator pages to the OS regardless of offload policy (unload / teardown).
    Best effort: unsupported allocators and failures return False."""
    global _host_memory_reclaim_warning_logged
    global _host_memory_reclaim_unsupported_logged
    try:
        reclaim = _resolve_host_memory_reclaimer()
        if reclaim is None:
            # The call site discards the result, so a permanent no-op is otherwise invisible.
            if logger is not None and not _host_memory_reclaim_unsupported_logged:
                _host_memory_reclaim_unsupported_logged = True
                try:
                    logger.info(
                        "diffusion.memory: no host allocator pressure API on this platform "
                        "(%s); freed host pages will not be returned early",
                        sys.platform,
                    )
                except Exception:  # noqa: BLE001
                    pass
            return False
        reclaim()
        _host_empty_cache()
        return True
    except Exception as exc:  # noqa: BLE001
        if logger is not None and not _host_memory_reclaim_warning_logged:
            _host_memory_reclaim_warning_logged = True
            try:
                logger.warning("diffusion.memory: host allocator reclamation failed: %s", exc)
            except Exception:  # noqa: BLE001
                pass
        return False


LOAD_HEAP_TRIM_ENV = "UNSLOTH_DIFFUSION_LOAD_HEAP_TRIM"
_load_heap_trim_warning_logged = False


def _rss_anon_mib() -> Optional[int]:
    try:
        with open("/proc/self/status", encoding = "utf-8") as status:
            for line in status:
                if line.startswith("RssAnon:"):
                    return int(line.split()[1]) >> 10
    except Exception:  # noqa: BLE001 - not Linux, or no procfs
        return None
    return None


def trim_host_heap_after_load(logger: Any = None) -> bool:
    """Return the heap a finished load freed to the OS; glibc otherwise keeps it (Qwen-Image-2.1, 8 GiB budget:
    0.65 GiB warm, 1.9 GiB on a first load). Live tensors and the pinned caches are untouched."""
    if _env_off(LOAD_HEAP_TRIM_ENV):
        return False
    try:
        reclaim = _resolve_host_memory_reclaimer()
        if reclaim is None:
            return False
        import time

        before = _rss_anon_mib()
        began = time.perf_counter()
        reclaim()
        took = time.perf_counter() - began
        after = _rss_anon_mib()
        if logger is not None:
            if before is not None and after is not None:
                logger.info(
                    "diffusion.memory: returned %d MiB of freed heap to the OS after the load (%.0f ms)",
                    before - after,
                    took * 1000,
                )
            else:
                logger.info(
                    "diffusion.memory: returned freed heap to the OS after the load (%.0f ms)",
                    took * 1000,
                )
        return True
    except Exception as exc:  # noqa: BLE001 - a trim never fails a load
        global _load_heap_trim_warning_logged
        if logger is not None and not _load_heap_trim_warning_logged:
            _load_heap_trim_warning_logged = True
            try:
                logger.warning("diffusion.memory: heap trim after the load failed: %s", exc)
            except Exception:  # noqa: BLE001
                pass
        return False


def normalize_memory_mode(value: Optional[str]) -> Optional[str]:
    """Lower/strip a requested mode (accepting dashes); None passes through. Raises ValueError
    for an unsupported mode so the route rejects it as a 4xx before any GPU work."""
    if value is None:
        return None
    normalized = str(value).strip().lower().replace("-", "_")
    if not normalized:
        return None
    if normalized not in MEMORY_MODES:
        valid = ", ".join(MEMORY_MODES)
        raise ValueError(f"Unsupported diffusion memory_mode '{value}'. Use one of: {valid}.")
    return normalized


@dataclass(frozen = True)
class DeviceMemory:
    """Point-in-time view of the active device's memory, in MiB.

    ``memory_kind`` distinguishes discrete VRAM (CPU offload helps) from unified / system memory
    (offload moves bytes within the same pool, so it does not)."""

    backend: str
    device: str
    memory_kind: str  # "discrete_vram" | "unified_memory" | "system_memory" | "unknown"
    free_mib: Optional[int] = None
    total_mib: Optional[int] = None
    # Metal's recommended working set when it bounds free / total (mps); None everywhere else.
    metal_working_set_mib: Optional[int] = None

    @property
    def is_unified(self) -> bool:
        return self.memory_kind in ("unified_memory", "system_memory")

    def as_public_dict(self) -> dict[str, Any]:
        return {
            "backend": self.backend,
            "device": self.device,
            "memory_kind": self.memory_kind,
            "free_mib": self.free_mib,
            "total_mib": self.total_mib,
        }


@dataclass(frozen = True)
class MemoryPlan:
    """The chosen runtime profile for one load."""

    requested_mode: str
    offload_policy: str
    vae_tiling: bool
    vae_slicing: bool
    device_memory: DeviceMemory
    estimates: dict[str, Optional[int]]
    reasons: tuple[str, ...] = ()
    # Off by default; set only when streaming text encoders is what makes group offload fit at all.
    stream_text_encoders: bool = False
    # False only on the tier keeping the transformer resident and streaming just the text encoders.
    stream_transformer: bool = True
    # MiB of the streamed denoiser / encoders kept resident (whole groups); None streams every group.
    resident_transformer_mib: Optional[int] = None
    resident_text_encoder_mib: Optional[int] = None
    secondary_ordinal: Optional[int] = None
    secondary_components: Optional[tuple[str, ...]] = None
    # The single-card plan a split replaced, applied on the primary if the split placement fails (a torchao denoiser
    # cannot take the whole-module offload a blind fallback would choose).
    fallback_plan: Optional["MemoryPlan"] = field(default = None, compare = False, repr = False)
    layer_split: Optional[Any] = None

    @property
    def engages_offload(self) -> bool:
        return self.offload_policy != OFFLOAD_NONE

    def as_public_dict(self) -> dict[str, Any]:
        return {
            "requested_mode": self.requested_mode,
            "offload_policy": self.offload_policy,
            "vae_tiling": self.vae_tiling,
            "vae_slicing": self.vae_slicing,
            "device_memory": self.device_memory.as_public_dict(),
            "estimates": dict(self.estimates),
            "reasons": list(self.reasons),
            "stream_text_encoders": self.stream_text_encoders,
            "stream_transformer": self.stream_transformer,
            "resident_transformer_mib": self.resident_transformer_mib,
            "resident_text_encoder_mib": self.resident_text_encoder_mib,
            "secondary_ordinal": self.secondary_ordinal,
            "secondary_components": (
                list(self.secondary_components) if self.secondary_components is not None else None
            ),
            **(
                {"layer_split": self.layer_split.as_public_dict()}
                if self.layer_split is not None
                else {}
            ),
        }


def snapshot_device_memory(target: Any) -> DeviceMemory:
    """Free / total memory for ``target``'s device. Never raises: a probe failure yields None
    counts, which the planner treats as "budget unknown" (stay resident)."""
    device = getattr(target, "device", "cpu")
    backend = getattr(target, "backend", device)

    if device == "cuda":
        free, total, kind = _cuda_memory(backend)
        return DeviceMemory(backend, device, kind, free, total)
    if device == "xpu":
        free, total = _xpu_memory()
        return DeviceMemory(backend, device, "discrete_vram", free, total)
    if device == "mps":
        # One CPU/GPU pool, so offload is pointless. The device is Metal's recommended working set (what the GPU holds
        # with good performance; allocations past it succeed and page), bounded by the RAM available right now.
        total, free = _system_memory_mib()
        ram = DeviceMemory(backend, device, "unified_memory", free, total)
        working_set, in_use = _metal_working_set_mib()
        if working_set is None:
            return ram
        ws_free = None if free is None else min(int(free), max(0, working_set - in_use))
        ws_total = working_set if total is None else min(int(total), working_set)
        metal = DeviceMemory(
            backend, device, "unified_memory", ws_free, ws_total, metal_working_set_mib = working_set
        )
        # The working set only widens the RAM reading: on an idle Mac with more RAM available than the working set,
        # 15% of it can leave less than RAM minus 20%, and a load that fit under the RAM reading must keep fitting.
        if free is not None and total is not None:
            ram_budget = int(free) - _reserve_mib("unified_memory", int(total))
            if ram_budget > int(ws_free) - _reserve_mib(METAL_WORKING_SET, ws_total):
                return ram
        return metal

    total, free = _system_memory_mib()
    return DeviceMemory(backend, device, "system_memory", free, total)


def _metal_working_set_mib() -> tuple[Optional[int], int]:
    """(Metal's recommended working set, what this process already holds in Metal) in MiB, or (None, 0) when torch
    cannot report it (``recommended_max_memory`` arrived in torch 2.5)."""
    try:
        import torch
        working_set = int(torch.mps.recommended_max_memory()) // (1024 * 1024)
        in_use = int(torch.mps.driver_allocated_memory()) // (1024 * 1024)
    except Exception:  # noqa: BLE001 - no reading: size against system RAM as before
        return None, 0
    return (working_set, in_use) if working_set > 0 else (None, 0)


def _graph_held_cache_bytes(device: Optional[int] = None) -> int:
    held = 0
    try:
        from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes
        held += int(_block_graph_pool_bytes(device))
    except Exception:  # noqa: BLE001
        pass
    try:
        # The whole-step pool reading is per current card, so a named card other than it is made current for the read.
        from .diffusion_cuda_graph import live_pool_free_bytes

        scope = contextlib.nullcontext()
        if device is not None:
            import torch
            try:
                current = int(torch.cuda.current_device())
            except Exception:  # noqa: BLE001 - no current card to compare: switch to the named one
                current = None
            if current != int(device):
                scope = torch.cuda.device(int(device))
        with scope:
            held += int(live_pool_free_bytes())
    except Exception:  # noqa: BLE001 -- no graph layer: nothing held
        pass
    return held


def cuda_reclaimable_bytes(device: Optional[int] = None) -> Optional[int]:
    """Bytes this process's caching allocator holds reserved-but-unallocated on CUDA ``device`` (the current card
    when None) that an ordinary allocation can take back, minus the CUDA graph pools that only their graphs may reuse
    (``_graph_held_cache_bytes``). None when the allocator cannot be read. May be negative; callers credit only a
    positive figure."""
    try:
        import torch
        if device is None:
            reclaimable = int(torch.cuda.memory_reserved()) - int(torch.cuda.memory_allocated())
        else:
            reclaimable = int(torch.cuda.memory_reserved(device)) - int(
                torch.cuda.memory_allocated(device)
            )
    except Exception:  # noqa: BLE001
        return None
    pool_device = device
    if pool_device is None:
        try:
            pool_device = torch.cuda.current_device()
        except Exception:  # noqa: BLE001 - no current CUDA device: count every pool
            pool_device = None
    return reclaimable - _graph_held_cache_bytes(pool_device)


def cuda_reusable_cache_bytes(device: Any = None) -> int:
    """``cuda_reclaimable_bytes`` (excludes CUDA graph pools) floored at 0 for any device form; resolved to an int
    index because ``_graph_held_cache_bytes`` would silently skip the graph-pool count for a torch.device."""
    index = None
    if device is not None:
        try:
            import torch
            index = device if isinstance(device, int) else torch.device(device).index
        except Exception:  # noqa: BLE001 - unparseable: read the current card
            index = None
    return max(0, cuda_reclaimable_bytes(index) or 0)


def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
    """Credits reserved-minus-allocated as free without flushing; cuda only, own allocator only, CUDA graph pool uncredited, plain snapshot on failure."""
    if getattr(target, "device", "cpu") != "cuda":
        return snapshot_device_memory(target)
    snapshot = snapshot_device_memory(target)
    if snapshot.free_mib is None:
        return snapshot
    reclaimable = cuda_reclaimable_bytes()
    if reclaimable is None:  # no allocator reading: the plain snapshot still stands
        return snapshot
    if reclaimable <= 0:
        return snapshot
    free = int(snapshot.free_mib) + reclaimable // (1024 * 1024)
    if snapshot.total_mib is not None:
        free = min(free, int(snapshot.total_mib))
    return DeviceMemory(
        snapshot.backend, snapshot.device, snapshot.memory_kind, free, snapshot.total_mib
    )


def _settle_delay(delay_s: float) -> float:
    """Spacing of memory re-reads so a transient undercount can clear; UNSLOTH_SETTLE_DELAY_S overrides."""
    override = os.environ.get("UNSLOTH_SETTLE_DELAY_S")
    if override is None:
        return delay_s
    try:
        return max(0.0, float(override))
    except (TypeError, ValueError):
        return delay_s  # a typo in the env must not change production behaviour


def settled_snapshot_device_memory(
    target: Any,
    attempts: int = 3,
    delay_s: float = 1.0,
    *,
    can_stop: Optional[Callable[[DeviceMemory], bool]] = None,
) -> DeviceMemory:
    """Max free over spaced reads (a transient only shrinks free); on mps, empty_cache first so freed pool is not budgeted as used memory."""
    device = getattr(target, "device", "cpu")
    if device == "mps":
        try:
            import torch
            empty_cache = getattr(getattr(torch, "mps", None), "empty_cache", None)
            if callable(empty_cache):
                empty_cache()
        except Exception:  # noqa: BLE001 - settle is best-effort; the snapshot below still runs
            pass
        return snapshot_device_memory(target)
    if device != "cuda":
        return snapshot_device_memory(target)
    try:
        import torch
        from .diffusion_cuda_graph import synchronize_outside_capture

        synchronize_outside_capture()
        torch.cuda.empty_cache()
    except Exception:  # noqa: BLE001 - settle is best-effort; the snapshot below still runs
        pass
    best = snapshot_device_memory(target)
    delay_s = _settle_delay(delay_s)
    for attempt in range(max(0, attempts - 1)):
        if best.free_mib is not None and best.total_mib is not None:
            # Free already within the reserve of total: nothing transient to wait out.
            if best.free_mib >= best.total_mib - max(2048, int(best.total_mib * 0.10)):
                break
        if attempt == 0 and can_stop is not None and can_stop(best):
            break
        try:
            import time
            time.sleep(delay_s)
        except Exception:  # noqa: BLE001
            break
        nxt = snapshot_device_memory(target)
        if nxt.free_mib is not None and (best.free_mib is None or nxt.free_mib > best.free_mib):
            best = nxt
    return best


def _cuda_memory(backend: str) -> tuple[Optional[int], Optional[int], str]:
    try:
        import torch

        # Not torch.cuda.mem_get_info: Windows ROCm over-reports free memory.
        from utils.hardware import trusted_mem_get_info

        free, total = trusted_mem_get_info()
        kind = "discrete_vram"
        try:
            # Current device, not 0: mem_get_info reports the current one.
            props = torch.cuda.get_device_properties(torch.cuda.current_device())
            if bool(getattr(props, "integrated", False) or getattr(props, "is_integrated", False)):
                kind = "unified_memory"  # e.g. Jetson / integrated SoC
        except Exception:
            pass
        free_mib, total_mib = int(free // (1024 * 1024)), int(total // (1024 * 1024))
        # ROCm APUs over-report free (free == total on Windows HIP): never credit host memory.
        if kind == "unified_memory" and not _torch_is_rocm(torch):
            free_mib, total_mib = _unified_reclaimable_memory_mib(free_mib, total_mib)
        return free_mib, total_mib, kind
    except Exception:
        return None, None, "discrete_vram"


def _torch_is_rocm(torch) -> bool:
    """Delegates to LlamaCppBackend so the two agree; version.hip alone misses some ROCm builds."""
    try:
        from core.inference.llama_cpp import LlamaCppBackend
        return LlamaCppBackend._torch_is_rocm(torch)
    except Exception:  # noqa: BLE001 - the answer still has to be right
        return is_rocm_torch(torch)


def _unified_reclaimable_memory_mib(free_mib: int, total_mib: int) -> tuple[int, int]:
    """Credits page cache via MemAvailable, clamped to driver reading and device total; also returns capacity (cgroup LIMIT, not remainder)."""
    available_mib = _available_system_memory_mib()
    cgroup_mib = _cgroup_available_memory_mib()
    if available_mib is None:
        credited = free_mib
    else:
        credited = max(free_mib, min(int(available_mib), total_mib))
    if cgroup_mib is not None and int(cgroup_mib) <= credited:
        # `<=`: the host reading is already cgroup-capped, so equality is normal in a container.
        credited = int(cgroup_mib)
    # Capacity is the cgroup LIMIT, never the remainder, which shrinks as the container fills.
    limit_mib = _cgroup_memory_limit_mib()
    if limit_mib is None:
        capacity = total_mib
    else:
        capacity = min(total_mib, int(limit_mib))
        # Memory above the limit cannot be charged, so it is not free either.
        credited = min(credited, capacity)
    return credited, capacity


def _available_system_memory_mib() -> Optional[int]:
    """Usable host RAM in MiB: system available capped by any enforcing cgroup's headroom."""
    from utils import host_memory
    try:
        from core.inference.llama_cpp import LlamaCppBackend
        return LlamaCppBackend._available_system_memory_mib()
    except Exception:  # noqa: BLE001 - the shared reader still applies the cgroup cap
        return host_memory.usable_mib(_system_memory_mib()[1], _cgroup_available_memory_mib())


def _cgroup_available_memory_mib() -> Optional[int]:
    """What an enforcing cgroup will still let this process charge, else None."""
    from utils import host_memory
    try:
        from core.inference.llama_cpp import LlamaCppBackend
        return LlamaCppBackend._cgroup_available_memory_mib()
    except Exception:  # noqa: BLE001 - read the cgroup directly rather than drop the cap
        try:
            return host_memory.cgroup_headroom_mib()
        except Exception:  # noqa: BLE001 - no readable limit is the same answer as none
            return None


def _cgroup_memory_limit_mib() -> Optional[int]:
    """The capacity an enforcing cgroup allows, else None. Not the remainder above."""
    from utils import host_memory
    try:
        from core.inference.llama_cpp import LlamaCppBackend
        return LlamaCppBackend._cgroup_memory_limit_mib()
    except Exception:  # noqa: BLE001 - read the cgroup directly rather than drop the cap
        try:
            return host_memory.cgroup_limit_mib()
        except Exception:  # noqa: BLE001 - no readable limit is the same answer as none
            return None


def _host_ram_capacity_mib() -> Optional[int]:
    """Host RAM this process may ever charge, in MiB: system total capped by any cgroup limit."""
    from utils import host_memory
    return host_memory.usable_mib(_system_memory_mib()[0], _cgroup_memory_limit_mib())


def _windows_commit_headroom_mib() -> Optional[int]:
    """(CommitLimit - CommitTotal) in MiB on Windows via ``K32GetPerformanceInfo``; 0 when exhausted, None when
    unreadable or not Windows. The diffusion guards' failure mode is commit exhaustion (#13432), which a
    physical `available` reading does not see."""
    if sys.platform != "win32":
        return None
    try:
        import ctypes

        class _PERFORMANCE_INFORMATION(ctypes.Structure):
            _fields_ = [
                ("cb", ctypes.c_ulong),
                ("CommitTotal", ctypes.c_size_t),
                ("CommitLimit", ctypes.c_size_t),
                ("CommitPeak", ctypes.c_size_t),
                ("PhysicalTotal", ctypes.c_size_t),
                ("PhysicalAvailable", ctypes.c_size_t),
                ("SystemCache", ctypes.c_size_t),
                ("KernelTotal", ctypes.c_size_t),
                ("KernelPaged", ctypes.c_size_t),
                ("KernelNonpaged", ctypes.c_size_t),
                ("PageSize", ctypes.c_size_t),
                ("HandleCount", ctypes.c_ulong),
                ("ProcessCount", ctypes.c_ulong),
                ("ThreadCount", ctypes.c_ulong),
            ]

        info = _PERFORMANCE_INFORMATION()
        info.cb = ctypes.sizeof(_PERFORMANCE_INFORMATION)
        # kernel32 exports the K32-prefixed name only; the unprefixed one lives in psapi
        kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
        if not kernel32.K32GetPerformanceInfo(ctypes.byref(info), info.cb):
            return None
        headroom = (int(info.CommitLimit) - int(info.CommitTotal)) * int(info.PageSize)
        return max(0, headroom // (1024 * 1024))
    except Exception:  # noqa: BLE001
        return None


def _xpu_memory() -> tuple[Optional[int], Optional[int]]:
    try:
        import torch
        mem_get_info = getattr(getattr(torch, "xpu", None), "mem_get_info", None)
        if callable(mem_get_info):
            free, total = mem_get_info()
            return int(free // (1024 * 1024)), int(total // (1024 * 1024))
    except Exception:
        pass
    return None, None


def _system_memory_mib() -> tuple[Optional[int], Optional[int]]:
    """(total, available) host RAM in MiB, via psutil then POSIX sysconf."""
    try:
        import psutil
        vm = psutil.virtual_memory()
        return int(vm.total // (1024 * 1024)), int(vm.available // (1024 * 1024))
    except Exception:
        pass
    try:
        page = os.sysconf("SC_PAGE_SIZE")
        total = os.sysconf("SC_PHYS_PAGES") * page
        avail = os.sysconf("SC_AVPHYS_PAGES") * page
        return int(total // (1024 * 1024)), int(avail // (1024 * 1024))
    except Exception:
        return None, None


def file_size_mib(path: Any) -> Optional[int]:
    """On-disk size of ``path`` in MiB, or None if it can't be stat'd."""
    try:
        from pathlib import Path
        return max(1, int(Path(path).expanduser().stat().st_size // (1024 * 1024)))
    except Exception:
        return None


def safetensors_prefix_mib(path: Any, prefix: str) -> Optional[int]:
    """MiB of the ``prefix*`` tensors, from the safetensors header alone; None if unreadable or unmatched."""
    from .diffusion_component_sizes import entry_nbytes, read_safetensors_header, tensor_entries

    header = read_safetensors_header(path)
    if header is None:
        return None
    try:
        total = 0
        for name, meta in tensor_entries(header):
            if name.startswith(prefix):
                total += entry_nbytes(meta)
    except Exception:  # noqa: BLE001 - not a readable safetensors header
        return None
    return -(-total // (1024 * 1024)) if total > 0 else None


def estimate_gguf_resident_mib(storage_mib: Optional[int]) -> Optional[int]:
    """GGUF weights stay packed, so resident size is about the on-disk size, not unpacked bf16."""
    if storage_mib is None:
        return None
    return int(storage_mib * 1.05)  # margin for allocator + bf16 norms/biases


def estimate_safetensors_dense_mib(
    storage_mib: Optional[int], *, fp8_upcast: bool = False
) -> Optional[int]:
    """Upcast fp8 checkpoints load as bf16 with no quantization_config, so resident size is about 2x."""
    if storage_mib is None:
        return None
    if fp8_upcast:
        return storage_mib * 2
    return storage_mib


# qwen-image-layered's layer_count / layer_resolution (pinned equal by a test; no families import here).
_QWEN_LAYERED_LAYERS = 2
_QWEN_LAYERED_CANVAS = 640


def estimate_image_runtime_mib(
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    family: Optional[str] = None,
    condition_pixels: int = 0,
) -> int:
    """Per-call activation / latent headroom for an image gen, scaled by pixel area and batch.
    Distilled / turbo models (few steps, no CFG) need less. ``condition_pixels`` (already weighted)
    join the target's token sequence once per batch image, so they count like target pixels."""
    w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
    h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
    batch = max(1, int(batch_size or 1))
    cond = max(0, int(condition_pixels or 0))
    if width is None and height is None and "qwen-image-layered" in (family or "").lower():
        # Planning call: reserve what the generation guard charges a decomposition, not one 1024x1024 frame.
        w = h = _QWEN_LAYERED_CANVAS
        cond = (_QWEN_LAYERED_LAYERS + 1) * w * h
    pixel_scale = ((w * h + cond) * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)
    return max(1024, int(8192 * max(0.25, pixel_scale) * _family_activation_multiplier(family)))


def estimate_video_runtime_mib(
    *, width: Optional[int], height: Optional[int], num_frames: Optional[int]
) -> int:
    """VAE decode is the video peak: decoded frames times 3 for intermediates and export."""
    w = max(64, int(width or 768))
    h = max(64, int(height or 512))
    frames = max(1, int(num_frames or 121))
    decoded_mib = (frames * w * h * 3 * 4) / float(1024 * 1024)
    return max(3072, int(4096 + 3.0 * decoded_mib))


@dataclass(frozen = True)
class CalibratedImageActivation:
    """MiB above the resident weights, margin included: each phase at 1024x1024, then the 2048x2048 worst case."""

    text_encoder_mib: int
    denoise_mib: int
    decode_mib: int
    tiled_decode_mib: int
    max_canvas_mib: int
    resident_only: bool = False

    def headroom(self, tiled: bool) -> int:
        return max(
            self.text_encoder_mib,
            self.denoise_mib,
            self.tiled_decode_mib if tiled else self.decode_mib,
        )


# NVIDIA worst case per (off / eager / default, max) tier; U-Nets unlisted (cannot stream encoders).
_ACTIVATION_MARGIN = 1.2
_MEASURED_IMAGE_ACTIVATION_MIB: dict[
    str, tuple[tuple[int, int, int, int, int], tuple[int, int, int, int, int]]
] = {
    # Text encoder, denoise, untiled/tiled decode at 1024x1024; max at 2048x2048; Qwen tiles are 512 px.
    "qwen-image-2.1": ((1_849, 666, 7_648, 1_697, 2_479), (1_849, 2_489, 7_648, 1_697, 9_602)),
    "flux.1": ((288, 892, 2_666, 2_456, 2_674), (288, 892, 2_666, 2_456, 2_674)),
    "flux.2-klein": ((1_516, 1_160, 2_645, 2_456, 3_876), (1_516, 1_205, 2_677, 2_456, 3_924)),
    # FLUX.3 Klein (distilled, no CFG), measured B200 bf16, 1024^2 t2i: TE 301 MiB at a ~560-token window, denoise eager
    # 594 bf16 / 901 int8 / 850 fp8, compiled 543-676 (int8 / fp8, regional dynamic=True), untiled AutoencoderKLFlux2
    # decode 2_437. Tiled decode is unmeasured (flux.2-klein's same VAE reads 2_456): capped at the untiled figure, which
    # a tiled decode cannot exceed. The compiled tier keeps the eager 901 rather than its measured 543-676, since the
    # table's tiers may only grow with speed. The 2048^2 column is max(denoise eager fp8 3_359, tiled decode) since the
    # untiled 2048 decode is 9_738. References are charged through ``condition_pixel_weight``. The base (CFG) is
    # unmeasured and keeps the flat estimate.
    "flux.3-klein": ((301, 901, 2_437, 2_437, 3_359), (301, 901, 2_437, 2_437, 3_359)),
    # FLUX.3 Klein base, B200 bf16: per column the larger of Klein and base (same DiT / VAE, one Flux2 formula in ComfyUI).
    "flux.3-klein-base": ((301, 901, 2_512, 2_512, 3_359), (301, 901, 2_512, 2_512, 3_359)),
    # MOSS-Image SFT, B200 bf16 measurements; both tiers take the eager worst.
    "moss-image": ((1_940, 528, 3_666, 3_662, 3_750), (1_940, 528, 3_666, 3_662, 3_750)),
    "z-image": ((744, 1_199, 2_666, 2_456, 4_327), (744, 1_271, 2_669, 2_456, 4_327)),
    # SDXL base, T4 fp16, 1024 (2048 tiled decode 2_455; untiled 7_706 left to the OOM retry); compiled unmeasured.
    "sdxl": ((9, 327, 2_434, 2_434, 2_455), (9, 327, 2_434, 2_434, 2_455)),
    # Lumina-Image-2.0, worst per column over tiers / A100 / L4; decode = the first compiled-tier decode (cudnn workspace).
    "lumina-2": ((1_130, 486, 4_497, 4_497, 4_497), (1_130, 486, 4_497, 4_497, 4_497)),
}
# U-Nets cannot stream encoders on group tiers: the measurement only decides all-resident vs whole-module offload.
_RESIDENT_ONLY_CALIBRATED = frozenset({"sdxl"})
# A plain bf16 denoiser keeps the whole-module tier the flat plan would take over the measured resident-transformer
# tier: Lumina-2's encoder is as large as its DiT, and streaming it per prompt was 1% slower per render (L4, 16 GiB).
_CALIBRATED_KEEPS_WHOLE_MODULE = frozenset({"lumina-2"})


def _resident_only_hint(family: Optional[str]) -> bool:
    parts = str(family or "").split()
    return bool(parts) and parts[0] in _RESIDENT_ONLY_CALIBRATED


def calibrated_image_activation(
    family: Optional[str],
    *,
    max_speed: bool = True,
    compute_bytes: int = 2,
) -> Optional[CalibratedImageActivation]:
    """``compute_bytes`` 4 (float32 compute) scales every column 2x, only for ``FP32_CALIBRATED_FAMILIES``: the
    tables were measured at 16-bit, and every other fp32-promoted family keeps the flat estimate (None)."""
    measured = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or ""))
    if measured is None:
        return None
    widen = 1.0
    if int(compute_bytes or 2) > 2:
        if not fp32_calibrated(family):
            return None
        widen = int(compute_bytes) / 2.0
    return CalibratedImageActivation(
        *(int(v * _ACTIVATION_MARGIN * widen) for v in measured[1 if max_speed else 0]),
        resident_only = str(family or "") in _RESIDENT_ONLY_CALIBRATED,
    )


# fp16-only GPUs: price float32 compute at 2x the 16-bit table (FLUX.3 T4 peaks measured below that); without it the
# flat 8 GiB estimate streams even a 3.5 GiB GGUF denoiser through pageable host memory.
FP32_CALIBRATED_ENV = "UNSLOTH_DIFFUSION_FP32_CALIBRATED"
FP32_CALIBRATED_FAMILIES = frozenset({"flux.3-klein", "flux.3-klein-base"})


def fp32_calibrated(family: Optional[str]) -> bool:
    return str(family or "") in FP32_CALIBRATED_FAMILIES and not _env_off(FP32_CALIBRATED_ENV)


# ComfyUI's minimum_inference_memory() (comfy/model_management.py); activations are priced separately here.
COMFY_MIN_INFERENCE_MIB = 800
COMFY_EXTRA_RESERVED_MIB = 400
COMFY_EXTRA_RESERVED_WINDOWS_MIB = 600
COMFY_EXTRA_RESERVED_WINDOWS_BIG_CARD_MIB = 100
# Placements measured on T4 / 8-16 GB cards are tuned against 2 GiB; ComfyUI's 1.2 GiB alone flips them.
DISCRETE_RESERVE_FLOOR_MIB = 2048


def comfy_min_inference_mib(total_mib: Optional[int] = None) -> int:
    """ComfyUI's ``minimum_inference_memory()`` in MiB: 1200 on Linux, 1400 / 1500 (card above 15 GiB) on Windows."""
    if sys.platform != "win32":
        return COMFY_MIN_INFERENCE_MIB + COMFY_EXTRA_RESERVED_MIB
    extra = COMFY_EXTRA_RESERVED_WINDOWS_MIB
    if int(total_mib or 0) > 15 * 1024:
        extra += COMFY_EXTRA_RESERVED_WINDOWS_BIG_CARD_MIB
    return COMFY_MIN_INFERENCE_MIB + extra


def discrete_reserve_mib(total_mib: Optional[int] = None) -> int:
    """A discrete card's fixed reserve: ComfyUI's, floored at 2 GiB."""
    return max(DISCRETE_RESERVE_FLOOR_MIB, comfy_min_inference_mib(total_mib))


_ROCM_APU_POOL = "rocm_apu_pool"


def _reserve_mib(memory_kind: str, base: int) -> int:
    if memory_kind == METAL_WORKING_SET:
        # Only other apps' growth during the load: the working set already sits below RAM. At least as strict as
        # llama.cpp (_APPLE_UNIFIED_MEMORY_FRACTION), which keeps 15% of the lower of available RAM and working set.
        return int(base * 0.15)
    if memory_kind == "unified_memory":
        return max(2048, int(base * 0.20))  # OS + CPU share this pool
    if memory_kind == "system_memory":
        return max(1024, int(base * 0.10))
    if memory_kind == _ROCM_APU_POOL:
        # The amdgpu GTT pool of an APU whose host RAM already covers the OS: still shared with the CPU side, so it
        # keeps the uncapped 10%.
        return max(2048, int(base * 0.10))
    # Fixed, not a share: a capped 10% kept 2.4-4 GiB idle on 24-40 GiB cards.
    return discrete_reserve_mib(base)


def _rocm_linux_apu_os_room_outside_pool(memory: DeviceMemory) -> bool:
    """Linux ROCm APU whose host RAM outside the pool's free part already covers the unified OS reserve: the pool is
    the amdgpu GTT cap, below physical RAM, so reserving 20% of it too reserves twice. Not Windows (HIP over-reports
    free, #7072); unknown readings answer False."""
    if (
        memory.memory_kind != "unified_memory"
        or str(getattr(memory, "device", None) or "").split(":")[0] != "cuda"
        or not sys.platform.startswith("linux")
        or memory.free_mib is None
    ):
        return False
    torch = sys.modules.get("torch")
    if torch is None or not _torch_is_rocm(torch):
        return False
    available = _available_system_memory_mib()
    if available is None:
        return False
    os_reserve = _reserve_mib("unified_memory", memory.total_mib or memory.free_mib)
    return int(available) - int(memory.free_mib) >= os_reserve


def _budget_reserve_kind(memory: DeviceMemory) -> str:
    if getattr(memory, "metal_working_set_mib", None) is not None:
        return METAL_WORKING_SET
    return _ROCM_APU_POOL if _rocm_linux_apu_os_room_outside_pool(memory) else memory.memory_kind


_LOAD_BUDGET_FRACTIONS: dict[str, float] = {}
_LOAD_BUDGET_LOCK = threading.Lock()
_BUDGET_SCOPE: contextvars.ContextVar[Optional[tuple]] = contextvars.ContextVar(
    "unsloth_vram_budget_scope", default = None
)
_UPCOMING_LOAD_FRACTION: contextvars.ContextVar[Optional[float]] = contextvars.ContextVar(
    "unsloth_vram_budget_upcoming_load", default = None
)


def _live_vram_budget_fraction() -> float:
    from utils.vram_budget_settings import get_vram_budget_fraction
    return float(get_vram_budget_fraction())


def _vram_budget_fraction() -> float:
    """The fraction the reserve readers charge."""
    scope = _BUDGET_SCOPE.get()
    with _LOAD_BUDGET_LOCK:
        if scope is not None and scope[0] == "load" and scope[2] is not None:
            return float(scope[2])
        if scope is not None and scope[1] in _LOAD_BUDGET_FRACTIONS:
            return _LOAD_BUDGET_FRACTIONS[scope[1]]
        if scope is None:
            upcoming = _UPCOMING_LOAD_FRACTION.get()
            if upcoming is not None:
                return float(upcoming)
        pinned = tuple(_LOAD_BUDGET_FRACTIONS.values())
    return min(pinned) if pinned else _live_vram_budget_fraction()


def vram_budget_scope(owner: str, *, load: bool = False) -> Callable:
    """Decorator: run a backend load (snapshots the live Budget) or generation under ``owner``'s budget scope."""

    def wrap(fn: Callable) -> Callable:
        @functools.wraps(fn)
        def scoped(*args: Any, **kwargs: Any) -> Any:
            outer = _BUDGET_SCOPE.get()
            if load and outer is not None and outer[0] == "load" and outer[1] == owner:
                return fn(*args, **kwargs)
            if load:
                fraction: Optional[float] = _UPCOMING_LOAD_FRACTION.get()
                if fraction is None:
                    try:
                        fraction = _live_vram_budget_fraction()
                    except Exception:  # noqa: BLE001 - settings are a convenience; a load must never fail on them
                        fraction = None
                scope = ("load", owner, fraction)
            else:
                scope = ("generate", owner, None)
            token = _BUDGET_SCOPE.set(scope)
            try:
                return fn(*args, **kwargs)
            finally:
                _BUDGET_SCOPE.reset(token)

        return scoped

    return wrap


def snapshot_upcoming_load_budget() -> Optional[float]:
    """Snapshot the live VRAM Budget for the load this request is preparing, before it ranks GPUs, so the ranking, the plan and the load itself agree."""
    fraction = _UPCOMING_LOAD_FRACTION.get()
    if fraction is not None:
        return fraction
    try:
        fraction = _live_vram_budget_fraction()
    except Exception:  # noqa: BLE001
        return None
    _UPCOMING_LOAD_FRACTION.set(fraction)
    return fraction


@contextlib.contextmanager
def upcoming_load_budget() -> Iterator[Optional[float]]:
    """``snapshot_upcoming_load_budget`` for a block only (a ranking helper that runs outside a request's own context)."""
    if _UPCOMING_LOAD_FRACTION.get() is not None:
        yield _UPCOMING_LOAD_FRACTION.get()
        return
    try:
        fraction: Optional[float] = _live_vram_budget_fraction()
    except Exception:  # noqa: BLE001
        fraction = None
    token = _UPCOMING_LOAD_FRACTION.set(fraction)
    try:
        yield fraction
    finally:
        _UPCOMING_LOAD_FRACTION.reset(token)


def carry_load_budget(fn: Callable) -> Callable:
    """Wrap a load's worker-thread target so it inherits the fraction the calling load began with."""
    scope = _BUDGET_SCOPE.get()
    fraction = (
        scope[2] if scope is not None and scope[0] == "load" else _UPCOMING_LOAD_FRACTION.get()
    )
    if fraction is None:
        return fn

    @functools.wraps(fn)
    def carried(*args: Any, **kwargs: Any) -> Any:
        token = _UPCOMING_LOAD_FRACTION.set(float(fraction))
        try:
            return fn(*args, **kwargs)
        finally:
            _UPCOMING_LOAD_FRACTION.reset(token)

    return carried


def commit_load_budget_fraction(owner: str) -> None:
    """Publish the fraction ``owner``'s load planned with, alongside its committed state. Never raises."""
    scope = _BUDGET_SCOPE.get()
    fraction = scope[2] if scope is not None and scope[0] == "load" and scope[1] == owner else None
    if fraction is None:
        try:
            fraction = _live_vram_budget_fraction()
        except Exception:  # noqa: BLE001
            return
    with _LOAD_BUDGET_LOCK:
        _LOAD_BUDGET_FRACTIONS[owner] = float(fraction)


def release_load_budget_fraction(owner: str) -> None:
    """Drop ``owner``'s snapshot when its committed pipeline is torn down."""
    with _LOAD_BUDGET_LOCK:
        _LOAD_BUDGET_FRACTIONS.pop(owner, None)


def _user_budget_reserve_mib(base: Optional[int]) -> int:
    """What the user's VRAM Budget setting keeps free of ``base``."""
    if not base or int(base) <= 0:
        return 0
    try:
        from utils.vram_budget_settings import VRAM_FRACTION_DEFAULT
        fraction = _vram_budget_fraction()
    except Exception:  # noqa: BLE001 - settings are a convenience; a load must never fail on them
        return 0
    if not fraction < VRAM_FRACTION_DEFAULT:
        return 0
    return int(base * (1.0 - fraction))


def user_budget_reserve_share() -> float:
    """The share of each card the user's VRAM Budget keeps free."""
    try:
        from utils.vram_budget_settings import VRAM_FRACTION_DEFAULT
        fraction = _vram_budget_fraction()
    except Exception:  # noqa: BLE001 - settings are a convenience; a load must never fail on them
        return 0.0
    return 1.0 - fraction if fraction < VRAM_FRACTION_DEFAULT else 0.0


def user_budget_reserve_bytes(device: Any, total_bytes: Optional[int]) -> int:
    """What the user's VRAM Budget keeps free on ``device`` (``total_bytes`` total), in bytes."""
    kind = str(getattr(device, "type", device) or "").split(":")[0]
    if kind in ("cpu", "mps") or not total_bytes:
        return 0
    try:
        return _user_budget_reserve_mib(int(total_bytes) >> 20) << 20
    except Exception:  # noqa: BLE001
        return 0


def _device_user_budget_reserve_mib(memory: DeviceMemory, base: Optional[int]) -> int:
    """:func:`_user_budget_reserve_mib` for ``memory``, 0 where the LLM path ignores the setting too."""
    backend = str(getattr(memory, "backend", "") or "").split(":")[0]
    if (
        backend in ("mps", "cpu")
        or getattr(memory, "memory_kind", None) == "system_memory"
        or getattr(memory, "metal_working_set_mib", None) is not None
    ):
        return 0
    return _user_budget_reserve_mib(base)


def _planner_reserve_mib(memory: DeviceMemory, base: int) -> int:
    return max(
        _reserve_mib(_budget_reserve_kind(memory), base),
        _device_user_budget_reserve_mib(memory, base),
    )


def _safe_device_budget_mib(memory: DeviceMemory) -> Optional[int]:
    """Free memory minus a headroom reserve (room for fragmentation + other tenants). None when
    free memory is unknown."""
    if memory.free_mib is None:
        return None
    base = memory.total_mib or memory.free_mib
    return max(0, int(memory.free_mib) - _planner_reserve_mib(memory, base))


def _fast_device_budget_mib(memory: DeviceMemory) -> Optional[int]:
    """Free memory minus half the standard reserve (min 2 GiB): an explicit ``fast`` offloads only when resident
    would not fit, not to keep ``auto``'s headroom for other tenants."""
    if memory.free_mib is None:
        return None
    base = memory.total_mib or memory.free_mib
    reserve = max(
        2048,
        _reserve_mib(_budget_reserve_kind(memory), base) // 2,
        _device_user_budget_reserve_mib(memory, base),
    )
    return max(0, int(memory.free_mib) - reserve)


def plan_keeps_transformer_resident(plan: Any) -> bool:
    """Whether ``plan`` never moves the denoiser after placement: torchao survives placement, not per-forward hooks."""
    policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
    if policy == OFFLOAD_NONE:
        return True
    return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))


PREQUANT_SEED_ON_HOST_ENV = "UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST"


def prequant_seed_device(
    plan: Any,
    device: str,
    scheme: Optional[str] = None,
) -> str:
    """Where a seeded pre-quantized denoiser is materialised: ``device`` when ``plan`` keeps it resident, else "cpu"
    for the schemes measured under offload. Loading it onto the GPU first adds a whole-denoiser spike that left the
    streaming hooks no room for their first block (Qwen-Image-2.1 int8 on 8 GB)."""
    if str(os.environ.get(PREQUANT_SEED_ON_HOST_ENV, "")).strip().lower() in (
        "0",
        "off",
        "false",
        "no",
    ):
        return device
    if plan is None or plan_keeps_transformer_resident(plan):
        return device
    if str(scheme) not in _TORCHAO_GROUP_OFFLOAD_MIN:
        return device
    return "cpu"


# Oldest torchao measured bit-exact under streamed group offload; 0.17 int8 (v1, no copy stream) ran 14x slower.
_TORCHAO_GROUP_OFFLOAD_MIN = {"int8": (0, 18), "fp8": (0, 17)}
# Without install_torchao_v1_int8_pin_ops, fresh torchao 0.17 installs got fp8 when offloading.
INT8_STREAM_TORCHAO17_ENV = "UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17"
_TORCHAO17_INT8_STREAM_MIN = (0, 17)


def _int8_tensor_pinnable() -> bool:
    """Whether the installed torchao's ``Int8Tensor`` implements the pin ops the stream needs (0.17+)."""
    try:
        from torchao.quantization.quantize_.workflows.int8 import int8_tensor
        import torch

        table = getattr(int8_tensor.Int8Tensor, "_ATEN_OP_TABLE", None)
        if isinstance(table, dict):
            ops = set()
            for value in table.values():
                ops.update(value.keys() if isinstance(value, dict) else ())
            if ops:
                return (
                    torch.ops.aten._pin_memory.default in ops
                    and torch.ops.aten.is_pinned.default in ops
                )
        return True  # no readable op table: the class exists, which is what 0.17 added
    except Exception:  # noqa: BLE001 - no Int8Tensor: keep the 0.18 floor
        return False


def _int8_stream_floor() -> tuple:
    if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (
        "0",
        "off",
        "false",
        "no",
    ):
        return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]
    if not _int8_tensor_pinnable():
        return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]
    return _TORCHAO17_INT8_STREAM_MIN


_TORCHAO_STREAM_SAFE_CLASSES = frozenset(("Int8Tensor", "Float8Tensor"))
# Before 0.38 diffusers moved only the torchao wrapper, leaving quantised data on the host.
_DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN = (0, 38)
_UNSET: Any = object()


def _installed_version(package: str) -> Optional[tuple[int, int]]:
    try:
        import re
        from importlib.metadata import version

        major, minor = re.match(r"(\d+)\.(\d+)", version(package)).groups()
        return int(major), int(minor)
    except Exception:  # noqa: BLE001 - absent / unreadable metadata
        return None


@functools.lru_cache(maxsize = 1)
def _installed_torchao_version() -> Optional[tuple[int, int]]:
    return _installed_version("torchao")


@functools.lru_cache(maxsize = 1)
def _installed_diffusers_version() -> Optional[tuple[int, int]]:
    return _installed_version("diffusers")


def _model_offload_fits_quantised(plan: Any) -> bool:
    """Quantised denoiser and encoders (SUM stands in for the largest) must each fit the budget whole."""
    try:
        est = plan.estimates
        budget = est.get("safe_device_budget_mib")
        model = est.get("model_dense_mib")
        companions = est.get("companion_dense_mib")
        if budget is None or model is None or companions is None:
            return False
        denoiser = max(0, int(model) - int(companions))
        overhead = int(est.get("runtime_headroom_mib") or 0) + int(
            est.get("base_overhead_mib") or 0
        )
        encoders = est.get("text_encoder_dense_mib")
        if encoders is None:
            encoders = companions
        return denoiser + overhead <= int(budget) and int(encoders) <= int(budget)
    except Exception:  # noqa: BLE001 - malformed plan: keep the resident rule
        return False


def torchao_offload_plan(
    plan: Any,
    scheme: Optional[str],
    *,
    torchao_version: Any = _UNSET,
    reclaimable_host_mib: int = 0,
) -> Optional[Any]:
    """Placement a torchao ``scheme`` denoiser survives under ``plan``, or None (sequential offload: never measured).
    ``reclaimable_host_mib``: host RAM the outgoing pipeline frees before this one pins."""
    if plan_keeps_transformer_resident(plan):
        return plan
    policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
    if policy == OFFLOAD_MODEL and _model_offload_fits_quantised(plan):
        return plan
    if policy not in (OFFLOAD_MODEL, OFFLOAD_GROUP, OFFLOAD_STREAMING):
        return None
    if not torchao_scheme_streams(scheme, torchao_version = torchao_version):
        return None
    if not _torchao_stream_pinnable(plan, reclaimable_host_mib):
        return None
    if policy != OFFLOAD_MODEL:
        return plan
    return torchao_streaming_plan(plan)


def _torchao_stream_pinnable(plan: Any, reclaimable_host_mib: int = 0) -> bool:
    """Lazy pinning refuses torchao and the stream-free fallback ran ~35x slower, so require an up-front pin."""
    forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
    if forced in ("0", "off", "false", "no"):
        return False
    if forced in ("1", "on", "true", "yes"):
        return True
    if _pinned_memory_capped():
        return False
    try:
        est = plan.estimates
        denoiser = int(est["model_dense_mib"]) - int(est["companion_dense_mib"])
    except Exception:  # noqa: BLE001 - an unsized plan cannot prove the pin fits
        return False
    budget = _pin_budget_mib()
    return budget is not None and 0 <= denoiser <= budget + max(0, int(reclaimable_host_mib))


def pipeline_host_mib(pipe: Any) -> int:
    """Host MiB held by ``pipe``'s components, which an unload hands back."""
    if pipe is None:
        return 0
    total = 0
    try:
        components = getattr(pipe, "components", None) or {}
        for module in components.values():
            if hasattr(module, "parameters"):
                total += _module_host_mib_on_cpu(module)
    except Exception:  # noqa: BLE001 - unsizeable: credit nothing
        return 0
    return total


def _module_host_mib_on_cpu(module: Any) -> int:
    try:
        seen: set[int] = set()
        nbytes = 0
        for tensor in (*module.parameters(), *module.buffers()):
            if id(tensor) in seen or getattr(tensor, "device", None) is None:
                continue
            seen.add(id(tensor))
            if tensor.device.type == "cpu":
                nbytes += sum(_storage_nbytes(tensor))
        return nbytes >> 20
    except Exception:  # noqa: BLE001
        return 0


def torchao_survives_plan(
    plan: Any,
    scheme: Optional[str],
    *,
    torchao_version: Any = _UNSET,
    reclaimable_host_mib: int = 0,
) -> bool:
    return (
        torchao_offload_plan(
            plan,
            scheme,
            torchao_version = torchao_version,
            reclaimable_host_mib = reclaimable_host_mib,
        )
        is not None
    )


def torchao_scheme_streams(scheme: Optional[str], *, torchao_version: Any = _UNSET) -> bool:
    floor = _TORCHAO_GROUP_OFFLOAD_MIN.get(str(scheme))
    if floor is None:
        return False
    if str(scheme) == "int8":
        floor = _int8_stream_floor()
    diffusers_version = _installed_diffusers_version()
    if diffusers_version is None or diffusers_version < _DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN:
        return False
    version = _installed_torchao_version() if torchao_version is _UNSET else torchao_version
    return version is not None and tuple(version)[:2] >= floor


def torchao_streaming_plan(plan: Any) -> Any:
    return replace(
        plan,
        offload_policy = OFFLOAD_STREAMING,
        reasons = tuple(plan.reasons)
        + (
            "the quantised transformer or a text encoder does not fit the device budget whole; streaming "
            "transformer blocks and text-encoder layers",
        ),
    )


def _torchao_weight_classes(module: Any) -> set[str]:
    classes: set[str] = set()
    try:
        for param in module.parameters():
            for tensor in (param, getattr(param, "data", None)):
                if tensor is not None and type(tensor).__module__.startswith("torchao"):
                    classes.add(type(tensor).__name__)
    except Exception:  # noqa: BLE001 - unreadable: report nothing, the caller keeps its kwargs
        return set()
    return classes


def _torchao_group_offload_kwargs(
    module: Any,
    kwargs: dict[str, Any],
    pinned_mib: Optional[list] = None,
) -> dict[str, Any]:
    """Group-offload kwargs a torchao ``module`` survives. Frozen first (swap_tensors on a requires_grad torchao weight
    hits an unimplemented ``aten.view``); the stream needs an up-front pin within ``pinned_mib``, a shared running total."""
    classes = _torchao_weight_classes(module)
    if not classes:
        return kwargs
    _freeze_torchao_weights(module)
    install_group_offload_torchao_swap_retry()
    if not kwargs.get("use_stream"):
        return kwargs
    if classes <= _TORCHAO_STREAM_SAFE_CLASSES or (
        classes <= _TORCHAO_V1_INT8_CLASSES and install_torchao_v1_int8_pin_ops()
    ):
        if not kwargs.get("low_cpu_mem_usage"):
            return kwargs
        # Same override the planner's _torchao_stream_pinnable honoured, so the two cannot disagree.
        forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
        if forced in ("1", "on", "true", "yes"):
            return {**kwargs, "low_cpu_mem_usage": False}
        budget = (
            None
            if forced in ("0", "off", "false", "no") or _pinned_memory_capped()
            else _pin_budget_mib()
        )
        need = _module_pin_mib(module)
        already = pinned_mib[0] if pinned_mib else 0
        if budget is not None and already + need <= budget:
            if pinned_mib is not None:
                pinned_mib[0] = already + need
            return {**kwargs, "low_cpu_mem_usage": False}
    safe = {
        k: v
        for k, v in kwargs.items()
        if k not in ("non_blocking", "record_stream", "low_cpu_mem_usage")
    }
    safe["use_stream"] = False
    return safe


# torchao <= 0.17's default int8 weight (LinearActivationQuantizedTensor over an AffineQuantizedTensor).
_TORCHAO_V1_INT8_CLASSES = frozenset(("LinearActivationQuantizedTensor", "AffineQuantizedTensor"))
_V1_INT8_PIN_OPS_INSTALLED = False


def install_torchao_v1_int8_pin_ops() -> bool:
    """Register ``is_pinned`` / ``pin_memory`` on torchao's v1 int8 classes so group offload streams them (else
    synchronous copies, 14x slower). Re-wraps the same int8 data, scale and zero point, so outputs stay bit-identical.
    Only where torchao ships the classes without the ops (<= 0.17)."""
    global _V1_INT8_PIN_OPS_INSTALLED
    if _V1_INT8_PIN_OPS_INSTALLED:
        return True
    if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (
        "0",
        "off",
        "false",
        "no",
    ):
        return False
    try:
        import torch
        from torchao.dtypes.affine_quantized_tensor import AffineQuantizedTensor as AQT
        from torchao.quantization.linear_activation_quantized_tensor import (
            LinearActivationQuantizedTensor as LAQT,
        )

        aten = torch.ops.aten
        pin_ops = [aten._pin_memory.default, aten.pin_memory.default]
        for cls in (LAQT, AQT):
            table = getattr(cls, "_ATEN_OP_TABLE", {}).get(cls, {})
            if aten.is_pinned.default in table or aten._pin_memory.default in table:
                return False  # this torchao implements them itself: never override its dispatch

        def _payload(t: Any) -> list:
            if isinstance(t, LAQT):
                return _payload(t.original_weight_tensor)
            if isinstance(t, AQT):
                impl = t.tensor_impl
                return [
                    x
                    for x in (getattr(impl, n, None) for n in ("int_data", "scale", "zero_point"))
                    if x is not None
                ]
            return [t]

        def _pinned(t: Any) -> Any:
            if isinstance(t, LAQT):
                return t._apply_fn_to_data(_pinned)
            if isinstance(t, AQT):
                return t._apply_fn_to_data(
                    lambda impl: impl._apply_fn_to_data(lambda x: x.pin_memory())
                )
            return t.pin_memory()

        # v1 lacks tensor_data_names, so diffusers could not restore it; payload stayed on GPU after offload.
        from diffusers.hooks import group_offloading as go

        restore = go._restore_torchao_tensor
        if not getattr(restore, "_unsloth_v1_int8", False):

            def _restore_v1(param: Any, source: Any) -> None:
                if isinstance(source, LAQT):
                    param.original_weight_tensor = source.original_weight_tensor
                elif isinstance(source, AQT):
                    param.tensor_impl = source.tensor_impl
                else:
                    restore(param, source)

            _restore_v1._unsloth_v1_int8 = True
            go._restore_torchao_tensor = _restore_v1

        record = go._record_stream_torchao_tensor
        if not getattr(record, "_unsloth_v1_int8", False):

            def _record_v1(param: Any, stream: Any) -> None:
                if isinstance(param, (LAQT, AQT)):
                    for x in _payload(param):
                        x.record_stream(stream)
                else:
                    record(param, stream)

            _record_v1._unsloth_v1_int8 = True
            go._record_stream_torchao_tensor = _record_v1

        for cls in (LAQT, AQT):

            @cls.implements([aten.is_pinned.default])
            def _is_pinned(func: Any, types: Any, args: Any, kwargs: Any) -> bool:
                return all(bool(x.is_pinned()) for x in _payload(args[0]))

            @cls.implements(pin_ops)
            def _pin(func: Any, types: Any, args: Any, kwargs: Any) -> Any:
                return _pinned(args[0])

        _V1_INT8_PIN_OPS_INSTALLED = True
        return True
    except Exception:  # noqa: BLE001 - no v1 classes / no dispatch table: keep the synchronous fallback
        return False


def _freeze_torchao_weights(module: Any) -> None:
    try:
        for param in module.parameters():
            if type(param).__module__.startswith("torchao") and getattr(
                param, "requires_grad", False
            ):
                param.requires_grad_(False)
    except Exception:  # noqa: BLE001 - an inference-only flag: leave the module as it is
        pass


def _holds_torchao_weights(module: Any) -> bool:
    """Whether any parameter of ``module`` is a torchao tensor subclass (GGUF and native int8 are not)."""
    try:
        for param in module.parameters():
            for tensor in (param, getattr(param, "data", None)):
                if tensor is not None and type(tensor).__module__.startswith("torchao"):
                    return True
    except Exception:  # noqa: BLE001 - an unreadable module is treated as movable, today's behaviour
        return False
    return False


def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:
    return any(
        _holds_torchao_weights(getattr(pipe, name, None))
        for name in ("transformer", "transformer_2", "unconditional_transformer")
        if getattr(pipe, name, None) is not None
    )


def _holds_fp8_stored(module: Any) -> bool:
    """Whether ``module`` holds fp8-STORED or packed nvfp4 / mxfp8 Linears; codes and scales are buffers, so a
    parameter scan reads it dense."""
    if module is None:
        return False
    try:
        from .diffusion_comfy_quant import fp8_stored_layers
        from .diffusion_packed_block import packed_block_layers
        return fp8_stored_layers(module) > 0 or packed_block_layers(module) > 0
    except Exception:  # noqa: BLE001 - unreadable: today's parameter verdict stands
        return False


def _pipe_denoisers_hold_packed(pipe: Any) -> bool:
    """GGUF / int8 / fp8-stored weights dequantize a whole Linear per forward: unmeasured by the dense eager table."""
    try:
        import torch
        for name in ("transformer", "transformer_2", "unconditional_transformer", "unet"):
            if _holds_fp8_stored(getattr(pipe, name, None)):
                return True
            params = getattr(getattr(pipe, name, None), "parameters", None)
            if callable(params) and any(
                type(p).__name__ == "GGUFParameter" or p.dtype in (torch.int8, torch.uint8)
                for p in params()
            ):
                return True
    except Exception:  # noqa: BLE001
        return True
    return False


def plan_fits_total_capacity(plan: Any) -> bool:
    """Separates a free-memory dip from a too-small device; False on any missing input so unknown sizes keep current behaviour."""
    try:
        required = plan.estimates.get("resident_required_mib")
        budget = total_capacity_budget_mib(plan.device_memory)
    except Exception:  # noqa: BLE001 - malformed plan: no retry
        return False
    if required is None or budget is None:
        return False
    return int(required) <= budget


def mps_plan_tight(plan: Any) -> bool:
    """Whether an Apple MPS plan, which stays resident whatever it needs (unified memory), needs more than the 0.85
    share of its budget a discrete plan stays resident in. False on every other device or when a size is unknown."""
    try:
        if plan.device_memory.device != "mps":
            return False
        required = plan.estimates.get("resident_required_mib")
        budget = plan.estimates.get("safe_device_budget_mib")
    except Exception:  # noqa: BLE001 - malformed plan: not tight
        return False
    if required is None or budget is None:
        return False
    return int(required) > int(int(budget) * 0.85)


def total_capacity_budget_mib(memory: DeviceMemory) -> Optional[int]:
    """TOTAL capacity minus the same reserve the free budget takes, times the 0.85 resident margin. None when the
    total is unknown. Shared with the dense prefetch gate so the two cannot disagree."""
    total = memory.total_mib
    if total is None:
        return None
    return int((int(total) - _planner_reserve_mib(memory, int(total))) * 0.85)


# Opt-in escape hatch for the unified-memory refusal below: the shortfall check is an estimate, so an operator who
# believes it is wrong can still attempt the load. Desktop / API callers send ``allow_oversized_load`` per load instead.
UNIFIED_OVERSIZE_ENV = "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD"
UNIFIED_OVERSIZE_REQUEST_FIELD = "allow_oversized_load"
# Must match the Settings switch label and location in frontend unified-memory-load-refusal.ts.
UNIFIED_OVERSIZE_SETTING_LABEL = "Allow oversized model loads"
UNIFIED_OVERSIZE_SETTING_LOCATION = "Settings > System > Model loading"
UNIFIED_OVERSIZE_REFUSAL_CODE = "unified-memory-load"

# Unlike the env var, the per-load flag does NOT lift the MPS allocator cap: that cap is process-wide and set once,
# and keeping it turns an over-commit into an MPS OOM rather than the OS killing the process.
_UNIFIED_OVERSIZE_REQUESTED: contextvars.ContextVar[bool] = contextvars.ContextVar(
    "unsloth_diffusion_allow_oversized_load", default = False
)


class UnifiedMemoryShortfallError(RuntimeError):
    """A RuntimeError so existing handlers still catch it."""

    code = UNIFIED_OVERSIZE_REFUSAL_CODE


def unified_memory_refusal_code(exc: Optional[BaseException]) -> Optional[str]:
    """``UNIFIED_OVERSIZE_REFUSAL_CODE`` when ``exc`` is, or was raised from, the unified-memory refusal."""
    seen: set[int] = set()
    while exc is not None and id(exc) not in seen:
        if isinstance(exc, UnifiedMemoryShortfallError):
            return UNIFIED_OVERSIZE_REFUSAL_CODE
        seen.add(id(exc))
        exc = exc.__cause__ or exc.__context__
    return None


@contextlib.contextmanager
def unified_oversize_requested(allowed: Any):
    """Per-load ``UNIFIED_OVERSIZE_ENV``; a ContextVar so concurrent loads / plans on other threads are unaffected."""
    token = _UNIFIED_OVERSIZE_REQUESTED.set(bool(allowed))
    try:
        yield
    finally:
        _UNIFIED_OVERSIZE_REQUESTED.reset(token)


def unified_oversize_scoped(allowed: Any, fn: Callable[..., Any]) -> Callable[..., Any]:
    @functools.wraps(fn)
    def _scoped(*args: Any, **kwargs: Any) -> Any:
        with unified_oversize_requested(allowed):
            return fn(*args, **kwargs)

    return _scoped


def scoped_allow_oversized_load(fn: Callable[..., Any]) -> Callable[..., Any]:
    """Kwarg is read, not popped, so ``fn`` keeps it in its signature; nested calls keep an outer True."""

    @functools.wraps(fn)
    def _wrapper(*args: Any, **kwargs: Any) -> Any:
        allowed = (
            bool(kwargs.get(UNIFIED_OVERSIZE_REQUEST_FIELD)) or _UNIFIED_OVERSIZE_REQUESTED.get()
        )
        with unified_oversize_requested(allowed):
            return fn(*args, **kwargs)

    return _wrapper


def _unified_oversize_env() -> bool:
    return os.environ.get(UNIFIED_OVERSIZE_ENV, "").strip().lower() in ("1", "true", "yes", "on")


def _unified_oversize_override() -> bool:
    return _UNIFIED_OVERSIZE_REQUESTED.get() or _unified_oversize_env()


def _unified_oversize_source() -> str:
    if _UNIFIED_OVERSIZE_REQUESTED.get():
        return UNIFIED_OVERSIZE_REQUEST_FIELD
    return f"{UNIFIED_OVERSIZE_ENV}=1"


def unified_capacity_memory(memory: DeviceMemory) -> DeviceMemory:
    """memory at capacity (free = total), so the verdict ignores what is loaded now; on mps the RAM
    view is re-read so the pick is not judged against a smaller working set just because
    something is resident."""
    capacity = replace(memory, free_mib = memory.total_mib)
    if getattr(memory, "device", None) != "mps" or memory.metal_working_set_mib is None:
        return capacity
    try:
        total, _free = _system_memory_mib()
    except Exception:  # noqa: BLE001 - no RAM reading: keep the working-set view
        return capacity
    if total is None:
        return capacity
    ram = DeviceMemory(memory.backend, memory.device, "unified_memory", int(total), int(total))
    ram_budget = _safe_device_budget_mib(ram)
    capacity_budget = _safe_device_budget_mib(capacity)
    if ram_budget is not None and (capacity_budget is None or ram_budget > capacity_budget):
        return ram
    return capacity


def _unified_memory_shortfall_text(plan: Any, family: Optional[str]) -> Optional[str]:
    """Ignores the overrides."""
    try:
        memory = plan.device_memory
        # system_memory excluded: the CPU path has swap and is not Metal-killed.
        if getattr(memory, "memory_kind", None) != "unified_memory":
            return None
        estimates = plan.estimates
        budget = estimates.get("safe_device_budget_mib")
        weights = estimates.get("model_dense_mib")
        overhead = estimates.get("base_overhead_mib")
        companions = estimates.get("companion_dense_mib")
        free = getattr(memory, "free_mib", None)
    except Exception:  # noqa: BLE001 - malformed plan: never block the load
        return None
    if budget is None or weights is None:
        return None
    required = int(weights) + int(overhead or 0)
    if required <= int(budget):
        return None
    what = f"'{family}'" if family else "This model"
    if free is None:
        free_note = "after reserving room for the operating system"
    elif getattr(memory, "metal_working_set_mib", None) is not None:
        free_note = (
            f"of the {int(free) / 1024:.0f} GB the GPU can use right now, after reserving room "
            "for other applications"
        )
    else:
        free_note = (
            f"of the {int(free) / 1024:.0f} GB currently free, after reserving room for the "
            "operating system"
        )
    # The text encoders and VAE load dense whatever quant the main model is, so past this floor no quant of it fits.
    floor = int(companions or 0) + int(overhead or 0)
    if companions and floor > int(budget):
        advice = (
            f"Its text encoders and VAE alone need about {floor / 1024:.0f} GB, so a lower quant of "
            "the main model will not fit either. Use a smaller model, or free memory by closing "
            "other applications."
        )
    elif companions and int(companions) < int(weights) and int(budget) - floor >= 1024:
        room = int(budget) - floor
        advice = (
            f"Its text encoders and VAE need about {floor / 1024:.0f} GB, which leaves room for a "
            f"main model of up to about {room / 1024:.0f} GB: use a lower GGUF quant (or another "
            "quantized build) of this model at or below that size, or free memory by closing "
            "other applications."
        )
    else:
        advice = (
            "Use a smaller or more quantized model (for example a lower GGUF quant), or free memory "
            "by closing other applications."
        )
    # One decimal when whole GB would print the same number twice ("needs about 14 GB, only about 14 GB usable").
    digits = 1 if f"{required / 1024:.0f}" == f"{int(budget) / 1024:.0f}" else 0
    return (
        f"{what} needs about {required / 1024:.{digits}f} GB of memory for its weights, but only "
        f"about {int(budget) / 1024:.{digits}f} GB is usable on this device ({free_note}). This device "
        "has unified memory, so the CPU and GPU share one pool: offloading weights to the CPU "
        "frees nothing, and the operating system stops an oversized load outright instead of "
        f"reporting an out-of-memory error. {advice} The size is an estimate. To try anyway, "
        f"choose Load anyway, or turn on '{UNIFIED_OVERSIZE_SETTING_LABEL}' in "
        f"{UNIFIED_OVERSIZE_SETTING_LOCATION} (API callers can send {UNIFIED_OVERSIZE_REQUEST_FIELD}; "
        f"server installs can set {UNIFIED_OVERSIZE_ENV}=1). An oversized load can exhaust memory, "
        "swap heavily, stop Unsloth or make the computer unresponsive."
    )


def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None) -> Optional[str]:
    """On UNIFIED device memory, a user-facing refusal when the WEIGHTS alone cannot fit the safe
    budget (else None).

    Unified memory is the one placement with no fallback left. On discrete VRAM an oversized model
    still loads, degrading to group / whole-module CPU offload and streaming from host RAM. On Apple
    Silicon (and integrated CUDA) the CPU and GPU share one pool, so offload moves bytes within that
    pool and frees nothing: ``plan_diffusion_memory`` correctly returns ``none`` and the load then
    allocates past physical memory. ``apply_mps_memory_cap`` bounds MPS allocations just below RAM,
    but that only turns the overrun into a late OOM after minutes of loading (and integrated CUDA has
    no such cap), so refusing up front is still the better failure.

    Weights only (``model_dense_mib`` plus the flat base overhead): the per-call runtime headroom is
    a coarse activation / VAE-decode estimate and the mps path already turns on VAE tiling and
    slicing, so counting it would refuse marginal loads that would in fact complete. The weights are
    unavoidable resident bytes, and if they alone do not fit then nothing at generation time can
    rescue the load. Same reasoning as the llama.cpp APU guard.

    None under the env var or a per-load ``allow_oversized_load``, as the loaders' speculative probes also see.

    Fail-open on anything unknown (no budget, no size), matching the planner's own "budget or model
    size unknown; staying resident".
    """
    if _unified_oversize_override():
        return None
    return _unified_memory_shortfall_text(plan, family)


def raise_on_unified_memory_shortfall(
    plan: Any,
    *,
    family: Optional[str] = None,
    logger: Any = None,
) -> None:
    """Refuse a load whose weights cannot fit unified device memory. No-op on every other
    placement, so the discrete-VRAM path is untouched.

    Lives outside ``plan_diffusion_memory`` on purpose: the planner is a pure sizing function
    that both loaders call SPECULATIVELY (the image loader re-plans candidate quantisations, and
    both re-plan against a settled snapshot), and a planner that raised would turn those probes
    into load failures instead of letting a smaller candidate win. Call this once, on the plan
    the loader has committed to, after the previous pipeline has been evicted so the free
    reading is the memory the load actually gets. Under an override the
    load proceeds with a warning."""
    message = _unified_memory_shortfall_text(plan, family)
    if message is None:
        return
    if _unified_oversize_override():
        if logger is not None:
            logger.warning(
                "diffusion.memory: loading past the unified-memory estimate (%s): %s",
                _unified_oversize_source(),
                message,
            )
        return
    if logger is not None:
        logger.error("diffusion.memory: refusing oversized unified-memory load: %s", message)
    raise UnifiedMemoryShortfallError(message)


def _sum_required(*values: Optional[int]) -> Optional[int]:
    total = 0
    for value in values:
        if value is None:
            return None
        total += int(value)
    return total


def plan_diffusion_memory(
    *,
    target: Any,
    device_memory: DeviceMemory,
    model_dense_mib: Optional[int],
    runtime_headroom_mib: int,
    companion_dense_mib: Optional[int] = None,
    text_encoder_dense_mib: Optional[int] = None,
    base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
    requested_mode: Optional[str] = None,
    explicit_offload: bool = False,
    calibrated_activation: Optional[CalibratedImageActivation] = None,
) -> MemoryPlan:
    """Encoders run once before step 0, so streaming them adds one transfer pass per call, not per step."""
    mode = normalize_memory_mode(requested_mode) or MEMORY_MODE_AUTO
    can_offload = bool(getattr(target, "supports_model_cpu_offload", False))
    budget = _safe_device_budget_mib(device_memory)
    required = _sum_required(model_dense_mib, runtime_headroom_mib, base_overhead_mib)
    # The resident floor under group offload: companions stay, the transformer streams.
    group_floor = _sum_required(companion_dense_mib, runtime_headroom_mib, base_overhead_mib)
    # Second floor with text encoders streamed too; computed only when both terms are known, and clamped at 0 since they can differ in source.
    group_floor_streamed_te = (
        _sum_required(
            max(0, int(companion_dense_mib) - int(text_encoder_dense_mib)),
            runtime_headroom_mib,
            base_overhead_mib,
        )
        if companion_dense_mib is not None and text_encoder_dense_mib is not None
        else None
    )
    # Floor for a resident transformer with streamed text encoders (they run once per call, not per step).
    resident_transformer_floor = (
        _sum_required(
            max(0, int(model_dense_mib) - int(text_encoder_dense_mib)),
            runtime_headroom_mib,
            base_overhead_mib,
        )
        if model_dense_mib is not None
        and text_encoder_dense_mib is not None
        and int(text_encoder_dense_mib) > 0
        else None
    )
    reasons: list[str] = []
    stream_text_encoders = False
    stream_transformer = True
    estimates: dict[str, Optional[int]] = {
        "safe_device_budget_mib": budget,
        "model_dense_mib": model_dense_mib,
        "companion_dense_mib": companion_dense_mib,
        "text_encoder_dense_mib": text_encoder_dense_mib,
        "runtime_headroom_mib": runtime_headroom_mib,
        "base_overhead_mib": base_overhead_mib,
        "resident_required_mib": required,
        "group_floor_mib": group_floor,
        "group_floor_streamed_te_mib": group_floor_streamed_te,
        "resident_transformer_floor_mib": resident_transformer_floor,
    }

    def _group_fits() -> bool:
        # Group offload only helps if the resident companions fit; a too-big text encoder needs whole-module offload.
        return group_floor is not None and budget is not None and group_floor <= budget

    def _group_fits_streamed_te() -> bool:
        return (
            group_floor_streamed_te is not None
            and budget is not None
            and group_floor_streamed_te <= budget
        )

    def _resident_transformer_fits() -> bool:
        return (
            resident_transformer_floor is not None
            and budget is not None
            and resident_transformer_floor <= budget
        )

    # Speed order: group, group + streamed encoders, then whole-module offload (per-step paging).
    def _offload_tier() -> tuple[str, bool, bool]:
        if _resident_transformer_fits():
            return OFFLOAD_GROUP, True, False
        if _group_fits():
            return OFFLOAD_GROUP, False, True
        if _group_fits_streamed_te():
            return OFFLOAD_GROUP, True, True
        return OFFLOAD_MODEL, False, True

    _STREAMED_TE_REASON = (
        "companions exceed budget, but they fit with the text encoders streamed too "
        "(they run once, before step 0); streaming them beats paging every component per step"
    )
    _RESIDENT_TRANSFORMER_REASON = (
        "the transformer fits resident once the text encoders are streamed (they run once, "
        "before step 0); every denoise step runs at resident speed"
    )

    if not can_offload or device_memory.is_unified:
        # No separate device to stream to on MPS/CPU; unified memory offload just shuffles bytes in one pool.
        policy = OFFLOAD_NONE
        if device_memory.is_unified:
            reasons.append("unified/system memory: CPU offload frees no device memory")
        else:
            reasons.append(f"{device_memory.backend}: CPU offload unavailable; staying resident")
    elif mode == MEMORY_MODE_FAST:
        policy = OFFLOAD_NONE
        fast_budget = _fast_device_budget_mib(device_memory)
        estimates["resident_budget_mib"] = fast_budget
        fast_required = required
        if calibrated_activation is not None and calibrated_activation.resident_only:
            # the flat 8 GiB guess demoted an explicit fast SDXL on 15-16 GB cards, where it fits with room to spare
            fast_required = _sum_required(
                model_dense_mib, calibrated_activation.headroom(False), base_overhead_mib
            )
            estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(False)
        if fast_budget is not None and fast_required is not None and fast_required > fast_budget:
            policy, stream_text_encoders, stream_transformer = _offload_tier()
            reasons.append(
                f"fast requested but weights do not fit resident ({fast_required} MiB needed, "
                f"{fast_budget} MiB free after the reserve); offloading"
            )
            if not stream_transformer:
                reasons.append(_RESIDENT_TRANSFORMER_REASON)
            elif stream_text_encoders:
                reasons.append(_STREAMED_TE_REASON)
        else:
            reasons.append("fast requested; weights resident on device")
    elif mode == MEMORY_MODE_BALANCED:
        policy = OFFLOAD_GROUP
        reasons.append("balanced requested; streamed block-level transformer offload")
    elif mode == MEMORY_MODE_LOW_VRAM:
        policy = OFFLOAD_MODEL
        reasons.append("low_vram requested; whole-module offload of every component")
    elif budget is None or required is None:
        policy = OFFLOAD_NONE
        reasons.append("device budget or model size unknown; staying resident")
    elif required <= int(budget * 0.85):
        policy = OFFLOAD_NONE
        reasons.append("weights fit resident with headroom")
    elif _resident_transformer_fits():
        policy = OFFLOAD_GROUP
        stream_text_encoders = True
        stream_transformer = False
        reasons.append(_RESIDENT_TRANSFORMER_REASON)
    elif _group_fits():
        policy = OFFLOAD_GROUP
        reasons.append("tight fit; stream the transformer, companions resident")
    elif _group_fits_streamed_te():
        policy = OFFLOAD_GROUP
        stream_text_encoders = True
        reasons.append(_STREAMED_TE_REASON)
    else:
        policy = OFFLOAD_MODEL
        # An UNKNOWN companion split fails the group tiers too; don't report it as 'exceeds budget'.
        if group_floor is None:
            reasons.append(
                "companion size unknown, so no streamed tier can be sized; "
                "whole-module offload of every component"
            )
        else:
            reasons.append("companions exceed budget; whole-module offload of every component")

    # The legacy cpu_offload flag applies only when no memory_mode was supplied, so an explicit `fast` stays resident.
    if explicit_offload and normalize_memory_mode(requested_mode) is None:
        estimates["explicit_cpu_offload"] = 1
    if (
        explicit_offload
        and normalize_memory_mode(requested_mode) is None
        and policy == OFFLOAD_NONE
        and can_offload
        and not device_memory.is_unified
    ):
        policy = OFFLOAD_MODEL
        reasons.append("explicit cpu_offload overrides resident placement")

    # VAE slicing is exact on any tier; tiling is bit-identical only for one tile (<= 1 MP).
    any_offload = policy != OFFLOAD_NONE or device_memory.backend in ("mps", "cpu")
    tile = policy in (OFFLOAD_MODEL, OFFLOAD_SEQUENTIAL) or device_memory.backend in ("mps", "cpu")
    if (
        calibrated_activation is not None
        and mode == MEMORY_MODE_AUTO
        and not (explicit_offload and normalize_memory_mode(requested_mode) is None)
        and can_offload
        and not device_memory.is_unified
    ):
        faster = _calibrated_faster_tier(
            calibrated_activation,
            budget = budget,
            model_dense_mib = model_dense_mib,
            companion_dense_mib = companion_dense_mib,
            text_encoder_dense_mib = text_encoder_dense_mib,
            base_overhead_mib = base_overhead_mib,
            free_mib = (
                None
                if device_memory.free_mib is None
                else int(device_memory.free_mib)
                - _device_user_budget_reserve_mib(
                    device_memory, device_memory.total_mib or device_memory.free_mib
                )
            ),
            policy = policy,
            stream_transformer = stream_transformer,
        )
        if faster is not None:
            flat_streamed = policy == OFFLOAD_GROUP and stream_transformer
            policy, stream_text_encoders, stream_transformer, tile, reason = faster
            if flat_streamed and policy == OFFLOAD_GROUP and not stream_transformer:
                estimates["calibrated_over_streamed_transformer"] = 1
            any_offload = policy != OFFLOAD_NONE
            reasons.append(reason)
            estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(tile)
    if calibrated_activation is not None and "calibrated_headroom_mib" not in estimates:
        from .diffusion_multigpu import current_selection
        if current_selection() is not None:
            # Untiled: a layer split keeps the VAE resident on the primary.
            estimates["calibrated_split_headroom_mib"] = calibrated_activation.headroom(False)
    split = _multigpu_companion_split(
        target = target,
        device_memory = device_memory,
        mode = mode,
        requested_mode = requested_mode,
        explicit_offload = explicit_offload,
        can_offload = can_offload,
        policy = policy,
        budget = budget,
        model_dense_mib = model_dense_mib,
        companion_dense_mib = companion_dense_mib,
        runtime_headroom_mib = runtime_headroom_mib,
        base_overhead_mib = base_overhead_mib,
        text_encoder_dense_mib = text_encoder_dense_mib,
        calibrated_activation = calibrated_activation,
    )
    single_card = MemoryPlan(
        requested_mode = mode,
        offload_policy = policy,
        vae_tiling = tile,
        vae_slicing = any_offload,
        device_memory = device_memory,
        estimates = dict(estimates),
        reasons = tuple(reasons),
        # only ever meaningful under group offload; every other tier already places the encoders
        stream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,
        stream_transformer = stream_transformer or policy != OFFLOAD_GROUP,
    )
    if split is not None:
        estimates["multigpu_primary_required_mib"] = split.primary_required_mib
        estimates["multigpu_secondary_required_mib"] = split.secondary_required_mib
        return MemoryPlan(
            requested_mode = mode,
            offload_policy = OFFLOAD_NONE,
            vae_tiling = False,
            vae_slicing = False,
            device_memory = device_memory,
            estimates = estimates,
            reasons = tuple(reasons) + (split.reason,),
            secondary_ordinal = split.secondary_ordinal,
            secondary_components = split.components,
            fallback_plan = single_card,
        )
    return single_card


def _multigpu_companion_split(
    *,
    target: Any,
    device_memory: DeviceMemory,
    mode: str,
    requested_mode: Optional[str],
    explicit_offload: bool,
    can_offload: bool,
    policy: str,
    budget: Optional[int],
    model_dense_mib: Optional[int],
    companion_dense_mib: Optional[int],
    runtime_headroom_mib: int,
    base_overhead_mib: int,
    text_encoder_dense_mib: Optional[int] = None,
    calibrated_activation: Optional[CalibratedImageActivation] = None,
) -> Any:
    """The multi-GPU companion split for this plan, or None (one card: every existing decision unchanged). Only for
    ``auto`` / ``fast`` on a discrete CUDA card that is the primary of a multi-card selection; explicit offload modes
    and the legacy ``cpu_offload`` flag keep their meaning."""
    from .diffusion_multigpu import (
        current_selection,
        plan_companion_split,
        secondary_resident_limit_mib,
    )

    selection = current_selection()
    if selection is None or not can_offload or device_memory.is_unified:
        return None
    if getattr(device_memory, "device", None) != "cuda" or budget is None:
        return None
    ordinal = getattr(target, "ordinal", None)
    if ordinal is None and selection.automatic:
        from .diffusion_device import placed_cuda_ordinal
        ordinal = placed_cuda_ordinal(target)
    if ordinal != selection.primary:
        return None
    if mode not in (MEMORY_MODE_AUTO, MEMORY_MODE_FAST):
        return None
    if explicit_offload and normalize_memory_mode(requested_mode) is None:
        return None
    if mode == MEMORY_MODE_FAST:
        limit = _fast_device_budget_mib(device_memory)
    else:
        limit = int(budget * 0.85)
    return plan_companion_split(
        single_card_policy = policy,
        model_dense_mib = model_dense_mib,
        companion_dense_mib = companion_dense_mib,
        runtime_headroom_mib = runtime_headroom_mib,
        base_overhead_mib = base_overhead_mib,
        primary_limit_mib = limit,
        secondary_limit_mib = lambda ordinal: secondary_resident_limit_mib(ordinal, mode),
        selection = selection,
        text_encoder_dense_mib = text_encoder_dense_mib,
        phase_headroom_mib = (
            (
                calibrated_activation.text_encoder_mib,
                calibrated_activation.denoise_mib,
                calibrated_activation.decode_mib,
            )
            if calibrated_activation is not None
            else None
        ),
    )


def _calibrated_faster_tier(
    act: CalibratedImageActivation,
    *,
    budget: Optional[int],
    model_dense_mib: Optional[int],
    companion_dense_mib: Optional[int],
    text_encoder_dense_mib: Optional[int],
    base_overhead_mib: int,
    free_mib: Optional[int],
    policy: str,
    stream_transformer: bool,
) -> Optional[tuple[str, bool, bool, bool, str]]:
    """Strictly faster tier than the flat pick, else None; resident tiers must fit the 2048 denoise (it cannot tile).
    A flat all-resident pick whose measured 2048 denoise does not fit is demoted to the fastest tier that does:
    nothing it holds has offload hooks a request could release. A resident-transformer pick (encoders streamed) is
    never demoted: like ComfyUI, which sizes residency per call, the generation guard handles an oversized request.
    Every check grows with the budget, so more VRAM never picks a slower tier."""
    if budget is None or free_mib is None or model_dense_mib is None or companion_dense_mib is None:
        return None
    te = max(0, int(text_encoder_dense_mib or 0))
    transformer = max(0, int(model_dense_mib) - int(companion_dense_mib))
    others = max(0, int(companion_dense_mib) - te)
    overhead = max(0, int(base_overhead_mib))
    budget = int(budget)
    free = int(free_mib) - overhead
    room = free - act.max_canvas_mib
    if act.resident_only:
        # Larger requests are still checked by the generation guard; an oversized decode is tiled by its OOM retry.
        weights = transformer + te + others
        if (
            policy != OFFLOAD_NONE
            and weights + act.headroom(False) + overhead <= budget
            and weights <= room
        ):
            return (
                OFFLOAD_NONE,
                False,
                True,
                False,
                "measured activations fit resident; the U-Net's only other tier uploads every component per call",
            )
        return None
    if policy == OFFLOAD_NONE:
        flat_rank = 0
    elif policy == OFFLOAD_GROUP and not stream_transformer:
        flat_rank = 1
    elif policy == OFFLOAD_MODEL:
        flat_rank = 4
    elif policy == OFFLOAD_GROUP:
        flat_rank = 5
    else:
        return None
    model_viable = max(transformer, te, others) <= budget and (
        flat_rank < 5
        or (
            transformer <= room
            and te + act.text_encoder_mib <= free
            and others + act.tiled_decode_mib <= free
        )
    )
    resident_streamed_te = te > 0 and transformer + others <= room
    candidates = (
        (
            transformer + te + others + act.headroom(False) + overhead <= int(budget * 0.85)
            and transformer + te + others <= room,
            (OFFLOAD_NONE, False, True, False, "measured activations fit resident with headroom"),
        ),
        (
            resident_streamed_te
            and transformer + others + act.headroom(False) + overhead <= budget,
            (
                OFFLOAD_GROUP,
                True,
                False,
                False,
                "measured activations keep the transformer resident with the text encoders streamed",
            ),
        ),
        (
            resident_streamed_te and transformer + others + act.headroom(True) + overhead <= budget,
            (
                OFFLOAD_GROUP,
                True,
                False,
                True,
                "measured activations keep the transformer resident with the text encoders "
                "streamed and the VAE decode tiled",
            ),
        ),
        (
            model_viable and others + act.decode_mib + overhead <= budget,
            (
                OFFLOAD_MODEL,
                False,
                True,
                False,
                "whole-module offload uploads the transformer once per call, and the measured "
                "VAE decode fits untiled",
            ),
        ),
        (
            model_viable,
            (
                OFFLOAD_MODEL,
                False,
                True,
                True,
                "whole-module offload uploads the transformer once per call instead of every step",
            ),
        ),
    )
    # Only the 2048 denoise is checked here: it cannot tile, while an oversized decode is tiled by the generation guard.
    if flat_rank == 0 and transformer + te + others > room:
        for fits, (tier_policy, stream_te, stream_tr, tile, reason) in candidates[flat_rank + 1 :]:
            if fits:
                return (
                    tier_policy,
                    stream_te,
                    stream_tr,
                    tile,
                    "the measured 2048 denoise does not fit beside the flat resident pick; "
                    + reason,
                )
        return None
    for rank, (fits, tier) in enumerate(candidates):
        if rank >= flat_rank:
            return None
        if fits:
            return tier
    return None


def _streamable_components(pipe: Any, torch: Any) -> dict[str, tuple[Any, str]]:
    """Only denoisers and text encoders can be streamed; the VAE and other components stay resident."""
    streamed: dict[str, tuple[Any, str]] = {}
    for name in ("transformer", "transformer_2", "unconditional_transformer"):
        module = getattr(pipe, name, None)
        if isinstance(module, torch.nn.Module):
            streamed[name] = (module, "block_level")
    for name, module in getattr(pipe, "components", {}).items():
        if str(name).startswith("text_encoder") and isinstance(module, torch.nn.Module):
            streamed[str(name)] = (module, "leaf_level")
    return streamed


# "0" keeps run-once companions resident under streaming offload.
STREAM_RUN_ONCE_COMPANIONS_ENV = "UNSLOTH_VIDEO_STREAM_RUN_ONCE_COMPANIONS"
# Run once per request before step 0 (LTX-2 text connectors).
_RUN_ONCE_COMPANION_CLASSES = ("LTX2TextConnectors",)


def run_once_companions(pipe: Any) -> dict[str, Any]:
    """Run-once companions to stream leaf by leaf with the encoders, keeping them off the card for the denoise."""
    if _env_off(STREAM_RUN_ONCE_COMPANIONS_ENV):
        return {}
    components = getattr(pipe, "components", None)
    if not isinstance(components, dict):
        return {}
    return {
        str(name): module
        for name, module in components.items()
        if type(module).__name__ in _RUN_ONCE_COMPANION_CLASSES
        and callable(getattr(module, "parameters", None))
    }


def _run_once_companions_pin(dit_mib: int, leaf_mib: int, companion_mib: int, device: Any) -> bool:
    """Pin budget fits the companions too; unpinned streaming is slower than resident."""
    forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
    if forced in ("0", "off", "false", "no") or not host_pinning_allowed(device):
        return False
    if forced in ("1", "on", "true", "yes"):
        return True
    budget = None if _pinned_memory_capped() else _pin_budget_mib()
    return budget is not None and budget > 0 and dit_mib + leaf_mib + companion_mib <= budget


def _module_storage_bytes(module: Any, seen: set[int]) -> int:
    storage_bytes = 0
    for tensor in list(module.parameters(recurse = True)) + list(module.buffers(recurse = True)):
        if id(tensor) in seen:
            continue
        seen.add(id(tensor))
        storage_bytes += int(tensor.numel()) * int(tensor.element_size())
    return storage_bytes


def largest_streamable_companion_mib(pipe: Any) -> Optional[int]:
    """MiB of the largest text encoder refinement could stream, as loaded, or None."""
    try:
        import torch
        mib = 1024 * 1024
        sizes = [
            (_module_storage_bytes(module, set()) + mib - 1) // mib
            for name, (module, offload_type) in _streamable_components(pipe, torch).items()
            if offload_type == "leaf_level"
        ]
    except Exception:  # noqa: BLE001 - a sizing aid; the caller treats unknown as unmeasured
        return None
    return max(sizes) if sizes else None


BALANCED_FIT_CHECK_ENV = "UNSLOTH_DIFFUSION_BALANCED_FIT_CHECK"


def _balanced_fit_check_enabled() -> bool:
    return str(os.environ.get(BALANCED_FIT_CHECK_ENV, "")).strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def refine_balanced_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:
    """Fit-checks balanced plans on the loaded encoders, since pre-load estimates can be the dense size."""
    if not _balanced_fit_check_enabled():
        return plan
    if getattr(plan, "requested_mode", None) != MEMORY_MODE_BALANCED:
        return plan
    if plan.offload_policy != OFFLOAD_GROUP or not bool(getattr(plan, "stream_transformer", True)):
        return plan
    estimates = dict(plan.estimates)
    budget = estimates.get("safe_device_budget_mib")
    if budget is None:
        return plan
    try:
        import torch

        components = getattr(pipe, "components", {})
        if not isinstance(components, dict):
            return plan
        streamed = _streamable_components(pipe, torch)
        mib = 1024 * 1024
        resident = 0
        encoders = 0
        for name, component in components.items():
            if not isinstance(component, torch.nn.Module):
                continue
            if name in streamed and streamed[name][1] == "block_level":
                continue  # the denoisers already stream under group offload
            size = (_module_storage_bytes(component, set()) + mib - 1) // mib
            resident += size
            if name in streamed:
                encoders += size
        overhead = int(estimates.get("runtime_headroom_mib") or 0) + int(
            estimates.get("base_overhead_mib") or 0
        )
    except Exception:  # noqa: BLE001 - a sizing aid; keep the plan the user asked for
        return plan

    estimates["balanced_resident_companions_mib"] = resident
    if resident + overhead <= int(budget):
        return plan
    if encoders > 0 and bool(getattr(plan, "stream_text_encoders", False)) is False:
        if (resident - encoders) + overhead <= int(budget):
            return replace(
                plan,
                stream_text_encoders = True,
                estimates = estimates,
                reasons = plan.reasons
                + (
                    f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget "
                    "beside the runtime headroom; streaming the text encoders too (they run once, before step 0)",
                ),
            )
    torchao = _pipe_denoisers_hold_torchao(pipe)
    return replace(
        plan,
        offload_policy = OFFLOAD_STREAMING if torchao else OFFLOAD_MODEL,
        stream_text_encoders = False,
        vae_tiling = True,
        vae_slicing = True,
        estimates = estimates,
        reasons = plan.reasons
        + (
            f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget even with "
            "the text encoders streamed; "
            + (
                "streaming transformer blocks and text-encoder layers"
                if torchao
                else "whole-module offload of every component"
            ),
        ),
    )


def refine_memory_plan_for_components(
    pipe: Any,
    plan: MemoryPlan,
    *,
    headroom_mib: int = 0,
) -> MemoryPlan:
    """Switches to granular streaming only when a loaded component cannot fit; keeps whole-module offload if all fit, and checks streamed parts fit together.

    ``headroom_mib``: what the component's own phase holds beside it (a whole-module onload holds the component AND
    that phase's activations at once); the caller prices it per phase. 0 (the image path) is the bare size check."""
    if plan.offload_policy != OFFLOAD_MODEL:
        return plan
    budget = plan.estimates.get("safe_device_budget_mib")
    if budget is None or int(budget) <= 0:
        return plan

    try:
        import torch

        components = getattr(pipe, "components", {})
        transformer = getattr(pipe, "transformer", None)
        if not isinstance(components, dict) or not isinstance(transformer, torch.nn.Module):
            return plan
        # Streaming cannot move torchao weights; the loader already checked fit (torchao numel reads bf16-sized here).
        if _pipe_denoisers_hold_torchao(pipe):
            return plan
        streamable = _streamable_components(pipe, torch)

        sizes: dict[str, int] = {}
        mib = 1024 * 1024
        for name, component in components.items():
            if not isinstance(component, torch.nn.Module):
                continue
            sizes[str(name)] = (_module_storage_bytes(component, set()) + mib - 1) // mib
    except Exception:  # noqa: BLE001 - runtime measurement is an optional refinement
        return plan

    streamed_sizes = {n: m for n, m in sizes.items() if n in streamable}
    if not streamed_sizes:
        return plan
    beside = max(0, int(headroom_mib or 0))

    def need(name: str) -> int:
        # the headroom is the denoise phase's: a text encoder runs in its own phase and keeps the bare check
        return streamed_sizes[name] + (beside if streamable[name][1] == "block_level" else 0)

    largest_name = max(streamed_sizes, key = need)
    largest_mib = streamed_sizes[largest_name]
    if need(largest_name) <= int(budget):
        return plan
    if streamable[largest_name][1] != "block_level":
        beside = 0
    # what streaming leaves resident, all at once: over budget here means streaming OOMs too
    resident_mib = sum(m for n, m in sizes.items() if n not in streamable)
    if resident_mib > int(budget):
        return plan

    estimates = dict(plan.estimates)
    estimates["largest_component_mib"] = largest_mib
    estimates["streaming_resident_mib"] = resident_mib
    return replace(
        plan,
        offload_policy = OFFLOAD_STREAMING,
        estimates = estimates,
        reasons = plan.reasons
        + (
            f"loaded {largest_name} is {largest_mib} MiB"
            + (f" (+{beside} MiB runtime headroom)" if beside else "")
            + f", above the {int(budget)} MiB device budget; streaming transformer blocks and text-encoder layers",
        ),
    )


# Flat planner's 8192 MiB/MP activation reserve overstates peaks, so it streams models that fit.
MEASURED_ACTIVATION_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION"
PARTIAL_RESIDENT_ENV = "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT"

# Worst measured CUDA MiB above the resident weights, one 1024x1024 image, encoder + every step + VAE decode, torchao
# int8 / fp8 denoisers on the compiled tiers (Qwen-Image-2.1: encoder 1849 streamed, denoise 1442, decode 1730).
# FLUX.1 / Z-Image: their worst phase is the untiled 16-channel VAE decode (_MEASURED_IMAGE_ACTIVATION_MIB).
# MOSS-Image: 2905 MiB worst-phase peak fully streamed (B200 capped at 16 GiB).
# Qwen-Image-Edit-2511: 6046 MiB whole render (1024^2, one 1024^2 source, 40 steps, true CFG 3), int8 DiT + encoder.
# Qwen-Image-2512 t2i: 6032 MiB whole render (1024^2, 20 steps, true CFG 4), int8 DiT + encoder, recorded on sm_120.
_MEASURED_IMAGE_PEAK_MIB: dict[str, int] = {
    "qwen-image-2.1": 1849,
    "flux.1": 2666,
    "z-image": 2666,
    "moss-image": 2976,
    "qwen-image-edit": 6046,
    "qwen-image": 6032,
}
# "0": Qwen-Image-Edit places from its learned peak only.
MEASURED_ACTIVATION_QWEN_EDIT_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_QWEN_EDIT"
# "0": Qwen-Image / Qwen-Image-2512 text-to-image places from its learned peak only.
MEASURED_ACTIVATION_QWEN_IMAGE_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_QWEN_IMAGE"
# Streamed tier: peak reserved above resident weights AND the prefetch window (the resident seed already pads it).
# Qwen-Image t2i has its own figure: the borrowed Edit figure (3824) left 3.5 GiB unused at a 16 GiB budget.
_MEASURED_STREAMED_IMAGE_PEAK_MIB: dict[str, int] = {
    "qwen-image-edit": 3824,
    "qwen-image": 2560,
}
MEASURED_ACTIVATION_STREAMED_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_STREAMED"
# "0": Qwen-Image t2i streams with the borrowed Edit figure again.
QWEN_IMAGE_STREAMED_SEED_ENV = "UNSLOTH_DIFFUSION_QWEN_IMAGE_STREAMED_SEED"
_QWEN_IMAGE_BORROWED_STREAMED_MIB = 3824
_MEASURED_PEAK_SPEED_MODES = ("default", "max")
_MEASURED_PEAK_MARGIN = 1.15
_MEASURED_PEAK_ROUND_MIB = 256

# Dense denoisers, eager tier: worst CUDA MiB above resident weights, fp16, 1024x1024, encode + steps + VAE decode.
# family -> (peak MiB, largest loaded DiT MiB it covers; a bigger DiT keeps the flat estimate)
MEASURED_ACTIVATION_DENSE_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE"
_MEASURED_DENSE_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
    "flux.2-klein": (2455, 7800),
    # FLUX.3 Klein: worst phase is the untiled VAE decode (2_437 MiB @1024^2, bf16); DiT 10_583 MiB bf16.
    "flux.3-klein": (2437, 10600),
    "flux.1": (2448, 23800),
    "qwen-image": (4248, 40900),
}
_MEASURED_DENSE_SPEED_MODES = ("off",)
# Dense fp16 / fp32 denoisers (pre-Ampere cards: T4, V100, Turing) on the default and max tiers, seeded from the
# family's measured peaks instead of the flat 8 GiB per megapixel estimate. "0" restores the flat estimate.
MEASURED_ACTIVATION_DENSE_TIERS_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE_TIERS"


# Dense bf16, eager tier: worst recorded whole-render peak at 1024x1024, all groups streamed (what a learned peak holds).
# family -> (peak MiB, largest loaded DiT MiB it covers). "0" = learned peak alone.
MEASURED_ACTIVATION_BF16_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16"
_MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
    # FLUX.1-schnell, Colab L4, 4 steps: 3417 (4 loads), 3581 (1 load).
    "flux.1": (3581, 23800),
    # Not seeded: FLUX.2-klein-4B's 2446 used a tiled decode; Qwen-Image-2.1's 5486 was recorded resident, unstreamed.
}


# Small-host int8 DiT, eager: (peak MiB above resident weights at 1024^2, largest int8 DiT MiB covered).
# T4 fp16, 4 steps, sliced VAE: FLUX.2-klein-4B 2805 (Kaggle), 9B 2459 (Colab); the 4B's covers both.
# "0" = learned peak alone.
MEASURED_ACTIVATION_SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"
_MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
    "flux.2-klein": (2805, 8700),
}


def _env_off(name: str) -> bool:
    return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")


def _small_host_int8_seed_mib(
    family: Optional[str], speed_mode: Optional[str], int8_dit_mib: int, widen: int
) -> Optional[int]:
    """Seed MiB for a small-host int8 denoiser (``widen`` = the compute element size), or None."""
    if _env_off(MEASURED_ACTIVATION_SMALL_HOST_ENV):
        return None
    entry = _MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB.get(str(family or "").lower())
    if entry is None or str(speed_mode or "") not in _MEASURED_DENSE_SPEED_MODES:
        return None
    if not 0 < int(int8_dit_mib) <= entry[1]:
        return None
    return entry[0] * max(2, int(widen)) // 2


def pipe_small_host_int8(pipe: Any) -> bool:
    """Whether the small-host route stored this pipe's denoiser as int8 weights (diffusion_small_host.mark)."""
    info = getattr(pipe, "_unsloth_small_host", None)
    comps = info.get("components") if isinstance(info, dict) else None
    if not isinstance(comps, dict):
        return False
    return any(
        str(comps.get(name, "")).startswith("int8 weights")
        for name in ("transformer", "transformer_2", "unconditional_transformer")
    )


def dense_bf16_seeded(family: Optional[str]) -> bool:
    return not _env_off(MEASURED_ACTIVATION_BF16_ENV) and (
        str(family or "").lower() in _MEASURED_DENSE_BF16_EAGER_PEAK_MIB
    )


def _dense_seed_mib(
    family: str,
    speed_mode: str,
    dense_mib: int,
    widen: int,
    tier_seeds: bool,
    bf16: bool = False,
) -> Optional[int]:
    """Seed MiB for a dense fp16 / fp32 denoiser (``widen`` = its element size), or None. Eager: the dense table. Default
    / max: the larger of the dense table and the tier's worst phase (encoder, denoise, untiled decode), only for a
    family the dense table measured at this DiT size. Families it lacks keep the flat estimate: on a T4, Z-Image's 11.5
    GiB DiT would then be kept partly resident, which ran slower per step than streaming it whole (3.8-4.4 vs 3.2-3.6
    s/step). ``tier_seeds``: the target is where the phase table was measured (NVIDIA with a sub-quadratic attention
    kernel, ``diffusion._measured_target``); elsewhere (math-only attention, ROCm) the tiers keep the flat estimate.
    ``bf16``: the bf16 table, eager tier and ``tier_seeds`` targets only (all it was measured on)."""
    if bf16:
        entry = _MEASURED_DENSE_BF16_EAGER_PEAK_MIB.get(family)
        if (
            entry is None
            or not tier_seeds
            or _env_off(MEASURED_ACTIVATION_BF16_ENV)
            or speed_mode not in _MEASURED_DENSE_SPEED_MODES
            or not 0 < dense_mib <= entry[1]
        ):
            return None
        return entry[0]
    entry = _MEASURED_DENSE_EAGER_PEAK_MIB.get(family)
    if entry is None or not 0 < dense_mib <= entry[1] * widen // 2:
        return None
    peak = entry[0] * widen // 2
    if speed_mode in _MEASURED_DENSE_SPEED_MODES:
        return peak
    phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(family)
    if (
        speed_mode not in _MEASURED_PEAK_SPEED_MODES
        or not tier_seeds
        or phases is None
        or _env_off(MEASURED_ACTIVATION_DENSE_TIERS_ENV)
    ):
        return None
    return max(peak, max(phases[1 if speed_mode == "max" else 0][:3]) * widen // 2)


def measured_image_runtime_mib(
    family: Optional[str],
    speed_mode: Optional[str],
    *,
    width: Optional[int] = None,
    height: Optional[int] = None,
    batch_size: int = 1,
    dense_transformer_mib: Optional[int] = None,
    compute_bytes: int = 2,
    quant: Optional[str] = None,
    phase_seeds: bool = False,
    static_seed: bool = True,
    dense_tier_seeds: bool = False,
    phase: Optional[str] = None,
    dense_bf16: bool = False,
    small_host_int8: bool = False,
    streamed: bool = False,
) -> Optional[int]:
    """Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).
    ``streamed``: use ``_MEASURED_STREAMED_IMAGE_PEAK_MIB`` (window excluded) where present.
    ``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.
    ``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248).

    Seeds: the peak tables above, then the per-phase calibration (``_MEASURED_IMAGE_ACTIVATION_MIB``, the same
    quantity per phase) for a quantised family the peak table lacks, only with ``phase_seeds`` (the target is where
    those were measured: NVIDIA with a sub-quadratic attention kernel, ``_calibrated_activation``). ``quant`` (``denoiser_quant_label``) adds the
    peak this machine recorded for (family, quant, speed tier, resolution bucket) at an earlier render
    (diffusion_measured_peaks): it raises a seed and stands in for a missing one, never lowers a seed.
    ``static_seed=False`` (a denoiser no static table was measured on: bf16 or GGUF / int8 packed dense weights)
    answers from the learned peak alone. ``dense_tier_seeds`` enables the dense default / max tier seeds (``_dense_seed_mib``)
    on the target the phase table was measured on."""
    if _env_off(MEASURED_ACTIVATION_ENV):
        return None
    seed: Optional[int] = None
    if dense_transformer_mib is not None and _env_off(MEASURED_ACTIVATION_DENSE_ENV):
        return None
    if static_seed and dense_transformer_mib is not None:
        seed = _dense_seed_mib(
            str(family or "").lower(),
            str(speed_mode or ""),
            int(dense_transformer_mib),
            max(2, int(compute_bytes or 2)),
            dense_tier_seeds,
            bf16 = dense_bf16,
        )
    elif small_host_int8 and dense_transformer_mib is not None and phase is None:
        seed = _small_host_int8_seed_mib(
            family, speed_mode, int(dense_transformer_mib), max(2, int(compute_bytes or 2))
        )
    elif static_seed and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:
        seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
        if streamed and phase is None and not _env_off(MEASURED_ACTIVATION_STREAMED_ENV):
            seed = _MEASURED_STREAMED_IMAGE_PEAK_MIB.get(str(family or "").lower(), seed)
            if str(family or "").lower() == "qwen-image" and _env_off(QWEN_IMAGE_STREAMED_SEED_ENV):
                seed = _QWEN_IMAGE_BORROWED_STREAMED_MIB
        if str(family or "").lower() == "qwen-image-edit" and _env_off(
            MEASURED_ACTIVATION_QWEN_EDIT_ENV
        ):
            seed = None
        if str(family or "").lower() == "qwen-image" and _env_off(
            MEASURED_ACTIVATION_QWEN_IMAGE_ENV
        ):
            seed = None
        if seed is None and phase_seeds:
            phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or "").lower())
            if phases is not None:
                # worst of encoder / denoise / untiled decode at 1024x1024 for this tier
                seed = max(phases[1 if speed_mode == "max" else 0][:3])
    w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
    h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
    batch = max(1, int(batch_size or 1))
    learned = None
    if quant is not None:
        from .diffusion_measured_peaks import lookup as learned_peak
        learned = learned_peak(
            family = family,
            quant = quant,
            speed_mode = speed_mode,
            width = w,
            height = h,
            batch_size = batch,
            phase = phase,
        )
    if seed is None and learned is None:
        return None
    scale = max(1.0, (w * h * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT))
    need = max(float(seed or 0) * scale, float(learned or 0)) * _MEASURED_PEAK_MARGIN
    step = _MEASURED_PEAK_ROUND_MIB
    return int(-(-need // step) * step)


def learned_peak_label(
    pipe: Any,
    attention: Optional[str] = None,
    model: Optional[str] = None,
    vae: Optional[str] = None,
    te: Optional[str] = None,
    cache: Optional[str] = None,
    graphs: Optional[str] = None,
) -> str:
    """``denoiser_quant_label`` plus the loaded model (variants of one family differ in size), the text encoders'
    precision (a dense streamed encoder peaks above a quantised one), the VAE savers (an untiled decode peaks far
    above a tiled one) and the engaged attention backend (backends with different workspaces): none shares a learned
    peak. The VAE savers and the backend can change after the load; ``with_label_part`` rewrites them."""
    label = denoiser_quant_label(pipe)
    for name, value in (
        ("model", model),
        ("te", te),
        ("cache", cache),
        ("vae", vae),
        ("attn", attention),
        ("graphs", graphs),
    ):
        if value is not None:
            label = f"{label}|{name}:{value}"
    return label


# Renders without guidance on a CFG-capable family; read only when no CFG peak exists.
CFG_OFF_PART = "cfg:off"
LEARNED_PEAKS_CFG_OFF_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS_CFG_OFF"


def cfg_off_peaks_enabled() -> bool:
    return not _env_off(LEARNED_PEAKS_CFG_OFF_ENV)


# Deferred profiles record and price their own tier's learned peak; 0 = flat estimate for compiled renders.
TIER_PEAKS_ENV = "UNSLOTH_DIFFUSION_TIER_PEAKS"
TIER_MOVED_ATTR = "_unsloth_peak_tier_moved"


def tier_peaks_enabled() -> bool:
    return not _env_off(TIER_PEAKS_ENV)


def without_cfg_off(label: str) -> str:
    return "|".join(part for part in str(label).split("|") if part != CFG_OFF_PART)


def _learned_read_label(
    pipe: Any, family: Optional[str], quant: str, speed_mode: Optional[str]
) -> tuple[str, bool]:
    """``(label, cfg_off_only)``: True means CFG requests must be sized from the flat estimate."""
    from .diffusion_measured_peaks import lookup_bucket

    if (
        not getattr(pipe, "_unsloth_peak_expects_cfg", False)
        or not cfg_off_peaks_enabled()
        or lookup_bucket(family = family, quant = quant, speed_mode = speed_mode) is not None
    ):
        return quant, False
    off = f"{quant}|{CFG_OFF_PART}"
    if lookup_bucket(family = family, quant = off, speed_mode = speed_mode) is None:
        return quant, False
    return off, True


def with_label_part(label: str, name: str, value: str) -> str:
    """``label`` with its ``name:`` part replaced by ``value`` (unchanged when it has no such part)."""
    return "|".join(
        f"{name}:{value}" if part.startswith(f"{name}:") else part for part in str(label).split("|")
    )


# Non-txt2img workflows record under ``wf:<workflow>`` so an edit's larger peak never inflates the next txt2img load.
WORKFLOW_PART = "wf"
TXT2IMG_WORKFLOW = "txt2img"


def with_workflow(label: str, workflow: Optional[str]) -> str:
    """``label`` keyed for ``workflow``: unchanged for text-to-image (and an unknown workflow), else ``|wf:<name>``."""
    name = str(workflow or "").strip().lower()
    if not name or name == TXT2IMG_WORKFLOW:
        return str(label)
    return f"{label}|{WORKFLOW_PART}:{name}"


def learned_phase_split(
    family: Optional[str], quant: Optional[str], speed_mode: Optional[str]
) -> Optional[dict]:
    """Learned phases (+ ``pixels`` / ``quadratic``) at the default bucket, or None unless every phase shares the
    whole-render bucket and the larger padded phase reaches the whole-render peak."""
    from .diffusion_measured_peaks import PHASE_DECODE, PHASE_PRE_DECODE, PHASES, lookup_bucket

    if quant is None:
        return None
    total = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)
    if total is None:
        return None
    found = {}
    for phase in PHASES:
        value = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode, phase = phase)
        if value is None or value[1] != total[1]:
            return None
        found[phase] = int(value[0])
    if max(found[PHASE_PRE_DECODE], found[PHASE_DECODE]) < int(total[0]):
        return None
    found.update(pixels = int(total[1]), quadratic = bool(total[2]))
    return found


_VAE_SAVERS = (
    ("use_tiling", "enable_vae_tiling", "enable_tiling"),
    ("use_slicing", "enable_vae_slicing", "enable_slicing"),
)


def _vae_saver_can(pipe: Any, pipe_method: str, vae_method: str) -> bool:
    return callable(getattr(pipe, pipe_method, None)) or callable(
        getattr(getattr(pipe, "vae", None), vae_method, None)
    )


def _vae_savers_observable(pipe: Any) -> bool:
    """False when a saver can be engaged but the VAE keeps no ``use_*`` flag to read it back: its state is unknown."""
    vae = getattr(pipe, "vae", None)
    return all(
        hasattr(vae, flag) or not _vae_saver_can(pipe, pm, vm) for flag, pm, vm in _VAE_SAVERS
    )


def planned_vae_saver_tag(pipe: Any, plan: Any) -> str:
    """The VAE savers ``plan`` asks for that this pipeline can engage (what ``apply_memory_plan`` will turn on);
    ``unknown`` (no learned peak used or recorded) when the VAE cannot report them."""
    if not _vae_savers_observable(pipe):
        return "unknown"
    return vae_saver_tag(
        getattr(plan, "vae_tiling", False) and _vae_saver_can(pipe, *_VAE_SAVERS[0][1:]),
        getattr(plan, "vae_slicing", False) and _vae_saver_can(pipe, *_VAE_SAVERS[1][1:]),
    )


def live_vae_saver_tag(pipe: Any) -> str:
    """The VAE savers in force right now (``unknown`` as in ``planned_vae_saver_tag``)."""
    if not _vae_savers_observable(pipe):
        return "unknown"
    vae = getattr(pipe, "vae", None)
    return vae_saver_tag(getattr(vae, "use_tiling", False), getattr(vae, "use_slicing", False))


def stream_window_pad_mib(pipe: Any) -> Optional[int]:
    """MiB a learned peak measured with no denoiser group streaming adds to reach the streamed baseline (the prefetch
    window a streamed placement holds above its start allocation), so every placement reads one conservative scale.
    None when that window cannot be sized, or the denoiser is neither group-streamed nor on the card (whole-module /
    sequential offload onloads it whole inside the render): the peak is then not recorded. Resident text encoders add
    their leaf-streaming window the same way (``_encoder_stream_window_mib``)."""
    encoders = _encoder_stream_window_mib(pipe)
    if encoders is None:
        return None
    streaming = partial = False
    for name, module in (getattr(pipe, "components", {}) or {}).items():
        if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
            continue
        if module is None:
            continue  # an empty slot (Wan2.2-TI2V-5B and Wan2.1 register transformer_2 = None)
        state = _streaming_state(module)
        if state is None:
            return None
        if state:
            streaming = True
            partial = partial or any(
                getattr(g, "_unsloth_resident", False) for g in _offload_groups(module)
            )
    if streaming and not partial:
        return encoders
    window = _stream_window_mib(pipe)
    if window < 0:
        return None
    if streaming:
        # some groups resident: the measured window is the largest STREAMED group's, a fully streamed load holds the
        # largest of all; pad the difference
        streamed = _streamed_group_window_mib(pipe)
        if streamed < 0:
            return None
        window = max(0, window - streamed)
    return window + encoders


def _streamed_group_window_mib(pipe: Any) -> int:
    """Device MiB the prefetcher holds for the denoiser groups that stream now: ``depth + 1`` of the largest, or -1."""
    try:
        from .diffusion_offload_prefetch import prefetch_depth

        largest = 0
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
                continue
            if module is None:
                continue
            for group in _offload_groups(module):
                if getattr(group, "_unsloth_resident", False):
                    continue
                largest = max(largest, _group_nbytes(group))
        if largest <= 0:
            return -1
        return -(-largest * (prefetch_depth() + 1) // (1024 * 1024))
    except Exception:  # noqa: BLE001
        return -1


def _group_nbytes(group: Any) -> int:
    """Storage bytes of one offload group's modules, parameters and buffers (shared tensors once)."""
    seen: set[int] = set()
    nbytes = 0
    for member in getattr(group, "modules", None) or ():
        for t in (*member.parameters(), *member.buffers()):
            if id(t) not in seen:
                seen.add(id(t))
                nbytes += sum(_storage_nbytes(t))
    for t in (
        *(getattr(group, "parameters", None) or ()),
        *(getattr(group, "buffers", None) or ()),
    ):
        if id(t) not in seen:
            seen.add(id(t))
            nbytes += sum(_storage_nbytes(t))
    return nbytes


def _streaming_state(module: Any) -> Optional[bool]:
    """True when ``module`` streams offload groups, False when it sits on the card, None when neither can be told
    (whole-module / sequential offload, weights off the card without hooks)."""
    try:
        groups = _offload_groups(module) or ()
        if any(not getattr(g, "_unsloth_resident", False) for g in groups):
            return True
        if not groups:
            if getattr(module, "_hf_hook", None) is not None:
                return None
            first = next(iter(module.parameters()), None)
            if first is not None and getattr(first.device, "type", None) != "cuda":
                return None
        return False
    except Exception:  # noqa: BLE001
        return None


def _encoder_stream_window_mib(pipe: Any) -> Optional[int]:
    """MiB a load that streams the text encoders (leaf-level groups) holds in flight that a render with them resident
    did not: ``depth + 1`` of the largest leaf of each resident encoder, less the streamed leaves' window of a
    partially resident one. None when an encoder's placement is unknown."""
    try:
        import torch

        from .diffusion_offload_prefetch import prefetch_depth

        total = 0
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):
                continue
            state = _streaming_state(module)
            if state is None:
                return None
            groups = list(_offload_groups(module) or ()) if state else []
            if state and not any(getattr(g, "_unsloth_resident", False) for g in groups):
                continue  # fully streamed during the render: its whole window is in the peak
            largest = 0
            for sub in module.modules():
                seen: set[int] = set()
                nbytes = 0
                for t in (*sub.parameters(recurse = False), *sub.buffers(recurse = False)):
                    if id(t) not in seen:
                        seen.add(id(t))
                        nbytes += sum(_storage_nbytes(t))
                largest = max(largest, nbytes)
            window = -(-largest * (prefetch_depth() + 1) // (1024 * 1024))
            if state:
                # some leaves resident: the peak holds the largest STREAMED leaf's window, a fully streamed encoder
                # the largest of all; pad the difference
                streamed = max(
                    (
                        _group_nbytes(g)
                        for g in groups
                        if not getattr(g, "_unsloth_resident", False)
                    ),
                    default = 0,
                )
                window = max(0, window - -(-streamed * (prefetch_depth() + 1) // (1024 * 1024)))
            total += window
        return total
    except Exception:  # noqa: BLE001
        return None


def vae_saver_tag(tiling: Any, slicing: Any) -> str:
    """The VAE savers in force, as the learned-peak key names them."""
    return "+".join(name for name, on in (("tiled", tiling), ("sliced", slicing)) if on) or "full"


def denoiser_quant_label(pipe: Any) -> str:
    """Stable label of the denoiser weights' format for the learned-peak key: torchao subclasses, GGUF, int8 / uint8
    packed, fp8-stored (codes in buffers, widened per layer: ``fp8_stored:<float dtypes>#<variant>``), or the dense
    dtype."""
    try:
        import torch

        dits = [
            getattr(pipe, name, None)
            for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
        ]
        dits = [d for d in dits if isinstance(d, torch.nn.Module)]
        if not dits:
            return "none"
        classes: set[str] = set()
        for dit in dits:
            classes |= _torchao_weight_classes(dit)
        if classes:
            return "torchao:" + "+".join(sorted(classes)) + _quant_variant_tag(dits)
        dtypes: set[str] = set()
        for dit in dits:
            for param in dit.parameters():
                if type(param).__name__ == "GGUFParameter":
                    return "gguf"
                if param.dtype in (torch.int8, torch.uint8):
                    dtypes.add("int8")
                elif param.is_floating_point():
                    dtypes.add(str(param.dtype).replace("torch.", ""))
        if "int8" in dtypes:
            return "packed:int8" + _quant_variant_tag(dits)
        if any(_holds_fp8_stored(dit) for dit in dits):
            # Never the dense key: same model file, same float dtype, but a per-layer widen the dense run lacks.
            return "fp8_stored:" + "+".join(sorted(dtypes)) + _quant_variant_tag(dits)
        return "dense:" + "+".join(sorted(dtypes)) if dtypes else "unknown"
    except Exception:  # noqa: BLE001
        return "unknown"


def _quant_variant_tag(dits: list) -> str:
    """``#`` plus a short digest of the denoisers' quant graph variant (``graph_variant``, the compile cache's key):
    plain vs ConvRot, W8A8 vs weight-only, hosted vs runtime scales share a weight class but not their kernels or
    workspaces. ``#unknown`` when it cannot be read (the label then neither reads nor records a peak)."""
    try:
        import hashlib
        import json

        from .diffusion_compile_cache import graph_variant

        variants = [graph_variant(dit) for dit in dits]
        if any(isinstance(v.get("weights"), str) for v in variants):
            return "#unknown"
        blob = json.dumps(variants, sort_keys = True, default = str)
        return "#" + hashlib.sha256(blob.encode()).hexdigest()[:12]
    except Exception:  # noqa: BLE001
        return "#unknown"


def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:
    """name -> (MiB, role in 'dit' / 'text_encoder' / 'other') from storage bytes (a torchao int8 weight counts its
    int8 data + scales); None when unreadable."""
    try:
        import torch

        out: dict[str, tuple[int, str]] = {}
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if not isinstance(module, torch.nn.Module):
                continue
            seen: set[int] = set()
            nbytes = 0
            for tensor in (*module.parameters(), *module.buffers()):
                if id(tensor) in seen:
                    continue
                seen.add(id(tensor))
                nbytes += sum(_storage_nbytes(tensor))
            name = str(name)
            role = (
                "dit"
                if name in ("transformer", "transformer_2", "unconditional_transformer")
                else "text_encoder"
                if name.startswith("text_encoder")
                else "other"
            )
            out[name] = (-(-nbytes // (1024 * 1024)), role)
        return out
    except Exception:  # noqa: BLE001 - an optional refinement
        return None


RESIDENT_DIT_ENV = "UNSLOTH_DIFFUSION_RESIDENT_DIT"
# Resident slack: 10% of card (min 1 GiB) for fragmentation and lazy kernels; peak covers the rest.
_RESIDENT_DIT_SLACK_FRACTION = 0.10
_RESIDENT_DIT_SLACK_MIN_MIB = 1024


def _resident_dit_slack_mib(memory: Any) -> int:
    """The measured-residency margin, or what the user's VRAM Budget keeps free when larger (as the planner reserve)."""
    base = getattr(memory, "total_mib", None) or getattr(memory, "free_mib", None) or 0
    return max(
        _RESIDENT_DIT_SLACK_MIN_MIB,
        int(int(base) * _RESIDENT_DIT_SLACK_FRACTION),
        _device_user_budget_reserve_mib(memory, int(base)),
    )


def _resident_dit_fits(memory: Any, dit_mib: int, headroom_mib: int, other_mib: int) -> bool:
    """Whole denoiser + non-encoder companions + measured peak x margin + slack within free memory, encoders streamed."""
    if _env_off(RESIDENT_DIT_ENV):
        return False
    free = getattr(memory, "free_mib", None)
    if free is None:
        return False
    return int(dit_mib) + int(other_mib) + int(headroom_mib) + _resident_dit_slack_mib(
        memory
    ) <= int(free)


STREAMED_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_STREAMED_RESIDENCY"


@functools.lru_cache(maxsize = 1)
def _diffusers_takes_block_modules() -> bool:
    """Whether the installed diffusers' ``apply_group_offloading`` accepts ``block_modules``."""
    try:
        import inspect

        from diffusers.hooks import apply_group_offloading
        return "block_modules" in inspect.signature(apply_group_offloading).parameters
    except Exception:  # noqa: BLE001 - no diffusers / unreadable: the stock grouping
        return False


def _block_stacks(module: Any) -> list:
    """The block stacks block-level group offload streams: ``ModuleList`` / ``Sequential`` children, plus the stacks of
    the ModuleDicts ``_dict_block_modules`` hands to diffusers as ``block_modules``."""
    import torch

    stacks = []
    dicts = set(_dict_block_modules(module)) if _diffusers_takes_block_modules() else set()
    for name, child in module.named_children():
        if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
            stacks.append(child)
        elif name in dicts:
            stacks.extend(child.values())
    return stacks


def _stream_window_mib(pipe: Any) -> int:
    """Device MiB the block prefetcher holds in flight: ``depth + 1`` of the largest denoiser block, or -1."""
    try:
        import torch

        from .diffusion_offload_prefetch import prefetch_depth

        largest = 0
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
                continue
            if not isinstance(module, torch.nn.Module):
                continue
            for child in _block_stacks(module):
                for block in child:
                    seen: set[int] = set()
                    nbytes = 0
                    for t in (*block.parameters(), *block.buffers()):
                        if id(t) not in seen:
                            seen.add(id(t))
                            nbytes += sum(_storage_nbytes(t))
                    largest = max(largest, nbytes * DEFAULT_GROUP_BLOCKS)
        if largest <= 0:
            return -1
        return -(-largest * (prefetch_depth() + 1) // (1024 * 1024))
    except Exception:  # noqa: BLE001
        return -1


def _streamed_dit_room_mib(memory: Any, headroom_mib: int, other_mib: int, window_mib: int) -> int:
    """Partial room: the whole-resident fit plus the prefetch window, not the flat reserve AND base overhead."""
    free = getattr(memory, "free_mib", None)
    if free is None or window_mib < 0 or _env_off(STREAMED_RESIDENCY_ENV):
        return 0
    return max(
        0,
        int(free)
        - _resident_dit_slack_mib(memory)
        - int(other_mib)
        - int(headroom_mib)
        - int(window_mib),
    )


# Size a streamed denoiser's resident blocks from the denoise phase (encode/decode release around it); "0" = whole-render peak.
DENOISE_PHASE_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_DENOISE_PHASE_RESIDENCY"
_PHASE_DENOISE_COLUMN = 1
_PHASE_UNTILED_DECODE_COLUMN = 2
_PHASE_TILED_DECODE_COLUMN = 3


def denoise_phase_residency_enabled() -> bool:
    return not _env_off(DENOISE_PHASE_RESIDENCY_ENV)


# (denoise, tiled decode) MiB, streamed int8, default tier, 1024^2, window excluded, no margin; measured on an L4 at
# 6 GiB. The resident table (901 / 2437) left ~2 GiB unused. "0" = the resident table.
STREAMED_PHASE_SEED_ENV = "UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED"
_MEASURED_STREAMED_PHASE_MIB: dict[str, tuple[int, int]] = {
    "flux.3-klein": (308, 904),
}


def _streamed_phase_seed(
    family: Optional[str], speed_mode: Optional[str], quant: Optional[str], tiled: bool
) -> Optional[tuple[int, int]]:
    if _env_off(STREAMED_PHASE_SEED_ENV) or not tiled or str(speed_mode or "") != "default":
        return None
    if "int8" not in str(quant or "").split("|", 1)[0].lower():
        return None
    return _MEASURED_STREAMED_PHASE_MIB.get(str(family or "").lower())


def _top_level_group_mib(pipe: Any) -> Optional[int]:
    """MiB of the largest denoiser's weights outside its block stacks (``ModuleList`` / ``Sequential`` children):
    block-level group offload keeps them in one top-level group that is onloaded for the whole forward when it is not
    resident, beside the block window. None when unreadable."""
    try:
        import torch

        largest = 0
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
                continue
            if not isinstance(module, torch.nn.Module):
                continue
            blocks: set[int] = set()
            for child in _block_stacks(module):
                blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))
            seen: set[int] = set()
            nbytes = 0
            for t in (*module.parameters(), *module.buffers()):
                if id(t) in seen or id(t) in blocks:
                    continue
                seen.add(id(t))
                nbytes += sum(_storage_nbytes(t))
            largest = max(largest, -(-nbytes // (1024 * 1024)))
        return largest
    except Exception:  # noqa: BLE001
        return None


def _denoise_phase_reserve(
    pipe: Any,
    *,
    family: Optional[str],
    speed_mode: Optional[str],
    quant: Optional[str],
    static_seed: bool,
    window_mib: int,
    tiled: bool,
) -> Optional[tuple[int, tuple[int, int, bool]]]:
    """``(denoise MiB, (decode peak MiB, bucket pixels, quadratic))`` for a 1024x1024 render, or None.

    The denoise MiB is what the steps allocate above the resident weights besides the prefetch window (margin
    included): the larger of the static denoise seed (``_MEASURED_IMAGE_ACTIVATION_MIB``, window excluded) and this
    build's learned pre-decode peak less the part of the window it is sure to hold (learned peaks are on the streamed
    scale, ``stream_window_pad_mib``, at the prefetch depth they were recorded with), so no window is charged twice. The decode peak (no margin) is the larger of the
    static decode seed for the VAE savers in force and the learned unpadded decode phase. None when neither source
    covers both phases, when a learned whole-render peak has no consistent phase split (its phases are unknown), or
    when the switch is off."""
    if not denoise_phase_residency_enabled() or _env_off(MEASURED_ACTIVATION_ENV):
        return None
    try:
        from .diffusion_measured_peaks import (
            PHASE_DECODE_UNPADDED,
            PHASE_PRE_DECODE,
            lookup_bucket,
        )

        name = str(family or "").lower()
        denoise = decode = None
        pixels, quadratic = DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT, False
        phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(name)
        if (
            static_seed
            and phases is not None
            and bool(getattr(pipe, "_unsloth_phase_seeds", False))
            and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES
        ):
            row = phases[1 if speed_mode == "max" else 0]
            streamed = _streamed_phase_seed(family, speed_mode, quant, tiled)
            if streamed is not None:
                row = (row[0], streamed[0], row[2], streamed[1], row[4])
            denoise = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))
            decode = int(row[_PHASE_TILED_DECODE_COLUMN if tiled else _PHASE_UNTILED_DECODE_COLUMN])
        if quant is not None:
            whole = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)
            if whole is not None:
                split = learned_phase_split(family, quant, speed_mode)
                if split is None:
                    return None
                from .diffusion_offload_prefetch import prefetch_depth

                # the sample holds the window of the depth it was recorded at, which the key does not name: credit
                # only the smallest one (depth 1, two blocks), so a later deeper prefetch is never double-credited
                held = max(0, int(window_mib)) * 2 // (prefetch_depth() + 1)
                learned_pre = int(math.ceil(split[PHASE_PRE_DECODE] * _MEASURED_PEAK_MARGIN)) - held
                denoise = max(int(denoise or 0), learned_pre, 0)
                decode = max(int(decode or 0), int(split[PHASE_DECODE_UNPADDED]))
                pixels, quadratic = int(split["pixels"]), bool(split["quadratic"])
        if denoise is None or not decode:
            return None
        step = _MEASURED_PEAK_ROUND_MIB
        return int(-(-int(denoise) // step) * step), (int(decode), int(pixels), bool(quadratic))
    except Exception:  # noqa: BLE001 - unreadable: the whole-render placement
        return None


def _denoiser_compute_bytes(pipe: Any) -> Optional[int]:
    """Denoiser compute element size for the dense table: 2 (fp16), 4 (fp32); None for bf16 / unreadable."""
    try:
        import torch
        for name in ("transformer", "unet"):
            module = getattr(pipe, name, None)
            dtype = getattr(module, "dtype", None) if module is not None else None
            if dtype is not None:
                return {torch.float16: 2, torch.float32: 4}.get(dtype)
    except Exception:  # noqa: BLE001
        pass
    return None


_DENOISER_FIRST_REASON = (
    "the text encoders stream instead of staying resident: that keeps {streamed} MiB of the denoiser resident "
    "rather than {resident} MiB (the denoiser runs every step, the encoders once per prompt)"
)


def refine_plan_from_loaded_weights(
    pipe: Any,
    plan: MemoryPlan,
    *,
    family: Optional[str],
    speed_mode: Optional[str],
    logger: Any = None,
    **labels: Optional[str],
) -> MemoryPlan:
    """Re-place a streamed auto load from its loaded weights, denoiser first: the flat plan keeps
    encoders resident once their floor fits; the encoders-streamed plan is taken only if it keeps
    strictly more of the denoiser resident."""
    refined = _refine_plan_from_loaded_weights(
        pipe, plan, family = family, speed_mode = speed_mode, logger = logger, **labels
    )
    if (
        getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO
        or plan.offload_policy != OFFLOAD_GROUP
        or bool(getattr(plan, "stream_text_encoders", False))
        or not bool(getattr(plan, "stream_transformer", True))
        or plan.estimates.get("text_encoder_dense_mib") is None
    ):
        return refined
    try:
        streamed = _refine_plan_from_loaded_weights(
            pipe,
            replace(plan, stream_text_encoders = True),
            family = family,
            speed_mode = speed_mode,
            logger = None,
            **labels,
        )
        resident_dit = int(getattr(refined, "resident_transformer_mib", None) or 0)
        streamed_dit = int(getattr(streamed, "resident_transformer_mib", None) or 0)
        if streamed_dit <= resident_dit:
            return refined
        reason = _DENOISER_FIRST_REASON.format(streamed = streamed_dit, resident = resident_dit)
        if logger is not None:
            logger.info("diffusion.memory: %s", reason)
        return replace(
            streamed,
            reasons = streamed.reasons + (reason,),
            estimates = {**streamed.estimates, "denoiser_first_resident_te_dit_mib": resident_dit},
        )
    except Exception as exc:  # noqa: BLE001 - keep the encoders-resident refinement
        if logger is not None:
            logger.debug("diffusion.memory: streamed-encoder comparison skipped (%s)", exc)
        return refined


def _denoiser_is_dense_bf16(pipe: Any) -> bool:
    """A plain bf16 denoiser: no torchao, packed (GGUF / int8) or fp8-stored weights."""
    try:
        import torch

        module = next(
            (
                getattr(pipe, n, None)
                for n in ("transformer", "unet")
                if getattr(pipe, n, None) is not None
            ),
            None,
        )
        if module is None or getattr(module, "dtype", None) != torch.bfloat16:
            return False
        if _pipe_denoisers_hold_torchao(pipe) or _pipe_denoisers_hold_packed(pipe):
            return False
        return not any(
            _holds_fp8_stored(getattr(pipe, n, None))
            for n in ("transformer", "transformer_2", "unet")
        )
    except Exception:  # noqa: BLE001 - unreadable: keep the streamed plan
        return False


_WHOLE_MODULE_KEPT_REASON = (
    "whole-module offload ran the {dit} MiB denoiser whole below the streamed tiers' {floor} MiB floor; it still "
    "fits, and uploading it once per call beats streaming it every step"
)
_WHOLE_MODULE_COST_REASON = (
    " (measured {h2d:.0f} GiB/s host-to-device and {tflops:.0f} bf16 TFLOPS: {model:.2f} s vs {group:.2f} s "
    "streamed for a {steps}-step {side}px render)"
)
_WHOLE_MODULE_DECLINED_REASON = (
    "streaming kept over whole-module offload of the {dit} MiB denoiser: measured {h2d:.0f} GiB/s host-to-device "
    "and {tflops:.0f} bf16 TFLOPS price a {steps}-step {side}px render at {model:.2f} s whole vs {group:.2f} s streamed"
)

# ``auto``: whole-module offload only where priced cheaper; ``1`` always (#15129); ``0`` never.
WHOLE_MODULE_OFFLOAD_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_OFFLOAD"
WHOLE_MODULE_PER_REQUEST_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_PER_REQUEST"
WHOLE_TIER_ATTR = "_unsloth_whole_tier"


def per_request_whole_enabled() -> bool:
    return not _env_off(WHOLE_MODULE_PER_REQUEST_ENV)


# Un-overlapped whole upload vs one streamed step's transfer: 1.47x B200, 1.27-1.58x RTX PRO 6000 (klein-4B bf16).
_WHOLE_UPLOAD_FACTOR = 1.5
# Resident step time / probe-priced FLOPs: 1.38 B200, 0.97 RTX PRO 6000; 1.0 picks right on both, 1.35 does not.
_DIT_COMPUTE_EFFICIENCY = 1.0
_COST_CANVAS_PX = 1024
# 2x2 patches of an 8x VAE latent
_DIT_PIXELS_PER_TOKEN = 16

_OFFLOAD_COST_PROBES: dict = {}


def _offload_cost_measurements() -> Optional[tuple[float, float]]:
    """(pinned host-to-device GiB/s, bf16 GEMM TFLOPS) of the current CUDA device, cached per device; None off CUDA.
    Best of a few runs, so a transient stall on a shared card is not read as the card."""
    result = None
    key = None
    try:
        import torch

        # integrated GPU: the probe's pinned buffer only takes RAM from the card; unpriced keeps the whole-module plan
        if not torch.cuda.is_available() or is_rocm_torch(torch) or not host_pinning_allowed():
            return None
        key = torch.cuda.current_device()
        if key in _OFFLOAD_COST_PROBES:
            return _OFFLOAD_COST_PROBES[key]
        device = torch.device("cuda", key)
        with torch.cuda.device(device):
            nbytes = 256 << 20
            host = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)
            dev = torch.empty(nbytes, dtype = torch.uint8, device = device)
            a = torch.randn(4096, 3072, dtype = torch.bfloat16, device = device)
            b = torch.randn(3072, 3072, dtype = torch.bfloat16, device = device)
            start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)

            def _best_ms(fn, reps: int, inner: int) -> float:
                fn()
                torch.cuda.synchronize()
                best = float("inf")
                for _ in range(reps):
                    start.record()
                    for _ in range(inner):
                        fn()
                    end.record()
                    end.synchronize()
                    best = min(best, start.elapsed_time(end) / inner)
                return best

            copy_ms = _best_ms(lambda: dev.copy_(host, non_blocking = True), 3, 2)
            gemm_ms = _best_ms(lambda: torch.mm(a, b), 3, 10)
            del host, dev, a, b
            torch.cuda.empty_cache()
        if copy_ms > 0 and gemm_ms > 0:
            result = (
                (nbytes / 2**30) / (copy_ms / 1000.0),
                2.0 * 4096 * 3072 * 3072 / (gemm_ms / 1000.0) / 1e12,
            )
    except Exception:  # noqa: BLE001 - unmeasurable: the caller keeps its unpriced choice
        result = None
    if key is not None:
        _OFFLOAD_COST_PROBES[key] = result
    return result


def _whole_module_offload_costs(
    dit_mib: int,
    *,
    steps: int,
    h2d_gib_s: float,
    bf16_tflops: float,
    side_px: int = _COST_CANVAS_PX,
    tokens: Optional[int] = None,
) -> tuple[float, float]:
    """(whole-module, streamed) denoiser seconds for one ``steps``-step render.

    streamed = ``steps * max(compute, transfer)``; transfer counts the whole denoiser even when partly resident (B200
    klein step stayed 0.145-0.170 s at 0-6751 of 7393 MiB resident). whole = one un-overlapped upload + resident steps.
    """
    gib = max(0, int(dit_mib)) / 1024.0
    transfer = gib / max(float(h2d_gib_s), 1e-6)
    params = max(0, int(dit_mib)) * 2**20 / 2.0  # bf16
    if tokens is None:
        tokens = (int(side_px) // _DIT_PIXELS_PER_TOKEN) ** 2
    compute = (
        _DIT_COMPUTE_EFFICIENCY * 2.0 * params * tokens / (max(float(bf16_tflops), 1e-6) * 1e12)
    )
    n = max(1, int(steps))
    whole = _WHOLE_UPLOAD_FACTOR * transfer + n * compute
    streamed = n * max(compute, transfer)
    return whole, streamed


def prefer_whole_module_offload(
    pipe: Any,
    refined: MemoryPlan,
    *,
    logger: Any = None,
    steps: Optional[int] = None,
    family: Optional[str] = None,
) -> MemoryPlan:
    """``refined``, or whole-module offload in its place where it beats streaming. With ``steps`` the switch is
    priced from this card's measured bandwidth and bf16 throughput (RTX PRO 6000 streams faster at 4 steps, B200 does
    not); unmeasurable or without ``steps`` it is kept. Plain bf16 only: an fp16 T4 run pointed the other way.
    A ``_CALIBRATED_KEEPS_WHOLE_MODULE`` family's measured plan is priced as the streamed plan it replaced.
    """
    switch = (os.environ.get(WHOLE_MODULE_OFFLOAD_ENV) or "auto").strip().lower()
    if switch in ("0", "off", "false", "no", "never"):
        return refined
    try:
        if (
            getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO
            or refined.offload_policy != OFFLOAD_GROUP
            or getattr(refined, "secondary_ordinal", None) is not None
        ):
            return refined
        est = refined.estimates
        calibrated = bool(
            str(family or "") in _CALIBRATED_KEEPS_WHOLE_MODULE
            and est.get("calibrated_over_streamed_transformer")
        )
        if not calibrated and not bool(getattr(refined, "stream_transformer", True)):
            return refined
        model_dense, companions = est.get("model_dense_mib"), est.get("companion_dense_mib")
        floors = [
            int(f)
            for f in (est.get("group_floor_mib"), est.get("group_floor_streamed_te_mib"))
            if f is not None
        ]
        if model_dense is None or companions is None or not floors:
            return refined
        transformer = int(model_dense) - int(companions)
        te = max(0, int(est.get("text_encoder_dense_mib") or 0))
        others = max(0, int(companions) - te)
        floor = min(floors)
        if transformer <= 0 or max(transformer, te, others) >= floor:
            return refined
        if not _denoiser_is_dense_bf16(pipe):
            return refined
        # A refined plan pinning the whole denoiser runs every step resident: kept.
        sizes = _loaded_component_mib(pipe) or {}
        dit = sum(m for m, r in sizes.values() if r == "dit") or transformer
        if not calibrated and int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:
            return refined
        reason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)
        priced: dict = {}
        if (
            steps is not None
            and switch not in ("1", "on", "true", "yes", "always")
            and getattr(pipe, "transformer", None) is not None
        ):
            measured = _offload_cost_measurements()
            if measured is not None:
                h2d, tflops = measured
                whole, streamed = _whole_module_offload_costs(
                    dit, steps = int(steps), h2d_gib_s = h2d, bf16_tflops = tflops
                )
                fields = dict(
                    dit = dit,
                    h2d = h2d,
                    tflops = tflops,
                    model = whole,
                    group = streamed,
                    steps = int(steps),
                    side = _COST_CANVAS_PX,
                )
                priced = {
                    "whole_module_cost_s": round(whole, 4),
                    "streamed_cost_s": round(streamed, 4),
                    "whole_module_cost_steps": int(steps),
                    "measured_h2d_gib_s": round(h2d, 2),
                    "measured_bf16_tflops": round(tflops, 1),
                }
                if whole >= streamed:
                    declined = _WHOLE_MODULE_DECLINED_REASON.format(**fields)
                    if logger is not None:
                        logger.info("diffusion.memory: %s", declined)
                    if calibrated:
                        return refined
                    if per_request_whole_enabled():
                        try:
                            setattr(
                                pipe,
                                WHOLE_TIER_ATTR,
                                {"dit_mib": int(dit), "h2d_gib_s": h2d, "bf16_tflops": tflops},
                            )
                        except Exception:  # noqa: BLE001 - the load-time plan then stays for every request
                            pass
                    return replace(
                        refined,
                        reasons = refined.reasons + (declined,),
                        estimates = {**refined.estimates, **priced},
                    )
                reason += _WHOLE_MODULE_COST_REASON.format(**fields)
        if logger is not None:
            logger.info("diffusion.memory: %s", reason)
        kept = dict(refined.estimates)
        if calibrated:
            # the generate guard prices it as main's flat whole-module plan
            kept.pop("calibrated_headroom_mib", None)
        return replace(
            refined,
            offload_policy = OFFLOAD_MODEL,
            stream_text_encoders = False,
            stream_transformer = True,
            vae_tiling = True,
            vae_slicing = True,
            resident_transformer_mib = None,
            resident_text_encoder_mib = None,
            reasons = refined.reasons + (reason,),
            estimates = {**kept, "whole_module_kept_below_floor_mib": floor, **priced},
        )
    except Exception as exc:  # noqa: BLE001 - keep the streamed refinement
        if logger is not None:
            logger.debug("diffusion.memory: whole-module comparison skipped (%s)", exc)
        return refined


def whole_tier_request_costs(
    pipe: Any,
    *,
    steps: Optional[int],
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    cfg: Optional[bool] = None,
    condition_pixels: int = 0,
) -> Optional[tuple[float, float]]:
    """(whole, streamed) seconds for this request's own tokens (canvas, condition images, batch, CFG); None when the
    load did not record :data:`WHOLE_TIER_ATTR` or the per-request choice is off."""
    tier = getattr(pipe, WHOLE_TIER_ATTR, None)
    if not isinstance(tier, dict) or steps is None or not per_request_whole_enabled():
        return None
    try:
        w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
        h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
        per_image = (w // _DIT_PIXELS_PER_TOKEN) * (h // _DIT_PIXELS_PER_TOKEN) + max(
            0, int(condition_pixels or 0)
        ) // (_DIT_PIXELS_PER_TOKEN**2)
        tokens = per_image * max(1, int(batch_size or 1)) * (2 if cfg else 1)
        return _whole_module_offload_costs(
            int(tier["dit_mib"]),
            steps = int(steps),
            h2d_gib_s = float(tier["h2d_gib_s"]),
            bf16_tflops = float(tier["bf16_tflops"]),
            tokens = tokens,
        )
    except Exception:  # noqa: BLE001 - unpriced: the load's streamed plan
        return None


_MISSING = object()


def pin_denoiser_for_call(
    pipe: Any,
    device: Any,
    *,
    activation_mib: int,
    logger: Any = None,
) -> Optional[Callable[[], None]]:
    """Pin every streamed offload group of ``pipe.transformer`` for one call; returns the release, or None.

    The hooks stay and blocks compile below them, so compiled code never sees the pin. Pinned only if the groups plus
    ``activation_mib`` fit free memory now (VAE reserve kept); all or nothing."""
    transformer = getattr(pipe, "transformer", None)
    try:
        import torch

        groups = _offload_groups(transformer) if transformer is not None else []
        if not groups or any(getattr(g, "offload_to_disk_path", None) for g in groups):
            return None
        streamed = [g for g in groups if not getattr(g, "_unsloth_resident", False)]
        if not streamed:
            return None
        onload = torch.device(device if device is not None else "cuda")
        if onload.type != "cuda":
            return None
        if onload.index is None:
            onload = torch.device("cuda", torch.cuda.current_device())
        unique: dict = {}
        for group in streamed:
            for t in _resident_tensors(group):
                unique.setdefault(id(t), t)
        need = sum(sum(_storage_nbytes(t)) for t in unique.values())
        free, total = (int(x) for x in torch.cuda.mem_get_info(onload))
        free -= user_budget_reserve_bytes(onload, total)
        cached = max(
            0,
            int(torch.cuda.memory_reserved(onload))
            - int(torch.cuda.memory_allocated(onload))
            - _graph_held_cache_bytes(onload.index),
        )
        room = (free + cached) >> 20
        if (need >> 20) + max(0, int(activation_mib)) > room:
            if logger is not None:
                logger.info(
                    "diffusion.memory: the denoiser keeps streaming: running it whole needs %d MiB + %d MiB of "
                    "activations, %d MiB free",
                    need >> 20,
                    int(activation_mib),
                    room,
                )
            return None
        every: dict = {}
        for group in groups:
            for t in _resident_tensors(group):
                every.setdefault(id(t), t)
        whole_mib = (sum(sum(_storage_nbytes(t)) for t in every.values()) >> 20) + 2
        saved = {
            name: transformer.__dict__.get(name, _MISSING)
            for name in ("_unsloth_resident_room", "_unsloth_resident_device")
        }
        _keep_groups_resident(transformer, whole_mib, onload, logger)
        pinned = [g for g in streamed if getattr(g, "_unsloth_resident", False)]
        done = [False]

        def release() -> None:
            if done[0]:
                return
            done[0] = True
            state = getattr(transformer, "_unsloth_stream_state", None)
            if isinstance(state, dict):
                state["streamed"] = (
                    1  # a release that fails part way still waits on the copy stream
                )
            try:
                for group in reversed(pinned):
                    if getattr(group, "_unsloth_resident", False):
                        _release_group(group)
                        _release_group_range(transformer, group)
            finally:
                if isinstance(state, dict):
                    state["streamed"] = _streamed_group_count(groups)
                for name, value in saved.items():
                    try:
                        if value is _MISSING:
                            transformer.__dict__.pop(name, None)
                        else:
                            setattr(transformer, name, value)
                    except Exception:  # noqa: BLE001
                        pass
                from .diffusion_cuda_graph import synchronize_outside_capture

                synchronize_outside_capture(onload)

        if len(pinned) != len(streamed):
            release()
            return None
        return release
    except Exception as exc:  # noqa: BLE001 - streaming every group is the safe state
        if logger is not None:
            logger.warning("diffusion.memory: whole denoiser for this call skipped (%s)", exc)
        return None


def _refine_plan_from_loaded_weights(
    pipe: Any,
    plan: MemoryPlan,
    *,
    family: Optional[str],
    speed_mode: Optional[str],
    logger: Any = None,
    attention: Optional[str] = None,
    model: Optional[str] = None,
    vae: Optional[str] = None,
    te: Optional[str] = None,
    cache: Optional[str] = None,
    graphs: Optional[str] = None,
) -> MemoryPlan:
    """Re-place a streamed ``auto`` load from its LOADED weights and the family's measured activation peak.

    Keeps the flat plan's offload hooks and makes whole groups resident (denoiser first, then encoders) within the
    safe budget left after the measured peak x margin and the base overhead. No-op for explicit modes, non-CUDA / unified memory, unmeasured families (dense denoisers: the eager-tier table)
    or speed tiers and ``model`` plans. A bf16, GGUF / int8 packed or fp8-stored denoiser (no static table measured it)
    places only from a learned peak recorded for its exact build."""
    try:
        if getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO:
            return plan
        policy = plan.offload_policy
        if policy not in (OFFLOAD_GROUP, OFFLOAD_STREAMING):
            return plan
        memory = plan.device_memory
        if memory.is_unified or getattr(memory, "device", None) != "cuda":
            return plan
        budget = plan.estimates.get("safe_device_budget_mib")
        if budget is None:
            return plan
        budget = int(budget)
        sizes = _loaded_component_mib(pipe)
        if not sizes:
            return plan
        dit = sum(m for m, r in sizes.values() if r == "dit")
        encoders = sum(m for m, r in sizes.values() if r == "text_encoder")
        other = sum(m for m, r in sizes.values() if r == "other")
        if dit <= 0:
            return plan
        dense_mib = None if _pipe_denoisers_hold_torchao(pipe) else dit
        compute_bytes = _denoiser_compute_bytes(pipe)
        # Dense tables cover unpacked fp16 / fp32 / seeded bf16 only; stored fp8 widens per forward, never priced.
        packed = dense_mib is not None and _pipe_denoisers_hold_packed(pipe)
        fp8_stored = any(
            _holds_fp8_stored(getattr(pipe, name, None))
            for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
        )
        dense_bf16 = bool(
            dense_mib is not None
            and compute_bytes is None
            and not packed
            and not fp8_stored
            and dense_bf16_seeded(family)
            and _denoiser_is_dense_bf16(pipe)
            and bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))
        )
        static_seed = not (
            fp8_stored
            or (dense_mib is not None and ((compute_bytes is None and not dense_bf16) or packed))
        )
        small_host_int8 = bool(
            packed and not fp8_stored and compute_bytes is not None and pipe_small_host_int8(pipe)
        )
        quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)
        try:
            pipe._unsloth_peak_identity = (family, quant, speed_mode)
        except Exception:  # noqa: BLE001 - a pipe refusing attributes only skips recording
            pass
        read_quant, cfg_off_only = _learned_read_label(pipe, family, quant, speed_mode)
        headroom = measured_image_runtime_mib(
            family,
            speed_mode,
            dense_transformer_mib = dense_mib,
            compute_bytes = compute_bytes or 2,
            quant = read_quant,
            phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
            dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
            static_seed = static_seed,
            dense_bf16 = dense_bf16,
            small_host_int8 = small_host_int8,
        )
        if headroom is None:
            return plan
        from .diffusion_measured_peaks import lookup_bucket

        learned, learned_pixels, learned_quadratic = lookup_bucket(
            family = family, quant = read_quant, speed_mode = speed_mode
        ) or (None, None, False)
        overhead = int(plan.estimates.get("base_overhead_mib") or DEFAULT_BASE_OVERHEAD_MIB)
        floor = headroom + overhead + other
        estimates = dict(plan.estimates)
        estimates.update(
            measured_runtime_headroom_mib = headroom,
            measured_learned_peak_mib = learned,
            measured_learned_peak_pixels = learned_pixels,
            measured_learned_peak_quadratic = learned_quadratic,
            measured_dense_transformer_mib = dense_mib,
            measured_compute_bytes = compute_bytes,
            measured_static_seed = static_seed,
            measured_dense_bf16 = dense_bf16,
            measured_small_host_int8 = small_host_int8,
            measured_learned_cfg_off = bool(cfg_off_only and learned),
            # tier / phase seed measured on text-to-image: unpriced references size flat (measured_request_extra_mib)
            measured_omits_references = bool(
                getattr(pipe, "_unsloth_unpriced_references", False)
                and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES
                and (
                    (dense_mib is not None and getattr(pipe, "_unsloth_dense_tier_seeds", False))
                    or (dense_mib is None and getattr(pipe, "_unsloth_phase_seeds", False))
                )
            ),
            loaded_transformer_mib = dit,
            loaded_text_encoder_mib = encoders,
            loaded_other_mib = other,
        )
        if _env_off(PARTIAL_RESIDENT_ENV):
            return plan
        # Flat hooks stay installed; release_resident_groups lets an oversized request stream groups again.
        stream_te = bool(getattr(plan, "stream_text_encoders", False))
        # The whole-render peak includes the stream prefetch window, which an all-resident placement never allocates:
        # all-resident is checked against the unpadded peak, else encoders against the padded pre-decode phase. Extra
        # residency is lent to the decode; release_for_decode re-streams it only if free memory is short then.
        pre_headroom = resident_headroom = None
        split = None
        if stream_te and policy == OFFLOAD_GROUP:
            split = learned_phase_split(family, read_quant, speed_mode)
        if split is not None:
            from .diffusion_measured_peaks import PHASE_PRE_DECODE, PHASE_UNPADDED

            def _phase_headroom(phase: str) -> Optional[int]:
                value = measured_image_runtime_mib(
                    family,
                    speed_mode,
                    dense_transformer_mib = dense_mib,
                    compute_bytes = compute_bytes or 2,
                    quant = read_quant,
                    phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
                    dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
                    static_seed = static_seed,
                    phase = phase,
                    dense_bf16 = dense_bf16,
                )
                return value if value is not None and value < headroom else None

            pre_headroom = _phase_headroom(PHASE_PRE_DECODE)
            if not bool(getattr(plan, "stream_transformer", True)):
                resident_headroom = _phase_headroom(PHASE_UNPADDED)
        room = budget - floor
        if policy == OFFLOAD_GROUP and not stream_te:
            room -= encoders  # resident companions
        whole_dit = wide_dit = False
        if not bool(getattr(plan, "stream_transformer", True)):
            room -= dit
            dit_room = 0
        else:
            dit_room = min(max(room, 0), dit)
            if (
                dit_room < dit
                # torchao denoisers only: the slack was measured on the int8 route, not the dense eager table
                and dense_mib is None
                and (policy == OFFLOAD_STREAMING or stream_te)
                and _resident_dit_fits(memory, dit, headroom, other)
            ):
                # pin it whole; during the encode it drops back to the flat room (install_encode_release)
                encode_room = int(dit_room)
                dit_room, room, whole_dit = dit, dit, True
            elif (
                dit_room < dit and dense_mib is None and (policy == OFFLOAD_STREAMING or stream_te)
            ):
                # partial: the encode keeps the flat room (install_encode_release)
                window = _stream_window_mib(pipe)
                streamed_headroom = headroom
                if static_seed and not small_host_int8:
                    streamed_headroom = min(
                        headroom,
                        measured_image_runtime_mib(
                            family,
                            speed_mode,
                            compute_bytes = compute_bytes or 2,
                            quant = read_quant,
                            phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
                            static_seed = static_seed,
                            streamed = True,
                        )
                        or headroom,
                    )
                streamed_room = min(
                    _streamed_dit_room_mib(memory, streamed_headroom, other, window), dit
                )
                phase_reserve = None
                if window >= 0:
                    phase_reserve = _denoise_phase_reserve(
                        pipe,
                        family = family,
                        speed_mode = speed_mode,
                        quant = read_quant,
                        static_seed = static_seed and not small_host_int8,
                        window_mib = window,
                        tiled = "tiled" in planned_vae_saver_tag(pipe, plan),
                    )
                    top = _top_level_group_mib(pipe) if phase_reserve is not None else None
                    if top is None:
                        phase_reserve = None
                    if phase_reserve is not None:
                        phase_room = min(
                            _streamed_dit_room_mib(memory, phase_reserve[0], other, window), dit
                        )
                        if (
                            phase_reserve[0] >= streamed_headroom
                            or phase_room <= streamed_room
                            # the top-level group goes resident first; one that misses is onloaded whole for every
                            # forward beside the block window, which the denoise seed (measured resident) omits
                            or phase_room < top
                        ):
                            phase_reserve = None
                        else:
                            streamed_room = phase_room
                if streamed_room > dit_room:
                    encode_room = int(dit_room)
                    dit_room, room, wide_dit = streamed_room, streamed_room, True
                    estimates["stream_window_mib"] = window
                    if streamed_headroom < headroom:
                        estimates["measured_streamed_headroom_mib"] = streamed_headroom
                        # seed covers denoise only; flat encode room on top overran at 16 GiB
                        encode_room = 0
                    if phase_reserve is not None:
                        decode_peak, decode_pixels, decode_quadratic = phase_reserve[1]
                        # the encode keeps the flat room, the decode streams the shortfall again
                        estimates.update(
                            measured_denoise_phase_headroom_mib = phase_reserve[0],
                            decode_lend_transformer_mib = int(streamed_room),
                            measured_decode_peak_mib = decode_peak,
                            measured_decode_peak_pixels = decode_pixels,
                            measured_decode_peak_quadratic = decode_quadratic,
                        )
        te_room = (
            min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0
        )
        lend = 0
        phase_headroom = None
        if split is not None and not whole_dit and not wide_dit:
            phase_te_room = te_room
            if (
                resident_headroom is not None
                and room + (headroom - resident_headroom) - dit_room >= encoders
            ):
                phase_headroom, phase_te_room = resident_headroom, encoders
            elif pre_headroom is not None:
                phase_headroom = pre_headroom
                phase_te_room = min(max(0, room + (headroom - pre_headroom) - dit_room), encoders)
            lend = max(0, phase_te_room - te_room)
            if lend > 0:
                from .diffusion_measured_peaks import PHASE_DECODE_UNPADDED
                te_room = phase_te_room
                estimates.update(
                    measured_phase_headroom_mib = phase_headroom,
                    measured_phase_all_resident = phase_headroom == resident_headroom,
                    measured_decode_peak_mib = split[PHASE_DECODE_UNPADDED],
                    measured_decode_peak_pixels = split["pixels"],
                    measured_decode_peak_quadratic = split["quadratic"],
                    decode_lend_text_encoder_mib = lend,
                )
        reference_cut = 0
        if (
            policy == OFFLOAD_GROUP
            and stream_te
            and not whole_dit
            and not wide_dit
            and (not bool(getattr(plan, "stream_transformer", True)) or dit_room >= dit)
        ):
            # The resident transformer must also hold an edit's denoise, else the first edit streams it mid-render.
            reference = _load_reference_need(
                pipe,
                family = family,
                speed_mode = speed_mode,
                quant = read_quant,
                static_seed = static_seed,
            )
            if reference is not None:
                basis = phase_headroom if lend > 0 and phase_headroom is not None else headroom
                spare = max(
                    0, int(room) + int(headroom) - int(basis) - int(dit_room) - int(te_room)
                )
                short = max(0, int(reference[1]) - int(basis) - spare)
                # Cut the encoder only when a render-time release of it could not cover the reference.
                reference_cut = int(te_room) if short > int(te_room) else 0
                te_room -= reference_cut
                if lend > 0:
                    lend = max(0, lend - reference_cut)
                    estimates["decode_lend_text_encoder_mib"] = lend
                estimates.update(
                    measured_reference_phase_mib = reference[0],
                    measured_reference_need_mib = reference[1],
                    measured_reference_reserve_mib = int(basis) + spare + reference_cut,
                    measured_reference_te_cut_mib = reference_cut,
                )
        if dit_room <= 0 and te_room <= 0 and not reference_cut:
            return plan
        if whole_dit or wide_dit:
            estimates["resident_dit_slack_mib"] = _resident_dit_slack_mib(memory)
            estimates["encode_resident_transformer_mib"] = encode_room
        new = replace(
            plan,
            resident_transformer_mib = int(dit_room) if dit_room > 0 else None,
            resident_text_encoder_mib = int(te_room) if te_room > 0 else None,
            estimates = {
                **estimates,
                "resident_transformer_mib": int(dit_room),
                "resident_text_encoder_mib": int(te_room),
            },
            reasons = plan.reasons
            + (
                (
                    f"the {dit} MiB transformer is resident (not streamed by the plan)"
                    if not bool(getattr(plan, "stream_transformer", True))
                    else f"{int(dit_room)} MiB of the {dit} MiB transformer"
                )
                + f" and {int(te_room)} MiB of the {encoders} MiB "
                f"encoders stay resident (measured peak {headroom} MiB"
                + (
                    (
                        f"; {phase_headroom} MiB without the streaming window, every weight resident"
                        if phase_headroom == resident_headroom
                        else f"; {phase_headroom} MiB before the VAE decode"
                    )
                    + f", the decode may stream {lend} MiB of them again"
                    if lend > 0
                    else ""
                )
                + "); the rest streams"
                + (
                    f" ({reference_cut} MiB of the encoders stream instead so the transformer also holds a "
                    f"{estimates['measured_reference_need_mib']} MiB edit denoise)"
                    if reference_cut
                    else ""
                )
                + (
                    f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"
                    f" {encode_room} MiB of it while the encoders run)"
                    if whole_dit
                    else f" (free memory less a {_resident_dit_slack_mib(memory)} MiB slack and a "
                    f"{estimates.get('stream_window_mib')} MiB prefetch window"
                    + (
                        f", sized from the {estimates['measured_denoise_phase_headroom_mib']} MiB denoise phase"
                        " (the decode may stream part of it again)"
                        if estimates.get("measured_denoise_phase_headroom_mib") is not None
                        else ""
                    )
                    + f"; {encode_room} MiB of it while the encoders run)"
                    if wide_dit
                    else ""
                ),
            ),
        )
        if logger is not None:
            logger.info(
                "diffusion.memory: measured-activation placement %s -> %s (%s)",
                policy,
                new.offload_policy,
                new.reasons[-1],
            )
        return new
    except Exception as exc:  # noqa: BLE001 - keep the flat plan
        if logger is not None:
            logger.debug("diffusion.memory: measured-activation placement skipped (%s)", exc)
        return plan


def _placed_on(tensor: Any, device_type: str, is_torchao: Callable[[Any], bool]) -> bool:
    """torchao subclasses are judged by inner tensors: after a streamed offload the wrapper can report the wrong device."""
    if is_torchao(tensor):
        try:
            names, _ = tensor.__tensor_flatten__()
            inner = [getattr(tensor, n) for n in names]
            return all(getattr(t, "device", tensor.device).type == device_type for t in inner)
        except Exception:  # noqa: BLE001 - unflattenable: move it, the swap is idempotent
            return False
    return tensor.device.type == device_type


def _keep_groups_resident(
    module: Any,
    room_mib: int,
    device: Any,
    logger: Any = None,
    only: Optional[set] = None,
) -> int:
    """Make whole offload groups of a streamed ``module`` resident within ``room_mib`` (top-level group first, then
    blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept.
    ``only`` (group ids): a restore pins back what its own release streamed, not an enclosing release's groups."""
    if room_mib is None or int(room_mib) <= 0 or _env_off(PARTIAL_RESIDENT_ENV):
        return 0
    try:
        import torch
        from diffusers.hooks import group_offloading as go

        groups = _offload_groups(module)
        if not groups:
            return 0
        top = [g for g in groups if getattr(g, "offload_leader", None) is module]
        ordered = top + [g for g in groups if getattr(g, "offload_leader", None) is not module]
        is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)
        onload = torch.device(device)
        left = int(room_mib) * 1024 * 1024
        kept = 0
        try:
            from .diffusion_stable_arena import pin_group_in_range, stable_repin_wanted
            stable = stable_repin_wanted(module)
        except Exception:  # noqa: BLE001 - re-pins move the stock way
            stable = False
        _tensors = _resident_tensors
        disable = getattr(getattr(torch, "compiler", None), "disable", None)

        def _noop(*args: Any, **kwargs: Any) -> None:
            return None

        # Groups of this module still on the copy stream; with none, nothing can be in flight to wait for.
        state = getattr(module, "_unsloth_stream_state", None)
        if not isinstance(state, dict):
            state = {"streamed": 1}
            try:
                module._unsloth_stream_state = state
            except AttributeError:
                pass

        def _resident_onload(stream: Any) -> Callable[[], None]:
            # a prefetching predecessor skips its own copy-stream wait and relies on this onload_ to do it
            def onload_(*args: Any, **kwargs: Any) -> None:
                # event-fenced prefetch: streamed groups wait on their own copy; a resident block may start the forward's fill
                kick = state.get("kick")
                if callable(kick):
                    kick()
                if stream is not None and state["streamed"] and not state.get("fenced"):
                    stream.synchronize()

            return disable(onload_) if callable(disable) else onload_

        noop = disable(_noop) if callable(disable) else _noop

        try:
            module._unsloth_resident_room = int(room_mib)
            module._unsloth_resident_device = onload
        except AttributeError:
            pass
        # Tied weights (an LM head sharing the embedding, as in Qwen3 text encoders) sit in two groups. A resident
        # group next to a streamed partner breaks: the partner's offload points the shared tensor back at its host
        # copy and the resident group's no-op onload never brings it back. Such groups go resident together or not.
        tied = _tied_group_closures(ordered, _tensors)
        done: set = set()
        for group in ordered:
            if getattr(group, "offload_to_disk_path", None):
                break
            if id(group) in done:
                continue
            closure = tied.get(id(group)) or [group]
            done.update(id(g) for g in closure)
            unique: dict = {}
            for member in closure:
                for t in _tensors(member):
                    unique.setdefault(id(t), t)
            tensors = list(unique.values())
            need = sum(sum(_storage_nbytes(t)) for t in tensors)
            if all(getattr(g, "_unsloth_resident", False) for g in closure):
                left -= need
                continue
            if (
                need > left
                or any(getattr(g, "offload_to_disk_path", None) for g in closure)
                or (only is not None and any(id(g) not in only for g in closure))
            ):
                # too large (a later, smaller group may still fit), or another release's group
                continue
            for member in closure:
                cpu = getattr(member, "cpu_param_dict", None) or {}
                members_tensors = _tensors(member)
                placed: set = set()
                if stable and onload.type == "cuda":
                    # The step graph recorded the resident groups' addresses: a re-pin lands where the last one did.
                    try:
                        placed = pin_group_in_range(
                            module,
                            member,
                            members_tensors,
                            lambda t: not _placed_on(t, onload.type, is_torchao),
                            lambda t, cpu = cpu: cpu.get(t, t),
                            onload,
                            go._swap_torchao_tensor,
                        )
                    except Exception as exc:  # noqa: BLE001 - the stock move still places the group
                        placed = set()
                        # Unmap an empty range so the stock move below does not pay for a second copy.
                        try:
                            from .diffusion_stable_arena import release_group_range
                            release_group_range(module, member, members_tensors)
                        except Exception:  # noqa: BLE001
                            pass
                        if logger is not None:
                            logger.warning(
                                "diffusion.memory: stable re-pin range unavailable (%s); the group moves the stock way",
                                exc,
                            )
                for t in members_tensors:
                    if id(t) in placed or _placed_on(t, onload.type, is_torchao):
                        continue
                    moved = cpu.get(t, t).to(onload)
                    if is_torchao(t):
                        go._swap_torchao_tensor(t, moved)
                    else:
                        t.data = moved
            for i, member in enumerate(closure):
                if getattr(member, "_unsloth_resident", False):
                    continue
                # host copies stay: release_resident_groups streams the group again for an oversized request
                member._unsloth_streamed_hooks = (
                    member.__dict__.get("onload_"),
                    member.__dict__.get("offload_"),
                )
                member.onload_ = _resident_onload(getattr(member, "stream", None))
                member.offload_ = noop
                member._unsloth_resident = True
                member._unsloth_resident_bytes = need if i == 0 else 0
                member._unsloth_tied_groups = [g for g in closure if g is not member]
            left -= need
            kept += need
        state["streamed"] = _streamed_group_count(ordered)
        if kept:
            _bump_placement_epoch()
        if kept and onload.type == "cuda":
            from .diffusion_cuda_graph import synchronize_outside_capture
            synchronize_outside_capture(onload)
        if logger is not None and kept:
            logger.info(
                "diffusion.memory: %s keeps %d MiB resident (%d of %d offload groups); the rest streams",
                type(module).__name__,
                kept >> 20,
                sum(1 for g in ordered if getattr(g, "_unsloth_resident", False)),
                len(ordered),
            )
        return kept >> 20
    except Exception as exc:  # noqa: BLE001 - streaming every group is the safe state
        if logger is not None:
            logger.warning("diffusion.memory: partial residency skipped (%s)", exc)
        return 0


def _tied_group_closures(groups: list, tensors_of: Callable[[Any], list]) -> dict:
    """id(group) -> every group that shares a tensor with it, transitively (only for groups that share any)."""
    owners: dict[int, list] = {}
    for group in groups:
        for t in tensors_of(group):
            owners.setdefault(id(t), []).append(group)
    closures: dict = {}
    for group in groups:
        if id(group) in closures:
            continue
        stack, seen = [group], {id(group): group}
        while stack:
            current = stack.pop()
            for t in tensors_of(current):
                for other in owners.get(id(t), ()):
                    if id(other) not in seen:
                        seen[id(other)] = other
                        stack.append(other)
        if len(seen) > 1:
            members = [g for g in groups if id(g) in seen]
            for g in members:
                closures[id(g)] = members
    return closures


def _bump_placement_epoch() -> None:
    try:
        from .diffusion_offload_prefetch import bump_placement_epoch
        bump_placement_epoch()
    except Exception:  # noqa: BLE001 - no graph layer to invalidate
        pass


def _streamed_group_count(groups: list) -> int:
    return sum(
        1
        for g in groups
        if getattr(g, "stream", None) is not None and not getattr(g, "_unsloth_resident", False)
    )


def _resident_tensors(group: Any) -> list:
    """Every tensor of an offload group, each once (its modules' parameters, their buffers, then the group's own)."""
    out: list = []
    seen: set[int] = set()
    for t in (
        [p for m in group.modules for p in m.parameters()]
        + [b for m in group.modules for b in m.buffers()]
        + list(group.parameters or [])
        + list(group.buffers or [])
    ):
        if id(t) not in seen:
            seen.add(id(t))
            out.append(t)
    return out


def _release_group(group: Any) -> None:
    # resident encoder groups with a dropped host copy get it back first
    rebuild_dropped_host_copy(group)
    onload_, offload_ = getattr(group, "_unsloth_streamed_hooks", (None, None))
    for name, fn in (("onload_", onload_), ("offload_", offload_)):
        if fn is not None:
            setattr(group, name, fn)
        else:
            group.__dict__.pop(name, None)
    group._unsloth_resident = False
    group.offload_()
    _bump_placement_epoch()


def _release_group_range(owner: Any, group: Any) -> None:
    """A released group's weights are on their host copies again: its stable re-pin range gives the physical memory
    back (the addresses stay reserved)."""
    if "_unsloth_repin_range" in getattr(group, "__dict__", {}):
        from .diffusion_stable_arena import release_group_range
        release_group_range(owner, group, _resident_tensors(group))


def release_resident_groups(
    pipe: Any,
    need_mib: int,
    logger: Any = None,
    denoisers_only: bool = False,
    reason: str = "an oversized request",
    device: Any = None,
) -> Optional[Callable[[], None]]:
    """Stream resident offload groups again until ``need_mib`` is freed (text encoders first, then the denoiser's
    blocks from the last); returns a callable pinning exactly those groups again, or None when nothing was resident.
    A request larger than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as
    the flat plan. ``denoisers_only`` leaves the text encoders' groups alone (the prompt encode needs them).
    ``device``: only modules resident on that card."""
    released: list = []
    streamed: set = set()

    def restore(strict: bool = False) -> None:
        """Pin the released groups again. ``strict``: raise when any of them is still not resident afterwards (the
        re-pin catches its own failures and leaves those groups streamed, which callers that must retry need to
        know)."""
        for module in released:
            _keep_groups_resident(
                module,
                module._unsloth_resident_room,
                module._unsloth_resident_device,
                logger,
                only = streamed,
            )
        if strict:
            left = [
                g
                for module in released
                for g in (_offload_groups(module) or [])
                if id(g) in streamed and not getattr(g, "_unsloth_resident", False)
            ]
            if left:
                raise RuntimeError(f"re-pinning resident offload groups left {len(left)} streamed")

    try:
        import torch

        modules = [
            m
            for m in (getattr(pipe, "components", {}) or {}).values()
            if getattr(m, "_unsloth_resident_room", None)
            and not (denoisers_only and _is_text_encoder_module(pipe, m))
        ]
        if device is not None:
            from .diffusion_vae_oom import _same_card
            modules = [
                m
                for m in modules
                if _same_card(getattr(m, "_unsloth_resident_device", None) or device, device)
            ]
        if not modules or int(need_mib) <= 0:
            return None
        modules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)
        left = int(need_mib) * 1024 * 1024
        # Groups the load kept no host copy of get theirs back in one batch (one slab, one sync) before any streams.
        pending: list = []
        budget = left
        for module in modules:
            for group in reversed(_offload_groups(module) or []):
                if budget <= 0:
                    break
                if not getattr(group, "_unsloth_resident", False):
                    continue
                for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:
                    if getattr(member, "_unsloth_resident", False) and member not in pending:
                        pending.append(member)
                        budget -= int(getattr(member, "_unsloth_resident_bytes", 0))
        try:
            rebuild_dropped_host_copies(pending)
        except Exception as exc:  # noqa: BLE001 - a group still without a host copy stays resident below
            if logger is not None:
                logger.warning(
                    "diffusion.memory: copying resident groups back to the host failed (%s)", exc
                )
        for module in modules:
            for group in reversed(_offload_groups(module) or []):
                if left <= 0:
                    break
                if not getattr(group, "_unsloth_resident", False):
                    continue
                if any(
                    getattr(m, _HOST_DROPPED_ATTR, None)
                    for m in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]
                ):
                    continue  # no host copy to stream from: it stays resident, later groups may still free room
                # Before demoting: a release that fails part way must still wait on the copy stream.
                state = getattr(module, "_unsloth_stream_state", None)
                if isinstance(state, dict):
                    state["streamed"] = 1
                if module not in released:
                    released.append(module)
                # tied partners stream together (a resident partner would keep reading the host copy)
                for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:
                    if not getattr(member, "_unsloth_resident", False):
                        continue
                    # Recorded before demoting: a release that fails part way is still re-pinned by ``restore``.
                    streamed.add(id(member))
                    _release_group(member)
                    _release_group_range(module, member)
                    left -= int(getattr(member, "_unsloth_resident_bytes", 0))
        if not released:
            return None
        for module in released:
            state = getattr(module, "_unsloth_stream_state", None)
            if isinstance(state, dict):
                state["streamed"] = _streamed_group_count(_offload_groups(module))
        device = getattr(released[0], "_unsloth_resident_device", None)
        if device is not None and torch.device(device).type == "cuda":
            from .diffusion_cuda_graph import synchronize_outside_capture
            synchronize_outside_capture(device)
        if logger is not None:
            logger.info(
                "diffusion.memory: streaming %d MiB of resident groups for %s",
                (int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,
                reason,
            )
        restore.released_mib = (int(need_mib) * 1024 * 1024 - left) >> 20  # take_deferred_restore
        return restore
    except Exception as exc:  # noqa: BLE001 - the guard still refuses what cannot fit
        if logger is not None:
            logger.warning("diffusion.memory: releasing resident groups failed (%s)", exc)
        # groups released before the failure stay streamed until the caller re-pins them: hand back their restore
        return restore if released else None


# Restoring after every oversized request costs a host-to-device round trip per render; wait for one that needs them.
DEFERRED_RESTORE_ATTR = "_unsloth_deferred_resident_restore"


def _phase_placement(pipe: Any) -> bool:
    reserve = getattr(pipe, "_unsloth_measured_reserve", None) or ()
    return bool(
        len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX
        and reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]
        and denoise_phase_residency_enabled()
    )


def chained_restore(first: Callable[[], None], second: Callable[[], None]) -> Callable[[], None]:
    def restore() -> None:
        try:
            first()
        finally:
            second()

    restore.released_mib = int(getattr(first, "released_mib", 0) or 0) + int(
        getattr(second, "released_mib", 0) or 0
    )
    return restore


def defer_resident_restore(pipe: Any, restore: Optional[Callable[[], None]]) -> bool:
    """Hold ``restore`` (an oversized request's ``release_resident_groups``) on ``pipe`` instead of running it, for a
    placement sized from the denoise phase. True when held; False: the caller restores now."""
    if restore is None or not _phase_placement(pipe):
        return False
    try:
        held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)
        setattr(
            pipe, DEFERRED_RESTORE_ATTR, restore if held is None else chained_restore(held, restore)
        )
    except Exception:  # noqa: BLE001 - unheld: restore now
        return False
    return True


# A held restore carries on while the next request's own release is at least this close to what it streamed out.
_DEFER_TOLERANCE_MIB = 256


def _unused_room_mib(pipe: Any) -> int:
    """MiB of the placement's resident rooms its whole groups left unused (free memory a request may count on)."""
    room = placed = 0
    for module in (getattr(pipe, "components", {}) or {}).values():
        if not getattr(module, "_unsloth_resident_room", None):
            continue
        room += int(module._unsloth_resident_room)
        placed += sum(
            int(getattr(g, "_unsloth_resident_bytes", 0) or 0)
            for g in (_offload_groups(module) or ())
        )
    return max(0, room - (placed >> 20))


def take_deferred_restore(pipe: Any, extra_mib: int) -> tuple[Optional[Callable[[], None]], int]:
    """At a request's guard: ``(restore to carry on, MiB the request must still release)``. For a placement sized from
    the denoise phase the request's MiB (priced against its room) first counts the room its whole groups left unused.
    A request releasing about as much as a held restore streamed out (or more) keeps those groups streamed and
    releases only the rest; any other request (one the placement was sized for, or a much smaller release) pins them
    back first."""
    extra = int(extra_mib or 0)
    if extra > 0 and _phase_placement(pipe):
        extra = max(0, extra - _unused_room_mib(pipe))
    held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)
    if held is None:
        return None, extra
    try:
        setattr(pipe, DEFERRED_RESTORE_ATTR, None)
    except Exception:  # noqa: BLE001
        pass
    streamed = int(getattr(held, "released_mib", 0) or 0)
    if extra > 0 and extra + _DEFER_TOLERANCE_MIB >= streamed:
        return held, max(0, extra - streamed)
    held()
    return None, extra


DECODE_LEND_ATTR = "_unsloth_decode_lend"
# Kept free beside the decode's learned peak x margin when deciding whether lent encoder groups must stream again.
_DECODE_SLACK_MIB = 256


def decode_lend_from_plan(plan: Any) -> Optional[tuple[int, int, int, bool]]:
    """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency, or denoiser
    residency sized from the denoise phase, to the decode."""
    estimates = getattr(plan, "estimates", None) or {}
    lend = estimates.get("decode_lend_text_encoder_mib")
    if not lend or not getattr(plan, "resident_text_encoder_mib", None):
        lend = estimates.get("decode_lend_transformer_mib")
        if not lend or not getattr(plan, "resident_transformer_mib", None):
            return None
    return (
        int(lend),
        int(estimates.get("measured_decode_peak_mib") or 0),
        int(
            estimates.get("measured_decode_peak_pixels")
            or DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
        ),
        bool(estimates.get("measured_decode_peak_quadratic")),
    )


def decode_need_mib(
    lend: Any,
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
) -> int:
    """MiB the decode of a ``width x height x batch`` request needs above the allocation at its start (margin
    included), scaled from the learned decode peak the way the store scales (``diffusion_measured_peaks``)."""
    from .diffusion_measured_peaks import resolution_bucket

    _, decode_mib, covered, quadratic = lend
    w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
    h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
    batch = max(1, int(batch_size or 1))
    covered = max(1, int(covered))
    scale = 1.0
    if quadratic:
        if w * h > covered:
            scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2
        scale *= batch
    elif w * h * batch > covered:
        scale = resolution_bucket(w, h, batch) / float(covered)
    return int(math.ceil(float(decode_mib) * scale * _MEASURED_PEAK_MARGIN))


def release_for_decode(
    pipe: Any,
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    device: Any = None,
    logger: Any = None,
) -> Optional[Callable[[], None]]:
    """If free memory (driver + the allocator cache outside CUDA graph pools) is short of the learned decode peak x margin, stream the shortfall of
    lent groups again (text encoders first) and return the restore; else None. Mirrors ComfyUI's
    ``load_models_gpu(memory_required=...)``. The decode is untouched, so output is bit-identical."""
    lend = getattr(pipe, DECODE_LEND_ATTR, None)
    if not lend:
        return None
    try:
        import torch

        dev = torch.device(device if device is not None else "cuda")
        if dev.type != "cuda":
            return None
        if dev.index is None:
            dev = torch.device("cuda", torch.cuda.current_device())
        need = decode_need_mib(lend, width = width, height = height, batch_size = batch_size)
        free, total = (int(x) for x in torch.cuda.mem_get_info(dev))
        # keep the VRAM Budget reserve free, like every other decode-time check
        free -= user_budget_reserve_bytes(dev, total)
        # only the cache an ordinary decode allocation can reuse: a CUDA graph pool's free blocks are not
        cached = max(
            0,
            int(torch.cuda.memory_reserved(dev))
            - int(torch.cuda.memory_allocated(dev))
            - _graph_held_cache_bytes(dev.index),
        )
        short = need + _DECODE_SLACK_MIB - ((free + cached) >> 20)
    except Exception as exc:  # noqa: BLE001 - unreadable: the decode's OOM retry still frees the card
        if logger is not None:
            logger.debug("diffusion.memory: decode residency check skipped (%s)", exc)
        return None
    if short <= 0:
        return None
    return release_resident_groups(pipe, short, logger, reason = "the VAE decode", device = dev)


def resident_group_mib(pipe: Any) -> int:
    """MiB an oversized request could release: every resident offload group of the pipeline's components."""
    total = 0
    for module in (getattr(pipe, "components", {}) or {}).values():
        if not getattr(module, "_unsloth_resident_room", None):
            continue
        for group in _offload_groups(module) or []:
            if getattr(group, "_unsloth_resident", False):
                total += int(getattr(group, "_unsloth_resident_bytes", 0))
    return total >> 20


def _any_layer_split(pipe: Any) -> bool:
    from .diffusion_layer_split import is_layer_split
    from .diffusion_multigpu import DENOISER_ATTRS
    return any(is_layer_split(getattr(pipe, name, None)) for name in DENOISER_ATTRS)


def hook_resident_denoiser(
    pipe: Any,
    device: Any,
    logger: Any = None,
    resident: bool = True,
) -> bool:
    """Give a hookless resident torchao transformer block-level offload hooks, every group kept resident, so
    release_resident_groups can stream part of it. On demand, not at load: hooks at load recompile every load's second
    render. True when installed. ``resident=False``: every group streams (the load's flat plan), for a transformer
    that cannot come back to the card whole. A failure whose own return to the card fails raises, the transformer
    split between host and card with no hooks (return_torchao_denoiser / hook_resident_denoiser(resident=False) are
    the ways out)."""
    transformer = getattr(pipe, "transformer", None)
    if (
        transformer is None
        or _offload_groups(transformer)
        or not _pipe_denoisers_hold_torchao(pipe)
        or _any_layer_split(pipe)
    ):
        return False
    import torch

    onload = torch.device(device)
    if onload.type != "cuda":
        return False
    dit_mib = _module_pin_mib(transformer)
    # the load's _torchao_stream_pinnable rule: unpinned, the apply copies the whole transformer to pageable host RAM
    if not _streamed_pin_plan(dit_mib, 0, device = onload)[0]:
        if logger is not None:
            logger.info(
                "diffusion.memory: the resident transformer keeps no offload hooks (%d MiB cannot be pinned)",
                dit_mib,
            )
        return False
    torch_hooks = _module_forward_hook_ids(transformer)
    try:
        import inspect

        from diffusers.hooks import apply_group_offloading

        install_group_offload_buffer_restore()
        install_group_offload_hooks_eager()
        gkwargs: dict[str, Any] = {
            "onload_device": onload,
            "offload_device": torch.device("cpu"),
            "offload_type": "block_level",
            "num_blocks_per_group": DEFAULT_GROUP_BLOCKS,
            "use_stream": True,
        }
        params = inspect.signature(apply_group_offloading).parameters
        for name in ("non_blocking", "record_stream"):
            if name in params:
                gkwargs[name] = True
        if "low_cpu_mem_usage" in params:
            gkwargs["low_cpu_mem_usage"] = False
        pinned_mib = [0]
        _apply_group_offloading_packed(
            apply_group_offloading,
            transformer,
            _torchao_group_offload_kwargs(transformer, gkwargs, pinned_mib),
            logger,
        )
        if not _pin_top_level_group(transformer, logger, pinned_mib, device = onload):
            _skip_top_level_copy_back(transformer, logger)
        if gkwargs.get("record_stream"):
            install_group_prefetch(transformer, onload, logger)
        kept = 0
        if resident:
            kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)
            if not all(
                getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)
            ):
                raise RuntimeError("not every offload group stayed resident")
    except Exception as exc:  # noqa: BLE001 - the guard still refuses what cannot fit
        # the apply already moved the weights to their host copies: back to the card, as before the call (a return
        # that itself fails raises: the caller owns the way back)
        _unhook_and_return(transformer, onload, torch_hooks)
        if logger is not None:
            logger.warning("diffusion.memory: resident transformer left without hooks (%s)", exc)
        return False
    if logger is not None:
        logger.info(
            "diffusion.memory: the resident transformer takes offload hooks (%d MiB kept resident) so this request "
            "can stream part of it",
            kept,
        )
    return True


_FORWARD_HOOK_DICTS = (
    "_forward_pre_hooks",
    "_forward_pre_hooks_with_kwargs",
    "_forward_hooks",
    "_forward_hooks_with_kwargs",
    "_forward_hooks_always_called",
)


def _module_forward_hook_ids(module: Any) -> dict[str, set]:
    return {name: set(getattr(module, name, None) or ()) for name in _FORWARD_HOOK_DICTS}


def _unhook_and_return(transformer: Any, onload: Any, torch_hooks: Optional[dict]) -> None:
    _remove_group_offload_hooks(transformer)
    if torch_hooks is not None:
        _drop_module_forward_hooks_since(transformer, torch_hooks)
    from .diffusion_offload_prefetch import PREFETCHER_ATTR

    for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):
        transformer.__dict__.pop(attr, None)
    _return_module_tensors(transformer, onload)


def return_torchao_denoiser(
    pipe: Any,
    device: Any,
    logger: Any = None,
    torch_hooks: Optional[dict] = None,
) -> None:
    """The way back to where a hookless resident torchao transformer started, from wherever a failed
    hook_resident_denoiser left it: streamed under offload hooks (settled) -> every group resident again, the state
    a successful hook_resident_denoiser leaves; split with no hooks -> whole on the card. Raises if it is not there."""
    import torch

    transformer = getattr(pipe, "transformer", None)
    if transformer is None:
        return
    onload = torch.device(device)
    if _offload_groups(transformer):
        _keep_groups_resident(transformer, _module_host_mib(transformer) + 1, onload, logger)
        if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):
            raise RuntimeError("the transformer's offload groups are not all resident again")
        return
    _unhook_and_return(transformer, onload, torch_hooks)


def module_forward_hook_ids(module: Any) -> dict[str, set]:
    return _module_forward_hook_ids(module)


def _drop_module_forward_hooks_since(module: Any, before: dict[str, set]) -> None:
    """Remove the torch forward hooks registered on ``module`` after ``before`` was taken (the group prefetcher's)."""
    for name, ids in before.items():
        hooks = getattr(module, name, None)
        if hooks is not None:
            for key in set(hooks) - ids:
                hooks.pop(key, None)


def _return_module_tensors(module: Any, onload: Any) -> None:
    """Move ``module``'s weights back to ``onload`` after its hooks are gone. A torchao weight on its host copy can
    still report the card (``Module.to`` skips or fails on it), so its inner tensors move under a new wrapper."""
    import torch
    from diffusers.hooks import group_offloading as go

    is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)
    for t in [*module.parameters(), *module.buffers()]:
        if _placed_on(t, onload.type, is_torchao):
            continue
        if is_torchao(t):
            names, ctx = t.__tensor_flatten__()
            moved = type(t).__tensor_unflatten__(
                {name: getattr(t, name).to(onload) for name in names}, ctx, t.size(), t.stride()
            )
            go._swap_torchao_tensor(t, moved)
        else:
            t.data = t.data.to(onload)
    if onload.type == "cuda":
        from .diffusion_cuda_graph import synchronize_outside_capture
        synchronize_outside_capture(onload)


def _unwind_group_offload(
    pipe: Any,
    snapshots: list,
    logger: Any = None,
) -> bool:
    """Undo a partial ``_apply_group_offload``: group hooks, the prefetchers' torch hooks and attributes, deferred
    pins. The weights stay where the hooks left them (host copies or the card); the caller's retry moves them."""
    try:
        from .diffusion_offload_prefetch import PREFETCHER_ATTR
        from .diffusion_small_host import ENCODER_PREFETCH_ATTR

        for module, before in snapshots:
            _remove_group_offload_hooks(module)
            _drop_module_forward_hooks_since(module, before)
            for attr in (PREFETCHER_ATTR, ENCODER_PREFETCH_ATTR, "_unsloth_stream_state"):
                module.__dict__.pop(attr, None)
            _drop_deferred_pinning(pipe, module)
        return bool(snapshots)
    except Exception as exc:  # noqa: BLE001 - the original failure propagates
        if logger is not None:
            logger.warning("diffusion.memory: could not unwind partial group offload (%s)", exc)
        return False


def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
    for name, component in (getattr(pipe, "components", {}) or {}).items():
        if component is module:
            return str(name).startswith("text_encoder")
    return False


# "0": pin the denoiser back after every text encoder instead of once per encode_prompt call.
ENCODE_RELEASE_ONCE_ENV = "UNSLOTH_DIFFUSION_ENCODE_RELEASE_ONCE"
# "0": the prompt encode always streams the whole surplus, even when free memory holds the encoder's need.
ENCODE_SHORTFALL_ENV = "UNSLOTH_DIFFUSION_ENCODE_SHORTFALL"
_ENCODE_SLACK_MIB = 512
# Per token, in hidden_size x 2 bytes (plus one per layer: output_hidden_states keeps every layer).
_ENCODE_TOKEN_HIDDEN_FACTOR = 32
# Vision inputs are not priced.
_ENCODE_VISUAL_KWARGS = (
    "pixel_values",
    "pixel_values_videos",
    "image_grid_thw",
    "video_grid_thw",
    "images",
)


def _encoder_width(module: Any) -> tuple[Optional[int], int, int]:
    config = getattr(module, "config", None)
    for cfg in (config, getattr(config, "text_config", None)):
        hidden = getattr(cfg, "hidden_size", None) or getattr(cfg, "d_model", None)
        if hidden:
            heads = (
                getattr(cfg, "num_attention_heads", None) or getattr(cfg, "num_heads", None) or 1
            )
            layers = (
                getattr(cfg, "num_hidden_layers", None) or getattr(cfg, "num_layers", None) or 0
            )
            return int(hidden), int(heads), int(layers)
    return None, 1, 0


def encode_need_mib(module: Any, args: Any, kwargs: Any) -> Optional[int]:
    """MiB one encoder call needs: the ``depth + 1`` largest streamed groups plus prompt activations, with margin.
    None when unpriceable (visual inputs, no token ids, unknown width)."""
    try:
        import torch

        from .diffusion_offload_prefetch import prefetch_depth

        kwargs = kwargs or {}
        if any(kwargs.get(name) is not None for name in _ENCODE_VISUAL_KWARGS):
            return None
        ids = kwargs.get("input_ids")
        if ids is None:
            ids = next(
                (
                    a
                    for a in (args or ())
                    if isinstance(a, torch.Tensor)
                    and a.dim() == 2
                    and not torch.is_floating_point(a)
                ),
                None,
            )
        if ids is None:
            embeds = kwargs.get("inputs_embeds")
            if not isinstance(embeds, torch.Tensor) or embeds.dim() != 3:
                return None
            batch, tokens = int(embeds.shape[0]), int(embeds.shape[1])
        elif isinstance(ids, torch.Tensor) and ids.dim() == 2:
            batch, tokens = int(ids.shape[0]), int(ids.shape[1])
        else:
            return None
        hidden, heads, layers = _encoder_width(module)
        if not hidden:
            return None
        streamed = sorted(
            (
                _group_nbytes(g)
                for g in (_offload_groups(module) or ())
                if not getattr(g, "_unsloth_resident", False)
            ),
            reverse = True,
        )
        window = sum(streamed[: prefetch_depth() + 1])
        acts = (
            batch
            * tokens
            * (hidden * 2 * (_ENCODE_TOKEN_HIDDEN_FACTOR + layers) + heads * tokens * 4)
        )
        return (
            int(math.ceil((window + acts) / float(1 << 20) * _MEASURED_PEAK_MARGIN))
            + _ENCODE_SLACK_MIB
        )
    except Exception:  # noqa: BLE001 - unpriced: the whole surplus streams
        return None


def encode_available_mib(device: Any = None) -> Optional[int]:
    """Driver free less the VRAM Budget reserve, plus allocator cache outside CUDA graph pools, in MiB."""
    try:
        import torch

        dev = torch.device(device if device is not None else "cuda")
        if dev.type != "cuda":
            return None
        if dev.index is None:
            dev = torch.device("cuda", torch.cuda.current_device())
        free, total = (int(x) for x in torch.cuda.mem_get_info(dev))
        free -= user_budget_reserve_bytes(dev, total)
        cached = max(
            0,
            int(torch.cuda.memory_reserved(dev))
            - int(torch.cuda.memory_allocated(dev))
            - _graph_held_cache_bytes(dev.index),
        )
        # clamp after adding the cache: a reserve larger than driver free still claims part of it
        return max(free + cached, 0) >> 20
    except Exception:  # noqa: BLE001
        return None


def _encode_device(pipe: Any) -> Any:
    for module in (getattr(pipe, "components", {}) or {}).values():
        device = getattr(module, "_unsloth_resident_device", None)
        if device is not None and getattr(module, "_unsloth_resident_room", None):
            return device
    return None


def encode_release_mib(pipe: Any, module: Any, args: Any, kwargs: Any, surplus: int) -> int:
    """Only the shortfall (ComfyUI's ``free_memory(memory_required - free)``), at most ``surplus``; unpriced: all."""
    if _env_off(ENCODE_SHORTFALL_ENV):
        return surplus
    need = encode_need_mib(module, args, kwargs)
    if need is None:
        return surplus
    available = encode_available_mib(_encode_device(pipe))
    if available is None:
        return surplus
    return max(0, min(int(surplus), int(need) - int(available)))


def install_encode_release(
    pipe: Any,
    plan: Any,
    logger: Any = None,
) -> int:
    """While a text encoder runs, stream the whole-resident (or widened partial) denoiser back to the flat room (the
    partial placement's encode state); pin it back on return, or once encode_prompt returns (one move per prompt with
    several encoders). Returns the number of encoders hooked."""
    estimates = getattr(plan, "estimates", None) or {}
    encode_room = estimates.get("encode_resident_transformer_mib")
    whole = getattr(plan, "resident_transformer_mib", None)
    if encode_room is None or not whole:
        return 0
    surplus = int(whole) - max(int(encode_room), 0)
    if surplus <= 0:
        return 0
    try:
        import weakref

        import torch

        try:
            pipe_ref = weakref.ref(pipe)  # the hooks live on the encoder, which the pipe owns
        except TypeError:
            pipe_ref = lambda: pipe  # noqa: E731 - not weak-referenceable
        disable = getattr(getattr(torch, "compiler", None), "disable", None)
        pending: list = []
        asked = [0]
        active = [0]

        def _before(
            module: Any,
            args: Any,
            kwargs: Any = None,
        ) -> None:
            active[0] += 1
            owner = pipe_ref()
            if owner is None or active[0] > 1:
                return  # nested encoder call: the outer one already released
            # Each encoder prices its own shortfall; a prompt never streams more than the surplus in total.
            need = min(
                encode_release_mib(owner, module, args, kwargs, surplus),
                surplus - asked[0],
            )
            if need <= 0:
                return
            asked[0] += need
            restore = release_resident_groups(
                owner, need, logger, denoisers_only = True, reason = "the prompt encode"
            )
            # No empty_cache: the allocator frees cached segments and retries on OOM; flushing per prompt cost a sync.
            if restore is not None:
                pending.append(restore)

        # encode_prompt calls in flight: their encoders keep one release and the outermost return pins back
        scope = [0]

        def _restore_pending() -> None:
            asked[0] = 0
            while pending:
                pending.pop()()

        def _after(module: Any, args: Any, output: Any) -> None:
            active[0] = max(0, active[0] - 1)
            if active[0] > 0 or scope[0] > 0:
                return  # the next encoder of this prompt reuses the room; encode_prompt's return pins back
            _restore_pending()

        before = disable(_before) if callable(disable) else _before
        after = disable(_after) if callable(disable) else _after
        hooked = 0
        for name, module in (getattr(pipe, "components", {}) or {}).items():
            if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):
                continue
            handles = (
                module.register_forward_pre_hook(before, with_kwargs = True),
                # always_call: an encode that raises (cancel, OOM) still pins the denoiser back
                module.register_forward_hook(after, always_call = True),
            )
            module._unsloth_encode_release = handles
            hooked += 1
        encode = getattr(pipe, "encode_prompt", None)
        if (
            hooked > 1
            and callable(encode)
            and not _env_off(ENCODE_RELEASE_ONCE_ENV)
            and not getattr(encode, "_unsloth_encode_release_scope", False)
        ):

            @functools.wraps(encode)
            def encode_prompt(*args: Any, **kwargs: Any) -> Any:
                scope[0] += 1
                try:
                    return encode(*args, **kwargs)
                finally:
                    scope[0] -= 1
                    if scope[0] == 0:
                        _restore_pending()

            encode_prompt._unsloth_encode_release_scope = True
            try:
                pipe.encode_prompt = encode_prompt
            except Exception:  # noqa: BLE001 - a read-only attribute keeps the per-encoder pin-back
                pass
        if hooked and logger is not None:
            logger.info(
                "diffusion.memory: the prompt encode streams %d MiB of the resident transformer (%d MiB stay), "
                "pinned again before step 0",
                surplus,
                max(int(encode_room), 0),
            )
        return hooked
    except Exception as exc:  # noqa: BLE001 - fall back to the partial placement rather than risk the encode
        if logger is not None:
            logger.warning(
                "diffusion.memory: encode release not installed (%s); the transformer streams past the flat room",
                exc,
            )
        release_resident_groups(
            pipe, surplus, logger, denoisers_only = True, reason = "the partial placement"
        )
        return 0


MEASURED_RESERVE_DENOISE_PHASE_INDEX = 13

DERIVED_REFERENCE_PRICE_ENV = "UNSLOTH_DIFFUSION_DERIVED_REFERENCE_PRICE"
# References priced by token count, not the flat 8192 MiB/MP (Qwen-Image-2.1 edit 1024^2: 3271-3447 MiB, 2061 K/V cache).
_DERIVED_REFERENCE_FAMILIES = frozenset({"qwen-image-2.1"})


def derived_reference_pricing_enabled() -> bool:
    return not _env_off(DERIVED_REFERENCE_PRICE_ENV)


def _reference_pixels_per_token(pipe: Any) -> Optional[int]:
    try:
        scale = int(getattr(pipe, "vae_scale_factor", 0) or 0)
        config = getattr(getattr(pipe, "transformer", None), "config", None)
        patch = int(getattr(config, "patch_size", 1) or 1) if config is not None else 0
    except Exception:  # noqa: BLE001
        return None
    if scale <= 0 or patch <= 0:
        return None
    return (scale * patch) ** 2


def _prefix_kv_bytes_per_token(pipe: Any, compute_bytes: int) -> Optional[int]:
    """Condition K/V cache bytes per cached token (0: no prefix cache; None: unreadable)."""
    transformer = getattr(pipe, "transformer", None)
    config = getattr(transformer, "config", None)
    if transformer is None or config is None:
        return None
    try:
        from .diffusion_prefix_kv import _takes_prefix_kv

        if not _takes_prefix_kv(transformer) or not getattr(config, "causal_condition", False):
            return 0
        layers = int(getattr(config, "num_layers", 0) or 0)
        heads = int(
            getattr(config, "num_key_value_heads", None)
            or getattr(config, "num_attention_heads", 0)
            or 0
        )
        head_dim = int(getattr(config, "attention_head_dim", 0) or 0)
    except Exception:  # noqa: BLE001
        return None
    if layers <= 0 or heads <= 0 or head_dim <= 0:
        return None
    return layers * 2 * heads * head_dim * max(1, int(compute_bytes or 2))


def derived_reference_need_mib(
    pipe: Any,
    *,
    family: Optional[str],
    phase_mib: int,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    reference_pixels: int = 0,
    cfg: Optional[bool] = None,
    compute_bytes: int = 2,
    quadratic_attention: bool = False,
) -> Optional[int]:
    """Denoise MiB of a reference request: ``phase_mib`` scaled by joint sequence / 1024^2 plus the condition K/V cache,
    doubled under CFG. None keeps the flat allowance (unmeasured family, kill switch, quadratic attention, unreadable)."""
    if not derived_reference_pricing_enabled() or quadratic_attention:
        return None
    name = (str(family or "").split() or [""])[0].lower()
    if name not in _DERIVED_REFERENCE_FAMILIES or not phase_mib or int(reference_pixels or 0) <= 0:
        return None
    per_token = _reference_pixels_per_token(pipe)
    kv_bytes = _prefix_kv_bytes_per_token(pipe, compute_bytes)
    if per_token is None or kv_bytes is None:
        return None
    batch = max(1, int(batch_size or 1))
    target = max(64, int(width or DEFAULT_IMAGE_WIDTH)) * max(
        64, int(height or DEFAULT_IMAGE_HEIGHT)
    )
    reference = int(reference_pixels)
    bucket = float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)
    branches = 1 if cfg is False else 2
    activation = float(phase_mib) * max(1.0, (target + reference) * batch / bucket) * branches
    cache = (reference // per_token) * batch * branches * kv_bytes / float(1 << 20)
    step = _MEASURED_PEAK_ROUND_MIB
    total = activation + cache * _MEASURED_PEAK_MARGIN
    return int(-(-total // step) * step)


# "0": the load places for text-to-image only; an edit's reference then streams resident groups mid-render.
EDIT_LOAD_PLACEMENT_ENV = "UNSLOTH_DIFFUSION_EDIT_LOAD_PLACEMENT"
# ``_unsloth_measured_reserve`` slot: (denoise phase MiB, MiB the placement covers).
MEASURED_RESERVE_REFERENCE_INDEX = 14


def edit_load_placement_enabled() -> bool:
    return derived_reference_pricing_enabled() and not _env_off(EDIT_LOAD_PLACEMENT_ENV)


def _default_reference_pixels(family: str) -> int:
    """Uses the request path's helpers so the load and the render-time guard price the same image."""
    try:
        from .diffusion_conditioning import default_reference_pixels
        from .diffusion_families import _FAMILIES

        fam = next((f for f in _FAMILIES if str(f.name).lower() == family), None)
        return default_reference_pixels(fam) if fam is not None else 0
    except Exception:  # noqa: BLE001 - unreadable: no edit placement
        return 0


def _load_reference_need(
    pipe: Any,
    *,
    family: Optional[str],
    speed_mode: Optional[str],
    quant: Optional[str],
    static_seed: bool,
) -> Optional[tuple[int, int]]:
    """(denoise phase MiB, denoise MiB) of a guidance-free default edit; None when unpriced or switched off."""
    name = (str(family or "").split() or [""])[0].lower()
    if not edit_load_placement_enabled() or name not in _DERIVED_REFERENCE_FAMILIES:
        return None
    phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(name)
    if (
        phases is None
        or not static_seed
        or not bool(getattr(pipe, "_unsloth_phase_seeds", False))
        or str(speed_mode or "") not in _MEASURED_PEAK_SPEED_MODES
    ):
        return None
    reference = _default_reference_pixels(name)
    if not reference:
        return None
    step = _MEASURED_PEAK_ROUND_MIB
    row = phases[1 if speed_mode == "max" else 0]
    phase = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))
    phase = int(-(-phase // step) * step)
    need = derived_reference_need_mib(
        pipe,
        family = family,
        phase_mib = phase,
        width = DEFAULT_IMAGE_WIDTH,
        height = DEFAULT_IMAGE_HEIGHT,
        reference_pixels = reference,
        cfg = False,
    )
    if need is None:
        return None
    if quant is not None:
        learned = measured_image_runtime_mib(
            family, speed_mode, quant = with_workflow(quant, "edit"), static_seed = False
        )
        if learned is not None:
            need = max(int(need), int(learned))
    return phase, int(need)


def load_covered_reference_need_mib(
    pipe: Any,
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    reference_pixels: int = 0,
    cfg: Optional[bool] = None,
    quadratic_attention: bool = False,
) -> Optional[int]:
    reserve = getattr(pipe, "_unsloth_measured_reserve", None) or ()
    if len(reserve) <= MEASURED_RESERVE_REFERENCE_INDEX or int(reference_pixels or 0) <= 0:
        return None
    covered = reserve[MEASURED_RESERVE_REFERENCE_INDEX]
    if not covered or not edit_load_placement_enabled():
        return None
    return derived_reference_need_mib(
        pipe,
        family = reserve[1],
        phase_mib = int(covered[0]),
        width = width,
        height = height,
        batch_size = batch_size,
        reference_pixels = int(reference_pixels),
        cfg = cfg,
        compute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2,
        quadratic_attention = quadratic_attention,
    )


def static_reference_release_mib(pipe: Any, need_mib: int) -> Optional[int]:
    reserve = getattr(pipe, "_unsloth_measured_reserve", None)
    if not reserve or len(reserve) <= MEASURED_RESERVE_REFERENCE_INDEX:
        return None
    covered = reserve[MEASURED_RESERVE_REFERENCE_INDEX]
    if not covered:
        return None
    return max(0, int(need_mib) - max(int(reserve[0]), int(covered[1])))


def live_reference_shortfall_mib(
    pipe: Any,
    need_mib: int,
    device: Any = None,
) -> Optional[int]:
    """MiB ``need_mib`` lacks in free memory now (driver free less budget reserve, plus reusable cache); None if unreadable."""
    try:
        import torch

        dev = torch.device(device if device is not None else "cuda")
        if dev.type != "cuda":
            return None
        if dev.index is None:
            dev = torch.device("cuda", torch.cuda.current_device())
        free, total = (int(x) for x in torch.cuda.mem_get_info(dev))
        free -= user_budget_reserve_bytes(dev, total)
        cached = max(
            0,
            int(torch.cuda.memory_reserved(dev))
            - int(torch.cuda.memory_allocated(dev))
            - _graph_held_cache_bytes(dev.index),
        )
        return max(0, int(need_mib) + _DECODE_SLACK_MIB - ((free + cached) >> 20))
    except Exception:  # noqa: BLE001 - unreadable: the planned release stands
        return None


def measured_request_extra_mib(
    pipe: Any,
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    condition_pixels: int = 0,
    cfg: Optional[bool] = None,
    unpriced_references: bool = False,
    workflow: Optional[str] = None,
    tier_peak_mib: Optional[int] = None,
    reference_pixels: int = 0,
    quadratic_attention: bool = False,
) -> int:
    """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made).  ``tier_peak_mib``:
    a deferred profile's tier peak (``TIER_MOVED_ATTR``), raising eager pricing; None keeps the flat estimate. ``workflow``:
    a workflow other than text-to-image also reads the peak recorded under its own key (``with_workflow``). ``cfg``:
    whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:
    the request carries reference images whose latents join the denoiser's sequence but are not in
    ``condition_pixels`` (FLUX.2 Klein's primary reference): a placement made from a dense default / max tier seed,
    measured on text-to-image, then sizes the request from the flat estimate.

    A placement that sized its resident blocks from the denoise phase (``MEASURED_RESERVE_DENOISE_PHASE_INDEX``) holds
    only that phase's reserve while the steps run: a request it was sized for (text-to-image, at most the 1024x1024
    bucket, no condition images or tier peak, within the whole-render reserve) releases nothing, its encode and decode
    priced by their own hooks; any other request is priced against the denoise reserve, which first streams the blocks
    the phase sizing added. ``reference_pixels`` (unweighted): priced by ``derived_reference_need_mib`` when set."""
    reserve = getattr(pipe, "_unsloth_measured_reserve", None)
    if not reserve:
        return 0
    phase = (
        reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]
        if len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX
        else None
    )
    headroom, family, speed_mode = reserve[:3]
    dense_mib = reserve[3] if len(reserve) > 3 else None
    compute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2
    identity = getattr(pipe, "_unsloth_peak_identity", None)
    sized = dict(
        width = width,
        height = height,
        batch_size = batch_size,
        dense_transformer_mib = dense_mib,
        compute_bytes = compute_bytes,
    )
    # A seeded family keeps the seed and the learned peak the load placed from: a peak this session's renders recorded
    # after the load only places the NEXT load, it must not stream resident groups for a request the same size as one
    # that just completed. An unseeded family was placed from its learned peak, which is all there is to check against.
    sized["phase_seeds"] = bool(getattr(pipe, "_unsloth_phase_seeds", False))
    sized["dense_tier_seeds"] = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))
    sized["static_seed"] = bool(reserve[8]) if len(reserve) > 8 and reserve[8] is not None else True
    sized["dense_bf16"] = bool(len(reserve) > 10 and reserve[10])
    sized["small_host_int8"] = bool(len(reserve) > 11 and reserve[11])
    sized["streamed"] = bool(len(reserve) > 12 and reserve[12])
    need = measured_image_runtime_mib(family, speed_mode, **sized)
    learned = reserve[5] if len(reserve) > 5 else None
    # placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement covers
    cfg_unmeasured = bool(len(reserve) > 9 and reserve[9]) and cfg is not False
    # tier and phase seeds were measured on text-to-image: an unsized reference is not covered
    tier_seeded = str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES and (
        (dense_mib is not None and sized["dense_tier_seeds"])
        or (dense_mib is None and sized["phase_seeds"])
    )
    tier_moved = bool(getattr(pipe, TIER_MOVED_ATTR, False))
    tier_peak = int(tier_peak_mib) if tier_moved and tier_peak_mib else None
    if (unpriced_references and tier_seeded) or (
        # eager-only seeds: same invalidation as a learned peak
        (learned or need is None or sized["dense_bf16"] or sized["small_host_int8"])
        and (
            getattr(pipe, "_unsloth_peak_static", False)
            or (tier_moved and tier_peak is None)
            or getattr(pipe, "_unsloth_loras", None)
            or cfg_unmeasured
        )
    ):
        # the learned peak no longer describes this pipeline (runtime LoRA adapters, a planned VAE saver that did not
        # engage): requests size from the flat estimate
        flat = estimate_image_runtime_mib(
            width = width,
            height = height,
            batch_size = batch_size,
            family = family,
            condition_pixels = condition_pixels,
        )
        return max(0, int(flat) - int(phase or headroom))
    if learned:
        # the learned peak the load reserved, scaled here (not reread from the store, which another worker may evict
        # or a failed write may leave empty while this pipeline stays loaded)
        from .diffusion_measured_peaks import resolution_bucket

        w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
        h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
        batch = max(1, int(batch_size or 1))
        covered = (
            reserve[6]
            if len(reserve) > 6 and reserve[6]
            else DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
        )
        # the learned store's own scaling: a request past the covered bucket takes its bucket's ceiling, squared over
        # the image's pixels for a math-attention entry (linear over the batch)
        scale = 1.0
        if len(reserve) > 7 and reserve[7]:
            if w * h > int(covered):
                scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2
            scale *= batch
        elif w * h * batch > int(covered):
            scale = resolution_bucket(w, h, batch) / float(covered)
        step = _MEASURED_PEAK_ROUND_MIB
        scaled = int(-(-(float(learned) * scale * _MEASURED_PEAK_MARGIN) // step) * step)
        seeded = need is not None
        need = scaled if need is None else max(need, scaled)
        if not seeded and isinstance(identity, tuple) and len(identity) == 3:
            stored = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
            if stored is not None:
                need = max(need, stored)
    if need is None and isinstance(identity, tuple) and len(identity) == 3:
        need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
    if tier_peak is not None:
        need = tier_peak if need is None else max(int(need), tier_peak)
    base_need = need
    own_learned = None
    cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))
    allowance = int(
        8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN
    )
    if need is not None:
        need += allowance
    sized_for = True  # the request is the one the placement was sized for (``phase`` pricing)
    if isinstance(identity, tuple) and len(identity) == 3:
        keyed = with_workflow(identity[1], workflow)
        if keyed != identity[1]:
            sized_for = False  # another workflow: its denoise phase was never measured against the placement
            own = None
            if (
                cfg is False
                and getattr(pipe, "_unsloth_peak_expects_cfg", False)
                and cfg_off_peaks_enabled()
            ):
                # a guidance-off render on a CFG-capable family records ``|cfg:off`` before the workflow part
                off = with_workflow(f"{without_cfg_off(identity[1])}|{CFG_OFF_PART}", workflow)
                own = measured_image_runtime_mib(family, speed_mode, quant = off, **sized)
            if own is None:
                own = measured_image_runtime_mib(family, speed_mode, quant = keyed, **sized)
            own_learned = own
            if own is not None:
                # the learned workflow peak is the other complete estimate: it was measured with the condition images
                # resident, so the allowance is not charged on top of it
                need = own if need is None else max(need, own)
    if need is None:
        return 0
    covered = (
        reserve[MEASURED_RESERVE_REFERENCE_INDEX]
        if len(reserve) > MEASURED_RESERVE_REFERENCE_INDEX
        else None
    )
    if (
        not phase
        and covered
        and allowance > 0
        and int(reference_pixels or 0) > 0
        and tier_peak is None
        and base_need is not None
        and int(base_need) <= int(headroom)
    ):
        # only the steps grow with the reference (encode and decode run under their own hooks)
        derived = load_covered_reference_need_mib(
            pipe,
            width = width,
            height = height,
            batch_size = batch_size,
            reference_pixels = int(reference_pixels),
            cfg = cfg,
            quadratic_attention = quadratic_attention,
        )
        if derived is not None:
            if own_learned is not None:
                derived = max(int(derived), int(own_learned))
            return max(0, int(derived) - max(int(headroom), int(covered[1])))
    if phase:
        pixels = (
            max(64, int(width or DEFAULT_IMAGE_WIDTH))
            * max(64, int(height or DEFAULT_IMAGE_HEIGHT))
            * max(1, int(batch_size or 1))
        )
        within = (
            tier_peak is None
            and pixels <= DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
            and base_need is not None
            and int(base_need) <= int(headroom)
        )
        if sized_for and allowance == 0 and within and int(need) <= int(headroom):
            return 0
        if within and allowance > 0 and int(reference_pixels or 0) > 0:
            # encode and decode run under their own hooks; only the steps grow with the reference
            derived = derived_reference_need_mib(
                pipe,
                family = family,
                phase_mib = int(phase),
                width = width,
                height = height,
                batch_size = batch_size,
                reference_pixels = int(reference_pixels),
                cfg = cfg,
                compute_bytes = compute_bytes,
                quadratic_attention = quadratic_attention,
            )
            if derived is not None:
                if own_learned is not None:
                    derived = max(int(derived), int(own_learned))
                return max(0, int(derived) - int(phase))
        return max(0, int(need) - int(phase))
    return max(0, int(need) - int(headroom))


def apply_memory_plan(
    pipe: Any,
    plan: MemoryPlan,
    *,
    device: str,
    placement_device: Optional[str] = None,
    logger: Any = None,
) -> tuple[str, bool]:
    """placement_device must be indexed (cuda:1); a bare cuda lets CPU offload fall back to cuda:0."""
    placement = placement_device or device
    try:
        pipe._unsloth_applied_plan = None
    except Exception:  # noqa: BLE001
        pass
    tiling_engaged = False
    if plan.vae_tiling:
        tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)
    if plan.vae_slicing:
        _enable_vae_saver(pipe, "enable_vae_slicing", "enable_slicing", logger)

    def _fallback_to_model_offload() -> None:
        # Group plan set vae_tiling=False; model-offload fallback can OOM on decode, so enable tiling.
        nonlocal tiling_engaged
        pipe.enable_model_cpu_offload(device = placement)
        keep_cpu_weights_on_offload(pipe, logger)
        if not tiling_engaged:
            tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)

    policy = plan.offload_policy
    if policy == OFFLOAD_MODEL and _stream_unet_instead_of_model_offload(pipe, placement, logger):
        policy = OFFLOAD_GROUP
        try:
            pipe._unsloth_applied_plan = replace(
                plan, offload_policy = OFFLOAD_GROUP, stream_text_encoders = True
            )
        except Exception:  # noqa: BLE001
            pass
    elif policy == OFFLOAD_MODEL:
        pipe.enable_model_cpu_offload(device = placement)
        keep_cpu_weights_on_offload(pipe, logger)
    elif policy == OFFLOAD_GROUP:
        # getattr, not attribute access: manually built / duck-typed plans predate this field.
        group_kwargs: dict[str, Any] = {
            "stream_text_encoders": bool(getattr(plan, "stream_text_encoders", False))
        }
        if not bool(getattr(plan, "stream_transformer", True)):
            group_kwargs["stream_transformer"] = False
        resident_mib = getattr(plan, "resident_transformer_mib", None)
        if resident_mib:
            group_kwargs["resident_transformer_mib"] = int(resident_mib)
        resident_te_mib = getattr(plan, "resident_text_encoder_mib", None)
        if resident_te_mib and group_kwargs["stream_text_encoders"]:
            group_kwargs["resident_text_encoder_mib"] = int(resident_te_mib)
        _set_group_offload_failure(pipe, None)
        applied = _apply_group_offload(pipe, placement, logger, **group_kwargs)
        failure = getattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, None)
        if not applied and failure is not None and _is_oom(failure):
            _set_group_offload_failure(pipe, None)  # the retry records its own failure, if any
            lowered = _retry_group_offload_lower(pipe, plan, placement, logger, failure)
            if lowered is not None:
                applied = True
                try:
                    pipe._unsloth_applied_plan = lowered
                except Exception:  # noqa: BLE001
                    pass
        if not applied:
            if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):
                raise RuntimeError(
                    "the text encoder could not be streamed beside the resident quantised "
                    "transformer, and whole-module offload cannot move torchao weights"
                )
            if _pipe_denoisers_hold_torchao(pipe) and not _model_offload_fits_quantised(plan):
                # Whole-module offload would onload the quantised transformer whole, which is what streaming avoided.
                raise RuntimeError(
                    "group offloading could not be set up for the quantised transformer, and it does not "
                    "fit the GPU whole for whole-module offload"
                )
            _fallback_to_model_offload()
            policy = OFFLOAD_MODEL
    elif policy == OFFLOAD_STREAMING:
        resident_mib = getattr(plan, "resident_transformer_mib", None)
        if resident_mib:
            _apply_streaming_offload(
                pipe, placement, logger, resident_transformer_mib = int(resident_mib)
            )
        else:
            _apply_streaming_offload(pipe, placement, logger)
    elif policy == OFFLOAD_SEQUENTIAL:
        try:
            pipe.enable_sequential_cpu_offload(device = placement)
        except Exception as exc:  # noqa: BLE001 - keep the model loadable
            if logger is not None:
                logger.warning(
                    "diffusion.memory: sequential offload failed (%s); "
                    "falling back to whole-module offload",
                    exc,
                )
            _fallback_to_model_offload()
            policy = OFFLOAD_MODEL
    elif (
        getattr(plan, "secondary_ordinal", None) is not None
        or getattr(plan, "layer_split", None) is not None
    ):
        if getattr(plan, "layer_split", None) is not None:
            from .diffusion_layer_split_plan import apply_layer_split_plan as _apply_split
        else:
            from .diffusion_multigpu import apply_split_plan as _apply_split
        if not _apply_split(pipe, plan, placement = placement, logger = logger):
            fallback = getattr(plan, "fallback_plan", None)
            if (
                fallback is not None
                and getattr(fallback, "secondary_ordinal", None) is None
                and getattr(fallback, "layer_split", None) is None
            ):
                # the policy one card would have used (group offload keeps a torchao denoiser loadable)
                result = apply_memory_plan(
                    pipe, fallback, device = device, placement_device = placement_device, logger = logger
                )
                # applied_plan() must report the fallback (or the lower re-plan it recorded), not the split
                if getattr(pipe, "_unsloth_applied_plan", None) is None:
                    try:
                        pipe._unsloth_applied_plan = fallback
                    except Exception:  # noqa: BLE001
                        pass
                return result
            _fallback_to_model_offload()
            policy = OFFLOAD_MODEL
            try:
                pipe._unsloth_applied_plan = replace(
                    plan,
                    offload_policy = OFFLOAD_MODEL,
                    secondary_ordinal = None,
                    secondary_components = None,
                    fallback_plan = None,
                    layer_split = None,
                )
            except Exception:  # noqa: BLE001
                pass
    else:
        from .diffusion_fast_load import fast_upload

        components = getattr(pipe, "components", None)
        modules = list(components.values()) if isinstance(components, dict) else []
        with fast_upload(modules, placement, logger = logger):
            pipe.to(placement)
    return policy, tiling_engaged


UNET_STREAM_ENV = "UNSLOTH_DIFFUSION_UNET_STREAM"
# Measured SDXL 1024^2, 25 steps, diffusers 0.41, L4 bf16 / T4 fp16; MiB above weights.
_MEASURED_UNET_OFFLOAD_DENOISE_MIB = 817
# Streamed: worst phase is the VAE decode (L4 bf16).
_MEASURED_UNET_STREAMED_PEAK_MIB = 2435
# force_upcast fp16 VAE (T4) decodes in fp32.
_MEASURED_UNET_STREAMED_FP32_DECODE_PEAK_MIB = 3228
_UNET_STREAM_SLACK_MIB = 512
# Streamed denoise: 333 MiB above the weights at 1024^2 (L4 bf16, CFG pair, incl. the ~168 MiB prefetch window), the
# activations scaled linearly to 2048^2 so a larger canvas still fits beside the resident leaves: 168 + 4 x 165.
_MEASURED_UNET_STREAMED_DENOISE_MIB = 828
# =0: size the resident leaves from the decode peak for the whole render (no lending, the #16295 sizing).
UNET_STREAM_LEND_ENV = "UNSLOTH_DIFFUSION_UNET_STREAM_LEND"


def unet_stream_lends_decode() -> bool:
    """Resident UNet leaves are sized from the denoise peak and lent to the VAE decode (release_for_decode streams the
    shortfall again before the decode and re-pins it after), as ComfyUI unloads part of the UNet for its decode."""
    return not _env_off(UNET_STREAM_LEND_ENV)


def unet_decode_lend(fp32_decode: bool = False) -> tuple[int, int, int, bool]:
    """The ``DECODE_LEND_ATTR`` tuple of a streamed UNet: the decode's measured peak above the weights at 1024^2,
    scaled linearly with the canvas (a tiled decode grows slower: it releases more than it needs, never less)."""
    peak = (
        _MEASURED_UNET_STREAMED_FP32_DECODE_PEAK_MIB
        if fp32_decode
        else _MEASURED_UNET_STREAMED_PEAK_MIB
    )
    return (peak, peak, DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT, False)


def _measured_reserve_mib(peak_mib: int) -> int:
    step = _MEASURED_PEAK_ROUND_MIB
    return int(-(-(peak_mib * _MEASURED_PEAK_MARGIN) // step) * step)


def unet_stream_room_mib(
    free_mib: int,
    unet_mib: int,
    vae_mib: int,
    *,
    fp32_decode: bool = False,
    lend: Optional[bool] = None,
) -> Optional[int]:
    """None if whole-module offload fits; else MiB of UNet leaves to keep resident. ``lend`` (default: the kill switch
    ``UNSLOTH_DIFFUSION_UNET_STREAM_LEND``): sized from the denoise peak, the decode borrows the shortfall through
    release_for_decode; off: sized so the decode fits beside every resident leaf for the whole render."""
    if unet_mib + _measured_reserve_mib(_MEASURED_UNET_OFFLOAD_DENOISE_MIB) <= free_mib:
        return None
    if lend is None:
        lend = unet_stream_lends_decode()
    if lend:
        peak = _MEASURED_UNET_STREAMED_DENOISE_MIB
    else:
        peak = (
            _MEASURED_UNET_STREAMED_FP32_DECODE_PEAK_MIB
            if fp32_decode
            else _MEASURED_UNET_STREAMED_PEAK_MIB
        )
    reserve = _measured_reserve_mib(peak) + _UNET_STREAM_SLACK_MIB
    return max(0, int(free_mib) - int(vae_mib) - reserve)


def _vae_decodes_in_fp32(vae: Any) -> bool:
    try:
        import torch
        first = next(iter(vae.parameters()), None)
        return (
            first is not None
            and first.dtype == torch.float16
            and bool(getattr(getattr(vae, "config", None), "force_upcast", False))
        )
    except Exception:  # noqa: BLE001
        return True


def _stream_unet_instead_of_model_offload(
    pipe: Any,
    placement: str,
    logger: Any = None,
) -> bool:
    """Stream a UNet too large for whole-module offload (which OOMs on step one) leaf by leaf."""
    if _env_off(UNET_STREAM_ENV):
        return False
    try:
        import torch

        unet = getattr(pipe, "unet", None)
        if not isinstance(unet, torch.nn.Module) or getattr(pipe, "transformer", None) is not None:
            return False
        onload = torch.device(placement)
        if onload.type != "cuda" or not torch.cuda.is_available():
            return False
        free, total = (int(x) for x in torch.cuda.mem_get_info(onload))
        free_mib = max(0, free - user_budget_reserve_bytes(onload, total)) >> 20

        def _mib(m: Any) -> int:
            if not isinstance(m, torch.nn.Module):
                return 0
            ts = {id(t): t for t in list(m.parameters()) + list(m.buffers())}
            return sum(t.numel() * t.element_size() for t in ts.values()) >> 20

        unet_mib = _mib(unet)
        vae = getattr(pipe, "vae", None)
        fp32_decode = vae is not None and _vae_decodes_in_fp32(vae)
        lend = unet_stream_lends_decode()
        room = unet_stream_room_mib(
            free_mib, unet_mib, _mib(vae), fp32_decode = fp32_decode, lend = lend
        )
        if room is None:
            return False
    except Exception:  # noqa: BLE001
        return False
    _set_group_offload_failure(pipe, None)
    applied = _apply_group_offload(
        pipe,
        placement,
        logger,
        stream_text_encoders = True,
        resident_transformer_mib = room or None,
        denoiser = "unet",
    )
    if applied:
        try:
            # deferred speed profile would rebuild weights outside the leaf hooks; read by _engage_deferred_speed
            pipe._unsloth_unet_streamed = True
        except Exception:  # noqa: BLE001
            pass
        try:
            # the leaves were sized from the denoise peak: the decode borrows its shortfall (release_for_decode)
            setattr(
                pipe, DECODE_LEND_ATTR, unet_decode_lend(fp32_decode) if lend and room else None
            )
        except Exception:  # noqa: BLE001
            pass
    if logger is not None:
        logger.info(
            "diffusion.memory: UNet (%d MiB) does not fit %d MiB free for whole-module offload; %s",
            unet_mib,
            free_mib,
            f"streaming it leaf by leaf, {room} MiB kept resident"
            if applied
            else "streaming failed, whole-module offload",
        )
    return applied


_GROUP_OFFLOAD_FAILURE_ATTR = "_unsloth_group_offload_failure"


class _GroupOffloadFailure(Exception):
    """Type and message of a failed group offload setup. The live exception is not kept: its traceback holds the
    failed frames, whose tensors would stay allocated through the very retry that needs the room."""

    def __init__(self, exc: BaseException) -> None:
        super().__init__(f"{type(exc).__name__}: {exc}")
        self.type_name = type(exc).__name__
        self.oom = _is_oom(exc)


def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:
    """What made ``_apply_group_offload`` return False (it absorbs the exception so the caller can fall back)."""
    if exc is not None and not isinstance(exc, _GroupOffloadFailure):
        exc = _GroupOffloadFailure(exc)
    try:
        setattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, exc)
    except Exception:  # noqa: BLE001
        pass


def _is_oom(exc: BaseException) -> bool:
    if isinstance(exc, _GroupOffloadFailure):
        return exc.oom
    from .diffusion_batched import is_oom_error
    return is_oom_error(exc)


def applied_plan(pipe: Any, plan: Any) -> Any:
    """The plan ``apply_memory_plan`` actually placed: ``plan``, or the lower-residency re-plan a transient setup OOM
    forced (callers gate graphs, fused GEMMs and the encode release on residency)."""
    return getattr(pipe, "_unsloth_applied_plan", None) or plan


def _retry_group_offload_lower(
    pipe: Any, plan: MemoryPlan, placement: str, logger: Any, exc: BaseException
) -> Optional[MemoryPlan]:
    """One retry after an OOM while setting up group offload (another process can move VRAM between the plan's
    snapshot and placement): return every denoiser and text encoder to the host, empty the cache, and stream the
    denoisers and encoders with nothing kept resident. Returns the re-plan, or None (the caller refuses as before)."""
    try:
        import gc

        import torch

        for name, comp in (getattr(pipe, "components", {}) or {}).items():
            name = str(name)
            if not isinstance(comp, torch.nn.Module):
                continue
            if name in (
                "transformer",
                "transformer_2",
                "unconditional_transformer",
            ) or name.startswith("text_encoder"):
                _return_module_tensors(comp, torch.device("cpu"))
        gc.collect()
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        if logger is not None:
            logger.warning(
                "diffusion.memory: group offload setup hit OOM (%s); emptied the cache and re-planning once with "
                "the transformer and text encoders streamed",
                str(exc).splitlines()[0][:200] if str(exc) else type(exc).__name__,
            )
    except Exception as retry_exc:  # noqa: BLE001 - the caller's refusal stands
        if logger is not None:
            logger.warning(
                "diffusion.memory: lower-residency group offload retry failed (%s)", retry_exc
            )
        return None
    # outside the catch: a raise here left hooks on part of the pipe, which the caller's whole-module fallback rejects
    if not _apply_group_offload(pipe, placement, logger, stream_text_encoders = True):
        return None
    estimates = dict(plan.estimates or {})
    estimates.update(resident_transformer_mib = 0, resident_text_encoder_mib = 0, setup_oom_retry = True)
    return replace(
        plan,
        stream_text_encoders = True,
        stream_transformer = True,
        resident_transformer_mib = None,
        resident_text_encoder_mib = None,
        estimates = estimates,
        reasons = tuple(plan.reasons)
        + (
            "group offload setup hit OOM; re-planned once with the transformer and text encoders streamed",
        ),
    )


def _enable_vae_saver(pipe: Any, pipe_method: str, vae_method: str, logger: Any) -> bool:
    """Turn on a VAE memory saver, trying the pipeline shortcut first then the VAE submodule
    (some pipelines, e.g. Z-Image, only expose it on ``pipe.vae``). Returns whether it engaged."""
    for owner, method in ((pipe, pipe_method), (getattr(pipe, "vae", None), vae_method)):
        fn = getattr(owner, method, None)
        if not callable(fn):
            continue
        try:
            fn()
            return True
        except Exception as exc:  # noqa: BLE001 - a VAE saver is an optimisation, never fatal
            if logger is not None:
                logger.warning("diffusion.memory: %s() failed: %s", method, exc)
    return False


def _pin_vision_embedding_device(module: Any) -> int:
    """Keeps Qwen3-VL's vision pos_embed on the compute device, which group offload otherwise leaves
    on CPU."""
    import types

    walk = getattr(module, "modules", None)
    if not callable(walk):
        return 0
    patched = 0
    for sub in walk():
        original = getattr(sub, "fast_pos_embed_interpolate", None)
        if not callable(original) or getattr(original, "_unsloth_device_pinned", False):
            continue

        def _on_grid_device(
            self: Any,
            grid_thw: Any,
            *,
            _original: Any = original,
        ) -> Any:
            out = _original(grid_thw)
            device = getattr(grid_thw, "device", None)
            return out.to(device) if device is not None and out.device != device else out

        _on_grid_device._unsloth_device_pinned = True  # type: ignore[attr-defined]
        sub.fast_pos_embed_interpolate = types.MethodType(_on_grid_device, sub)
        patched += 1
    return patched


# ``0`` never pins the streamed-encoder tiers, ``1`` always does.
GROUP_OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"
_PIN_RESERVE_MIN_MIB = _PIN_RESERVE_MIN_BYTES >> 20


def _module_host_mib(module: Any) -> int:
    """Pinned host MiB for ``module``; per tensor rounded to a power of two like torch's pinned allocator."""
    try:
        seen: set[int] = set()
        total = 0
        tensors = list(module.parameters(recurse = True)) + list(module.buffers(recurse = True))
        for tensor in tensors:
            if id(tensor) in seen:
                continue
            seen.add(id(tensor))
            for nbytes in _storage_nbytes(tensor):
                if nbytes > 0:
                    total += 1 << (nbytes - 1).bit_length()
        return total // (1024 * 1024)
    except Exception:  # noqa: BLE001 - an unsizeable module is priced as nothing to pin
        return 0


_HOST_PAGE = 4096


def _page_bytes(nbytes: int) -> int:
    return -(-int(nbytes) // _HOST_PAGE) * _HOST_PAGE


def _module_registered_mib(module: Any) -> int:
    """Pinned host MiB of ``module`` in registered host memory (``registered_host_copy``): each tensor page-rounded."""
    try:
        total = sum(
            _page_bytes(nbytes)
            for tensor in _unique_tensors(module)
            for nbytes in _storage_nbytes(tensor)
            if nbytes > 0
        )
        return -(-total // (1024 * 1024))
    except Exception:  # noqa: BLE001 - priced as the per-tensor pin
        return _module_host_mib(module)


def _module_offload_registered_mib(module: Any) -> int:
    """Registered MiB a group offload pins: page-rounded, counted once per owning submodule (UMT5's tied embedding pins twice)."""
    try:
        total = 0
        for sub in module.modules():
            seen: set[int] = set()
            for tensor in list(sub.parameters(recurse = False)) + list(sub.buffers(recurse = False)):
                if id(tensor) in seen:
                    continue
                seen.add(id(tensor))
                total += sum(
                    _page_bytes(nbytes) for nbytes in _storage_nbytes(tensor) if nbytes > 0
                )
        return -(-total // (1024 * 1024))
    except Exception:  # noqa: BLE001 - priced as the per-tensor pin
        return _module_host_mib(module)


def _unique_tensors(module: Any) -> list:
    seen: set[int] = set()
    out = []
    for tensor in list(module.parameters(recurse = True)) + list(module.buffers(recurse = True)):
        if id(tensor) not in seen:
            seen.add(id(tensor))
            out.append(tensor)
    return out


def _module_pin_mib(module: Any) -> int:
    """Pinned host MiB of an up-front pin of ``module``: slab packing when the arena is on, else per-tensor rounding."""
    from .diffusion_pinned_arena import arena_reserved_bytes, pin_arena_enabled

    if not pin_arena_enabled():
        return _module_host_mib(module)
    try:
        return -(-arena_reserved_bytes(_unique_tensors(module)) // (1024 * 1024))
    except Exception:  # noqa: BLE001 - priced as the per-tensor pin
        return _module_host_mib(module)


# Block-level offload groups only direct ModuleList children; split FLUX.3's ModuleDict of block stacks too ("0": off).
GROUP_OFFLOAD_DICT_BLOCKS_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_DICT_BLOCKS"
_DICT_BLOCK_CLASSES = frozenset({"Flux3Transformer2DModel"})


def _dict_block_modules(module: Any) -> list[str]:
    """Direct children of ``module`` that are ModuleDicts of block stacks."""
    if _env_off(GROUP_OFFLOAD_DICT_BLOCKS_ENV) or type(module).__name__ not in _DICT_BLOCK_CLASSES:
        return []
    try:
        import torch

        names = []
        for name, child in module.named_children():
            if (
                isinstance(child, torch.nn.ModuleDict)
                and len(child) > 0
                and all(
                    isinstance(v, (torch.nn.ModuleList, torch.nn.Sequential))
                    for v in child.values()
                )
            ):
                names.append(name)
        return names
    except Exception:  # noqa: BLE001 - unreadable: the stock grouping
        return []


def _with_dict_block_modules(
    apply: Callable[..., Any], module: Any, kwargs: dict[str, Any]
) -> dict[str, Any]:
    if kwargs.get("offload_type", "block_level") != "block_level" or "block_modules" in kwargs:
        return kwargs
    names = _dict_block_modules(module)
    if not names:
        return kwargs
    try:
        import inspect
        if "block_modules" not in inspect.signature(apply).parameters:
            return kwargs
    except (TypeError, ValueError):
        return kwargs
    return {**kwargs, "block_modules": names}


def _apply_group_offloading_packed(
    apply: Callable[..., Any],
    module: Any,
    kwargs: dict[str, Any],
    logger: Any = None,
) -> Any:
    """``apply(module, **kwargs)`` with an up-front pin (stream, ``low_cpu_mem_usage=False``) packed into slabs.

    diffusers pins each tensor through torch's host allocator, which rounds it up to a power of two: a 23.8 GB bf16
    Flux DiT held 37.2 GB pinned. Slabs hold the same tensors in ~weight bytes; the H2D copies read the same pinned
    pages, so overlap is unchanged. Lazy (``low_cpu_mem_usage``) and unstreamed applies run as before."""
    from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload

    kwargs = _with_dict_block_modules(apply, module, kwargs)
    pack = (
        bool(kwargs.get("use_stream"))
        and not kwargs.get("low_cpu_mem_usage", False)
        and host_pinning_allowed(kwargs.get("onload_device"))
    )
    expected = None
    if pack:
        try:
            expected = packable_bytes(_unique_tensors(module))
        except Exception:  # noqa: BLE001 - whole slabs, as without a size
            expected = None
    with pinned_arena_for_group_offload(
        enabled = None if pack else False, expected_bytes = expected
    ) as arena:
        out = apply(module, **kwargs)
    if arena is not None and arena.payload_bytes:
        try:
            module._unsloth_pin_arena_bytes = (arena.payload_bytes, arena.reserved_bytes)
        except Exception:  # noqa: BLE001 - only a diagnostic
            pass
        if logger is not None:
            logger.info(
                "diffusion.memory: %s pinned in %.2f GB of slabs (%.2f GB of weights)",
                type(module).__name__,
                arena.reserved_bytes / 1e9,
                arena.payload_bytes / 1e9,
            )
    return out


def _storage_nbytes(tensor: Any, depth: int = 0) -> list[int]:
    """Bytes per allocation; torchao subclasses report their logical bf16 size, so size the packed inner tensors."""
    flatten = getattr(type(tensor), "__tensor_flatten__", None)
    if flatten is not None and depth < 4:
        try:
            names, _ctx = tensor.__tensor_flatten__()
            inner = [getattr(tensor, name, None) for name in names]
            if inner and all(t is not None for t in inner):
                return [n for t in inner for n in _storage_nbytes(t, depth + 1)]
        except Exception:  # noqa: BLE001 - fall back to the logical size, which only over-counts
            pass
    return [int(tensor.numel()) * int(tensor.element_size())]


def _pin_budget_mib() -> Optional[int]:
    """Pinnable host MiB leaving ``max(4 GiB, 15%)`` of the cgroup-capped total free, or None if unreadable."""
    total = _host_ram_capacity_mib()
    available = _available_system_memory_mib()
    if total is None or available is None:
        return None
    available = _pin_available_mib(int(available))
    reserve = max(_PIN_RESERVE_MIN_MIB, int(int(total) * _PIN_RESERVE_FRACTION))
    return max(0, available - reserve)


# 0 restores the stricter cgroup reading that counts mapped file pages on the active list as used.
PIN_CREDIT_MAPPED_ENV = "UNSLOTH_DIFFUSION_PIN_CREDIT_MAPPED"


def _pin_available_mib(available_mib: int) -> int:
    """Under a cgroup limit, the reading with clean mapped page cache credited (pins replace the checkpoint's mmap
    views), taken whole rather than added to ``available_mib``: the lists move between two samples."""
    if str(os.environ.get(PIN_CREDIT_MAPPED_ENV, "")).strip().lower() in (
        "0",
        "off",
        "false",
        "no",
    ):
        return available_mib
    try:
        from utils import host_memory

        headroom = host_memory.credited_cgroup_headroom_mib()
        if headroom is None:
            return available_mib
        return int(host_memory.usable_mib(host_memory.system_available_mib(), headroom))
    except Exception:  # noqa: BLE001 - the uncredited reading still stands
        return available_mib


def _remove_group_offload_hooks(module: Any) -> None:
    """Undo a group offloading apply that raised part way (best effort)."""
    try:
        from diffusers.hooks import group_offloading as go
        from diffusers.hooks.hooks import HookRegistry

        registry = HookRegistry.check_if_exists_or_initialize(module)
        for name in (
            "_GROUP_OFFLOADING",
            "_LAZY_PREFETCH_GROUP_OFFLOADING",
            "_LAYER_EXECUTION_TRACKER",
        ):
            hook = getattr(go, name, None)
            if isinstance(hook, str):
                registry.remove_hook(hook, recurse = True)
    except Exception:  # noqa: BLE001 - the fallback reports its own failure
        pass


def _pinned_memory_capped() -> bool:
    """Windows (WDDM) and WSL2 cap pinned host memory near 1 GiB (NVIDIA CUDA on WSL known limitation)."""
    if sys.platform == "win32":
        return True
    try:
        with open("/proc/version", "r", encoding = "utf-8") as f:
            return "microsoft" in f.read().lower()
    except Exception:  # noqa: BLE001 - not Linux, or unreadable: not WSL
        return False


def _streamed_pin_plan(
    transformer_mib: int,
    encoder_mib: int,
    logger: Any = None,
    device: Any = None,
) -> tuple[bool, bool]:
    """(pin transformer, pin encoders), transformer first. ``device``: the onload target the pins serve."""
    forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
    if forced in ("0", "off", "false", "no") or not host_pinning_allowed(device):
        return False, False
    if forced in ("1", "on", "true", "yes"):
        return True, True
    budget = None if _pinned_memory_capped() else _pin_budget_mib()
    if budget is None or budget <= 0:
        pin_transformer = pin_encoders = False
    else:
        pin_transformer = transformer_mib <= budget
        pinned = transformer_mib if pin_transformer else 0
        pin_encoders = pinned + encoder_mib <= budget
    if logger is not None:
        try:
            logger.info(
                "diffusion.memory: streamed-encoder tier pins transformer=%s (%d MiB) encoders=%s (%d MiB) "
                "against %s MiB of pinnable host RAM",
                pin_transformer,
                transformer_mib,
                pin_encoders,
                encoder_mib,
                budget,
            )
        except Exception:  # noqa: BLE001
            pass
    return pin_transformer, pin_encoders


def install_group_offload_torchao_swap_retry() -> bool:
    """``swap_tensors`` refuses a weakref'd tensor; retry once after gc so uncollected compile garbage does not fail."""
    try:
        from diffusers.hooks import group_offloading as go
    except Exception:  # noqa: BLE001 - no group offload in this diffusers
        return False
    original = getattr(go, "_swap_torchao_tensor", None)
    if original is None or getattr(original, "_unsloth_swap_retry", False):
        return False

    @functools.wraps(original)
    def _swap_torchao_tensor(param: Any, source: Any) -> None:
        try:
            original(param, source)
        except RuntimeError as exc:
            if "weakref" not in str(exc):
                raise
            import gc

            gc.collect()
            original(param, source)

    _swap_torchao_tensor._unsloth_swap_retry = True
    go._swap_torchao_tensor = _swap_torchao_tensor
    return True


BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_BACKGROUND_PIN"
_BG_PIN_ATTR = "_unsloth_background_pin"
_PENDING_PINS_ATTR = "_unsloth_background_pins"
_BACKGROUND_PIN_REQUEST_ATTR = "_unsloth_background_pin_requested"


def request_background_pins(pipe: Any) -> None:
    """Ask the next group offload apply on ``pipe`` to pin off the load path (see _GroupPinner)."""
    try:
        setattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, True)
    except Exception:  # noqa: BLE001 - a pipe refusing attributes pins eagerly, as before
        pass


def _background_pin_enabled(device: Any = None) -> bool:
    return (os.environ.get(BACKGROUND_PIN_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    ) and host_pinning_allowed(device)


# "0": image-load streamed encoders pin into slabs up front instead of on a background worker.
ENCODER_BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_ENCODER_BACKGROUND_PIN"
# "0": an encoder that does not fit the pin budget whole pins nothing instead of a fitting prefix.
PARTIAL_ENCODER_PIN_ENV = "UNSLOTH_DIFFUSION_PARTIAL_ENCODER_PIN"
_ENCODER_PIN_REQUEST_ATTR = "_unsloth_background_encoder_pin_requested"
_PARTIAL_PIN_MIN_MIB = 256
# "0": old behaviour (power-of-two pin price, no partial encoder prefix).
VIDEO_EXACT_PIN_PLAN_ENV = "UNSLOTH_VIDEO_EXACT_PIN_PLAN"
_CAST_WRITES_PENDING_ATTR = "_unsloth_cast_writes_pending"


def request_background_encoder_pins(pipe: Any) -> None:
    """Mark ``pipe`` so the next group offload defers its streamed encoder pins; denoisers still pin up front."""
    try:
        setattr(pipe, _ENCODER_PIN_REQUEST_ATTR, True)
    except Exception:  # noqa: BLE001 - a pipe refusing attributes pins eagerly, as before
        pass


def finish_encoder_pins_at_load(pipe: Any, logger: Any = None) -> float:
    """Wait for the encoder pins; left running into the first prompt they cost what they saved (FLUX.1 B200)."""
    if pipe is None:
        return 0.0
    start_background_pins(pipe)
    waited = finish_background_pins(pipe)
    if waited >= 0.5 and logger is not None:
        logger.info("diffusion.memory: the load waited %.1f s for the encoder pins", waited)
    return waited


# "0": an image load pins (and keeps) the host copy of every text-encoder offload group, resident ones too.
RESIDENT_HOST_DROP_ENV = "UNSLOTH_DIFFUSION_DROP_RESIDENT_HOST_COPY"
_HOST_DROPPED_ATTR = "_unsloth_host_dropped"


def _resident_host_drop_enabled() -> bool:
    return not _env_off(RESIDENT_HOST_DROP_ENV)


def drop_resident_host_copies(
    groups: list,
    logger: Any = None,
    label: str = "",
) -> tuple[list, int]:
    """Free the pageable host copy of each resident group (its weights live on the card); returns ``(kept, bytes)``:
    the groups that still need a pin, and the bytes freed. A group whose eviction ever streams it again gets an exact
    copy back from the card first (``rebuild_dropped_host_copy``). Tied partners drop together or not at all."""
    import torch

    def droppable(group: Any) -> bool:
        if not getattr(group, "_unsloth_resident", False) or getattr(
            group, "offload_to_disk_path", None
        ):
            return False
        cpu = getattr(group, "cpu_param_dict", None)
        if not isinstance(cpu, dict) or not cpu:
            return False
        for t, src in cpu.items():
            if (
                type(src) is not torch.Tensor
                or src.device.type != "cpu"
                or src.is_pinned()
                # the weight itself must be on the card, not reading this host copy
                or getattr(t, "device", None) is None
                or t.device.type == "cpu"
            ):
                return False
        return True

    kept: list = []
    dropped: set = set()
    freed = 0
    for group in groups:
        if id(group) in dropped:
            continue
        closure = [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]
        if not all(droppable(g) for g in closure):
            kept.append(group)
            continue
        seen: set = set()
        for member in closure:
            cpu = member.cpu_param_dict
            for src in cpu.values():
                key = (src.data_ptr(), src.nbytes)
                if key not in seen:
                    seen.add(key)
                    freed += int(src.nbytes)
            setattr(member, _HOST_DROPPED_ATTR, list(cpu.keys()))
            cpu.clear()
            dropped.add(id(member))
    if freed and logger is not None:
        try:
            logger.info(
                "diffusion.memory: %s keeps no host copy of its %d resident offload groups (%.1f GiB not pinned); "
                "an eviction copies them back from the card",
                label,
                len(dropped),
                freed / 2**30,
            )
        except Exception:  # noqa: BLE001
            pass
    return kept, freed


def rebuild_dropped_host_copy(group: Any) -> int:
    """Give ``group`` (and its tied partners) host copies of their device weights again; returns bytes copied."""
    return rebuild_dropped_host_copies([group])


_REBUILD_ALIGN = 512


def rebuild_dropped_host_copies(groups: Iterable[Any]) -> int:
    """Host copies of the device weights of every dropped group in ``groups`` (tied partners included), in one
    registered slab filled by device-to-host copies; returns bytes copied. Each tensor shared by tied partners is
    copied once. Without a registered slab (refused, or no CUDA) each tensor gets its own pageable copy."""
    import torch

    members: list = []
    seen_groups: set = set()
    for group in groups:
        for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:
            if id(member) not in seen_groups and getattr(member, _HOST_DROPPED_ATTR, None):
                seen_groups.add(id(member))
                members.append(member)
    if not members:
        return 0
    tensors: dict = {}
    for member in members:
        for t in getattr(member, _HOST_DROPPED_ATTR, None) or ():
            tensors.setdefault(id(t), t)
    lives = {key: (t.data if hasattr(t, "data") else t) for key, t in tensors.items()}
    offsets: dict = {}
    total = 0
    for key, live in lives.items():
        offsets[key] = total
        total += -(-int(live.nbytes) // _REBUILD_ALIGN) * _REBUILD_ALIGN
    device = next((live.device for live in lives.values() if live.device.type == "cuda"), None)
    slab = None
    # Pinned only where pinning is allowed now and the copy fits what host RAM can pin now; else pageable copies
    # (the group streams slower, the release itself never waits on a pin).
    budget = None if device is None or _pinned_memory_capped() else _pin_budget_mib()
    if (
        device is not None
        and host_pinning_allowed(device)
        and budget is not None
        and (total >> 20) <= budget
    ):
        try:
            slab = registered_pinned_empty(total)
        except Exception:  # noqa: BLE001 - per-tensor pageable copies below
            slab = None
    hosts: dict = {}
    with torch.no_grad():
        if device is not None:
            torch.cuda.current_stream(device).synchronize()
        for key, live in lives.items():
            if slab is not None:
                off = offsets[key]
                host = slab[off : off + int(live.nbytes)].view(live.dtype).view(live.shape)
                host.copy_(live, non_blocking = True)
            else:
                host = live.detach().to("cpu", copy = True)
            hosts[key] = host
        if slab is not None:
            torch.cuda.current_stream(device).synchronize()
    for member in members:
        cpu = member.cpu_param_dict
        for t in getattr(member, _HOST_DROPPED_ATTR, None) or ():
            cpu[t] = hosts[id(t)]
        setattr(member, _HOST_DROPPED_ATTR, None)
    return sum(int(live.nbytes) for live in lives.values())


def _encoder_background_pin_wanted(pipe: Any, device: Any = None) -> bool:
    # Needs the registered pin: per-tensor pin_memory() rounds to a power of two, past the weight-byte budget.
    return (
        bool(getattr(pipe, _ENCODER_PIN_REQUEST_ATTR, False))
        and not _env_off(ENCODER_BACKGROUND_PIN_ENV)
        and _background_pin_enabled(device)
        and _fast_pin_supported(device)
    )


def note_cast_cache_writes(pipe: Any, writes: Any) -> None:
    """Record the cast-cache writes a first load in a Studio home deferred; the pin plan reads them (``done()``)."""
    try:
        setattr(pipe, _CAST_WRITES_PENDING_ATTR, writes)
    except Exception:  # noqa: BLE001 - a pipe refusing attributes keeps the exact plan
        pass


def _cast_cache_write_pending(pipe: Any) -> bool:
    writes = getattr(pipe, _CAST_WRITES_PENDING_ATTR, None)
    if writes is None:
        return False
    try:
        return not writes.done()
    except Exception:  # noqa: BLE001 - unreadable: treat as still writing
        return True


def _video_exact_pins(
    pipe: Any,
    device: Any = None,
    logger: Any = None,
) -> bool:
    """Video load priced at page-rounded bytes; a first load still writing its cast cache keeps the default plan (extra pins cost it 3-11 s)."""
    if not (
        bool(getattr(pipe, _FAST_PIN_REQUEST_ATTR, False))
        and not _env_off(VIDEO_EXACT_PIN_PLAN_ENV)
        and _fast_pin_supported(device)
    ):
        return False
    if _cast_cache_write_pending(pipe):
        if logger is not None:
            logger.info(
                "diffusion.memory: this load writes its cast cache (first load in this home): its pins keep the "
                "default plan, no encoder prefix"
            )
        return False
    return True


def _late_encoder_pin_cap(
    pipe: Any,
    module: Any,
    device: Any = None,
) -> Callable[[], int]:
    """Encoder pin cap read when the pinner starts, not at plan time (the load still holds source shards then), minus earlier pinners' unplaced bytes."""

    def cap() -> int:
        budget = _streamed_pin_leftover_mib(0, False, device)
        owed = 0
        for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
            if pinner.module is module:
                break
            owed += pinner.outstanding_bytes()
        return max(0, (int(budget) << 20) - owed)

    return cap


def _streamed_pin_leftover_mib(
    transformer_mib: int,
    pin_transformer: bool,
    device: Any = None,
) -> int:
    """Pin budget MiB left after the transformer; 0 wherever ``_streamed_pin_plan`` would not pin by budget."""
    if _env_off(PARTIAL_ENCODER_PIN_ENV) or not host_pinning_allowed(device):
        return 0
    forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
    if forced in ("0", "off", "false", "no", "1", "on", "true", "yes") or _pinned_memory_capped():
        return 0
    budget = _pin_budget_mib()
    if budget is None:
        return 0
    left = int(budget) - (int(transformer_mib) if pin_transformer else 0)
    return left if left >= _PARTIAL_PIN_MIN_MIB else 0


_DENOISER_NAMES = ("transformer", "transformer_2", "unconditional_transformer", "unet")

DENOISER_RESIDENT = "resident"
DENOISER_PARTIAL = "partial"
DENOISER_STREAMED = "streamed"


def denoiser_residency(pipe: Any) -> Optional[tuple[int, int]]:
    """``(resident, total)`` offload groups across the denoisers, read off the live hooks: a denoiser another offload
    hook moves counts as one streamed group, an unhooked one as one resident group, None when unreadable."""
    resident = total = 0
    try:
        for name in _DENOISER_NAMES:
            module = getattr(pipe, name, None)
            if module is None:
                continue
            groups = _offload_groups(module)
            if groups:
                total += len(groups)
                resident += sum(1 for g in groups if getattr(g, "_unsloth_resident", False))
                continue
            registry = getattr(module, "_diffusers_hook", None)
            total += 1
            if not (
                any(getattr(sub, "_hf_hook", None) is not None for sub in module.modules())
                or any("offload" in str(key) for key in (getattr(registry, "hooks", None) or {}))
            ):
                resident += 1
    except Exception:  # noqa: BLE001 - a status read must never fail the load or the render
        return None
    return resident, total


def denoiser_offload(pipe: Any, offload_policy: Any) -> Optional[str]:
    """resident | partial | streamed for a group-offloaded pipeline; None for other policies or an unreadable one."""
    if offload_policy != OFFLOAD_GROUP or pipe is None:
        return None
    counts = denoiser_residency(pipe)
    if counts is None:
        return None
    resident, total = counts
    if resident >= total:
        return DENOISER_RESIDENT
    return DENOISER_PARTIAL if resident else DENOISER_STREAMED


def agreed_placement(placements: Iterable[Optional[str]]) -> Optional[str]:
    """The placement every denoise of one image ran with; None when they differ."""
    distinct = set(placements)
    return distinct.pop() if len(distinct) == 1 else None


def denoisers_pinned_resident(pipe: Any) -> bool:
    """Every denoiser unhooked or with all its offload groups pinned, at least one pinned. The hooks stay so an
    oversized request can stream it again (``release_resident_groups``)."""
    pinned = False
    for name in _DENOISER_NAMES:
        module = getattr(pipe, name, None)
        if module is None:
            continue
        try:
            if any(getattr(sub, "_hf_hook", None) is not None for sub in module.modules()):
                return False  # accelerate (model / sequential) offload moves it
        except Exception:  # noqa: BLE001 - unreadable: assume it moves
            return False
        groups = _offload_groups(module)
        if not groups:
            registry = getattr(module, "_diffusers_hook", None)
            if any("offload" in str(key) for key in (getattr(registry, "hooks", None) or {})):
                return False
            continue
        if not all(getattr(g, "_unsloth_resident", False) for g in groups):
            return False
        pinned = True
    return pinned


def _has_offload_groups(module: Any) -> bool:
    try:
        return bool(_offload_groups(module))
    except Exception:  # noqa: BLE001 - an unreadable module counts as unhooked
        return False


def _offload_groups(module: Any) -> list:
    """The diffusers offload groups hooked under ``module``, in registration (block) order."""
    try:
        from diffusers.hooks import group_offloading as go
        name = getattr(go, "_GROUP_OFFLOADING", "group_offloading")
    except Exception:  # noqa: BLE001
        return []
    groups: list = []
    seen: set = set()
    for sub in module.modules():
        registry = getattr(sub, "_diffusers_hook", None)
        get_hook = getattr(registry, "get_hook", None)
        group = getattr(get_hook(name), "group", None) if callable(get_hook) else None
        if group is not None and id(group) not in seen:
            seen.add(id(group))
            groups.append(group)
    return groups


# Video loads only. "0" restores the per-tensor ``pin_memory()``.
FAST_PIN_ENV = "UNSLOTH_VIDEO_FAST_PIN"
_FAST_PIN_REQUEST_ATTR = "_unsloth_fast_pin_requested"
_FAST_PIN_THREADS = 8


def request_fast_pins(pipe: Any) -> None:
    """Ask the background pinners of ``pipe`` to use registered host memory."""
    try:
        setattr(pipe, _FAST_PIN_REQUEST_ATTR, True)
    except Exception:  # noqa: BLE001 - a pipe refusing attributes pins the default way
        pass


def _fast_pin_supported(device: Any = None) -> bool:
    if (os.environ.get(FAST_PIN_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return False
    # WDDM / WSL2 cap page-locked memory; ROCm's register path is unmeasured.
    if not sys.platform.startswith("linux") or _pinned_memory_capped():
        return False
    try:
        import torch

        if (
            is_rocm_torch(torch)
            or not torch.cuda.is_available()
            or not host_pinning_allowed(device)
        ):
            return False
        cudart = torch.cuda.cudart()
        return hasattr(cudart, "cudaHostRegister") and hasattr(cudart, "cudaHostUnregister")
    except Exception:  # noqa: BLE001
        return False


class _RegisteredHostBuffer:
    """Anonymous mapping page-locked with ``cudaHostRegister``; tensor views keep it alive, the last one unregisters it.

    Torch's pinned allocator pins on one thread (10.9-13.1 s for LTX-2.3's 24.6 GiB Gemma3); 8 threads + register: 2.3 s."""

    def __init__(self, nbytes: int):
        import ctypes
        import mmap

        import torch

        self._cudart = torch.cuda.cudart()
        self._size = max(int(nbytes), 1)
        self._map = mmap.mmap(-1, self._size)
        self._anchor = ctypes.c_char.from_buffer(self._map)
        self.ptr = ctypes.addressof(self._anchor)
        self._registered = False
        self._device: Optional[int] = None
        self.__array_interface__ = {
            "shape": (int(nbytes),),
            "typestr": "|u1",
            "data": (self.ptr, False),
            "version": 3,
        }

    def register(self) -> None:
        err = self._cudart.cudaHostRegister(self.ptr, self._size, 0)
        if int(err) != 0:
            raise RuntimeError(f"cudaHostRegister failed ({err})")
        import torch

        # A bare synchronize() on another thread would wait on (and open a context on) device 0.
        self._device = torch.cuda.current_device()
        self._registered = True

    def __del__(self) -> None:
        try:
            if self._registered:
                import torch

                # An async upload may still read these pages; never unmap under it.
                torch.cuda.synchronize(self._device)
                self._cudart.cudaHostUnregister(self.ptr)
                self._registered = False
        except Exception:  # noqa: BLE001 - interpreter teardown
            pass
        try:
            del self._anchor
            self._map.close()
        except Exception:  # noqa: BLE001
            pass


# "0" restores torch's pinned allocator for up-front pinned slabs.
REGISTERED_SLAB_ENV = "UNSLOTH_DIFFUSION_REGISTERED_SLABS"


def registered_pinned_empty(
    nbytes: int, fill: Optional[Callable[[Any], None]] = None
) -> Optional[Any]:
    """Page-locked uint8 host tensor in registered memory, or None (use torch's pinned allocator).
    Torch's allocator faults pages on one thread (~18 s for MiniMax-H3); ``zero_`` here faults them in parallel.
    ``fill(buf)``, when given, writes every byte before registering, in place of the zero fill: the copy faults the
    pages, saving a full write pass (an 8 GB H3 conditioner pin: 3.8 -> 3.4 s hot, 6.1 -> 4.1 s cold, A100 VM).
    Its errors propagate."""
    if (os.environ.get(REGISTERED_SLAB_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return None
    if int(nbytes) <= 0 or not _fast_pin_supported():
        return None
    import numpy as np
    import torch

    try:
        holder = _RegisteredHostBuffer(int(nbytes))
        buf = torch.from_numpy(np.asarray(holder))
    except Exception:  # noqa: BLE001
        return None
    if fill is not None:
        fill(buf)
    try:
        if fill is None:
            buf.zero_()
        holder.register()
    except Exception:  # noqa: BLE001
        return None
    return buf


def registered_host_copy(src: Any) -> Any:
    """A contiguous copy of host tensor ``src`` in CUDA-registered (pinned) memory."""
    import numpy as np
    import torch

    holder = _RegisteredHostBuffer(src.numel() * src.element_size())
    out = torch.from_numpy(np.asarray(holder)).view(src.dtype).view(src.shape)
    out.copy_(src)
    holder.register()
    return out


def _registered_pin(src: Any, device: Any = None) -> Any:
    """``src.pin_memory()`` in registered host memory where the type allows (GGUF re-wrapped, avoiding pow2 rounding)."""
    import torch

    if not host_pinning_allowed(device):
        return src  # integrated GPU: a pinned copy only takes RAM from the card
    if type(src) is torch.Tensor:
        return registered_host_copy(src)
    gguf = _gguf_parameter_class()
    if gguf is not None and type(src) is gguf and _exact_pin_supported(device):
        pinned = registered_host_copy(src.as_tensor())
        return gguf(pinned, requires_grad = False, quant_type = src.quant_type)
    return src.pin_memory()


class _GroupPinner:
    """Pins a streamed module's offload groups on a worker thread, first block first.

    Pinning at load reads every streamed byte from disk before the load returns: 125-139 s of LTX-2.3's 37 GB DiT,
    20-44 s of Wan2.2-5B's text encoder on Colab. Off the load path the read overlaps the prompt encode and the
    first compile. A group's ``onload_`` waits for that group only, and the swap happens before the wait releases,
    so no group is ever onloaded while its host copy is being replaced."""

    def __init__(
        self,
        module: Any,
        groups: list,
        device: Any,
        logger: Any = None,
        fast: bool = False,
        byte_cap: Optional[int] = None,
    ):
        import threading

        self.fast = bool(fast)
        # Pin whole groups in order while they fit; the rest stay pageable.
        if byte_cap is None or callable(byte_cap):
            self.byte_cap = byte_cap
        else:
            self.byte_cap = max(0, int(byte_cap))
        self.module = module
        self.label = type(module).__name__
        self.groups = groups
        self.device = device
        self.logger = logger
        self.pinned = 0
        self.copied = 0
        self.waited_s = 0.0
        self._done = {id(g): threading.Event() for g in groups}
        self._stop = threading.Event()
        self._lock = threading.Lock()
        self.after: list = []
        self.late = False
        # image-load text encoders: groups resident when the pin starts drop their host copy instead of pinning it
        self.lazy_resident = False
        self._thread = threading.Thread(target = self._entry, name = "unsloth-offload-pin", daemon = True)
        self._started = False
        for group in groups:
            setattr(group, _BG_PIN_ATTR, self)

    def start(self) -> None:
        with self._lock:
            if not self._started:
                self._started = True
                self._thread.start()

    def _entry(self) -> None:
        import time

        began = time.perf_counter()
        for dep in self.after:
            dep.start()
        while self.after and not self._stop.is_set():
            if all(dep.join(0) for dep in self.after):
                break
            elapsed = time.perf_counter() - began
            # Fast (not disk-bound) pins: pinning beside the others wins; overlapping the render only moves the cost.
            rate = sum(max(dep.copied, dep.pinned) for dep in self.after) / max(elapsed, 1e-3)
            if elapsed >= _LATE_PIN_PROBE_S and rate >= _LATE_PIN_PARALLEL_BPS:
                self.late = False
                if self.logger is not None:
                    self.logger.info(
                        "diffusion.memory: host weights pin at %.1f GiB/s; %s pins beside the others",
                        rate / 2**30,
                        self.label,
                    )
                break
            time.sleep(0.1)
        if self._stop.is_set():
            for event in self._done.values():
                event.set()
            return
        self._run()

    def wait(self, group: Any) -> None:
        import threading

        event = self._done.get(id(group))
        if event is None or threading.current_thread() is self._thread:
            return
        self.start()
        if not event.is_set():
            import time

            began = time.perf_counter()
            event.wait()
            self.waited_s += time.perf_counter() - began

    def join(self, timeout: Optional[float] = None) -> bool:
        """Wait for the worker; True once it has exited (or when called from the worker itself)."""
        import threading

        if threading.current_thread() is self._thread:
            return True
        self._thread.join(timeout)
        return not self._thread.is_alive()

    def stop(self, timeout: Optional[float] = None) -> None:
        self._stop.set()
        if self._started:
            self._thread.join(timeout)

    def _unpinned(self, group: Any) -> list:
        import torch

        # GGUF host copies are GGUFParameter (packed bytes + quant type): pin_memory keeps the subclass, which the
        # per-onload fallback already relies on. Leaving them unpinned kept every CUDA graph off the GGUF stream.
        gguf = _gguf_parameter_class()
        return [
            (tensor, src)
            for tensor, src in list(group.cpu_param_dict.items())
            if (type(src) is torch.Tensor or (gguf is not None and type(src) is gguf))
            and src.device.type == "cpu"
            and src.numel() > 0
            and not src.is_pinned()
        ]

    def _submit(self, pool: Any, src: Any) -> Any:
        future = pool.submit(_registered_pin, src, self.device)
        future.add_done_callback(functools.partial(self._count_copy, int(src.nbytes)))
        return future

    def _count_copy(self, nbytes: int, future: Any) -> None:
        if not future.cancelled():
            self.copied += nbytes

    @staticmethod
    def _fast_result(future: Any, src: Any) -> Any:
        try:
            return future.result()
        except Exception:  # noqa: BLE001 - registration refused: the allocator's pin for this tensor
            return src.pin_memory()

    def outstanding_bytes(self) -> int:
        """Bytes this pinner may still place: unpinned groups, capped by what its byte cap has left."""
        try:
            need = sum(
                _page_bytes(src.nbytes)
                for group in self.groups
                for _t, src in self._unpinned(group)
            )
        except Exception:  # noqa: BLE001 - unreadable: owes nothing
            return 0
        cap = self.byte_cap
        if callable(cap) or cap is None:
            return need
        return min(need, max(0, int(cap) - int(self.pinned)))

    def _capped_groups(self) -> list:
        if self.byte_cap is None:
            return list(self.groups)
        left = self.byte_cap
        out: list = []
        for group in self.groups:
            need = sum(_page_bytes(src.nbytes) for _t, src in self._unpinned(group))
            if need > left:
                break
            left -= need
            out.append(group)
        return out

    def _run(self) -> None:
        import time

        import torch

        start = time.perf_counter()
        failed = None
        pool = None
        try:
            if self.lazy_resident and _resident_host_drop_enabled():
                # Resident groups are read from the card; pinning their host copy only cost RAM and load time.
                kept, _freed = drop_resident_host_copies(self.groups, self.logger, self.label)
                kept_ids = {id(g) for g in kept}
                for group in self.groups:
                    if id(group) not in kept_ids:
                        self._done[id(group)].set()
                self.groups = kept
            if callable(self.byte_cap):
                self.byte_cap = max(0, int(self.byte_cap()))
                if self.logger is not None:
                    self.logger.info(
                        "diffusion.memory: %s pins up to %.1f GiB of its offload groups (cap read at pin start)",
                        self.label,
                        self.byte_cap / 2**30,
                    )
            if self.byte_cap is not None:
                kept = self._capped_groups()
                kept_ids = {id(g) for g in kept}
                # A group past the cap never pins here: its onload must not wait for the prefix.
                for group in self.groups:
                    if id(group) not in kept_ids:
                        self._done[id(group)].set()
                self.groups = kept
            if (
                getattr(self.device, "type", None) == "cuda"
                and getattr(self.device, "index", None) is not None
            ):
                torch.cuda.set_device(self.device)
            copies: dict = {}
            if self.fast:
                from concurrent.futures import ThreadPoolExecutor

                # Queued in group order, so the first group is ready first.
                pool = ThreadPoolExecutor(
                    max_workers = _FAST_PIN_THREADS,
                    thread_name_prefix = "unsloth-fast-pin",
                    initializer = torch.cuda.set_device,
                    initargs = (torch.cuda.current_device(),),
                )
                for group in self.groups:
                    copies[id(group)] = [
                        (
                            tensor,
                            src,
                            self._submit(pool, src),
                        )
                        for tensor, src in self._unpinned(group)
                    ]
            for group in self.groups:
                if self._stop.is_set():
                    break
                if self.fast:
                    placed = [
                        (tensor, src, self._fast_result(future, src))
                        for tensor, src, future in copies.pop(id(group))
                    ]
                else:
                    # One pin per tensor, as the eager apply does; chunk views are slower to stream on LTX-2.3.
                    placed = [
                        (tensor, src, src.pin_memory()) for tensor, src in self._unpinned(group)
                    ]
                cpu = group.cpu_param_dict
                for tensor, src, pinned in placed:
                    if cpu.get(tensor) is src:
                        cpu[tensor] = pinned
                        if tensor.device.type == "cpu" and tensor.data_ptr() == src.data_ptr():
                            tensor.data = pinned
                        self.pinned += src.nbytes
                # Done before the epoch: a capture check that sees the new epoch must also see this group ready.
                self._done[id(group)].set()
                _bump_placement_epoch()
        except Exception as exc:  # noqa: BLE001 - diffusers pins what is left on each onload
            failed = exc
        finally:
            if pool is not None:
                pool.shutdown(wait = True, cancel_futures = True)
            for event in self._done.values():
                event.set()
            if failed is not None:
                _bump_placement_epoch()
        if self.logger is not None:
            try:
                if failed is not None:
                    self.logger.warning(
                        "diffusion.memory: background pinning of %s stopped (%s); the rest pins on each onload",
                        self.label,
                        failed,
                    )
                self.logger.info(
                    "diffusion.memory: pinned %.1f GiB of %s host weights in the background in %.1f s "
                    "(onloads waited %.1f s%s)",
                    self.pinned / 2**30,
                    self.label,
                    time.perf_counter() - start,
                    self.waited_s,
                    ", registered" if self.fast else "",
                )
            except Exception:  # noqa: BLE001
                pass


def install_group_pin_wait() -> bool:
    """Make a diffusers offload group wait for its background pin before it onloads."""
    try:
        import torch
        from diffusers.hooks import group_offloading as go
    except Exception:  # noqa: BLE001
        return False
    group_cls = getattr(go, "ModuleGroup", None)
    original = getattr(group_cls, "onload_", None)
    if original is None or getattr(original, "_unsloth_pin_wait", False):
        return False

    @functools.wraps(original)
    def onload_(self, *args: Any, **kwargs: Any) -> Any:
        pinner = getattr(self, _BG_PIN_ATTR, None)
        if pinner is not None:
            pinner.wait(self)
        return original(self, *args, **kwargs)

    disable = getattr(getattr(torch, "compiler", None), "disable", None)
    wrapped = disable(onload_) if callable(disable) else onload_
    wrapped._unsloth_pin_wait = True
    group_cls.onload_ = wrapped
    return True


def start_background_pins(pipe: Any) -> int:
    """Start the pinners the load deferred; returns how many."""
    pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
    for pinner in pinners:
        pinner.start()
    return len(pinners)


def finish_background_pins(
    pipe: Any,
    cancel: Any = None,
    include_late: bool = True,
) -> float:
    """Blocks until deferred pinners finish, since a render overlapping one runs measurably slower."""
    import time

    start = time.perf_counter()
    pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
    for pinner in pinners:
        pinner.start()
    # Late pinners last: one may still turn early until the pinners before it finish.
    for pinner in sorted(pinners, key = lambda p: bool(getattr(p, "late", False))):
        if not include_late and getattr(pinner, "late", False):
            continue
        # Polled so a cancelled request leaves now; the pin itself keeps running for the next render.
        while not pinner.join(0.25):
            if cancel is not None and cancel.is_set():
                return time.perf_counter() - start
    return time.perf_counter() - start


# "0": a two-expert denoiser's second expert pins beside the other weights, before the first render.
LATE_EXPERT_PIN_ENV = "UNSLOTH_VIDEO_LATE_EXPERT_PIN"
_LATE_PIN_PROBE_S = 0.2
_LATE_PIN_PARALLEL_BPS = 3 << 30


def defer_late_expert_pins(
    pipe: Any,
    late_modules: Any,
    logger: Any = None,
) -> int:
    """When disk-bound, pin late_modules after the other pinners so the render waits only for those;
    a late group onload still waits for its own pin."""
    if _env_off(LATE_EXPERT_PIN_ENV):
        return 0
    late_ids = {id(m) for m in late_modules or () if m is not None}
    pending = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
    late = [p for p in pending if id(p.module) in late_ids]
    early = [p for p in pending if id(p.module) not in late_ids]
    if not late or not early:
        return 0
    for pinner in late:
        pinner.after = list(early)
        pinner.late = True
    if logger is not None:
        logger.info(
            "diffusion.memory: %d late expert(s) (%.1f GiB) pin after the other %d pinner(s) (%.1f GiB); "
            "the first render waits for those only",
            len(late),
            sum(p.outstanding_bytes() for p in late) / 2**30,
            len(early),
            sum(p.outstanding_bytes() for p in early) / 2**30,
        )
    return len(late)


def stop_background_pins(pipe: Any, timeout: Optional[float] = 30.0) -> None:
    for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
        try:
            pinner.stop(timeout)
        except Exception:  # noqa: BLE001 - teardown is best effort
            pass


def _drop_deferred_pinning(pipe: Any, module: Any) -> None:
    pending = getattr(pipe, _PENDING_PINS_ATTR, None)
    if pending:
        pending[:] = [pinner for pinner in pending if pinner.module is not module]


def _defer_pinning(
    pipe: Any,
    module: Any,
    device: Any,
    logger: Any,
    *,
    fast: Optional[bool] = None,
    byte_cap: Optional[int] = None,
    lazy_resident: bool = False,
) -> bool:
    groups = _offload_groups(module)
    if not groups:
        return False
    if fast is None:
        fast = bool(getattr(pipe, _FAST_PIN_REQUEST_ATTR, False))
    fast = bool(fast) and _fast_pin_supported(device)
    pinner = _GroupPinner(module, groups, device, logger, fast = fast, byte_cap = byte_cap)
    pinner.lazy_resident = bool(lazy_resident)
    pending = getattr(pipe, _PENDING_PINS_ATTR, None)
    if pending is None:
        pending = []
        try:
            setattr(pipe, _PENDING_PINS_ATTR, pending)
        except Exception:  # noqa: BLE001 - nowhere to park it: pin now, like the eager path
            pinner.start()
            return True
    pending.append(pinner)
    return True


EAGER_OFFLOAD_HOOKS_ENV = "UNSLOTH_DIFFUSION_EAGER_OFFLOAD_HOOKS"


def install_group_offload_hooks_eager() -> bool:
    """Keep diffusers' group-offload hooks out of compiled blocks: traced, the first-forward layer tracker guards on the
    block name and recompiles per block (44 recompiles on Qwen-Image-2.1 at 16 GB, 5 eager). Must run before the hooks register."""
    if _env_off(EAGER_OFFLOAD_HOOKS_ENV):
        return False
    try:
        import torch
        from diffusers.hooks import group_offloading as go
    except Exception:  # noqa: BLE001 - no group offload in this diffusers
        return False
    disable = getattr(getattr(torch, "compiler", None), "disable", None)
    if not callable(disable):
        return False
    patched = False
    for cls_name, methods in (
        ("GroupOffloadingHook", ("pre_forward", "post_forward")),
        ("LayerExecutionTrackerHook", ("pre_forward",)),
        ("LazyPrefetchGroupOffloadingHook", ("post_forward",)),
    ):
        cls = getattr(go, cls_name, None)
        for name in methods:
            fn = getattr(cls, "__dict__", {}).get(name) if cls is not None else None
            if fn is None or getattr(fn, "_unsloth_eager", False):
                continue
            eager = disable(fn)
            eager._unsloth_eager = True
            eager._unsloth_orig = fn
            setattr(cls, name, eager)
            patched = True
    return patched


def _install_group_offload_torchao_host_copy(go: Any) -> bool:
    """Give aliased torchao host copies a separate wrapper so ``swap_tensors`` cannot move them to CUDA."""
    group_cls = getattr(go, "ModuleGroup", None)
    original = getattr(group_cls, "_to_cpu", None)
    is_torchao = getattr(go, "_is_torchao_tensor", None)
    if (
        original is None
        or not callable(is_torchao)
        or getattr(original, "_unsloth_host_copy", False)
    ):
        return False

    def _to_cpu(tensor: Any, low_cpu_mem_usage: bool) -> Any:
        copy = original(tensor, low_cpu_mem_usage)
        if copy is not tensor or not is_torchao(tensor):
            return copy
        names, ctx = tensor.__tensor_flatten__()
        return type(tensor).__tensor_unflatten__(
            {name: getattr(tensor, name) for name in names}, ctx, tensor.size(), tensor.stride()
        )

    _to_cpu._unsloth_host_copy = True
    group_cls._to_cpu = staticmethod(_to_cpu)
    return True


def install_group_offload_buffer_restore() -> bool:
    """Fix buffer restoration and torchao host-copy aliasing in diffusers stream group offload."""
    try:
        from diffusers.hooks import group_offloading as go
    except Exception:  # noqa: BLE001 - no group offload in this diffusers
        return False
    _install_group_offload_torchao_host_copy(go)
    group_cls = getattr(go, "ModuleGroup", None)
    original = getattr(group_cls, "_offload_to_memory", None)
    if original is None or getattr(original, "_unsloth_buffer_restore", False):
        return False

    @functools.wraps(original)
    def _offload_to_memory(self, *args: Any, **kwargs: Any) -> Any:
        out = original(self, *args, **kwargs)
        cpu_copies = getattr(self, "cpu_param_dict", None)
        if getattr(self, "stream", None) is None or not cpu_copies:
            return out
        restore_torchao = getattr(go, "_restore_torchao_tensor", None)
        is_torchao = getattr(go, "_is_torchao_tensor", None)
        for group_module in getattr(self, "modules", None) or ():
            for buffer in group_module.buffers():
                cpu = cpu_copies.get(buffer)
                if cpu is None:
                    continue
                if callable(is_torchao) and callable(restore_torchao) and is_torchao(buffer):
                    restore_torchao(buffer, cpu)
                else:
                    buffer.data = cpu
        return out

    _offload_to_memory._unsloth_buffer_restore = True
    group_cls._offload_to_memory = _offload_to_memory
    return True


PIN_TOP_GROUP_ENV = "UNSLOTH_DIFFUSION_PIN_TOP_GROUP"
# read-only: diffusers leaves the top-level group's cpu_param_dict empty
TOP_GROUP_HOST_ATTR = "_unsloth_top_host"


TOP_GROUP_GGUF_ENV = "UNSLOTH_DIFFUSION_TOP_GROUP_GGUF"


def _top_group_tensor_ok(t: Any) -> bool:
    """Plain tensors and GGUFParameter (a ``.data`` re-point keeps its quant type); other subclasses keep diffusers' path."""
    import torch

    if type(t) in (torch.Tensor, torch.nn.Parameter):
        return True
    gguf = _gguf_parameter_class()
    return gguf is not None and type(t) is gguf and not _env_off(TOP_GROUP_GGUF_ENV)


def _pin_top_level_group(
    module: Any,
    logger: Any = None,
    pinned_mib: Optional[list] = None,
    device: Any = None,
) -> bool:
    """Onloads from one pinned host copy, async; skipped for torchao weights (not re-pointable) and when the copy does not fit pinnable host RAM.
    ``device``: the group's onload target (None: the current device)."""
    if (os.environ.get(PIN_TOP_GROUP_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return False
    if not host_pinning_allowed(device):
        return False
    try:
        import torch
        from diffusers.hooks import group_offloading as go

        registry = getattr(module, "_diffusers_hook", None)
        get_hook = getattr(registry, "get_hook", None)
        hook = (
            get_hook(getattr(go, "_GROUP_OFFLOADING", "group_offloading"))
            if callable(get_hook)
            else None
        )
        group = getattr(hook, "group", None)
        if (
            group is None
            or getattr(group, "stream", None) is not None
            or getattr(group, "offload_to_disk_path", None)
            or getattr(getattr(group, "onload_device", None), "type", None) != "cuda"
            or getattr(group, "_unsloth_pinned_top", False)
            or getattr(group, "_unsloth_top_no_copy_back", False)
        ):
            return False
        tensors: list = []
        seen: set = set()
        for tensor in (
            [p for m in group.modules for p in m.parameters()]
            + [b for m in group.modules for b in m.buffers()]
            + list(group.parameters or [])
            + list(group.buffers or [])
        ):
            if id(tensor) not in seen:
                seen.add(id(tensor))
                tensors.append(tensor)
        is_torchao = getattr(go, "_is_torchao_tensor", None)
        if not tensors or (callable(is_torchao) and any(is_torchao(t) for t in tensors)):
            return False
        if any(not _top_group_tensor_ok(t) for t in tensors):
            return False
        # the user's "pin nothing" override wins, as on every other streamed path
        if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in (
            "0",
            "off",
            "false",
            "no",
        ):
            return False
        # per tensor rounded to a power of two, like torch's pinned allocator (and _module_host_mib)
        need_mib = sum(
            1 << (int(t.numel()) * int(t.element_size()) - 1).bit_length()
            for t in tensors
            if int(t.numel()) * int(t.element_size()) > 0
        ) // (1024 * 1024)
        budget = None if _pinned_memory_capped() else _pin_budget_mib()
        # on the running total the encoders and torchao denoisers already pinned (or will, deferred) count against
        already = pinned_mib[0] if pinned_mib else 0
        if budget is None or already + need_mib > budget:
            return False
        host = {
            t: (t.data if t.data.device.type == "cpu" else t.data.cpu()).pin_memory()
            for t in tensors
        }
        device = group.onload_device

        def onload_() -> None:
            for tensor, pinned in list(host.items()):
                current = tensor.data
                if current.device.type == "cpu" and current.data_ptr() != pinned.data_ptr():
                    # replaced while offloaded (a .to() conversion, an adapter fused on the host): re-pin what is there now
                    pinned = current if current.is_pinned() else current.pin_memory()
                    host[tensor] = pinned
                tensor.data = pinned.to(device, non_blocking = True)

        def offload_() -> None:
            for tensor, pinned in host.items():
                tensor.data = pinned

        disable = getattr(getattr(torch, "compiler", None), "disable", None)
        if callable(disable):
            onload_, offload_ = disable(onload_), disable(offload_)
        offload_()
        group.onload_ = onload_
        group.offload_ = offload_
        group._unsloth_pinned_top = True
        # capture_refusal keeps streamed CUDA graphs off for GGUF: their pool starved a 6 GiB T4 decode
        gguf = _gguf_parameter_class()
        group._unsloth_top_gguf = gguf is not None and any(type(t) is gguf for t in tensors)
        setattr(group, TOP_GROUP_HOST_ATTR, host)
        if pinned_mib is not None:
            pinned_mib[0] = already + need_mib
        if logger is not None:
            logger.info(
                "diffusion.memory: %s top-level weights (%d MiB) onload from a pinned copy, no copy back",
                type(module).__name__,
                need_mib,
            )
        return True
    except Exception as exc:  # noqa: BLE001 - diffusers keeps its own (slower) path
        if logger is not None:
            logger.debug("diffusion.memory: top-level group left as diffusers built it (%s)", exc)
        return False


def _skip_top_level_copy_back(module: Any, logger: Any = None) -> bool:
    """Fallback when ``_pin_top_level_group`` cannot pin: offload re-points to the group's existing host tensors
    instead of copying the (inference-constant) weights back on the compute stream. Same kill switch as the pinned path."""
    if (os.environ.get(PIN_TOP_GROUP_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
        return False
    try:
        import torch
        from diffusers.hooks import group_offloading as go

        registry = getattr(module, "_diffusers_hook", None)
        get_hook = getattr(registry, "get_hook", None)
        hook = (
            get_hook(getattr(go, "_GROUP_OFFLOADING", "group_offloading"))
            if callable(get_hook)
            else None
        )
        group = getattr(hook, "group", None)
        if (
            group is None
            or getattr(group, "stream", None) is not None
            or getattr(group, "offload_to_disk_path", None)
            or getattr(getattr(group, "onload_device", None), "type", None) != "cuda"
            or getattr(group, "_unsloth_pinned_top", False)
            or getattr(group, "_unsloth_top_no_copy_back", False)
        ):
            return False
        tensors: list = []
        seen: set = set()
        for tensor in (
            [p for m in group.modules for p in m.parameters()]
            + [b for m in group.modules for b in m.buffers()]
            + list(group.parameters or [])
            + list(group.buffers or [])
        ):
            if id(tensor) not in seen:
                seen.add(id(tensor))
                tensors.append(tensor)
        is_torchao = getattr(go, "_is_torchao_tensor", None)
        if not tensors or (callable(is_torchao) and any(is_torchao(t) for t in tensors)):
            return False
        if any(not _top_group_tensor_ok(t) for t in tensors):
            return False
        host = {t: (t.data if t.data.device.type == "cpu" else t.data.cpu()) for t in tensors}
        device = group.onload_device

        def onload_() -> None:
            for tensor, cpu in list(host.items()):
                current = tensor.data
                if current.device.type == "cpu" and current.data_ptr() != cpu.data_ptr():
                    # replaced while offloaded (a .to() conversion, an adapter fused on the host): upload that instead
                    cpu = current
                    host[tensor] = cpu
                tensor.data = cpu.to(device)

        def offload_() -> None:
            for tensor, cpu in host.items():
                tensor.data = cpu

        disable = getattr(getattr(torch, "compiler", None), "disable", None)
        if callable(disable):
            onload_, offload_ = disable(onload_), disable(offload_)
        offload_()
        group.onload_ = onload_
        group.offload_ = offload_
        group._unsloth_top_no_copy_back = True
        setattr(group, TOP_GROUP_HOST_ATTR, host)
        if logger is not None:
            logger.info(
                "diffusion.memory: %s top-level weights (%d MiB) not pinnable here; offload re-points to the host "
                "copy instead of copying them back",
                type(module).__name__,
                sum(int(t.numel()) * int(t.element_size()) for t in tensors) >> 20,
            )
        return True
    except Exception as exc:  # noqa: BLE001 - diffusers keeps its own path
        if logger is not None:
            logger.debug(
                "diffusion.memory: top-level group copy-back left as diffusers built it (%s)", exc
            )
        return False


def _apply_group_offload(
    pipe: Any,
    device: str,
    logger: Any,
    *,
    stream_text_encoders: bool = False,
    stream_transformer: bool = True,
    background_pin: Optional[bool] = None,
    resident_transformer_mib: Optional[int] = None,
    resident_text_encoder_mib: Optional[int] = None,
    denoiser: str = "transformer",
) -> bool:
    """Streams transformer blocks in groups; encoders stay resident unless stream_text_encoders is set; returns False on any failure, caller falls back.
    ``denoiser="unet"`` streams leaf by leaf (UNet blocks are too coarse for groups)."""
    transformer = getattr(pipe, denoiser, None)
    if transformer is None:
        return False
    if not stream_transformer and not stream_text_encoders:
        return False
    if _any_layer_split(pipe):
        from .diffusion_layer_split_plan import undo_layer_split
        undo_layer_split(pipe)
        if logger is not None:
            logger.info("diffusion.memory: removed the layer split before applying group offload")
    installed = 0
    # every component's torch forward hooks before this call: an OOM after hooks exist unwinds to this state
    hook_snapshots: list = []
    grouped_before: set = set()
    try:
        import inspect

        import torch
        from diffusers.hooks import apply_group_offloading

        install_group_offload_buffer_restore()
        install_group_offload_hooks_eager()
        hook_snapshots = [
            (comp, _module_forward_hook_ids(comp))
            for comp in (getattr(pipe, "components", {}) or {}).values()
            if isinstance(comp, torch.nn.Module)
        ]
        grouped_before = {id(comp) for comp, _ in hook_snapshots if _has_offload_groups(comp)}

        # Dual-DiT pipelines (Ideogram 4): second DiT is as large as the first, so stream all DiTs.
        streamed: dict[str, Any] = {}
        if stream_transformer and denoiser != "transformer":
            streamed[denoiser] = transformer
        elif stream_transformer:
            streamed["transformer"] = transformer
            for extra in ("transformer_2", "unconditional_transformer"):
                module = getattr(pipe, extra, None)
                if isinstance(module, torch.nn.Module):
                    streamed[extra] = module
        # Encoders stream separately from DiTs, in their own dict so the resident placement loop skips them.
        streamed_encoders: dict[str, Any] = {}
        if stream_text_encoders:
            # Encoders run once before step 0, so stream them; match components by name prefix text_encoder.
            for name, comp in getattr(pipe, "components", {}).items():
                if name.startswith("text_encoder") and isinstance(comp, torch.nn.Module):
                    streamed_encoders[name] = comp

        onload = torch.device(device)
        # overlap H2D copies with compute; a stream makes diffusers pin, so not on an integrated GPU
        use_stream = onload.type == "cuda" and host_pinning_allowed(onload)
        gkwargs: dict[str, Any] = {
            "onload_device": onload,
            "offload_device": torch.device("cpu"),
            "offload_type": "block_level",
            "num_blocks_per_group": DEFAULT_GROUP_BLOCKS,
            "use_stream": use_stream,
        }
        if denoiser != "transformer":
            gkwargs["offload_type"] = "leaf_level"
            gkwargs.pop("num_blocks_per_group")
        # Gated on the signature so older diffusers still works.
        _params = inspect.signature(apply_group_offloading).parameters
        if use_stream:
            if "non_blocking" in _params:
                gkwargs["non_blocking"] = True
            if "record_stream" in _params:
                gkwargs["record_stream"] = True
        # background_pin: applied unpinned; a _GroupPinner pins it once the load commits (start_background_pins).
        if background_pin is None:
            background_pin = bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
        defer = (
            background_pin
            and use_stream
            and "low_cpu_mem_usage" in _params
            and _background_pin_enabled(onload)
        )
        # an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
        pin_mib = _module_host_mib if defer else _module_pin_mib
        # Power-of-two pricing read Wan's 12.5 GiB UMT5 as 17 GiB, so it never pinned on 31-51 GiB hosts.
        exact_defer = defer and _video_exact_pins(pipe, onload, logger)
        if exact_defer:
            _host_price = pin_mib

            def pin_mib(module: Any) -> int:
                if _torchao_weight_classes(module):
                    return _host_price(module)
                return _module_offload_registered_mib(module)

        # Image loads: only the streamed encoders defer (registered pins, priced at weight bytes); torchao ones cannot.
        encoders_defer = (
            not defer
            and stream_text_encoders
            and use_stream
            and "low_cpu_mem_usage" in _params
            and _encoder_background_pin_wanted(pipe, onload)
        )

        def _encoder_defers(module: Any) -> bool:
            return encoders_defer and not _torchao_weight_classes(module)

        def enc_pin_mib(module: Any) -> int:
            return _module_registered_mib(module) if _encoder_defers(module) else pin_mib(module)

        pin_streamed = (True, True)
        dit_pin_mib = 0
        if stream_text_encoders and "low_cpu_mem_usage" in _params:
            # The streamed path PINS every offloaded parameter in host RAM when a copy stream is in use (diffusers
            # group_offloading `_init_cpu_param_dict`), which is a fine trade when group offload was already the plan.
            # Pin only what host RAM covers (unbounded pinning hurt #8188); unpinned re-pins on every onload.
            dit_pin_mib = sum(pin_mib(m) for m in streamed.values())
            pin_streamed = _streamed_pin_plan(
                dit_pin_mib,
                sum(enc_pin_mib(m) for m in streamed_encoders.values()),
                logger,
                device = onload,
            )
            gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]
        small_host = bool(getattr(pipe, "_unsloth_small_host", None))
        if small_host and "low_cpu_mem_usage" in _params:
            # pinning would copy the small-host route's memory-mapped bytes back into host RAM
            pin_streamed = (False, False)
            gkwargs["low_cpu_mem_usage"] = True
        encoder_cap_mib = 0
        late_prefix = (
            exact_defer
            and stream_text_encoders
            and not pin_streamed[1]
            and not small_host
            and not _env_off(PARTIAL_ENCODER_PIN_ENV)
        )
        if late_prefix and logger is not None:
            logger.info(
                "diffusion.memory: streamed encoders pin the prefix of their offload groups that fits the host RAM "
                "free when the load returns"
            )
        if encoders_defer and stream_text_encoders and not pin_streamed[1] and not small_host:
            encoder_cap_mib = _streamed_pin_leftover_mib(dit_pin_mib, pin_streamed[0], onload)
            if encoder_cap_mib and logger is not None:
                logger.info(
                    "diffusion.memory: streamed encoders pin a %d MiB prefix of their offload groups in the background",
                    encoder_cap_mib,
                )
        if defer or encoders_defer:
            install_group_pin_wait()
        # Place companions BEFORE group hooks: a companion OOM then leaves no hooks, and
        # diffusers rejects enable_model_cpu_offload once group hooks exist.
        for name, comp in getattr(pipe, "components", {}).items():
            if name in streamed or name in streamed_encoders:
                continue
            if isinstance(comp, torch.nn.Module):
                comp.to(onload)
        # Encoders the plan pins count against the same budget as any torchao denoiser pinned below.
        pinned_mib = [
            sum(enc_pin_mib(m) for m in streamed_encoders.values())
            if stream_text_encoders and pin_streamed[1]
            else encoder_cap_mib
        ]
        for module in streamed.values():
            # torchao weights need their up-front pin (lazy pinning refuses them), so they never defer.
            if (
                defer
                and not gkwargs.get("low_cpu_mem_usage", False)
                and not _torchao_weight_classes(module)
            ):
                apply_group_offloading(
                    module,
                    **_with_dict_block_modules(
                        apply_group_offloading, module, {**gkwargs, "low_cpu_mem_usage": True}
                    ),
                )
                _defer_pinning(pipe, module, onload, logger)
            else:
                _apply_group_offloading_packed(
                    apply_group_offloading,
                    module,
                    _torchao_group_offload_kwargs(module, gkwargs, pinned_mib),
                    logger,
                )
            installed += 1
            if use_stream and not _pin_top_level_group(module, logger, pinned_mib, device = onload):
                _skip_top_level_copy_back(module, logger)
            if use_stream and gkwargs.get("record_stream"):
                install_group_prefetch(module, onload, logger)
        if resident_transformer_mib:
            room = int(resident_transformer_mib)
            for module in streamed.values():
                room -= _keep_groups_resident(module, room, onload, logger)
        # Encoders after the DiTs, one by one, failures absorbed: a refusing encoder stays resident
        # since hooks already exist. Leaf level: encoders are not uniform block stacks.
        ekwargs = {k: v for k, v in gkwargs.items() if k != "num_blocks_per_group"}
        ekwargs["offload_type"] = "leaf_level"
        if "low_cpu_mem_usage" in gkwargs:
            ekwargs["low_cpu_mem_usage"] = not pin_streamed[1]
        transformer_demoted = False
        te_room = int(resident_text_encoder_mib or 0)
        encoder_left_mib = encoder_cap_mib
        for name, module in streamed_encoders.items():
            try:
                whole = not ekwargs.get("low_cpu_mem_usage", False)
                if defer and whole:
                    apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
                    _defer_pinning(pipe, module, onload, logger)
                elif late_prefix and use_stream and not _torchao_weight_classes(module):
                    apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
                    _defer_pinning(
                        pipe,
                        module,
                        onload,
                        logger,
                        fast = True,
                        byte_cap = _late_encoder_pin_cap(pipe, module, onload),
                    )
                elif _encoder_defers(module) and (whole or encoder_left_mib > 0):
                    apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
                    cap = None if whole else int(encoder_left_mib) << 20
                    _defer_pinning(
                        pipe, module, onload, logger, fast = True, byte_cap = cap, lazy_resident = True
                    )
                    if not whole:
                        encoder_left_mib = max(0, encoder_left_mib - _module_registered_mib(module))
                else:
                    _apply_group_offloading_packed(apply_group_offloading, module, ekwargs, logger)
                installed += 1
                _pin_vision_embedding_device(module)
                if te_room > 0:
                    te_room -= _keep_groups_resident(module, te_room, onload, logger)
                if getattr(pipe, "_unsloth_small_host", None):
                    from .diffusion_small_host import install_encoder_prefetch
                    install_encoder_prefetch(module, onload, logger)
            except Exception as exc:  # noqa: BLE001 -- degrade this encoder, never fail the load
                if not stream_transformer and installed == 0:
                    # Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.
                    _remove_group_offload_hooks(module)
                    raise
                if not stream_transformer and not transformer_demoted:
                    if _pipe_denoisers_hold_torchao(pipe):
                        raise
                    # Model offload is gone once hooks exist: stream the transformer before this encoder goes resident.
                    for dit_name in ("transformer", "transformer_2", "unconditional_transformer"):
                        dit = getattr(pipe, dit_name, None)
                        if isinstance(dit, torch.nn.Module):
                            dkwargs = dict(gkwargs)
                            if "low_cpu_mem_usage" in _params and use_stream:
                                dkwargs["low_cpu_mem_usage"] = True
                            apply_group_offloading(
                                dit,
                                **_with_dict_block_modules(
                                    apply_group_offloading,
                                    dit,
                                    _torchao_group_offload_kwargs(dit, dkwargs, pinned_mib),
                                ),
                            )
                            installed += 1
                    transformer_demoted = True
                    if logger is not None:
                        logger.warning(
                            "diffusion.memory: %s refused group offload on the resident-transformer tier; "
                            "streaming the transformer instead",
                            name,
                        )
                if logger is not None:
                    logger.warning(
                        "diffusion.memory: group offload unavailable for %s (%s); "
                        "keeping it resident",
                        name,
                        exc,
                    )
                # A failed leaf apply may leave partial hooks; resident must mean no hooks.
                _remove_group_offload_hooks(module)
                _drop_deferred_pinning(pipe, module)
                module.to(onload)
        return True
    except Exception as exc:  # noqa: BLE001 - fall back to whole-module offload
        # apply_group_offloading can raise after hooking part of a module, before ``installed`` counts it
        if not installed and any(
            id(comp) not in grouped_before and _has_offload_groups(comp)
            for comp, _ in hook_snapshots
        ):
            installed = 1
        if installed and _is_oom(exc) and _unwind_group_offload(pipe, hook_snapshots, logger):
            # An OOM after some hooks exist (a later DiT, an encoder moved resident): every hook is gone again, so
            # the caller's one lower-residency retry can run instead of the load failing.
            installed = 0
        if installed:
            # Partial group-offload hooks make enable_model_cpu_offload reject the pipe; raise the real failure.
            if logger is not None:
                logger.warning(
                    "diffusion.memory: group offload failed after installing hooks on %d "
                    "module(s) (%s); cannot fall back to whole-module offload",
                    installed,
                    exc,
                )
            raise
        _set_group_offload_failure(pipe, exc)
        if logger is not None:
            logger.warning(
                "diffusion.memory: group offload failed (%s); falling back to "
                "whole-module offload",
                exc,
            )
        return False


def stream_text_encoders_onto(
    pipe: Any,
    device: Any,
    logger: Any = None,
) -> list[str]:
    """Leaf-level group offload of every ``text_encoder*`` onto ``device`` and nothing else: the denoisers and the other
    components are not touched (a layer split places them; ``_apply_group_offload`` would move every unstreamed
    component onto one card). A text encoder runs once, before step 0, so it pages in for that encode. Returns the
    streamed names; raises (with this call's hooks removed) when an encoder refuses, and the caller falls back."""
    import inspect

    import torch
    from diffusers.hooks import apply_group_offloading

    install_group_offload_buffer_restore()
    install_group_offload_hooks_eager()
    onload = torch.device(device)
    use_stream = onload.type == "cuda" and host_pinning_allowed(onload)
    kwargs: dict[str, Any] = {
        "onload_device": onload,
        "offload_device": torch.device("cpu"),
        "offload_type": "leaf_level",
        "use_stream": use_stream,
    }
    params = inspect.signature(apply_group_offloading).parameters
    if use_stream:
        if "non_blocking" in params:
            kwargs["non_blocking"] = True
        if "record_stream" in params:
            kwargs["record_stream"] = True
    encoders = {
        str(n): m
        for n, m in (getattr(pipe, "components", {}) or {}).items()
        if str(n).startswith("text_encoder") and isinstance(m, torch.nn.Module)
    }
    if "low_cpu_mem_usage" in params:
        pin = _streamed_pin_plan(
            0, sum(_module_pin_mib(m) for m in encoders.values()), logger, device = onload
        )
        kwargs["low_cpu_mem_usage"] = not pin[1]
    done: list[str] = []
    try:
        for name, module in encoders.items():
            module.to("cpu")
            done.append(name)
            _apply_group_offloading_packed(apply_group_offloading, module, kwargs, logger)
            _pin_vision_embedding_device(module)
    except Exception:
        for name in done:
            _remove_group_offload_hooks(encoders[name])
        raise
    if logger is not None and done:
        logger.info(
            "diffusion.memory: streaming %s onto %s for the encode", ", ".join(done), onload
        )
    return done


# Generate-time activation guard. The load-time plan cannot know the output resolution: a model is loaded once and
# then generates at whatever size the sliders say, so ``_plan_memory`` budgets the 1024x1024 default. That is the
# right call for PLACEMENT, but it means a request for a much larger frame is never checked against anything, so this
# re-checks per generation with the real dimensions. Opt-in escape hatch, mirroring the load-time one: the activation
# estimate is coarse, so an operator who believes it is wrong keeps a way through. Also sent per request
# (``allow_oversized``): a desktop install has no terminal.
OVERSIZED_GENERATE_ENV = "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_GENERATE"

# Must match the Images page setting label (frontend memory-refusal.ts).
OVERSIZED_GENERATE_SETTING_LABEL = "Allow oversized generations"

# Exposed through CORS in main.py: the desktop app is cross-origin (tauri://localhost).
IMAGE_REFUSAL_HEADER = "X-Unsloth-Refusal"
IMAGE_REFUSAL_MEMORY_ESTIMATE = "memory-estimate"

ACTIVATION_RUN = "run"
ACTIVATION_TILE = "tile"
ACTIVATION_REFUSE = "refuse"

# Denoiser MiB per output megapixel per image; linear only for sub-quadratic (flash / mem-efficient) attention.
DENOISE_MIB_PER_MEGAPIXEL = 1024

DEFAULT_VAE_TILE_SIDE = 1024


class ImageActivationShortfallError(ValueError):
    """This generation's activations cannot fit the free device budget.

    A ValueError subclass so ``/images/generate``'s existing mapping still turns it into a 400
    with the reason. The distinct type is what lets the OpenAI-compatible route, whose boundary
    sanitises every other exception into a bare 500, recognise the one message here that is
    written FOR the caller and hand it back instead of "Image generation failed." The Images page
    recognises it too (the route tags the 400), so it can offer a retry with the override.
    """


@dataclass(frozen = True)
class ImageActivationVerdict:
    """Generate-time guard decision (MiB figures); ACTIVATION_TILE = tile and slice the VAE for this call."""

    action: str
    message: Optional[str] = None
    needed_mib: Optional[int] = None
    tiled_needed_mib: Optional[int] = None
    budget_mib: Optional[int] = None
    overridden: bool = False


def _oversized_generate_override() -> bool:
    return os.environ.get(OVERSIZED_GENERATE_ENV, "").strip().lower() in ("1", "true", "yes", "on")


def vae_tile_side(vae: Any) -> Optional[int]:
    """The pixel side a VAE tiles at, or None when it cannot tile at all."""
    if vae is None or not callable(getattr(vae, "enable_tiling", None)):
        return None
    decode_side = getattr(vae, "_unsloth_decode_tile_side", None)
    if isinstance(decode_side, int) and not isinstance(decode_side, bool) and decode_side > 0:
        # diffusion_vae_tiling decodes in wider tiles than the attributes
        return max(64, min(4096, decode_side))
    sides: list[int] = []
    for attr in ("tile_sample_min_size", "tile_sample_min_height", "tile_sample_min_width"):
        value = getattr(vae, attr, None)
        values = value if isinstance(value, (list, tuple)) else (value,)
        for item in values:
            if isinstance(item, bool):
                continue
            try:
                side = int(item)
            except (TypeError, ValueError):
                continue
            if side > 0:
                sides.append(side)
    if not sides:
        return DEFAULT_VAE_TILE_SIDE
    return max(64, min(4096, max(sides)))


def vae_can_slice(vae: Any) -> bool:
    if vae is None:
        return False
    return bool(getattr(vae, "use_slicing", False)) or callable(
        getattr(vae, "enable_slicing", None)
    )


def vae_is_sliced(vae: Any) -> bool:
    if vae is None:
        return False
    return bool(getattr(vae, "use_slicing", callable(getattr(vae, "enable_slicing", None))))


def engage_vae_tiling_for_call(pipe: Any, logger: Any = None) -> Optional[Callable[[], None]]:
    """Tile and slice ``pipe``'s VAE for one generation; returns the undo, or None if nothing changed."""
    return engage_vae_tiling(pipe, logger = logger)[0]


def engage_vae_tiling(pipe: Any, logger: Any = None) -> tuple[Optional[Callable[[], None]], bool]:
    """``engage_vae_tiling_for_call`` plus whether tiling is actually on (``enable_tiling()`` may fail)."""
    vae = getattr(pipe, "vae", None)
    if vae is None:
        return None, False
    tiled = bool(getattr(vae, "use_tiling", False))
    undo: list[str] = []
    for flag, enable, disable in (
        ("use_tiling", "enable_tiling", "disable_tiling"),
        ("use_slicing", "enable_slicing", "disable_slicing"),
    ):
        if bool(getattr(vae, flag, False)):
            continue
        fn = getattr(vae, enable, None)
        if not callable(fn):
            continue
        try:
            fn()
        except Exception as exc:  # noqa: BLE001 - a VAE saver is an optimisation, never fatal
            if logger is not None:
                logger.warning("diffusion.memory: %s() failed: %s", enable, exc)
            continue
        undo.append(disable)
        if enable == "enable_tiling":
            tiled = bool(getattr(vae, flag, True))
    if not undo:
        return None, tiled

    def _restore() -> None:
        for method in undo:
            fn = getattr(vae, method, None)
            if not callable(fn):
                continue
            try:
                fn()
            except Exception as exc:  # noqa: BLE001 - restoring is best-effort
                if logger is not None:
                    logger.warning("diffusion.memory: %s() failed: %s", method, exc)

    return _restore, tiled


def _family_activation_multiplier(family: Optional[str]) -> float:
    fam = (family or "").lower()
    multiplier = 1.0
    if "edit" in fam:
        multiplier *= 1.35
    if "turbo" in fam or "distilled" in fam or "schnell" in fam:
        multiplier *= 0.85
    return multiplier


def estimate_tiled_image_runtime_mib(
    *,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    family: Optional[str] = None,
    condition_pixels: int = 0,
    tile_side: Optional[int] = None,
    vae_sliced: bool = False,
) -> int:
    """Per-call MiB with the VAE tiled: max(one-tile decode, denoiser); unsliced tiles carry the whole batch.
    Invalid under the SDPA math fallback, whose score matrix grows with the square of the tokens."""
    w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
    h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
    batch = max(1, int(batch_size or 1))
    cond = max(0, int(condition_pixels or 0))
    side = int(tile_side or DEFAULT_VAE_TILE_SIDE)
    megapixel = float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)
    vae = estimate_image_runtime_mib(
        width = min(w, side),
        height = min(h, side),
        batch_size = 1 if vae_sliced else batch,
        family = family,
    )
    denoise = batch * (
        DENOISE_MIB_PER_MEGAPIXEL * (w * h / megapixel) * _family_activation_multiplier(family)
        + 8192 * (cond / megapixel)
    )
    return max(1024, vae, int(denoise))


def _activation_refusal_message(
    *,
    width: int,
    height: int,
    batch: int,
    total_mib: int,
    overhead_mib: int,
    budget_mib: int,
    free_mib: int,
    source_driven: bool,
    condition_pixels: int,
    tiled: bool,
    controlnet: bool = False,
    calibrated: bool = False,
) -> str:
    batch_note = f" at a batch of {batch}" if batch > 1 else ""
    cond_note = (
        " with ControlNet" if controlnet else " with its input images" if condition_pixels else ""
    )
    if source_driven:
        remedy = (
            "Upload a smaller source image (this workflow takes its output size from the image, "
            "not the Resolution setting)"
        )
    else:
        remedy = "Generate at a smaller resolution"
    # Two decimals and overhead included: refusals are decided by tens of MiB and must not contradict themselves.
    return (
        f"Generating at {width}x{height}{batch_note}{cond_note} needs about {total_mib / 1024:.2f} GB "
        f"of working memory{' even with tiled VAE decoding' if tiled else ''} (including about "
        f"{overhead_mib / 1024:.2f} GB of fixed overhead), but only about {budget_mib / 1024:.2f} GB "
        f"is usable on this device (of the {free_mib / 1024:.2f} GB currently free, after reserving "
        "room for fragmentation and other processes). Working memory holds the image being "
        "generated, so unlike model weights it cannot be moved to the CPU. "
        # Smaller-batch hint only when batch > 1: a one-image refusal cannot be fixed by asking for fewer.
        f"{remedy}"
        f"{' or a smaller batch size' if batch > 1 else ''}"
        f"{', use fewer input images or a lower reference detail' if condition_pixels else ''}"
        f"{', generate without ControlNet' if controlnet else ''}"
        f"{', reload the model with the balanced memory mode' if calibrated else ''}, "
        "or free GPU memory by closing other applications. To try anyway, turn on "
        f"'{OVERSIZED_GENERATE_SETTING_LABEL}' under Advanced on the Images page "
        "(API callers of /api/inference/images/generate can send allow_oversized; server installs "
        f"can set {OVERSIZED_GENERATE_ENV}=1)."
    )


def image_activation_verdict(
    *,
    device_memory: DeviceMemory,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    family: Optional[str] = None,
    base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
    source_driven: bool = False,
    condition_pixels: int = 0,
    vae_tile_side: Optional[int] = None,
    vae_sliced: bool = False,
    quadratic_attention: bool = False,
    allow_oversized: bool = False,
    calibrated_placement: bool = False,
    controlnet: bool = False,
) -> ImageActivationVerdict:
    """Activations cannot be offloaded, so tile the VAE decode or refuse; refuse here, not in torch, since Windows WDDM spills silently; fail open when unknown."""
    override = bool(allow_oversized) or _oversized_generate_override()
    try:
        # Unified memory skips the check: 'free' moves with the OS, and load-time refusal already owns it.
        if getattr(device_memory, "is_unified", False):
            return ImageActivationVerdict(ACTIVATION_RUN)
        # CUDA / ROCm only (ROCm reports "cuda"); the estimate was measured on discrete VRAM.
        if getattr(device_memory, "device", None) != "cuda":
            return ImageActivationVerdict(ACTIVATION_RUN)
        free = getattr(device_memory, "free_mib", None)
        if free is None:
            return ImageActivationVerdict(ACTIVATION_RUN)
        budget = _safe_device_budget_mib(device_memory)
        if budget is None:
            return ImageActivationVerdict(ACTIVATION_RUN)
        # Calibrated tiers measured only the unconditioned denoise; a ControlNet counts as one output-sized image.
        # Not on a U-Net: its ControlNet adds no tokens and stays under the measured decode peak.
        if controlnet and _resident_only_hint(family):
            controlnet = False
        input_pixels = max(0, int(condition_pixels or 0))
        conditioned = bool(calibrated_placement) and (input_pixels > 0 or bool(controlnet))
        if calibrated_placement and controlnet:
            condition_pixels = input_pixels + max(64, int(width or DEFAULT_IMAGE_WIDTH)) * max(
                64, int(height or DEFAULT_IMAGE_HEIGHT)
            )
        needed = estimate_image_runtime_mib(
            width = width,
            height = height,
            batch_size = batch_size,
            family = family,
            condition_pixels = condition_pixels,
        )
        # Load budget: same estimator and family hint as _plan_memory, at default resolution.
        planned = estimate_image_runtime_mib(
            width = None,
            height = None,
            batch_size = 1,
            family = family,
        )
        tiled = None
        if vae_tile_side is not None and not quadratic_attention:
            tiled = estimate_tiled_image_runtime_mib(
                width = width,
                height = height,
                batch_size = batch_size,
                family = family,
                condition_pixels = condition_pixels,
                tile_side = vae_tile_side,
                vae_sliced = vae_sliced,
            )
    except Exception:  # noqa: BLE001 -- a broken probe must never block a generation
        return ImageActivationVerdict(ACTIVATION_RUN)
    overhead = max(0, int(base_overhead_mib))
    # Base overhead counts here too: the load plan sums it with the activations.
    numbers = dict(needed_mib = int(needed), tiled_needed_mib = tiled, budget_mib = int(budget))
    if int(needed) + overhead <= int(budget):
        return ImageActivationVerdict(ACTIVATION_RUN, **numbers)
    if not conditioned and needed <= planned:
        if (
            tiled is not None
            and int(tiled) + overhead <= int(budget)
            and _device_user_budget_reserve_mib(
                device_memory, device_memory.total_mib or device_memory.free_mib
            )
            > _reserve_mib(
                _budget_reserve_kind(device_memory),
                device_memory.total_mib or device_memory.free_mib,
            )
        ):
            return ImageActivationVerdict(ACTIVATION_TILE, **numbers)
        return ImageActivationVerdict(ACTIVATION_RUN, **numbers)
    if tiled is not None and (
        int(tiled) + overhead <= int(budget) or (not conditioned and tiled <= planned)
    ):
        return ImageActivationVerdict(ACTIVATION_TILE, **numbers)
    if override:
        # Tile even under quadratic attention: the estimate is untrusted there, but tiling still lowers the peak.
        return ImageActivationVerdict(
            ACTIVATION_TILE if vae_tile_side is not None else ACTIVATION_RUN,
            overridden = True,
            **numbers,
        )
    w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
    h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
    message = _activation_refusal_message(
        width = w,
        height = h,
        batch = max(1, int(batch_size or 1)),
        total_mib = int(tiled if tiled is not None else needed) + overhead,
        overhead_mib = overhead,
        budget_mib = int(budget),
        free_mib = int(free),
        source_driven = source_driven,
        condition_pixels = input_pixels,
        tiled = tiled is not None,
        controlnet = bool(controlnet),
        calibrated = conditioned,
    )
    return ImageActivationVerdict(ACTIVATION_REFUSE, message = message, **numbers)


def image_activation_shortfall_message(
    *,
    device_memory: DeviceMemory,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    family: Optional[str] = None,
    base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
    source_driven: bool = False,
    condition_pixels: int = 0,
    vae_tile_side: Optional[int] = None,
    vae_sliced: bool = False,
    quadratic_attention: bool = False,
    allow_oversized: bool = False,
    calibrated_placement: bool = False,
    controlnet: bool = False,
) -> Optional[str]:
    return image_activation_verdict(
        device_memory = device_memory,
        width = width,
        height = height,
        batch_size = batch_size,
        family = family,
        base_overhead_mib = base_overhead_mib,
        source_driven = source_driven,
        condition_pixels = condition_pixels,
        vae_tile_side = vae_tile_side,
        vae_sliced = vae_sliced,
        quadratic_attention = quadratic_attention,
        allow_oversized = allow_oversized,
        calibrated_placement = calibrated_placement,
        controlnet = controlnet,
    ).message


def raise_on_image_activation_shortfall(
    *,
    device_memory: DeviceMemory,
    width: Optional[int],
    height: Optional[int],
    batch_size: int = 1,
    family: Optional[str] = None,
    base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
    source_driven: bool = False,
    condition_pixels: int = 0,
    vae_tile_side: Optional[int] = None,
    vae_sliced: bool = False,
    quadratic_attention: bool = False,
    allow_oversized: bool = False,
    calibrated_placement: bool = False,
    controlnet: bool = False,
    logger: Any = None,
) -> ImageActivationVerdict:
    """Raises ValueError so the route answers 400 with the reason; RuntimeError becomes an opaque 500."""
    verdict = image_activation_verdict(
        device_memory = device_memory,
        width = width,
        height = height,
        batch_size = batch_size,
        family = family,
        base_overhead_mib = base_overhead_mib,
        source_driven = source_driven,
        condition_pixels = condition_pixels,
        vae_tile_side = vae_tile_side,
        vae_sliced = vae_sliced,
        quadratic_attention = quadratic_attention,
        allow_oversized = allow_oversized,
        calibrated_placement = calibrated_placement,
        controlnet = controlnet,
    )
    if verdict.action == ACTIVATION_REFUSE:
        if logger is not None:
            logger.error("diffusion.memory: refusing oversized generation: %s", verdict.message)
        raise ImageActivationShortfallError(verdict.message)
    if logger is not None and (verdict.action == ACTIVATION_TILE or verdict.overridden):
        logger.warning(
            "diffusion.memory: %dx%d needs ~%s MiB untiled, ~%s MiB tiled, budget %s MiB: %s",
            int(width or DEFAULT_IMAGE_WIDTH),
            int(height or DEFAULT_IMAGE_HEIGHT),
            verdict.needed_mib,
            verdict.tiled_needed_mib,
            verdict.budget_mib,
            (
                "running anyway (allow_oversized)"
                if verdict.overridden
                else "tiling the VAE for this generation"
            ),
        )
    return verdict


STREAMING_PREFETCH_ENV = "UNSLOTH_DIFFUSION_STREAMING_PREFETCH"


def install_group_prefetch(
    module: Any,
    device: Any,
    logger: Any = None,
) -> int:
    """Event-fenced, deeper prefetch for a block-streamed module's offload groups (diffusion_offload_prefetch)."""
    from .diffusion_offload_prefetch import install_group_prefetch as _install
    return _install(module, device, logger)


def _streaming_prefetch_enabled() -> bool:
    """Kill switch for the streaming tier's overlapped onload (pinned host copies + record_stream); default on."""
    return (os.environ.get(STREAMING_PREFETCH_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def _apply_streaming_offload(
    pipe: Any,
    device: str,
    logger: Any,
    *,
    resident_transformer_mib: Optional[int] = None,
) -> None:
    """Streams blocks and encoder leaves; a whole-component onload would OOM, so setup failure is fatal."""
    installed = 0
    try:
        import inspect

        import torch
        from diffusers.hooks import apply_group_offloading

        install_group_offload_buffer_restore()
        install_group_offload_hooks_eager()

        components = getattr(pipe, "components", {})
        if not isinstance(components, dict):
            raise RuntimeError("pipeline does not expose its components")

        # same selection the planner sized against, so what it promised to stream is what streams
        streamed = _streamable_components(pipe, torch)
        if "transformer" not in streamed:
            raise RuntimeError("pipeline has no transformer to stream")

        onload = torch.device(device)
        offload = torch.device("cpu")
        params = inspect.signature(apply_group_offloading).parameters
        use_stream = (
            onload.type == "cuda" and "low_cpu_mem_usage" in params and host_pinning_allowed(onload)
        )

        companions = run_once_companions(pipe)
        # Small companions (tiled VAE) stay resident; transformers and encoders stay on CPU behind hooks.
        for name, component in components.items():
            if str(name) in streamed or str(name) in companions:
                continue
            if isinstance(component, torch.nn.Module):
                component.to(onload)

        pinned_mib = [0]
        # Overlap needs a pinned host copy and record_stream (record_stream=False syncs the compute stream per group).
        prefetch = use_stream and _streaming_prefetch_enabled()
        pin_dits, pin_encoders = False, False
        defer = (
            prefetch
            and bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
            and _background_pin_enabled(onload)
        )
        # an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
        pin_mib = _module_host_mib if defer else _module_pin_mib
        exact_defer = defer and _video_exact_pins(pipe, onload, logger)
        if exact_defer:
            _host_price = pin_mib

            def pin_mib(module: Any) -> int:
                if _torchao_weight_classes(module):
                    return _host_price(module)
                return _module_offload_registered_mib(module)

        encoders_defer = not defer and prefetch and _encoder_background_pin_wanted(pipe, onload)

        def _encoder_defers(name: Any, module: Any, offload_type: str) -> bool:
            return (
                encoders_defer
                and offload_type != "block_level"
                and str(name).startswith("text_encoder")
                and not _torchao_weight_classes(module)
            )

        def leaf_pin_mib(name: Any, module: Any, offload_type: str) -> int:
            if _encoder_defers(name, module, offload_type):
                return _module_registered_mib(module)
            return pin_mib(module)

        dit_pin_mib = sum(pin_mib(m) for m, t in streamed.values() if t == "block_level")
        leaf_mib = 0
        if prefetch:
            leaf_mib = sum(
                leaf_pin_mib(n, m, t) for n, (m, t) in streamed.items() if t != "block_level"
            )
            companion_mib = sum(pin_mib(m) for m in companions.values())
            if companions and _run_once_companions_pin(
                dit_pin_mib, leaf_mib, companion_mib, onload
            ):
                for comp_name, module in companions.items():
                    streamed[comp_name] = (module, "leaf_level")
                leaf_mib += companion_mib
                if logger is not None:
                    logger.info(
                        "diffusion.memory: streaming %s (%d MiB) leaf by leaf: run once before step 0, off the "
                        "card for the denoise",
                        ", ".join(type(m).__name__ for m in companions.values()),
                        companion_mib,
                    )
        for comp_name, module in companions.items():
            if comp_name not in streamed:
                module.to(onload)
        if prefetch:
            pin_dits, pin_encoders = _streamed_pin_plan(
                dit_pin_mib, leaf_mib, logger, device = onload
            )
        late_prefix = (
            exact_defer
            and prefetch
            and not pin_encoders
            and not getattr(pipe, "_unsloth_small_host", None)
            and not _env_off(PARTIAL_ENCODER_PIN_ENV)
            and host_pinning_allowed(onload)
        )
        if late_prefix and logger is not None:
            logger.info(
                "diffusion.memory: streamed encoders pin the prefix of their offload groups that fits the host RAM "
                "free when the load returns"
            )
        encoder_left_mib = 0
        if encoders_defer and not pin_encoders:
            encoder_left_mib = _streamed_pin_leftover_mib(dit_pin_mib, pin_dits, onload)
            if encoder_left_mib and logger is not None:
                logger.info(
                    "diffusion.memory: streamed encoders pin a %d MiB prefix of their offload groups in the background",
                    encoder_left_mib,
                )
        if defer or encoders_defer:
            install_group_pin_wait()
        # pinned encoders count against the budget a torchao denoiser pins within, as on the group tier
        if pin_encoders:
            pinned_mib[0] = leaf_mib
        else:
            pinned_mib[0] = encoder_left_mib

        for comp_name, (module, offload_type) in streamed.items():
            kwargs: dict[str, Any] = {
                "onload_device": onload,
                "offload_device": offload,
                "offload_type": offload_type,
            }
            if offload_type == "block_level":
                kwargs["num_blocks_per_group"] = DEFAULT_GROUP_BLOCKS
            if "use_stream" in params:
                kwargs["use_stream"] = use_stream
            if use_stream and "non_blocking" in params:
                kwargs["non_blocking"] = True
            if use_stream and "record_stream" in params:
                kwargs["record_stream"] = prefetch
            pin = pin_dits if offload_type == "block_level" else pin_encoders
            enc_defer = _encoder_defers(comp_name, module, offload_type) and (
                pin or encoder_left_mib > 0
            )
            if use_stream and "low_cpu_mem_usage" in params:
                kwargs["low_cpu_mem_usage"] = not (pin and not defer and not enc_defer)
            kwargs = _torchao_group_offload_kwargs(module, kwargs, pinned_mib)
            _apply_group_offloading_packed(apply_group_offloading, module, kwargs, logger)
            if pin and defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
                _defer_pinning(pipe, module, onload, logger)
            elif (
                late_prefix
                and not pin
                and offload_type != "block_level"
                and str(comp_name).startswith("text_encoder")
                and not _torchao_weight_classes(module)
                and kwargs.get("use_stream")
                and kwargs.get("low_cpu_mem_usage")
            ):
                _defer_pinning(
                    pipe,
                    module,
                    onload,
                    logger,
                    fast = True,
                    byte_cap = _late_encoder_pin_cap(pipe, module, onload),
                )
            elif enc_defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
                cap = None if pin else int(encoder_left_mib) << 20
                _defer_pinning(pipe, module, onload, logger, fast = True, byte_cap = cap)
                if not pin:
                    encoder_left_mib = max(0, encoder_left_mib - _module_registered_mib(module))
            installed += 1
            if (
                use_stream
                and offload_type == "block_level"
                and not _pin_top_level_group(module, logger, pinned_mib, device = onload)
            ):
                _skip_top_level_copy_back(module, logger)
            if use_stream and prefetch and offload_type == "block_level":
                install_group_prefetch(module, onload, logger)
            if offload_type == "leaf_level":
                _pin_vision_embedding_device(module)
        if resident_transformer_mib:
            room = int(resident_transformer_mib)
            for module, offload_type in streamed.values():
                if offload_type == "block_level":
                    room -= _keep_groups_resident(module, room, onload, logger)
    except Exception as exc:
        if logger is not None:
            logger.warning(
                "diffusion.memory: granular streaming offload failed after installing hooks "
                "on %d module(s): %s",
                installed,
                exc,
            )
        raise RuntimeError(
            "A diffusion component is larger than the available GPU memory, and granular "
            f"streaming offload could not be enabled: {exc}"
        ) from exc