Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Per-block CUDA graphs keyed by inputs and weight addresses; moved weights re-record once; offload
hooks stay eager; static inputs shared since replays never overlap"""
from __future__ import annotations
import os
import traceback
from collections import OrderedDict
from typing import Any, Callable, Optional
from .diffusion_bg_compile import capture_suppressed as _bg_capture_suppressed
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
def _step_recording() -> bool:
try:
from .diffusion_cuda_graph import step_recording
except Exception: # noqa: BLE001
return False
return step_recording()
BLOCK_GRAPHS_ENV = "UNSLOTH_DIFFUSION_BLOCK_GRAPHS"
# Recordings kept per block (input layouts x weight placements); the least recently replayed goes first.
MAX_GRAPHS_PER_BLOCK = 6
# Distinct weight placements a block may see without one replay before it stops trying (address churn: a streaming
# path without the slot ring, or disk offload). Each attempt costs one eager call, never a recording.
MAX_UNREPLAYED_PLACEMENTS = 6
_OFF = ("0", "off", "false", "no")
def _torch():
import torch
return torch
def block_graphs_disabled() -> bool:
return (os.environ.get(BLOCK_GRAPHS_ENV) or "").strip().lower() in _OFF
def block_graphs_requested() -> bool:
"""Forced on everywhere, streamed and model-offloaded denoisers included."""
return (os.environ.get(BLOCK_GRAPHS_ENV) or "").strip().lower() in ("1", "on", "true", "yes")
# Streamed steps are copy-bound; model offload re-records every render.
OPT_IN_REASON = (
"offloaded denoiser streams its blocks: per-block CUDA graphs measured no faster (streamed steps are "
"copy-bound) and hold extra VRAM; set " + BLOCK_GRAPHS_ENV + "=1 to record per block"
)
MODEL_OFFLOAD_REASON = (
"model offload re-uploads the denoiser every render, so every block would record again; set "
+ BLOCK_GRAPHS_ENV
+ "=1 to record per block"
)
# Auto engages FBCache only at 20+ steps, since its win scales with step count: "dev" schedules qualify, distilled turbo never does
def _kv_layer_cache(obj: Any) -> bool:
name = type(obj).__name__
return name.endswith("KVLayerCache") and hasattr(obj, "k") and hasattr(obj, "v")
def _flatten(obj: Any, out: list) -> tuple:
torch = _torch()
if torch.is_tensor(obj):
out.append(obj)
return ("t", len(out) - 1)
if isinstance(obj, (list, tuple)):
return ("l", isinstance(obj, tuple), [_flatten(o, out) for o in obj])
if isinstance(obj, dict):
return ("d", [(k, _flatten(v, out)) for k, v in obj.items()])
if isinstance(obj, (int, float, bool, str, bytes)) or obj is None:
return ("v", obj)
if _kv_layer_cache(obj) and torch.is_tensor(obj.k) and torch.is_tensor(obj.v):
return ("kv", type(obj), _flatten(obj.k, out), _flatten(obj.v, out))
raise TypeError(f"cannot make a static buffer for a {type(obj).__name__} in the call tree")
def _rebuild(spec: tuple, tensors: list) -> Any:
kind = spec[0]
if kind == "t":
return tensors[spec[1]]
if kind == "l":
values = [_rebuild(s, tensors) for s in spec[2]]
return tuple(values) if spec[1] else values
if kind == "d":
return {k: _rebuild(s, tensors) for k, s in spec[1]}
if kind == "kv":
cache = spec[1].__new__(spec[1])
cache.k = _rebuild(spec[2], tensors)
cache.v = _rebuild(spec[3], tensors)
return cache
return spec[1]
def _walk(obj: Any, live: list) -> tuple:
"""Hashable key of a call tree, appending its tensors to ``live`` (compiled blocks guard on inference mode)."""
torch = _torch()
if torch.is_tensor(obj):
live.append(obj)
return (
"t",
tuple(obj.shape),
tuple(obj.stride()),
obj.dtype,
obj.device.type,
obj.device.index,
bool(obj.is_inference()),
)
if isinstance(obj, (list, tuple)):
return ("l", isinstance(obj, tuple), tuple(_walk(o, live) for o in obj))
if isinstance(obj, dict):
return ("d", tuple((k, _walk(v, live)) for k, v in obj.items()))
if isinstance(obj, (int, float, bool, str, bytes)) or obj is None:
return ("v", obj)
if _kv_layer_cache(obj) and torch.is_tensor(obj.k) and torch.is_tensor(obj.v):
return ("kv", type(obj), _walk(obj.k, live), _walk(obj.v, live))
return ("o", type(obj).__name__, id(obj))
def _unwalk(key: tuple, tensors: Any) -> Any:
kind = key[0]
if kind == "t":
return next(tensors)
if kind == "l":
values = [_unwalk(k, tensors) for k in key[2]]
return tuple(values) if key[1] else values
if kind == "d":
return {k: _unwalk(v, tensors) for k, v in key[1]}
if kind == "kv":
cache = key[1].__new__(key[1])
cache.k = _unwalk(key[2], tensors)
cache.v = _unwalk(key[3], tensors)
return cache
return key[1]
def graph_key(obj: Any) -> tuple:
return _walk(obj, [])
def _refusal(key: tuple, kwargs: dict) -> Optional[str]:
def walk(k: tuple) -> Optional[str]:
kind = k[0]
if kind == "o":
return "object"
if kind == "v" and isinstance(k[1], float):
return "float"
if kind == "l":
for sub in k[2]:
why = walk(sub)
if why:
return why
if kind == "d":
for _, sub in k[1]:
why = walk(sub)
if why:
return why
return None
why = walk(key)
if why:
return why
# A prefix K/V cache is only read on a cached step; the prefill writes it from Python.
if _has_kv(key) and kwargs.get("kv_cache_mode") != "cached":
return "kv_write"
return None
def _has_kv(key: tuple) -> bool:
kind = key[0]
if kind == "kv":
return True
if kind == "l":
return any(_has_kv(k) for k in key[2])
if kind == "d":
return any(_has_kv(k) for _, k in key[1])
return False
def _is_wrapper_subclass(t: Any) -> bool:
try:
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
return bool(is_traceable_wrapper_subclass(t))
except Exception: # noqa: BLE001
return False
class _WeightView:
"""The block's weights as device addresses, through every level of torchao inner tensors (a wrapper's own
``data_ptr`` is 0; torchao 0.17's int8 nests two)."""
__slots__ = ("tensors",)
def __init__(self, module: Any) -> None:
seen: set = set()
tensors: list = []
for t in list(module.parameters()) + list(module.buffers()):
if id(t) not in seen:
seen.add(id(t))
tensors.append(t)
self.tensors = tensors
def placement(self, device_index: Optional[int]) -> Optional[tuple]:
"""Data pointers of every plain tensor under the weights, or None when one is off ``device_index``."""
ptrs: list = []
for t in self.tensors:
for p in _leaves(t):
dev = p.device
if dev.type != "cuda" or (device_index is not None and dev.index != device_index):
return None
ptrs.append(p.data_ptr())
return tuple(ptrs)
def _leaves(t: Any) -> list:
if not _is_wrapper_subclass(t):
return [t]
try:
names, _ = t.__tensor_flatten__()
except Exception: # noqa: BLE001 - unflattenable: key on what it reports
return [t]
out: list = []
for n in names:
x = getattr(t, n, None)
if x is not None:
out.extend(_leaves(x))
return out
_UNSEEN = object()
class _Entry:
__slots__ = ("graph", "static_in", "static_out", "out_spec", "replays", "slots", "pool")
import weakref
_LIVE: "weakref.WeakSet" = weakref.WeakSet()
def pool_bytes(device: Optional[int] = None) -> int:
"""Unused graph-private bytes; retained outputs already count as allocated."""
total = 0
tracked = 0
pools: set = set()
for shared in tuple(_LIVE):
index = getattr(shared, "device_index", None)
if device is not None and index is not None and index != device:
continue
recorded = int(getattr(shared, "pool_bytes", 0) or 0)
handles = getattr(shared, "pools", None)
if handles:
tracked += recorded
pools.update(tuple(h) for h in handles)
else:
total += recorded
if not pools:
return total
try:
free = 0
from .diffusion_cuda_graph import _allocator_segments
for seg in _allocator_segments():
if device is not None and seg.get("device", device) != device:
continue
if tuple(seg.get("segment_pool_id") or ()) in pools:
free += int(seg.get("total_size", 0)) - int(seg.get("allocated_size", 0))
return total + max(0, free)
except Exception: # noqa: BLE001
return total + tracked
class _Shared:
"""Pool, capture stream and static buffers per (block class, layout), shared by one denoiser's blocks."""
def __init__(
self,
device_index: Optional[int],
logger: Any = None,
) -> None:
self.device_index = device_index
self.logger = logger
self.root: Any = None # the denoiser, for its NVFP4 per-step precision branch
self.pool = None
self.stream = None
self.static_in: dict = {}
self.static_out: dict = {}
self.static_refs: dict = {} # slot -> recordings reading its buffers
self.warmed: set = set()
self.pools: list = [] # Includes retired pools.
self.pool_bytes = 0
self.static_bytes = 0
self.strict_device = False
_LIVE.add(self)
def capture_stream(self) -> Any:
if self.stream is None:
torch = _torch()
self.stream = torch.cuda.Stream(device = self.device_index)
return self.stream
def graph_pool(self) -> Any:
if self.pool is None:
torch = _torch()
self.pool = torch.cuda.graph_pool_handle()
self.pools.append(self.pool)
return self.pool
def statics_for(self, slot: tuple, live: list) -> list:
buffers = self.static_in.get(slot)
if buffers is None:
buffers = [_static_like(t) for t in live]
self.static_in[slot] = buffers
self.static_bytes += sum(_nbytes(b) for b in buffers)
return buffers
def outputs_for(self, slot: tuple, metas: list) -> list:
buffers = self.static_out.get(slot)
if buffers is None:
torch = _torch()
buffers = []
for shape, stride, dtype, device, inference in metas:
with torch.inference_mode(inference):
buffers.append(torch.empty_strided(shape, stride, dtype = dtype, device = device))
self.static_out[slot] = buffers
self.static_bytes += sum(_nbytes(b) for b in buffers)
return buffers
def hold(self, slots: tuple) -> None:
for slot in slots:
self.static_refs[slot] = self.static_refs.get(slot, 0) + 1
def drop(self, slots: tuple) -> None:
"""Free the static buffers of ``slots`` no recording reads any more (an evicted layout)."""
for slot in slots:
left = self.static_refs.get(slot, 0) - 1
if left > 0:
self.static_refs[slot] = left
continue
self.static_refs.pop(slot, None)
buffers = self.static_in.pop(slot, None) or self.static_out.pop(slot, None) or ()
self.static_bytes = max(0, self.static_bytes - sum(_nbytes(b) for b in buffers))
self.warmed.discard(slot)
def release(self) -> None:
self.static_in.clear()
self.static_out.clear()
self.static_refs.clear()
self.warmed.clear()
self.pool = None
self.pools = []
self.pool_bytes = 0
self.static_bytes = 0
def _static_like(t: Any) -> Any:
"""Buffer must match t in shape, stride, dtype, inference mode and offset, or the block recompiles"""
torch = _torch()
shape, stride, offset = tuple(t.shape), tuple(t.stride()), int(t.storage_offset())
extent = 0 if t.numel() == 0 else 1 + sum((n - 1) * st for n, st in zip(shape, stride) if n > 0)
with torch.inference_mode(bool(t.is_inference())):
base = torch.empty(offset + extent, dtype = t.dtype, device = t.device)
return base.as_strided(shape, stride, offset)
def _nbytes(t: Any) -> int:
try:
return int(t.untyped_storage().nbytes())
except Exception: # noqa: BLE001
return 0
def _meta(t: Any) -> tuple:
return (tuple(t.shape), tuple(t.stride()), t.dtype, t.device, bool(t.is_inference()))
_LIVE_BLOCKS: "weakref.WeakSet" = weakref.WeakSet()
def live_block_graph_count(device_index: Optional[int]) -> int:
"""Per-block graphs alive on CUDA device ``device_index`` (``diffusion_cuda_graph.live_graph_count``)."""
return sum(
len(b.cache)
for b in tuple(_LIVE_BLOCKS)
if getattr(b, "_unsloth_capture_device", None) == device_index
)
class BlockGraph:
"""Replaces one block's compute callable (below its offload hooks) with a recording keyed by inputs and weight
placement. Any refusal or failure runs ``compute`` itself, so the block is never worse than ungraphed."""
def __init__(
self,
block: Any,
compute: Callable,
shared: _Shared,
*,
max_graphs: int = MAX_GRAPHS_PER_BLOCK,
) -> None:
self.block = block
self.compute = compute
self.shared = shared
self.cls = type(block).__name__
self.weights = _WeightView(block)
self.max_graphs = int(max_graphs)
self.cache: "OrderedDict[tuple, _Entry]" = OrderedDict()
self.enabled = True
self.bypassed = False
self.poisoned = False
self.capture_error: Optional[dict] = None
# (layout, placement) -> output meta of its first call; several, since CFG alternates two text lengths
self.seen: "OrderedDict[tuple, tuple]" = OrderedDict()
self.unreplayed_placements = 0
self.churned = False
self.protect: Any = None # None until resolved, then False or the NVFP4 step controller
self.segment: Any = None
self.seg_index = 0
self.owner: Any = None
self.placements: set = set()
self.refusals: dict = {}
self.stats = {
"captures": 0,
"recaptures": 0,
"replays": 0,
"eager_calls": 0,
"warmups": 0,
"fallbacks": 0,
"evictions": 0,
"refused_float": 0,
"refused_object": 0,
"refused_host_input": 0,
"refused_kv_write": 0,
"refused_host_weight": 0,
"refused_grad": 0,
"refused_output": 0,
"address_churn": 0,
}
try:
from functools import update_wrapper
update_wrapper(self, compute)
except Exception: # noqa: BLE001
pass
def set_bypass(self, on: bool) -> "BlockGraph":
self.bypassed = bool(on)
return self
def reset(self) -> "BlockGraph":
"""Drop every recording (the weights changed: a LoRA load or scale, an unload) and re-read which tensors the
block owns, since an adapter adds parameters the placement key must cover."""
self._forget_all()
try:
self.weights = _WeightView(self.block)
except Exception: # noqa: BLE001
pass
self.seen.clear()
self.unreplayed_placements = 0
self.placements.clear()
self.refusals.clear()
return self
def _eager(self, args: tuple, kwargs: dict) -> Any:
self.stats["eager_calls"] += 1
return self.compute(*args, **kwargs)
def __call__(self, *args: Any, **kwargs: Any) -> Any:
if _bg_capture_suppressed() or _step_recording():
# a whole-step graph above records this block's kernels (diffusion_cuda_graph), not a nested replay
return self.compute(*args, **kwargs)
if (
not self.enabled
or self.bypassed
or self.poisoned
or self.churned
or _bg_eager_forced()
or _step_cache_live(self.shared.root)
):
return self._eager(args, kwargs)
segment = self.segment
if segment is not None:
out = segment.call(self.seg_index, args, kwargs)
if out is not _NOT_HANDLED:
return out
torch = _torch()
if torch.is_grad_enabled():
self.stats["refused_grad"] += 1
return self._eager(args, kwargs)
live: list = []
try:
key = _walk((args, kwargs), live)
why = self.refusals.get(key, _UNSEEN)
if why is _UNSEEN:
why = _refusal(key, kwargs)
if len(self.refusals) < 64:
self.refusals[key] = why
except Exception: # noqa: BLE001
why = "object"
key = None
if why:
self.stats["refused_" + why] += 1
return self._eager(args, kwargs)
placement = self.weights.placement(self.shared.device_index)
if placement is None:
self.stats["refused_host_weight"] += 1
return self._eager(args, kwargs)
if any(t.device.type != "cuda" for t in live):
# A capture never records a host read, so a replay would keep the recorded value.
self.stats["refused_host_input"] += 1
return self._eager(args, kwargs)
if self.shared.strict_device and (
torch.cuda.current_device() != self.shared.device_index
or any(t.device.index != self.shared.device_index for t in live)
):
self.stats["refused_host_input"] += 1
return self._eager(args, kwargs)
if self.protect is None:
self.protect = self._protect_controller()
if self.protect:
# One recording per precision branch: a W4A4 replay at a W4A16 step would skip the protection.
from .diffusion_cuda_graph import protect_graph_key
placement = (placement, protect_graph_key(self.protect))
try:
extra = _root_key_extra(self.shared.root)
except Exception: # noqa: BLE001 - a branch flag that cannot be read: nothing safe to replay
self.stats["refused_object"] += 1
return self._eager(args, kwargs)
if extra is not None:
placement = (placement, extra)
full = (key, placement)
entry = self.cache.get(full)
if entry is None:
if self.owner is not None and self.owner.scope_sealed():
self.stats["length_eager"] = self.stats.get("length_eager", 0) + 1
return self._eager(args, kwargs)
if full not in self.seen:
# First call at this layout and placement: run it for real (the warm-up), record on the next one.
return self._first_sighting(full, args, kwargs)
entry = self._record(full, live)
if entry is None:
return self._eager(args, kwargs)
else:
self.cache.move_to_end(full)
return self._replay(entry, live)
def _protect_controller(self) -> Any:
root = self.shared.root
try:
from .diffusion_cuda_graph import _protect_keyed
if root is None or not _protect_keyed(root):
return False
from .diffusion_nvfp4_protect import module_controller
return module_controller(root)
except Exception: # noqa: BLE001
return False
def _first_sighting(self, full: tuple, args: tuple, kwargs: dict) -> Any:
placement = full[1]
if placement not in self.placements:
self.placements.add(placement)
self.unreplayed_placements += 1
if self.unreplayed_placements > MAX_UNREPLAYED_PLACEMENTS:
self.churned = True
self.stats["address_churn"] += 1
return self._eager(args, kwargs)
out = self._eager(args, kwargs)
try:
flat: list = []
spec = _flatten(out, flat)
self.seen[full] = (spec, [_meta(t) for t in flat])
while len(self.seen) > 4:
self.seen.popitem(last = False)
except Exception: # noqa: BLE001 - an output the graph cannot hand back (a dataclass, an object)
self.stats["refused_output"] += 1
return out
def _record(self, full: tuple, live: list) -> Optional[_Entry]:
from .diffusion_cuda_graph import (
CaptureRefused,
_capturing,
refuse_if_exhausted,
retire_failed_capture,
)
torch = _torch()
key = full[0]
slot = (self.cls, key)
out_spec, metas = self.seen.pop(full)
entry = _Entry()
out_slot = (self.cls, key, "out", tuple(metas))
try:
entry.static_in = self.shared.statics_for(slot, live)
entry.static_out = self.shared.outputs_for(out_slot, metas)
for dst, src in zip(entry.static_in, live):
dst.copy_(src)
static_args, static_kwargs = _unwalk(key, iter(entry.static_in))
if slot not in self.shared.warmed:
self.compute(*static_args, **static_kwargs)
self.shared.warmed.add(slot)
self.stats["warmups"] += 1
current = torch.cuda.current_stream()
stream = self.shared.capture_stream()
stream.wait_stream(current)
refuse_if_exhausted(self.shared.logger)
graph = torch.cuda.CUDAGraph()
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
with _capturing():
before = torch.cuda.memory_reserved()
try:
# Inside the try: a capture_begin that raised after taking the pool is abandoned too.
graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
try:
out = self.compute(*static_args, **static_kwargs)
flat: list = []
spec = _flatten(out, flat)
if spec != out_spec or len(flat) != len(entry.static_out):
raise RuntimeError("block output layout changed between calls")
for dst, src in zip(entry.static_out, flat):
dst.copy_(src)
finally:
graph.capture_end()
except BaseException as exc:
# Later blocks record into a fresh pool; this one stays with the graphs already in it.
retire_failed_capture(graph, pool, exc)
if self.shared.pool == pool:
self.shared.pool = None
raise
after = torch.cuda.memory_reserved()
del out, flat
current.wait_stream(stream)
self.shared.pool_bytes += max(0, after - before)
except CaptureRefused:
# Refused before capture_begin: nothing recorded, the pool untouched. Eager now, record on a later call.
self.seen[full] = (out_spec, metas)
self.stats["capture_refusals"] = self.stats.get("capture_refusals", 0) + 1
return None
except Exception as exc: # noqa: BLE001 - a block that cannot record runs its compute for the load's life
self._poison(exc)
return None
entry.slots = (slot, out_slot)
self.shared.hold(entry.slots)
entry.graph = graph
entry.pool = pool
entry.out_spec = out_spec
entry.replays = 0
if any(k[0] == full[0] for k in self.cache):
self.stats["recaptures"] += 1
self.cache[full] = entry
self.stats["captures"] += 1
if self.owner is not None:
self.owner.note_capture()
from .diffusion_cuda_graph import capture_device, note_capture
self._unsloth_capture_device = capture_device()
# counted live before the generation moves, so a reader seeing an unchanged generation never misses it
_LIVE_BLOCKS.add(self)
note_capture(self._unsloth_capture_device)
while len(self.cache) > self.max_graphs:
_drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)
self.stats["evictions"] += 1
return entry
def _forget(self, entry: _Entry) -> None:
entry.graph = None
self.shared.drop(getattr(entry, "slots", ()))
def _forget_all(self) -> None:
entries = list(self.cache.values())
self.cache.clear()
_drop_entries(self, entries)
def _replay(self, entry: _Entry, live: list) -> Any:
for dst, src in zip(entry.static_in, live):
if dst.data_ptr() != src.data_ptr():
dst.copy_(src)
entry.graph.replay()
entry.replays += 1
self.stats["replays"] += 1
self.unreplayed_placements = 0
self.placements.clear()
return _rebuild(entry.out_spec, [t.clone() for t in entry.static_out])
def _poison(self, exc: BaseException) -> None:
self.capture_error = {
"type": type(exc).__name__,
"msg": str(exc)[:4000],
"traceback": traceback.format_exc()[-6000:],
}
self.poisoned = True
self._forget_all()
self.stats["fallbacks"] += 1
logger = self.shared.logger
if logger is not None:
logger.warning(
"diffusion.block_graph: recording %s failed (%s: %s); this block runs ungraphed",
self.cls,
type(exc).__name__,
exc,
)
exc.__traceback__ = None
SEGMENT_GRAPHS_ENV = "UNSLOTH_DIFFUSION_SEGMENT_GRAPHS"
_NOT_HANDLED = object()
def segment_graphs_disabled() -> bool:
return (os.environ.get(SEGMENT_GRAPHS_ENV) or "").strip().lower() in _OFF
# 0: a segment whose blocks carry an armed NVFP4 step schedule stays per block for the whole run (the old behaviour).
SEGMENT_PROTECT_ENV = "UNSLOTH_DIFFUSION_SEGMENT_GRAPHS_PROTECT"
def segment_protect_fallback_enabled() -> bool:
"""Default on: only the protected (W4A16) steps leave the segment; the W4A4 steps keep replaying it."""
return (os.environ.get(SEGMENT_PROTECT_ENV) or "").strip().lower() not in _OFF
def _protect_live(controller: Any) -> bool:
"""The layers run W4A16 exactly when ``armed and protected``."""
return bool(getattr(controller, "armed", False) and getattr(controller, "protected", False))
class _SegRecord:
"""One eager pass over a segment: each block's call layout, where each of its input tensors came from (an earlier
block's output, or one of the first block's inputs), and its output layout."""
__slots__ = ("keys", "prov", "out_specs")
def __init__(self) -> None:
self.keys: list = []
self.prov: list = []
self.out_specs: list = []
class _SegEntry:
__slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays", "pool")
def _root_key_extra(root: Any) -> Any:
"""Branch state (GRAPH_KEY_EXTRA_ATTR) keys block graphs, so branches never replay each other's."""
from .diffusion_cuda_graph import GRAPH_KEY_EXTRA_ATTR
extra = getattr(root, GRAPH_KEY_EXTRA_ATTR, None) if root is not None else None
return ("x", extra()) if callable(extra) else None
def _step_cache_live(root: Any) -> bool:
"""Whether a step cache (First-Block-Cache) is engaged on the denoiser: its hooks decide inside the forward, per
step and per cond / uncond context, which a recording cannot see. Belt to the caller's per-chunk ``set_bypass``,
as ``GraphedForward`` does: an auto cache that turns on at generate time keeps every block eager."""
try:
return bool(getattr(root, "_unsloth_step_cache", None)) if root is not None else False
except Exception: # noqa: BLE001
return False
def _lazy_chain(root: Any) -> bool:
"""Whether this forward of a split ``root`` routes lazily: its class is one the split verified to pass each block's
outputs straight to the next block, and the call carries no residual input. Only such a chain may replay as one
segment, since a segment cannot see an op between blocks that rewrites a tensor in place."""
try:
from .diffusion_layer_split import layer_split_of
split = layer_split_of(root)
except Exception: # noqa: BLE001
return False
return bool(split is not None and split.lazy_now)
def _version(t: Any) -> Any:
# an inference tensor carries no version counter
try:
return None if t.is_inference() else t._version
except Exception: # noqa: BLE001
return None
class SegmentGraph:
"""One CUDA graph over a run of blocks a layer split keeps on one card, back to back in one block list: the first
block's call replays the whole run, and each later block's call checks that it got exactly the tensors the run
handed on (the previous block's outputs, the first block's other inputs) and returns its recorded outputs. Any
other call (an op between blocks rewrote a tensor, an eager boundary copied it) leaves the segment to the per-block
path for the rest of that forward, which computes the block itself.
Lifecycle per (layout of the first block's inputs, weight placement): the first forward records the run eagerly,
the second warms it up once and captures it, every later one replays. All on the card's own stream and pool."""
def __init__(
self,
graphs: list,
shared: _Shared,
max_graphs: int = MAX_GRAPHS_PER_BLOCK,
) -> None:
self.graphs = list(graphs)
self.shared = shared
self.max_graphs = int(max_graphs)
self.cache: "OrderedDict[tuple, _SegEntry]" = OrderedDict()
self.seen: "OrderedDict[tuple, _SegRecord]" = OrderedDict()
self.refused: set = set()
self.poisoned = False
self.enabled = True # off: every block takes its own graph's path; recordings are kept
self.capture_error: Optional[dict] = None
self.owner: Any = None
self._reset_forward()
self.stats = {
"segment_records": 0,
"segment_captures": 0,
"segment_replays": 0,
"segment_warmups": 0,
"segment_breaks": 0,
"segment_refusals": 0,
"segment_fallbacks": 0,
"segment_protected_steps": 0,
}
for i, g in enumerate(self.graphs):
g.segment = self
g.seg_index = i
_LIVE_BLOCKS.add(self)
def _reset_forward(self) -> None:
self.pos: Optional[int] = None
self.mode: Optional[str] = None
self.cur: Any = None
self.ext: list = []
self.returned: list = []
self.versions: list = []
def reset(self) -> None:
entries = list(self.cache.values())
self.cache.clear()
_drop_entries(self, entries)
self.seen.clear()
self.refused.clear()
self._reset_forward()
def detach(self) -> None:
self.reset()
for g in self.graphs:
if g.segment is self:
g.segment = None
def _forget(self, entry: _SegEntry) -> None:
entry.graph = None
entry.outputs = None
self.shared.drop(getattr(entry, "slots", ()))
def _placement(self) -> Optional[tuple]:
out = []
for g in self.graphs:
p = g.weights.placement(self.shared.device_index)
if p is None:
return None
out.append(p)
return tuple(out)
def call(self, i: int, args: tuple, kwargs: dict) -> Any:
if self.poisoned or not self.enabled:
return _NOT_HANDLED
if i == 0:
return self._start(args, kwargs)
if self.pos != i:
return _NOT_HANDLED
return self._next(i, args, kwargs)
def _start(self, args: tuple, kwargs: dict) -> Any:
self._reset_forward()
torch = _torch()
if (
torch.is_grad_enabled()
or not _lazy_chain(self.shared.root)
or _step_cache_live(self.shared.root)
):
return _NOT_HANDLED
live: list = []
try:
key = _walk((args, kwargs), live)
extra = _root_key_extra(self.shared.root)
except Exception: # noqa: BLE001
return _NOT_HANDLED
if _refusal(key, kwargs) or any(t.device.type != "cuda" for t in live):
return _NOT_HANDLED
if torch.cuda.current_device() != self.shared.device_index or any(
t.device.index != self.shared.device_index for t in live
):
return _NOT_HANDLED
controller = None
for g in self.graphs:
if g.protect is None:
g.protect = g._protect_controller()
if g.poisoned or g.churned or g.bypassed:
return _NOT_HANDLED
if g.protect:
if not segment_protect_fallback_enabled():
return _NOT_HANDLED
controller = controller or g.protect
if controller is not None and any(
_protect_live(g.protect) for g in self.graphs if g.protect
):
# W4A16 step: per block for this forward only; it writes no state a W4A4 recording reads.
self.stats["segment_protected_steps"] += 1
return _NOT_HANDLED
placement = self._placement()
if placement is None:
return _NOT_HANDLED
if controller is not None:
# Keyed by branch so a recording only replays at the branch it recorded.
from .diffusion_cuda_graph import protect_graph_key
placement = (placement, protect_graph_key(controller))
full = (key, placement) if extra is None else (key, placement, extra)
if full in self.refused:
return _NOT_HANDLED
entry = self.cache.get(full)
if entry is None and self.owner is not None and self.owner.scope_sealed():
return _NOT_HANDLED # its blocks run ungraphed too (BlockGraph.owner)
if entry is None and full in self.seen:
entry = self._capture(full, live)
if entry is None:
return _NOT_HANDLED
if entry is not None:
self.cache.move_to_end(full)
for dst, src in zip(entry.static_in, live):
if dst.data_ptr() != src.data_ptr():
dst.copy_(src)
entry.graph.replay()
entry.replays += 1
self.stats["segment_replays"] += 1
self.mode, self.cur, self.ext = "replay", entry, live
return self._hand_out(0, entry)
rec = _SegRecord()
out = self.graphs[0].compute(*args, **kwargs)
flat: list = []
try:
spec = _flatten(out, flat)
except Exception: # noqa: BLE001
self.refused.add(full)
self.stats["segment_refusals"] += 1
return out
rec.keys.append(key)
rec.prov.append(tuple(("e", e) for e in range(len(live))))
rec.out_specs.append(spec)
self.mode, self.cur, self.ext = "record", (full, rec), live
self.returned = [flat]
self.versions = [[_version(t) for t in flat]]
self.pos = 1
self.graphs[0].stats["eager_calls"] += 1
return out
def _provenance(self, live: list) -> Optional[tuple]:
out = []
for t in live:
found = None
for j in range(len(self.returned) - 1, -1, -1):
for m, r in enumerate(self.returned[j]):
if t is r:
if _version(t) != self.versions[j][m]:
return None # rewritten in place since its block handed it on
found = ("o", j, m)
break
if found is not None:
break
if found is None:
for e, x in enumerate(self.ext):
if t is x:
found = ("e", e)
break
if found is None:
return None
out.append(found)
return tuple(out)
def _next(self, i: int, args: tuple, kwargs: dict) -> Any:
live: list = []
try:
key = _walk((args, kwargs), live)
except Exception: # noqa: BLE001
self._abandon()
return _NOT_HANDLED
prov = self._provenance(live)
last = i == len(self.graphs) - 1
if self.mode == "record":
full, rec = self.cur
if prov is None or _refusal(key, kwargs):
self.refused.add(full)
self.stats["segment_refusals"] += 1
self._abandon()
return _NOT_HANDLED
out = self.graphs[i].compute(*args, **kwargs)
flat: list = []
try:
spec = _flatten(out, flat)
except Exception: # noqa: BLE001
self.refused.add(full)
self.stats["segment_refusals"] += 1
self._abandon()
return out
rec.keys.append(key)
rec.prov.append(prov)
rec.out_specs.append(spec)
self.returned.append(flat)
self.versions.append([_version(t) for t in flat])
self.graphs[i].stats["eager_calls"] += 1
if last:
self.seen[full] = rec
while len(self.seen) > 4:
self.seen.popitem(last = False)
self.stats["segment_records"] += 1
self._reset_forward()
else:
self.pos = i + 1
return out
entry = self.cur
if prov is None or key != entry.record.keys[i] or prov != entry.record.prov[i]:
self.stats["segment_breaks"] += 1
self._abandon()
return _NOT_HANDLED
return self._hand_out(i, entry)
def _hand_out(self, i: int, entry: _SegEntry) -> Any:
tensors = entry.outputs[i]
last = i == len(self.graphs) - 1
if last:
tensors = [t.clone() for t in tensors]
self._reset_forward()
else:
if i == 0:
self.returned = [tensors]
self.versions = [[_version(t) for t in tensors]]
else:
self.returned.append(tensors)
self.versions.append([_version(t) for t in tensors])
self.pos = i + 1
self.graphs[i].stats["segment_hits"] = self.graphs[i].stats.get("segment_hits", 0) + 1
return _rebuild(entry.record.out_specs[i], tensors)
def _abandon(self) -> None:
self._reset_forward()
def _chain(self, rec: _SegRecord, ext: list) -> list:
results: list = []
for i, g in enumerate(self.graphs):
tensors = [ext[p[1]] if p[0] == "e" else results[p[1]][p[2]] for p in rec.prov[i]]
call_args, call_kwargs = _unwalk(rec.keys[i], iter(tensors))
out = g.compute(*call_args, **call_kwargs)
flat: list = []
if _flatten(out, flat) != rec.out_specs[i]:
raise RuntimeError("segment block output layout changed between calls")
results.append(flat)
return results
def _capture(self, full: tuple, live: list) -> Optional[_SegEntry]:
from .diffusion_cuda_graph import (
CaptureRefused,
_capturing,
capture_device,
note_capture,
refuse_if_exhausted,
retire_failed_capture,
)
torch = _torch()
rec = self.seen.pop(full)
entry = _SegEntry()
slot = ("segment", id(self), full[0])
try:
entry.static_in = self.shared.statics_for(slot, live)
for dst, src in zip(entry.static_in, live):
dst.copy_(src)
# once, on the compute stream: a guard the static buffers trip recompiles here, never inside the capture
self._chain(rec, entry.static_in)
self.stats["segment_warmups"] += 1
current = torch.cuda.current_stream()
stream = self.shared.capture_stream()
stream.wait_stream(current)
refuse_if_exhausted(self.shared.logger)
graph = torch.cuda.CUDAGraph()
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
with _capturing():
before = torch.cuda.memory_reserved()
try:
graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
try:
outputs = self._chain(rec, entry.static_in)
finally:
graph.capture_end()
except BaseException as exc:
retire_failed_capture(graph, pool, exc)
if self.shared.pool == pool:
self.shared.pool = None
raise
after = torch.cuda.memory_reserved()
current.wait_stream(stream)
self.shared.pool_bytes += max(0, after - before)
except CaptureRefused:
self.seen[full] = rec
self.stats["segment_refusals"] = self.stats.get("segment_refusals", 0) + 1
return None
except Exception as exc: # noqa: BLE001 - this layout stays per block for the load's life
self.capture_error = {"type": type(exc).__name__, "msg": str(exc)[:4000]}
self.refused.add(full)
self.stats["segment_fallbacks"] += 1
if self.shared.logger is not None:
self.shared.logger.warning(
"diffusion.block_graph: recording a %d-block segment failed (%s: %s); its blocks record one by one",
len(self.graphs),
type(exc).__name__,
exc,
)
return None
entry.graph = graph
entry.pool = pool
entry.outputs = outputs
entry.record = rec
entry.slots = (slot,)
entry.replays = 0
self.shared.hold(entry.slots)
self.cache[full] = entry
self.stats["segment_captures"] += 1
if self.owner is not None:
self.owner.note_capture()
self._unsloth_capture_device = capture_device()
note_capture(self._unsloth_capture_device)
while len(self.cache) > self.max_graphs:
_drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)
return entry
def _segments(transformer: Any, graphs: list) -> list:
"""Runs of two or more armed blocks on one card, in forward order, as (graphs, shared) pairs."""
try:
from .diffusion_layer_split import find_blocks
except Exception: # noqa: BLE001
return []
by_block = {id(g.block): g for g in graphs}
runs: list = []
current: list = []
current_key = None
for _qual, group, block in find_blocks(transformer):
g = by_block.get(id(block))
key = (group, g.shared.device_index, id(g.shared)) if g is not None else None
if g is not None and key == current_key:
current.append(g)
continue
if len(current) > 1:
runs.append(current)
current = [g] if g is not None else []
current_key = key
if len(current) > 1:
runs.append(current)
return [(run, run[0].shared) for run in runs]
def _group_offload_hook(module: Any) -> Any:
try:
from diffusers.hooks import group_offloading as go
registry = getattr(module, "_diffusers_hook", None)
if registry is None:
return None
return registry.get_hook(getattr(go, "_GROUP_OFFLOADING", "group_offloading"))
except Exception: # noqa: BLE001
return None
def _is_original_forward(fn: Any, module: Any) -> bool:
return getattr(fn, "__self__", None) is module and getattr(fn, "__func__", None) is getattr(
type(module), "forward", None
)
def _own_forward(fn: Any, module: Any) -> Any:
"""Live class forward for a hook that captured the original before a class-level patch, else None."""
if getattr(fn, "__self__", None) is not module:
return None
func = getattr(fn, "__func__", None)
cls = type(module)
live = getattr(cls, "forward", None)
if func is None or live is None:
return None
if func is live:
return fn
qualname = getattr(func, "__qualname__", None)
owner = getattr(func, "__module__", None)
for base in cls.__mro__:
if qualname == f"{base.__qualname__}.forward" and owner == base.__module__:
try:
return live.__get__(module, cls)
except Exception: # noqa: BLE001 - not a descriptor: leave the hook's forward alone
return None
return None
def _inner_hook_reason(block: Any) -> Optional[str]:
"""Why a block's own subtree moves weights mid-forward (per-layer offload), which no recording can hold."""
for name, sub in block.named_modules():
if sub is block:
continue
if getattr(sub, "_hf_hook", None) is not None:
return "per-layer offload hooks inside the block"
if _group_offload_hook(sub) is not None:
return "per-layer group offload inside the block"
hook = getattr(block, "_hf_hook", None)
if hook is not None:
return "an accelerate hook on the block"
return None
def repeated_blocks(transformer: Any) -> list:
names = set(getattr(transformer, "_repeated_blocks", None) or ())
if not names:
return []
return [m for m in transformer.modules() if type(m).__name__ in names and m is not transformer]
class BlockGraphSet:
"""Every ``BlockGraph`` of one denoiser, behind the handle interface of ``diffusion_cuda_graph``."""
def __init__(
self,
transformer: Any,
shared: _Shared,
logger: Any = None,
) -> None:
self.module = transformer
self.shared = shared
self.card_shared: dict = {}
self.segments: list = []
self.logger = logger
self.graphs: list = []
self.restores: list = []
self.enabled = True
self.bypassed = False
self.slots_mib = 0
self._prefetcher = None
self.mode = "blocks"
self.max_graphs = MAX_GRAPHS_PER_BLOCK
self.length_varies = False
self.scope: Any = None
self._sealed_scopes: set = set()
self._scope_captures: dict = {}
self._render_scopes: set = set()
def scope_sealed(self) -> bool:
return self.length_varies and self.scope in self._sealed_scopes
def note_capture(self) -> None:
if self.length_varies:
self._scope_captures[self.scope] = self._scope_captures.get(self.scope, 0) + 1
def track_scopes(self) -> None:
"""Read each denoiser call's image scope (resolution, batch, references) before its blocks run."""
from .diffusion_cuda_graph import _image_scope
def _pre(module: Any, args: Any, kwargs: Any) -> None:
try:
self.scope = _image_scope(args, kwargs)
except Exception: # noqa: BLE001 - an unkeyable call: no scope, nothing seals
self.scope = None
self._render_scopes.add(self.scope)
self.length_varies = True
handle = self.module.register_forward_pre_hook(_pre, with_kwargs = True)
self.restores.append(handle.remove)
def shared_for(self, index: int) -> _Shared:
"""The pool, stream and static buffers of card ``index``: the primary's own, else one per other card."""
if self.shared.device_index is None:
self.shared.device_index = index
if self.shared.device_index == index:
self.shared.strict_device = True
return self.shared
state = self.card_shared.get(index)
if state is None:
state = _Shared(index, self.logger)
state.root = self.shared.root
state.strict_device = True
self.card_shared[index] = state
return state
def cards(self) -> set:
return {s.device_index for s in self._all_shared()}
def _all_shared(self) -> list:
return [self.shared, *self.card_shared.values()]
@property
def cache(self) -> dict:
out: dict = {}
for i, g in enumerate(self.graphs):
for k, v in g.cache.items():
out[(i, k)] = v
return out
@property
def stats(self) -> dict:
total: dict = {}
for g in self.graphs:
for k, v in g.stats.items():
total[k] = total.get(k, 0) + int(v)
for seg in self.segments:
for k, v in seg.stats.items():
total[k] = total.get(k, 0) + int(v)
total["captures"] = total.get("captures", 0) + total.get("segment_captures", 0)
total["warmups"] = total.get("warmups", 0) + total.get("segment_warmups", 0)
total["replays"] = total.get("replays", 0) + total.get("segment_replays", 0)
# The whole-forward wrapper's names, so diffusion_cuda_graph.stats aggregates both kinds alike.
total["refused_host_tensor"] = total.get("refused_host_weight", 0)
total["cap_skips"] = total.get("evictions", 0)
return total
@property
def poisoned(self) -> bool:
return bool(self.graphs) and all(g.poisoned or g.churned for g in self.graphs)
@property
def capture_error(self) -> Optional[dict]:
for g in self.graphs:
if g.capture_error:
return g.capture_error
return None
def set_bypass(self, on: bool) -> "BlockGraphSet":
self.bypassed = bool(on)
for g in self.graphs:
g.set_bypass(on)
return self
def reset(self) -> "BlockGraphSet":
self._sealed_scopes.clear()
self._scope_captures.clear()
self._render_scopes.clear()
for seg in self.segments:
seg.reset()
for g in self.graphs:
g.reset()
for state in self._all_shared():
state.release()
_release_cached()
return self
def free(self) -> "BlockGraphSet":
for seg in self.segments:
seg.detach()
self.segments.clear()
for g in self.graphs:
g.reset()
g.enabled = False
for restore in reversed(self.restores):
try:
restore()
except Exception: # noqa: BLE001
pass
self.restores.clear()
self.graphs.clear()
for state in self._all_shared():
state.release()
self.enabled = False
try:
from .diffusion_offload_prefetch import module_prefetcher
pf = module_prefetcher(self.module)
if pf is not None:
pf.disable_slots()
except Exception: # noqa: BLE001
pass
_release_cached()
return self
def describe(self) -> dict:
s = self.stats
return {
"module": type(self.module).__name__,
"mode": self.mode,
"blocks": len(self.graphs),
"enabled": bool(self.enabled),
"bypassed": bool(self.bypassed),
"poisoned": bool(self.poisoned),
"graphs": sum(len(g.cache) for g in self.graphs),
"max_graphs": self.max_graphs,
"stats": s,
"slot_ring_mib": int(self.slots_mib),
"slot_ring_allocated_mib": int(
getattr(getattr(self, "_prefetcher", None), "slot_bytes", 0) or 0
)
>> 20,
"pool_mib": int(sum(st.pool_bytes for st in self._all_shared()) >> 20),
"static_mib": int(sum(st.static_bytes for st in self._all_shared()) >> 20),
**({"cards": sorted(self.cards())} if self.card_shared else {}),
**(
{
"segments": len(self.segments),
"segment_blocks": sum(len(seg.graphs) for seg in self.segments),
"segment_graphs": sum(len(seg.cache) for seg in self.segments),
}
if self.segments
else {}
),
"capture_error": None
if not self.capture_error
else {
"type": str(self.capture_error.get("type")),
"msg": str(self.capture_error.get("msg")),
},
}
def why_off(self) -> Optional[str]:
"""Why no block has replayed, once a render has run; None while any replays or before the first call."""
s = self.stats
if s.get("replays") or not (s.get("eager_calls") or s.get("captures")):
return None
churned = sum(1 for g in self.graphs if g.churned)
poisoned = sum(1 for g in self.graphs if g.poisoned)
parts = []
if churned:
parts.append(f"{churned} block(s) find their weights at a new address every call")
if poisoned:
err = self.capture_error or {}
parts.append(f"{poisoned} block(s) failed to record ({err.get('type') or 'error'})")
for field, label in (
("refused_host_weight", "weights not on the GPU"),
("refused_kv_write", "prefix K/V prefill steps"),
("refused_float", "a float argument"),
("refused_object", "a non-tensor argument"),
("refused_output", "an output the graph cannot hand back"),
):
if s.get(field):
parts.append(f"{s[field]} call(s) with {label}")
detail = "; ".join(parts) or "every call so far was a first sighting"
return f"armed per block, but all {s.get('eager_calls', 0)} block call(s) ran their compute ({detail})"
def held_bytes(self) -> int:
"""Device memory this layer holds that the caching allocator cannot hand to anything else."""
pf = getattr(self, "_prefetcher", None)
slots = int(getattr(pf, "slot_bytes", 0) or 0) if pf is not None else 0
return int(sum(st.pool_bytes + st.static_bytes for st in self._all_shared()) + slots)
def _drop_entries(
owner: Any,
entries: list,
evicting: bool = False,
) -> None:
"""The one way a block or segment graph drops recordings (eviction, poison, reset): forget them, then count their
pool as retired, so no capture records beside its cached segments before a flush. An eviction from the current
pool is not counted (the entry just recorded into it keeps it alive); any other drop is."""
if not entries:
return
for entry in entries:
owner._forget(entry)
current = getattr(getattr(owner, "shared", None), "pool", None)
if (
evicting
and current is not None
and all(getattr(e, "pool", None) == current for e in entries)
):
return
try:
from .diffusion_cuda_graph import note_pool_retired
note_pool_retired()
except Exception: # noqa: BLE001
pass
def _release_cached() -> None:
try:
from .diffusion_cuda_graph import release_cached_outside_capture
if _torch().cuda.is_available():
release_cached_outside_capture()
except Exception: # noqa: BLE001
pass
def _device_index(transformer: Any, fallback: Any = None) -> Optional[int]:
torch = _torch()
for cand in (getattr(transformer, "_unsloth_resident_device", None), fallback):
if cand is None:
continue
try:
dev = torch.device(cand)
except Exception: # noqa: BLE001
continue
if dev.type == "cuda":
return dev.index # None: whichever CUDA device is current when the block runs
return None
def _block_card(block: Any) -> Optional[int]:
"""The CUDA ordinal every weight of ``block`` sits on (a layer split places each block whole), else None."""
index: Any = _UNSEEN
for t in _WeightView(block).tensors:
for leaf in _leaves(t):
dev = leaf.device
if dev.type != "cuda" or dev.index is None:
return None
if index is _UNSEEN:
index = dev.index
elif dev.index != index:
return None
return None if index is _UNSEEN else int(index)
def install_block_graphs(
transformer: Any,
*,
device: Any = None,
logger: Any = None,
slots: bool = True,
) -> tuple[Optional[BlockGraphSet], str]:
"""Arm one ``BlockGraph`` per repeated block of ``transformer``. Returns (handle or None, reason)."""
if block_graphs_disabled():
return None, f"block graphs disabled by {BLOCK_GRAPHS_ENV}"
blocks = repeated_blocks(transformer)
if not blocks:
return None, "no repeated blocks to record"
from .diffusion_layer_split import is_layer_split
torch = _torch()
split = is_layer_split(transformer)
shared = _Shared(_device_index(transformer, device), logger)
shared.root = transformer
handle = BlockGraphSet(transformer, shared, logger)
compile_below_offload_hooks(transformer, logger)
refused: dict = {}
for block in blocks:
why = _inner_hook_reason(block)
if why:
refused[why] = refused.get(why, 0) + 1
continue
compiled = getattr(block, "_compiled_call_impl", None)
hook = _group_offload_hook(block)
block_shared = shared
if split:
index = _block_card(block)
if index is None:
refused["weights off the GPU"] = refused.get("weights off the GPU", 0) + 1
continue
block_shared = handle.shared_for(index)
if hook is not None and compiled is not None:
# compiled through its hooks (UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS=0): the compute is inside a dynamo frame
why = "compiled through its offload hooks"
refused[why] = refused.get(why, 0) + 1
continue
try:
if hook is not None:
# Below the hook: the onload / offload stay eager Python and the recording holds only the compute.
registry = getattr(block, "_diffusers_hook", None)
refs = list(getattr(registry, "_fn_refs", None) or ())
target = getattr(block, "_unsloth_below_hook_ref", None)
compute = None
if target in refs:
compute = target.forward
else:
target = None
for r in refs:
compute = _own_forward(getattr(r, "forward", None), block)
if compute is not None:
target = r
break
if target is None:
refused["a stacked hook chain"] = refused.get("a stacked hook chain", 0) + 1
continue
original_fwd = target.forward
graph = BlockGraph(block, compute, block_shared)
target.forward = graph
def restore(ref: Any = target, fn: Any = original_fwd) -> None:
if ref.forward is not fn:
ref.forward = fn
elif compiled is not None:
graph = BlockGraph(block, compiled, block_shared)
block._compiled_call_impl = graph
def restore(
m: Any = block,
c: Any = compiled,
g: Any = graph,
) -> None:
if getattr(m, "_compiled_call_impl", None) is g:
m._compiled_call_impl = c
else:
slot = block.__dict__.get("forward")
fwd = slot if slot is not None else type(block).forward.__get__(block)
graph = BlockGraph(block, fwd, block_shared)
block.__dict__["forward"] = graph
def restore(
m: Any = block,
s: Any = slot,
g: Any = graph,
) -> None:
if m.__dict__.get("forward") is g:
if s is None:
m.__dict__.pop("forward", None)
else:
m.__dict__["forward"] = s
except Exception as exc: # noqa: BLE001
refused[f"install failed ({type(exc).__name__})"] = (
refused.get(f"install failed ({type(exc).__name__})", 0) + 1
)
continue
handle.graphs.append(graph)
handle.restores.append(restore)
if not handle.graphs:
reason = (
", ".join(f"{n} block(s): {w}" for w, n in refused.items()) or "no block could be armed"
)
return None, reason
if split and not segment_graphs_disabled():
handle.segments = [
SegmentGraph(run, run_shared)
for run, run_shared in _segments(transformer, handle.graphs)
]
handle.mode = "segments" if handle.segments else handle.mode
if split and _prompt_length_varies(transformer):
handle.track_scopes()
for unit in (*handle.graphs, *handle.segments):
unit.owner = handle
if slots and torch.cuda.is_available():
try:
from .diffusion_offload_prefetch import module_prefetcher
pf = module_prefetcher(transformer)
if pf is not None:
planned = pf.enable_slots(logger = logger)
handle._prefetcher = pf
handle.slots_mib = planned if getattr(pf, "slot_streamed", 1) else 0
except Exception as exc: # noqa: BLE001 - streamed blocks then simply churn and run their compute
if logger is not None:
logger.warning("diffusion.block_graph: slot ring unavailable (%s)", exc)
if logger is not None:
logger.info(
"diffusion.block_graph: armed %d of %d %s block(s)%s%s%s",
len(handle.graphs),
len(blocks),
type(transformer).__name__,
(
f" over cards {sorted(handle.cards())}"
+ (
f" as {len(handle.segments)} per-card segment graph(s)"
if handle.segments
else ""
)
)
if split
else "",
f"; streamed blocks read a {handle.slots_mib} MiB slot ring"
if handle.slots_mib
else "",
("; refused " + ", ".join(f"{n}: {w}" for w, n in refused.items())) if refused else "",
)
return handle, "armed"
def _prompt_length_varies(transformer: Any) -> bool:
try:
from .diffusion_cuda_graph import _length_varies
return _length_varies(transformer)
except Exception: # noqa: BLE001
return False
COMPILE_BELOW_HOOKS_ENV = "UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS"
def compile_below_hooks_enabled() -> bool:
return (os.environ.get(COMPILE_BELOW_HOOKS_ENV) or "").strip().lower() not in _OFF
def compile_below_offload_hooks(transformer: Any, logger: Any = None) -> int:
"""Compiles each block's own forward, keeping hooks eager; traced hooks recompile on every release."""
if not compile_below_hooks_enabled():
return 0
kwargs = getattr(transformer, "_unsloth_regional_compile_kwargs", None)
if not isinstance(kwargs, dict):
return 0
torch = _torch()
guard = getattr(transformer, "_unsloth_compile_guard", None)
moved = 0
for block in repeated_blocks(transformer):
compiled = getattr(block, "_compiled_call_impl", None)
if compiled is None or _group_offload_hook(block) is None:
continue
registry = getattr(block, "_diffusers_hook", None)
target = live = None
for r in list(getattr(registry, "_fn_refs", None) or ()):
live = _own_forward(getattr(r, "forward", None), block)
if live is not None:
target = r
break
if target is None:
continue
# compiles the live class forward: a per-arch patch installed after the hook captured the original runs too
fn = torch.compile(live, **dict(kwargs))
if guard is not None and callable(getattr(guard, "wrap", None)):
fn = guard.wrap(fn, live, transformer)
# A failed compile falls back to the live forward, so a class patch keeps running eager below the hook.
guard.restores.append(lambda ref = target, f = live: setattr(ref, "forward", f))
from . import diffusion_block_restride
if diffusion_block_restride.is_wrapped(compiled):
# FLUX.1's block 0 keeps the slice layout of the others, so all of them share one compiled graph.
fn = diffusion_block_restride.wrap(fn)
rewrap = getattr(compiled, "_unsloth_rewrap", None)
if callable(rewrap):
# A family adapter around the compiled call (LTX-2's STG flag) must wrap the moved compile too.
fn = rewrap(fn)
target.forward = fn
block._compiled_call_impl = None
block._unsloth_below_hook_ref = target
moved += 1
if moved and logger is not None:
logger.info(
"diffusion.speed: %d offloaded %s blocks compile below their offload hooks",
moved,
type(transformer).__name__,
)
return moved
def compile_pipe_below_offload_hooks(pipe: Any, logger: Any = None) -> int:
try:
from .diffusion_speed import _denoiser_dits
except Exception: # noqa: BLE001
return 0
moved = 0
for transformer in _denoiser_dits(pipe):
try:
moved += compile_below_offload_hooks(transformer, logger)
except Exception as exc: # noqa: BLE001
if logger is not None:
logger.warning("diffusion.speed: compile below offload hooks failed (%s)", exc)
return moved