Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Several CUDA/ROCm cards per pipeline without NCCL, split only at component or denoiser-call
boundaries, never inside a forward; outputs bit-identical; UNSLOTH_DIFFUSION_MULTIGPU=0 disables"""
from __future__ import annotations
import contextlib
import contextvars
import copy
import dataclasses
import os
import queue
import threading
import time
import types
import weakref
from collections import OrderedDict
from typing import Any, Callable, Iterator, Mapping, Optional, Sequence
MULTIGPU_ENV = "UNSLOTH_DIFFUSION_MULTIGPU"
CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL"
DENOISER_ATTRS: tuple[str, ...] = (
"transformer",
"transformer_2",
"unconditional_transformer",
"unet",
)
_KEEP_WITH_DENOISER_PREFIXES: tuple[str, ...] = ("controlnet",)
_ENTRY_METHODS: tuple[str, ...] = (
"forward",
"encode",
"decode",
"get_text_features",
"get_image_features",
)
# Sentinel for ``MemoryPlan.secondary_components``: only the text encoders move (the VAE decodes beside the denoiser).
TEXT_ENCODERS_ONLY: tuple[str, ...] = ("<text_encoders>",)
TEXT_ENCODER_HEADROOM_MIB = 1024
_RESIDENT_FRACTION = 0.85
def multigpu_mode() -> str:
"""``off`` / ``auto`` (default) / ``force``."""
raw = (os.environ.get(MULTIGPU_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no", "disable", "disabled"):
return "off"
if raw == "force":
return "force"
return "auto"
MULTIGPU_ALL_ENV = "UNSLOTH_DIFFUSION_MULTIGPU_ALL"
def multigpu_all_enabled() -> bool:
"""Multi-GPU paths for the remaining families and formats; ``0`` keeps the one-card choices."""
if multigpu_mode() == "off":
return False
return (os.environ.get(MULTIGPU_ALL_ENV) or "").strip().lower() not in (
"0",
"off",
"false",
"no",
"disable",
"disabled",
)
UNET_CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET"
# Automatic (no gpu_ids) arms CFG-parallel on an idle card of the same model; 0 keeps the replica to explicit picks.
AUTO_CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL_AUTO"
# A card counts as idle when no more than this is in use on it (driver + another process's context).
AUTO_CFG_PARALLEL_IDLE_USED_MIB = 1024
def automatic_cfg_parallel_off() -> bool:
return (os.environ.get(AUTO_CFG_PARALLEL_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
)
# ... and computes at most this much (nvidia-smi utilization.gpu, percent): a card running another job is not lent.
AUTO_CFG_PARALLEL_IDLE_UTIL_PCT = 5.0
def _idle_card(probe: Any) -> bool:
free, total = getattr(probe, "free_mib", None), getattr(probe, "total_mib", None)
if free is None or total is None:
return False
util = getattr(probe, "util_pct", None)
if util is not None and float(util) > AUTO_CFG_PARALLEL_IDLE_UTIL_PCT:
return False
return int(total) - int(free) <= AUTO_CFG_PARALLEL_IDLE_USED_MIB
def automatic_partner(
primary: Any,
cards: Sequence[Any],
need_mib: int,
*,
p2p: Optional[dict[int, bool]] = None,
) -> tuple[Optional[int], str]:
"""The card Automatic lends a load's replica to, from ``cards`` (DeviceProbe-like: ordinal, name, total_mib,
free_mib, util_pct), or None and the reason. Pure: no device is touched.
One partner per load, on any number of cards: CFG-parallel and the batch split each need exactly one second
copy, and a third copy would have no second call to run (a step has at most two guidance branches, and a batch is
cut in two). The cards left over stay free for another load (keep multiple models loaded) or another app.
A card is a candidate only when it is (in this order of rejection):
- not the primary;
- the same card model as the primary (same kernels, so the branches agree and neither card waits on a slower
one; a mixed pair such as a 4090 with a 3060 is never paired automatically, ``force`` overrides);
- idle: at most ``AUTO_CFG_PARALLEL_IDLE_USED_MIB`` in use and at most ``AUTO_CFG_PARALLEL_IDLE_UTIL_PCT``
utilisation (another process's work is never slowed and its memory never taken);
- able to hold the replica plus its reserve.
Among candidates: the most usable free memory, then a peer-to-peer link to the primary, then the lowest ordinal.
A static rule, not a trial: on identical idle cards the measured gains are 1.46x to 1.79x (well beyond A/A)."""
p2p = p2p or {}
pidx = getattr(primary, "ordinal", None)
pname = getattr(primary, "name", None)
rejected: dict[str, list[int]] = {}
fits: list[tuple[int, int, int]] = []
for card in cards:
o = int(card.ordinal)
if o == pidx:
continue
if not pname or card.name != pname:
rejected.setdefault("a different card model", []).append(o)
continue
if not _idle_card(card):
rejected.setdefault("busy", []).append(o)
continue
free = int(card.free_mib)
usable = free - _replica_reserve_mib(free, card.total_mib)
if usable < int(need_mib):
rejected.setdefault(f"short of the {int(need_mib)} MiB replica", []).append(o)
continue
fits.append((usable, 1 if p2p.get(o) else 0, -o))
if fits:
return -max(fits)[2], "idle card of the same model"
if not rejected:
return None, "no other card"
return None, "; ".join(f"cuda:{','.join(map(str, v))} {k}" for k, v in rejected.items())
def cfg_parallel_mode() -> str:
"""``off`` / ``auto`` (default) / ``force`` (also across different card models)."""
if multigpu_mode() == "off":
return "off"
raw = (os.environ.get(CFG_PARALLEL_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no", "disable", "disabled"):
return "off"
if raw == "force":
return "force"
return "auto"
@dataclasses.dataclass(frozen = True)
class MultiGpuSelection:
primary: int
secondaries: tuple[int, ...]
# No gpu_ids (the UI's Automatic): only the companion split runs, and only when one card would offload.
automatic: bool = False
_SELECTION: contextvars.ContextVar[Optional[MultiGpuSelection]] = contextvars.ContextVar(
"unsloth_diffusion_multigpu_selection", default = None
)
AUTOMATIC_ORDINALS = "automatic"
def make_selection(
primary: Optional[int],
ordinals: Optional[Sequence[int]],
*,
automatic: bool = False,
) -> Optional[MultiGpuSelection]:
"""A selection when at least two distinct cards were picked and the lever is on, else None (one card)."""
if primary is None or multigpu_mode() == "off":
return None
others = tuple(dict.fromkeys(int(o) for o in (ordinals or ()) if int(o) != int(primary)))
try:
import torch
# nvidia-smi can list a card this process's torch cannot address (a mask applied after the listing).
if torch.cuda.is_available():
visible = int(torch.cuda.device_count())
others = tuple(o for o in others if 0 <= o < visible)
except Exception: # noqa: BLE001 - unknown count: the capability filter below still applies
pass
if not others:
return None
if _rocm_torch():
# ROCm's capability is the gfx target (gfx942 (9, 4)), which does not order; match the primary's load dtype
# instead (arch-pinned attention kernels are gated per card in diffusion_layer_split_plan).
others = tuple(o for o in others if _rocm_dtype_parity(int(primary), o))
return (
MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)
if others
else None
)
# The load dtype and the device-specific patches are resolved for the primary (bf16 only from Ampere, FP8 kernels
# from Ada): a card of a lower compute capability may not run them, so it takes no component.
need = _capability(int(primary))
if need is not None:
others = tuple(o for o in others if (_capability(o) or need) >= need)
if not others:
return None
return MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)
def models_kept_loaded() -> bool:
"""Keep multiple models loaded is on (``fit`` / ``always``). Unreadable reads as off, like the arbiter."""
try:
from .gpu_residency import current_policy
return current_policy() != "off"
except Exception: # noqa: BLE001
return False
def automatic_selection(primary: Optional[int]) -> Optional[MultiGpuSelection]:
"""The selection of an automatic load: the ranked card (or the current one) plus every other parent-visible CUDA
card. None on one GPU, ROCm, XPU, MPS and CPU (the same cheap gate automatic ranking uses)."""
if multigpu_mode() == "off":
return None
try:
from .diffusion_device import auto_cuda_ranking_possible, resolve_diffusion_device_target
if not auto_cuda_ranking_possible() or resolve_diffusion_device_target().device != "cuda":
return None
import torch
if primary is None:
primary = int(torch.cuda.current_device())
count = int(torch.cuda.device_count())
except Exception: # noqa: BLE001 -- an unreadable host keeps one card
return None
return make_selection(primary, range(count), automatic = True)
def _capability(ordinal: int) -> Optional[tuple[int, int]]:
try:
import torch
major, minor = torch.cuda.get_device_capability(int(ordinal))
return int(major), int(minor)
except Exception: # noqa: BLE001
return None
def _rocm_torch() -> bool:
"""A ROCm torch build (AMD SDK / Radeon wheels included, which leave ``torch.version.hip`` unset)."""
try:
import torch
from .rocm_bf16 import is_rocm_torch
return is_rocm_torch(torch)
except Exception: # noqa: BLE001
return False
def _rocm_bf16(ordinal: int) -> Optional[bool]:
try:
import torch
from .rocm_bf16 import rocm_bf16_supported
return bool(rocm_bf16_supported(torch, int(ordinal)))
except Exception: # noqa: BLE001
return None
def _rocm_dtype_parity(primary: int, ordinal: int) -> bool:
"""A ROCm card runs the dtype the load resolves for the primary: bf16 when the primary has native bf16 (RDNA1/2,
GCN and Vega emulate it), anything otherwise. An unreadable card is kept, like an unknown CUDA capability."""
if _rocm_bf16(primary) is not True:
return True
return _rocm_bf16(ordinal) is not False
@contextlib.contextmanager
def selection_scope(
selection: Optional[MultiGpuSelection],
) -> Iterator[Optional[MultiGpuSelection]]:
"""Make ``selection`` visible to the planner and placement for the duration of one load (thread / task local)."""
if selection is None:
yield None
return
# Only a CFG replica reads the recorded compile arguments; automatic placement builds one unless its lever is off.
if (
not selection.automatic
or cfg_parallel_mode() == "force"
or not automatic_cfg_parallel_off()
):
install_compile_recorder()
token = _SELECTION.set(selection)
try:
yield selection
finally:
_SELECTION.reset(token)
def with_load_selection(run_load: Callable[..., Any]) -> Callable[..., Any]:
"""Decorate a backend's ``_run_load(self, **kwargs)``: pop ``_gpu_ordinals`` (every selected card, from
begin_load, or :data:`AUTOMATIC_ORDINALS`) and run the load inside :func:`selection_scope` with ``gpu_ordinal``
as the primary. Without a multi-card selection the load runs exactly as before."""
import functools
@functools.wraps(run_load)
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
ordinals = kwargs.pop("_gpu_ordinals", None)
if ordinals and models_kept_loaded():
selection = None
elif ordinals == AUTOMATIC_ORDINALS:
selection = automatic_selection(kwargs.get("gpu_ordinal"))
else:
selection = make_selection(kwargs.get("gpu_ordinal"), ordinals)
with selection_scope(selection):
return run_load(self, *args, **kwargs)
return wrapper
def current_selection() -> Optional[MultiGpuSelection]:
if multigpu_mode() == "off":
return None
return _SELECTION.get()
@dataclasses.dataclass(frozen = True)
class DeviceProbe:
ordinal: int
name: str
total_mib: Optional[int]
free_mib: Optional[int]
util_pct: Optional[float] = None
_PROBE_LOCK = threading.Lock()
_NAMES: dict[int, str] = {}
def _free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
selection = _SELECTION.get()
if selection is not None and selection.automatic and int(ordinal) != selection.primary:
# Sized from nvidia-smi: an automatic load must not open a CUDA context on a card it may never use.
from .diffusion_device import smi_free_total_mib
return smi_free_total_mib(int(ordinal))
try:
import torch
from utils.hardware import trusted_mem_get_info
free, total = trusted_mem_get_info(int(ordinal), module = torch.cuda)
return int(free) // (1 << 20), int(total) // (1 << 20)
except Exception: # noqa: BLE001
return None, None
def probe_devices(ordinals: Sequence[int]) -> dict[int, DeviceProbe]:
"""Name (cached per process) and free / total memory of each card."""
import torch
ords = [int(o) for o in ordinals]
with _PROBE_LOCK:
for o in ords:
if o not in _NAMES:
try:
_NAMES[o] = str(torch.cuda.get_device_name(o))
except Exception: # noqa: BLE001
_NAMES[o] = ""
out: dict[int, DeviceProbe] = {}
for o in ords:
free, total = _free_total_mib(o)
out[o] = DeviceProbe(
ordinal = o, name = _NAMES.get(o, ""), total_mib = total, free_mib = free, util_pct = _util_pct(o)
)
return out
def _util_pct(ordinal: int) -> Optional[float]:
"""nvidia-smi utilisation of a card an automatic load may borrow; None elsewhere (unknown counts as idle)."""
selection = _SELECTION.get()
if selection is None or not selection.automatic or int(ordinal) == selection.primary:
return None
return _smi_util_pct(ordinal)
def _smi_util_pct(ordinal: int) -> Optional[float]:
"""nvidia-smi utilisation of ``ordinal`` (no CUDA context), read outside any load scope; None when unknown."""
try:
from .diffusion_device import smi_util_pct
return smi_util_pct(int(ordinal))
except Exception: # noqa: BLE001
return None
def best_secondary(
selection: MultiGpuSelection,
probes: Optional[dict[int, DeviceProbe]] = None,
*,
mode: Optional[str] = None,
) -> Optional[int]:
if not selection.secondaries:
return None
from .diffusion_memory import (
DeviceMemory,
_fast_device_budget_mib,
_safe_device_budget_mib,
_user_budget_reserve_mib,
)
budgeted = _user_budget_reserve_mib(1 << 20) > 0
limit_of = _fast_device_budget_mib if mode == "fast" else _safe_device_budget_mib
def usable(free: Optional[int], total: Optional[int]) -> int:
if free is None:
return -1
if not budgeted:
return int(free)
memory = DeviceMemory("cuda", "cuda", "discrete_vram", int(free), total)
return int(limit_of(memory) or 0)
if probes is None and selection.automatic:
free = {o: _free_total_mib(o) for o in selection.secondaries}
return max(selection.secondaries, key = lambda o: (usable(*free[o]), -o))
if probes is None:
probes = probe_devices((selection.primary, *selection.secondaries))
def key(o: int) -> tuple[int, int]:
p = probes.get(o)
return (usable(p.free_mib, p.total_mib) if p is not None else -1, -o)
return max(selection.secondaries, key = key)
@dataclasses.dataclass(frozen = True)
class CompanionSplit:
secondary_ordinal: int
primary_required_mib: int
primary_limit_mib: int
secondary_required_mib: int
secondary_limit_mib: int
reason: str
# None = every companion moves; TEXT_ENCODERS_ONLY = only the encoders (the VAE stays with the denoiser)
components: Optional[tuple[str, ...]] = None
def plan_companion_split(
*,
single_card_policy: str,
model_dense_mib: Optional[int],
companion_dense_mib: Optional[int],
runtime_headroom_mib: int,
base_overhead_mib: int,
primary_limit_mib: Optional[int],
secondary_limit_mib: Callable[[int], Optional[int]],
selection: Optional[MultiGpuSelection],
text_encoder_dense_mib: Optional[int] = None,
phase_headroom_mib: Optional[tuple[int, int, int]] = None,
) -> Optional[CompanionSplit]:
"""Pure decision: return a split when the single-card plan offloads (or ``force``), the denoiser fits the primary
under the resident rule, and the moved companions fit a secondary. Tried in order: every companion (text encoders
+ VAE + the rest, the secondary priced with the full runtime headroom since it runs the decode), then the text
encoders alone (the VAE decodes on the primary; the secondary needs only an encoder forward's room).
``primary_limit_mib`` / ``secondary_limit_mib(ordinal)`` are each card's resident limit under the caller's memory
mode rule (auto: 85% of the safe budget; fast: the fast budget). ``phase_headroom_mib`` = measured (encode,
denoise, decode) MiB above the weights for families Studio has calibrated: each card is then priced only for the
phases it runs instead of the flat runtime headroom."""
mode = multigpu_mode()
if selection is None or mode == "off":
return None
if single_card_policy == "none" and mode != "force":
return None
if model_dense_mib is None or not companion_dense_mib or primary_limit_mib is None:
return None
limits: dict[int, int] = {}
for ordinal in selection.secondaries:
limit = secondary_limit_mib(ordinal)
if limit is not None:
limits[ordinal] = int(limit)
if not limits:
return None
runtime = int(runtime_headroom_mib)
if phase_headroom_mib is not None:
encode, denoise, decode = (int(v) for v in phase_headroom_mib)
# (moved MiB, primary headroom, secondary headroom, components, label)
companions_split = (denoise, max(encode, decode))
encoders_split = (max(denoise, decode), encode)
else:
companions_split = (runtime, runtime)
encoders_split = (runtime, TEXT_ENCODER_HEADROOM_MIB)
candidates: list[tuple[int, int, int, Optional[tuple[str, ...]], str]] = [
(int(companion_dense_mib), *companions_split, None, "text encoders + VAE"),
]
if text_encoder_dense_mib and 0 < int(text_encoder_dense_mib) <= int(companion_dense_mib):
candidates.append(
(int(text_encoder_dense_mib), *encoders_split, TEXT_ENCODERS_ONLY, "text encoders")
)
for moved_mib, pri_headroom, sec_headroom, components, label in candidates:
primary_required = (
max(0, int(model_dense_mib) - moved_mib) + int(pri_headroom) + int(base_overhead_mib)
)
if primary_required > int(primary_limit_mib):
continue
secondary_required = moved_mib + sec_headroom + int(base_overhead_mib)
fits = [o for o, limit in limits.items() if secondary_required <= limit]
if not fits:
continue
best = max(fits, key = lambda o: (limits[o], -o))
return CompanionSplit(
secondary_ordinal = best,
primary_required_mib = primary_required,
primary_limit_mib = int(primary_limit_mib),
secondary_required_mib = secondary_required,
secondary_limit_mib = limits[best],
components = components,
reason = (
f"multi-GPU: denoiser resident on cuda:{selection.primary} ({primary_required} of "
f"{int(primary_limit_mib)} MiB), {label} on cuda:{best} ({secondary_required} of {limits[best]} MiB); "
f"single card would use '{single_card_policy}'"
),
)
return None
@dataclasses.dataclass(frozen = True)
class AssignmentTrial:
"""One companion assignment the planner tried: what moves, and each card's need against its resident limit."""
moved: tuple[str, ...]
moved_mib: int
primary_need_mib: int
primary_limit_mib: int
secondary_need_mib: int
secondary_limit_mib: int
@property
def fits(self) -> bool:
return (
self.primary_need_mib <= self.primary_limit_mib
and self.secondary_need_mib <= self.secondary_limit_mib
)
def why_not(self, primary: int, secondary: int) -> str:
moved = "+".join(self.moved)
if self.primary_need_mib > self.primary_limit_mib:
return (
f"{moved} on cuda:{secondary}: cuda:{primary} still needs {_gib(self.primary_need_mib)} of "
f"{_gib(self.primary_limit_mib)}"
)
return (
f"{moved} on cuda:{secondary}: needs {_gib(self.secondary_need_mib)} there, "
f"{_gib(self.secondary_limit_mib)} allowed"
)
def _gib(mib: Optional[int]) -> str:
return "unknown" if mib is None else f"{int(mib) / 1024:.1f} GiB"
def component_assignment_trials(
sizes: dict[str, tuple[int, str]],
*,
headroom_mib: int,
overhead_mib: int,
primary_limit_mib: int,
secondary_limit_mib: int,
move_denoisers: bool = True,
encoders_only_limit_mib: Optional[int] = None,
) -> list[AssignmentTrial]:
"""Every assignment :func:`choose_component_assignment` tries, in its order, priced (the first that fits wins).
``encoders_only_limit_mib``: what a secondary holding nothing but text encoders may take (its free memory minus
ComfyUI's per-card floor); such a card runs one encoder forward per prompt, so it is priced with the weights and
an encoder forward's room instead of the base overhead and the 85% resident margin of a card that denoises."""
dits = [n for n, (_m, r) in sizes.items() if r == "dit"]
if not dits:
return []
first = "transformer" if "transformer" in dits else dits[0]
seconds = [n for n in dits if n != first]
companions = [
n
for n, (_m, r) in sizes.items()
if r != "dit" and not n.startswith(_KEEP_WITH_DENOISER_PREFIXES)
]
encoders = [n for n in companions if sizes[n][1] == "text_encoder"]
candidates: list[tuple[str, ...]] = []
if companions:
candidates.append(tuple(companions))
if encoders and len(encoders) < len(companions):
candidates.append(tuple(encoders))
if seconds and move_denoisers:
candidates.append(tuple(seconds))
if companions:
candidates.append(tuple(seconds + companions))
def need(names: list) -> int:
weights = sum(sizes[n][0] for n in names)
busy = any(sizes[n][1] != "text_encoder" for n in names)
return weights + (headroom_mib if busy else TEXT_ENCODER_HEADROOM_MIB) + overhead_mib
def secondary(moved: tuple[str, ...]) -> tuple[int, int]:
standard = (need(list(moved)), int(secondary_limit_mib))
if encoders_only_limit_mib is None or any(sizes[n][1] != "text_encoder" for n in moved):
return standard
encoders = (
sum(sizes[n][0] for n in moved) + TEXT_ENCODER_HEADROOM_MIB,
int(encoders_only_limit_mib),
)
return encoders if encoders[1] - encoders[0] > standard[1] - standard[0] else standard
trials = []
for moved in candidates:
sec_need, sec_limit = secondary(moved)
trials.append(
AssignmentTrial(
moved = moved,
moved_mib = sum(sizes[n][0] for n in moved),
primary_need_mib = need([n for n in sizes if n not in moved]),
primary_limit_mib = int(primary_limit_mib),
secondary_need_mib = sec_need,
secondary_limit_mib = sec_limit,
)
)
return trials
def choose_component_assignment(
sizes: dict[str, tuple[int, str]],
*,
headroom_mib: int,
overhead_mib: int,
primary_limit_mib: int,
secondary_limit_mib: int,
move_denoisers: bool = True,
encoders_only_limit_mib: Optional[int] = None,
) -> Optional[tuple[str, ...]]:
"""Names to place on the secondary so each card holds its share resident, or None. ``sizes`` maps component name
-> (MiB, role) with role 'dit' / 'text_encoder' / 'other'. A card running a denoiser or the VAE (any 'other'
component) is priced with the runtime headroom; a card holding only text encoders with an encoder forward's room.
Tried in order: every companion; the text encoders alone; the second denoiser; the second denoiser plus the
companions. The first denoiser always stays on the primary; ``move_denoisers=False`` (the secondary cannot run the
denoiser's attention backend) keeps every denoiser there."""
for trial in component_assignment_trials(
sizes,
headroom_mib = headroom_mib,
overhead_mib = overhead_mib,
primary_limit_mib = primary_limit_mib,
secondary_limit_mib = secondary_limit_mib,
move_denoisers = move_denoisers,
encoders_only_limit_mib = encoders_only_limit_mib,
):
if trial.fits:
return trial.moved
return None
def _secondary_runs_denoiser(
pipe: Any, selection: MultiGpuSelection, secondary: int, sizes: dict, logger: Any
) -> bool:
"""Whether a second denoiser may move wholesale to ``secondary``. ROCm admits mixed gfx targets (make_selection),
so the card must pass the layer split's per-card attention gate: an arch-pinned kernel (flash_attn, AITER) only on
the primary's gfx target, native SDPA not math-only beside a fused primary. CUDA's capability order already holds."""
if not _rocm_torch() or sum(1 for _m, role in sizes.values() if role == "dit") < 2:
return True
try:
from .diffusion_layer_split_plan import _rocm_backend_cards
kept, _backend = _rocm_backend_cards(
pipe, int(selection.primary), (int(secondary),), logger
)
except Exception: # noqa: BLE001 - unverifiable: keep the denoisers on the primary
return False
if int(secondary) in kept:
return True
if logger is not None:
logger.info(
"diffusion.multigpu: second denoiser stays on cuda:%s (attention backend not verified on cuda:%s)",
selection.primary,
secondary,
)
return False
@dataclasses.dataclass(frozen = True)
class PlacementCandidate:
"""One placement the automatic planner considered: ``kind`` is one of ``one_card`` (everything resident on the
primary), ``companion_split``, ``layer_split``, ``offload`` (one card, streamed) or ``cfg_parallel``."""
kind: str
reason: str
@dataclasses.dataclass(frozen = True)
class PlacementDecision:
"""What the multi-GPU refinement chose for a load under a multi-card selection, and why every other candidate
was rejected. Recorded per pipeline; ``status()["multi_gpu"]["decision"]`` shows it (``"placement"`` there stays
the companion split's component-to-device map). ``fallback_resident``: the one-card plan a split would fall back
to keeps everything resident (a forced split of a model that fits one card)."""
chosen: str
summary: str
primary: int
automatic: bool
rejected: tuple[PlacementCandidate, ...] = ()
fallback_resident: bool = False
def describe(self) -> dict[str, Any]:
return {
"chosen": self.chosen,
"summary": self.summary,
"primary": f"cuda:{self.primary}",
"automatic": self.automatic,
"rejected": [{"kind": c.kind, "reason": c.reason} for c in self.rejected],
}
_DECISIONS: "weakref.WeakKeyDictionary[Any, PlacementDecision]" = weakref.WeakKeyDictionary()
def placement_decision(pipe: Any) -> Optional[PlacementDecision]:
try:
return _DECISIONS.get(pipe) if pipe is not None else None
except TypeError:
return None
def _record_decision(
pipe: Any,
selection: MultiGpuSelection,
chosen: str,
summary: str,
rejected: Sequence[PlacementCandidate] = (),
*,
fallback_resident: bool = False,
) -> None:
with contextlib.suppress(TypeError):
_DECISIONS[pipe] = PlacementDecision(
chosen = chosen,
summary = f"placement: {summary}",
primary = int(selection.primary),
automatic = bool(selection.automatic),
rejected = tuple(rejected),
fallback_resident = bool(fallback_resident),
)
def forget_placement_decision(pipe: Any) -> None:
with contextlib.suppress(TypeError):
_DECISIONS.pop(pipe, None)
def _denoiser_mib(sizes: dict[str, tuple[int, str]]) -> int:
return sum(m for m, role in sizes.values() if role == "dit")
def _one_card_rejection(plan: Any, primary: int) -> PlacementCandidate:
required = plan.estimates.get("resident_required_mib")
budget = plan.estimates.get("safe_device_budget_mib")
return PlacementCandidate(
"one_card",
f"everything resident on cuda:{primary} needs {_gib(required)}, {_gib(budget)} free after the reserve; "
f"one card would use '{plan.offload_policy}'",
)
# Automatic placement order: one card (plus a second-card copy for guided steps / even batches) before a layer split,
# when a copy fits and streaming the denoiser on one card is hidden behind its compute: a layer split adds no
# parallelism (its blocks run one card after the other), so it only removes the stream, and below this exposed share
# the cards' own speed spread (T4s 10% apart under the same load) decides instead.
STREAM_HIDDEN_FRACTION = 0.05
STREAM_GATE_ENV = "UNSLOTH_DIFFUSION_SPLIT_STREAM_GATE"
_GEMM_PROBES: dict = {}
def _gemm_tflops(device_index: int, dtype: Any) -> Optional[float]:
"""Dense GEMM TFLOPS of ``device_index`` in ``dtype`` (the denoiser's compute dtype: fp16 on a T4, where bf16 runs
emulated), best of a few runs, cached."""
key = (int(device_index), str(dtype))
if key in _GEMM_PROBES:
return _GEMM_PROBES[key]
result = None
try:
torch = _torch()
with torch.cuda.device(int(device_index)):
a = torch.randn(4096, 3072, dtype = dtype, device = "cuda")
b = torch.randn(3072, 3072, dtype = dtype, device = "cuda")
start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
torch.mm(a, b)
torch.cuda.synchronize()
best = float("inf")
for _ in range(3):
start.record()
for _ in range(10):
torch.mm(a, b)
end.record()
end.synchronize()
best = min(best, start.elapsed_time(end) / 10)
del a, b
if best > 0:
result = 2.0 * 4096 * 3072 * 3072 / (best / 1000.0) / 1e12
except Exception: # noqa: BLE001 - unmeasurable: no gate
result = None
_GEMM_PROBES[key] = result
return result
def stream_exposure(
streamed_mib: int,
dense_params: float,
*,
h2d_gib_s: float,
tflops: float,
side_px: int = 1024,
) -> tuple[float, float, float]:
"""(transfer s, compute s, exposed share) of one denoise step streaming ``streamed_mib`` while computing a DiT of
``dense_params`` parameters at ``side_px``. Exposed share = time the step waits on the stream / step time. Pure."""
from .diffusion_memory import _DIT_COMPUTE_EFFICIENCY, _DIT_PIXELS_PER_TOKEN
transfer = max(0, int(streamed_mib)) / 1024.0 / max(float(h2d_gib_s), 1e-6)
tokens = (int(side_px) // _DIT_PIXELS_PER_TOKEN) ** 2
compute = (
_DIT_COMPUTE_EFFICIENCY
* 2.0
* float(dense_params)
* tokens
/ (max(float(tflops), 1e-6) * 1e12)
)
step = max(transfer, compute)
exposed = max(0.0, transfer - compute) / step if step > 0 else 0.0
return transfer, compute, exposed
def _hidden_stream_reason(pipe: Any, plan: Any, sizes: dict, selection: Any) -> Optional[str]:
"""Why Automatic keeps the one-card streamed plan instead of a layer split, or None (try the split as before).
Only for an automatic selection whose one-card plan streams the denoiser in groups (a capacity plan such as
sequential / whole-module offload keeps the split), priced at Studio's cost canvas from this card's measured
pinned H2D bandwidth and its GEMM throughput in the denoiser's dtype. ``UNSLOTH_DIFFUSION_SPLIT_STREAM_GATE=0``
restores the old choice."""
if not getattr(selection, "automatic", False):
return None
if (os.environ.get(STREAM_GATE_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return None
if getattr(plan, "offload_policy", None) != "group":
return None
denoiser_cls = type(getattr(pipe, "transformer", None)).__name__
if denoiser_cls not in _DIT_BATCH_SPLIT_KWARGS:
# Only where a second-card copy can serve every kind of request this family runs (an even batch splits, a
# guided step pairs): elsewhere (Z-Image, measured 2.3-3.1x faster split) the split stays as on main.
return None
partner = _copy_partner(pipe, selection)
if partner is None:
# the copy does not fit an idle same-model card: the split stays (FLUX.2-klein-9B: 0-14% faster split)
return None
try:
from .diffusion_memory import _COST_CANVAS_PX, _offload_cost_measurements
measured = _offload_cost_measurements()
denoiser = getattr(pipe, "transformer", None)
if measured is None or denoiser is None:
return None
dtype = next((p.dtype for p in denoiser.parameters() if p.is_floating_point()), None)
dtype = getattr(denoiser, "dtype", None) or dtype
if dtype is None or not getattr(dtype, "is_floating_point", False) or dtype.itemsize < 2:
dtype = _torch().float16
if dtype == _torch().float32:
dtype = _torch().float16
tflops = _gemm_tflops(int(selection.primary), dtype)
if tflops is None:
return None
dit = _denoiser_mib(sizes)
resident = int(getattr(plan, "resident_transformer_mib", None) or 0)
streamed = max(0, dit - resident)
params = sum(p.numel() for p in denoiser.parameters())
transfer, compute, exposed = stream_exposure(
streamed, params, h2d_gib_s = measured[0], tflops = tflops, side_px = _COST_CANVAS_PX
)
except Exception: # noqa: BLE001 - unpriced: the split is tried as before
return None
try:
_STREAM_ESTIMATES[pipe] = {
"streamed_mib": streamed,
"h2d_gib_s": round(measured[0], 2),
"tflops": round(tflops, 1),
"transfer_s": round(transfer, 3),
"compute_s": round(compute, 3),
"exposed": round(exposed, 3),
}
except TypeError:
pass
if exposed >= STREAM_HIDDEN_FRACTION:
return None
if streamed > 0:
# The primary would stream part of the denoiser while the partner could hold it whole: a single image then
# depends on which card runs cooler (FLUX.1-dev split -7.7% to +4.4%, Klein-9B 0% to +14% across T4 VMs), so
# main's split stays and one image is never slower than main. The order applies when the primary holds the
# whole denoiser (only encoders stream), where one card is the resident speed and the copy adds batches.
return None
return (
f"one card plus a copy on cuda:{partner}: streaming is hidden behind compute ({_gib(streamed)} per step in "
f"{transfer:.2f} s against {compute:.2f} s, {exposed:.0%} exposed at {_COST_CANVAS_PX}px), so a split would "
f"not be faster, while the copy splits even batches and guided steps; {STREAM_GATE_ENV}=0 tries the split"
)
def _copy_partner(pipe: Any, selection: Any) -> Optional[int]:
"""The idle same-model card a whole copy of the denoiser fits on (:func:`automatic_partner`), or None. Sized from
nvidia-smi for the secondaries, so no context is opened on them."""
try:
denoiser = getattr(pipe, "transformer", None)
need = (
sum(t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers()))
>> 20
)
probes = probe_devices((selection.primary, *selection.secondaries))
primary = probes.get(int(selection.primary))
cards = [
probes[o]
for o in selection.secondaries
if probes.get(o) is not None and probes[o].free_mib is not None
]
if primary is None:
return None
pick, _why = automatic_partner(primary, cards, need)
return pick
except Exception: # noqa: BLE001 - unsized: the split is tried as before
return None
_STREAM_ESTIMATES: "weakref.WeakKeyDictionary[Any, dict]" = weakref.WeakKeyDictionary()
def refine_plan_for_multigpu(
pipe: Any,
plan: Any,
logger: Any = None,
*,
card_reserve_mib: Optional[Mapping[int, int]] = None,
) -> Any:
"""Post-load: an offloading plan becomes a resident split when the LOADED components can be shared by the cards.
``card_reserve_mib`` (ordinal -> MiB): memory a sidecar process holds on a card, uncounted by the loaded sizes.
No-op without a selection, for explicit offload modes, a plan already resident (unless forced) or split, unified memory
or an unknown budget. When no component assignment fits, a layer split (diffusion_layer_split_plan) is tried before
keeping the offload plan. Returns the plan (a ``dataclasses.replace`` copy when it changed). Under a selection the
decision and every rejected candidate are recorded for :func:`placement_status`."""
selection = current_selection()
if selection is None or plan is None:
return plan
primary = int(selection.primary)
def keep(chosen: str, summary: str, *rejected: PlacementCandidate) -> Any:
fallback = getattr(plan, "fallback_plan", None)
_record_decision(
pipe,
selection,
chosen,
summary,
rejected,
fallback_resident = getattr(fallback, "offload_policy", None) == "none",
)
return plan
def one_card_plan(why: str) -> Any:
if plan.offload_policy == "none":
return keep("one_card", f"one card: everything resident on cuda:{primary} ({why})")
return keep(
"offload", f"one card with '{plan.offload_policy}' offload on cuda:{primary} ({why})"
)
if getattr(plan, "secondary_ordinal", None) is not None:
return keep(
"companion_split",
f"companion split: {', '.join(plan.secondary_components or ('companions',))} on "
f"cuda:{plan.secondary_ordinal}, the denoiser resident on cuda:{primary} (planned before the load)",
)
if getattr(plan, "layer_split", None) is not None:
return keep("layer_split", f"layer split across {_cards(plan.layer_split.cards)}")
from .diffusion_layer_split_plan import layer_split_mode, refine_plan_for_layer_split
layer_force = layer_split_mode() == "force"
if plan.offload_policy == "none" and multigpu_mode() != "force" and not layer_force:
required = plan.estimates.get("resident_required_mib")
budget = plan.estimates.get("safe_device_budget_mib")
not_needed = f"not needed: the model fits cuda:{primary}"
return keep(
"one_card",
f"one card: everything resident on cuda:{primary} (needs {_gib(required)}, {_gib(budget)} free after "
"the reserve)",
PlacementCandidate("companion_split", not_needed),
PlacementCandidate("layer_split", not_needed),
)
if getattr(plan, "requested_mode", None) not in ("auto", "fast"):
return one_card_plan(
f"memory mode '{getattr(plan, 'requested_mode', None)}' was chosen explicitly"
)
if plan.estimates.get("explicit_cpu_offload"):
return one_card_plan("CPU offload was requested explicitly")
memory = plan.device_memory
if memory.is_unified or getattr(memory, "device", None) != "cuda":
if memory.is_unified:
from .diffusion_layer_split_plan import note_unified_primary
note_unified_primary(pipe, selection, logger)
return one_card_plan("unified memory" if memory.is_unified else "not a CUDA card")
budget = plan.estimates.get("safe_device_budget_mib")
if budget is None:
return one_card_plan(f"free memory of cuda:{primary} is unknown")
from .diffusion_memory import _loaded_component_mib
sizes = _loaded_component_mib(pipe)
if not sizes:
return one_card_plan("component sizes are unreadable")
mode = getattr(plan, "requested_mode", None)
secondary = best_secondary(selection, mode = mode)
if secondary is None:
return one_card_plan("no second card")
reserve = {int(o): max(0, int(m)) for o, m in (card_reserve_mib or {}).items()}
def card_limit(ordinal: int) -> Optional[int]:
limit = secondary_resident_limit_mib(ordinal, mode)
return None if limit is None else int(limit) - reserve.get(int(ordinal), 0)
sec_limit = card_limit(secondary)
if sec_limit is None:
return one_card_plan(f"free memory of cuda:{secondary} is unknown")
from .diffusion_memory import _fast_device_budget_mib
primary_limit = (
_fast_device_budget_mib(memory) if mode == "fast" else int(int(budget) * _RESIDENT_FRACTION)
)
if primary_limit is None:
return one_card_plan(f"free memory of cuda:{primary} is unknown")
from .diffusion_memory import plan_keeps_transformer_resident
if mode != "fast" and plan.offload_policy != "none" and plan_keeps_transformer_resident(plan):
# Denoiser stays resident on one card, so the 85% margin must not refuse a lighter split.
primary_limit = max(int(primary_limit), int(budget))
primary_limit = int(primary_limit) - reserve.get(primary, 0)
measured_headroom = plan.estimates.get("measured_runtime_headroom_mib")
flat_headroom = (
plan.estimates.get("calibrated_headroom_mib")
or plan.estimates.get("calibrated_split_headroom_mib")
or plan.estimates.get("runtime_headroom_mib")
)
headroom = int(measured_headroom or flat_headroom or 0)
if plan.estimates.get("measured_omits_references"):
# A split has no resident groups to stream back, so a reference request needs the flat headroom.
headroom = max(headroom, int(flat_headroom or 0))
overhead = int(plan.estimates.get("base_overhead_mib") or 0)
rejected: list[PlacementCandidate] = []
if plan.offload_policy != "none":
rejected.append(_one_card_rejection(plan, primary))
def layer_split_or_plan() -> Any:
hidden = None if layer_force else _hidden_stream_reason(pipe, plan, sizes, selection)
if hidden is not None:
rejected.append(PlacementCandidate("layer_split", hidden))
_record_decision(
pipe,
selection,
"offload",
f"one card with '{plan.offload_policy}' offload on cuda:{primary} ({hidden})",
rejected,
)
if logger is not None:
logger.info("diffusion.multigpu: layer split not used (%s)", hidden)
return plan
refined = refine_plan_for_layer_split(
pipe,
plan,
selection = selection,
sizes = sizes,
primary_limit_mib = int(primary_limit),
secondary_limit_mib = card_limit,
headroom_mib = headroom,
overhead_mib = overhead,
logger = logger,
)
split = getattr(refined, "layer_split", None)
if split is not None:
_record_decision(
pipe,
selection,
"layer_split",
f"layer split: denoiser blocks across {_cards(split.cards)} (denoiser "
f"{_gib(_denoiser_mib(sizes))} does not fit one card with its working memory)",
rejected,
fallback_resident = plan.offload_policy == "none",
)
return refined
from .diffusion_layer_split_plan import (
LAYER_SPLIT_ENV,
engine_available,
layer_split_status,
plan_offloads_denoiser,
)
if layer_split_mode() == "off":
declined = f"{LAYER_SPLIT_ENV}=0"
elif not engine_available():
declined = "the layer split engine is unavailable"
elif not plan_offloads_denoiser(plan) and not layer_force:
declined = f"not needed: the denoiser stays resident on cuda:{primary}"
else:
declined = (layer_split_status(pipe) or {}).get("declined") or "no split fits"
rejected.append(PlacementCandidate("layer_split", str(declined)))
if plan.offload_policy == "none":
# a forced split of a model one card already holds: the plan returned is that resident one-card plan
_record_decision(
pipe,
selection,
"one_card",
f"one card: everything resident on cuda:{primary} (no forced split fits)",
rejected,
)
else:
_record_decision(
pipe,
selection,
"offload",
f"one card with '{plan.offload_policy}' offload on cuda:{primary} (no split fits)",
rejected,
)
return refined
if layer_force:
return layer_split_or_plan()
move_denoisers = _secondary_runs_denoiser(pipe, selection, secondary, sizes, logger)
encoders_limit = encoder_card_limit_mib(secondary)
trials = component_assignment_trials(
sizes,
headroom_mib = headroom,
overhead_mib = overhead,
primary_limit_mib = int(primary_limit),
secondary_limit_mib = int(sec_limit),
move_denoisers = move_denoisers,
encoders_only_limit_mib = (
None if encoders_limit is None else int(encoders_limit) - reserve.get(int(secondary), 0)
),
)
winner = next((t for t in trials if t.fits), None)
moved = winner.moved if winner is not None else None
for trial in trials:
if trial is winner:
break
rejected.append(PlacementCandidate("companion_split", trial.why_not(primary, secondary)))
if moved is None:
if not trials:
rejected.append(
PlacementCandidate("companion_split", "no denoiser among the loaded components")
)
return layer_split_or_plan()
reason = (
f"multi-GPU (loaded sizes): {', '.join(moved)} on cuda:{secondary}, the rest resident on "
f"cuda:{selection.primary}; single card would use '{plan.offload_policy}'"
)
if logger is not None:
logger.info("diffusion.multigpu: %s", reason)
on_primary = winner.primary_need_mib
_record_decision(
pipe,
selection,
"companion_split",
f"companion split: {', '.join(moved)} on cuda:{secondary} (denoiser {_gib(_denoiser_mib(sizes))} fits "
f"cuda:{primary}, {_gib(on_primary)} of {_gib(int(primary_limit))} with its working memory; "
f"{', '.join(moved)} {_gib(winner.moved_mib)} does not fit beside it)",
rejected,
fallback_resident = plan.offload_policy == "none",
)
return dataclasses.replace(
plan,
offload_policy = "none",
# Kept: the fit above may be priced with the tiled decode peak (calibrated_headroom_mib), so the savers the
# single-card plan chose must stay on.
vae_tiling = plan.vae_tiling,
vae_slicing = plan.vae_slicing,
stream_text_encoders = False,
stream_transformer = True,
resident_transformer_mib = None,
resident_text_encoder_mib = None,
reasons = tuple(plan.reasons) + (reason,),
secondary_ordinal = secondary,
secondary_components = tuple(moved),
fallback_plan = plan,
)
def _cards(ordinals: Sequence[int]) -> str:
return ", ".join(f"cuda:{int(o)}" for o in ordinals)
def encoder_card_limit_mib(ordinal: int) -> Optional[int]:
"""What a secondary that only runs text encoders may hold: free memory minus ComfyUI's per-card floor (the layer
split prices its block-only cards the same way). None when the card cannot be read."""
from .diffusion_layer_split_plan import secondary_reserve_mib
from .diffusion_memory import _user_budget_reserve_mib
free, total = _free_total_mib(int(ordinal))
if free is None:
return None
return max(0, int(free) - max(secondary_reserve_mib(), _user_budget_reserve_mib(total or free)))
def _replica_reserve_mib(free: Optional[int], total: Optional[int]) -> int:
"""Room a CFG replica leaves free on its card: 1 GiB, or what the user's VRAM Budget keeps free when larger."""
from .diffusion_memory import _user_budget_reserve_mib
return max(1024, _user_budget_reserve_mib(total or free))
def secondary_device_memory(ordinal: int) -> Any:
from .diffusion_memory import DeviceMemory
free, total = _free_total_mib(int(ordinal))
if free is None:
return None
return DeviceMemory(
backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
)
def secondary_safe_budget_mib(ordinal: int) -> Optional[int]:
from .diffusion_memory import _safe_device_budget_mib
memory = secondary_device_memory(ordinal)
return None if memory is None else _safe_device_budget_mib(memory)
def secondary_resident_limit_mib(ordinal: int, mode: str) -> Optional[int]:
from .diffusion_memory import _fast_device_budget_mib
if mode == "fast":
memory = secondary_device_memory(ordinal)
return None if memory is None else _fast_device_budget_mib(memory)
budget = secondary_safe_budget_mib(ordinal)
return None if budget is None else int(int(budget) * _RESIDENT_FRACTION)
def _torch():
import torch
return torch
def move_to_device(
obj: Any,
device: Any,
_depth: int = 0,
*,
tensor_fn: Optional[Callable[[Any], Any]] = None,
objects: bool = True,
) -> Any:
"""Tensors in ``obj`` (containers, dataclasses, ModelOutputs, plain attribute holders) on ``device``.
``tensor_fn(t)`` replaces ``t.to(device)`` for a tensor that is elsewhere. ``objects=False`` leaves dataclasses
and attribute holders as they are (a per-layer KV cache the callee writes into must keep its identity)."""
torch = _torch()
if isinstance(obj, torch.Tensor):
if obj.device == device:
return obj
return obj.to(device) if tensor_fn is None else tensor_fn(obj)
if (
_depth > 6
or obj is None
or isinstance(obj, (str, bytes, int, float, bool, torch.device, torch.dtype))
):
return obj
if isinstance(obj, dict):
# ModelOutput is an OrderedDict that also mirrors its keys as attributes; __setitem__ keeps both in sync.
try:
out = copy.copy(obj)
for key, value in obj.items():
moved = move_to_device(
value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects
)
if moved is not value:
out[key] = moved
return out
except Exception: # noqa: BLE001
return type(obj)(
(k, move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects))
for k, v in obj.items()
)
if isinstance(obj, tuple):
moved = [
move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj
]
if all(a is b for a, b in zip(moved, obj)):
return obj
if hasattr(obj, "_fields"):
return type(obj)(*moved)
return type(obj)(moved) if type(obj) is not tuple else tuple(moved)
if isinstance(obj, list):
return [
move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj
]
if type(obj).__name__ == "BlockMask" and callable(getattr(obj, "to", None)):
return obj.to(device)
if isinstance(obj, torch.nn.Module) or not objects:
return obj
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
changes = {}
for f in dataclasses.fields(obj):
value = getattr(obj, f.name, None)
moved = move_to_device(value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects)
if moved is not value:
changes[f.name] = moved
if not changes:
return obj
try:
return dataclasses.replace(obj, **changes)
except Exception: # noqa: BLE001 - init=False fields
out = copy.copy(obj)
for k, v in changes.items():
object.__setattr__(out, k, v)
return out
state = getattr(obj, "__dict__", None)
if (
isinstance(state, dict)
and state
and any(isinstance(v, torch.Tensor) for v in state.values())
):
if all(
isinstance(
v, (torch.Tensor, bool, int, float, str, type(None), torch.dtype, torch.device)
)
for v in state.values()
):
out = copy.copy(obj)
for k, v in state.items():
if isinstance(v, torch.Tensor) and v.device != device:
setattr(out, k, v.to(device) if tensor_fn is None else tensor_fn(v))
return out
return obj
def _module_device(module: Any) -> Any:
torch = _torch()
for tensor in module.parameters():
return tensor.device
for tensor in module.buffers():
return tensor.device
return None
def _execution_device(module: Any) -> Any:
"""The card ``module`` computes on: its first tensor's device, or, for a group-offloaded denoiser whose first
tensors sit in host memory between calls, the CUDA card its offload groups load onto."""
device = _module_device(module)
if device is not None and getattr(device, "type", None) == "cuda":
return device
try:
from .diffusion_memory import _offload_groups
for group in _offload_groups(module):
onload = getattr(group, "onload_device", None)
if onload is not None and getattr(onload, "type", None) == "cuda":
onload = _torch().device(onload)
if onload.index is None:
# an automatic load on the current card onloads to bare "cuda": name the card, so the replica
# search compares it with the secondaries by ordinal
onload = _torch().device("cuda", _torch().cuda.current_device())
return onload
except Exception: # noqa: BLE001 - unknown: the parameter device stands
pass
return device
def _device_context(device: Any):
torch = _torch()
if getattr(device, "type", None) == "cuda" and device.index is not None:
return torch.cuda.device(device)
return contextlib.nullcontext()
def _crosses(device: Any, home: Any) -> bool:
"""A call must move tensors: the module sits on a different accelerator card than the pipeline's home. A module on
the CPU (offloaded, or between loads) is left to its own hooks."""
return device is not None and device != home and device.type == "cuda"
def install_boundary(module: Any, home: Any) -> list[str]:
"""Wrap ``module``'s entry points: arguments go to the module's CURRENT card, results come back to ``home``."""
torch = _torch()
home = torch.device(home)
wrapped_names: list[str] = []
for name in _ENTRY_METHODS:
bound = getattr(module, name, None)
if not callable(bound) or getattr(bound, "_unsloth_boundary", None) is not None:
continue
def _make(orig: Callable[..., Any]) -> Callable[..., Any]:
def boundary(*args: Any, **kwargs: Any) -> Any:
device = _module_device(module)
if not _crosses(device, home):
return orig(*args, **kwargs)
args, kwargs = move_to_device((args, kwargs), device)
with _device_context(device):
out = orig(*args, **kwargs)
return move_to_device(out, home)
boundary._unsloth_boundary = orig # type: ignore[attr-defined]
boundary.__wrapped__ = orig # type: ignore[attr-defined]
boundary._unsloth_had_override = name in module.__dict__ # type: ignore[attr-defined]
return boundary
setattr(module, name, _make(bound))
wrapped_names.append(name)
return wrapped_names
def remove_boundary(module: Any) -> None:
for name in _ENTRY_METHODS:
current = module.__dict__.get(name) if hasattr(module, "__dict__") else None
if current is not None and getattr(current, "_unsloth_boundary", None) is not None:
if getattr(current, "_unsloth_had_override", False):
module.__dict__[name] = current._unsloth_boundary
continue
try:
delattr(module, name)
except AttributeError:
pass
_EXEC_PATCH_LOCK = threading.Lock()
# Set once this process places a split, so a one-card load never walks its components looking for one.
_SPLIT_EVER_PLACED = False
def _split_home(pipe: Any) -> Any:
if not _SPLIT_EVER_PLACED:
return None
try:
components = pipe.components
except Exception: # noqa: BLE001
return None
if not isinstance(components, dict):
return None
from .diffusion_layer_split import layer_split_of
for module in components.values():
layered = layer_split_of(module)
if layered is not None:
return layered.current_home()
for module in components.values():
home = getattr(module, "_unsloth_split_home", None)
if home is not None:
return home
return None
def _pipeline_bases() -> list[type]:
"""``DiffusionPipeline`` plus ``ModularPipeline`` (FLUX.3 / MiniMax-H3; not a subclass) when this diffusers has it."""
bases: list[type] = []
try:
from diffusers import DiffusionPipeline
bases.append(DiffusionPipeline)
except Exception: # noqa: BLE001
pass
try:
from diffusers import ModularPipeline
bases.append(ModularPipeline)
except Exception: # noqa: BLE001 - diffusers before modular pipelines
pass
return bases
def _patch_execution_device(cls: type) -> bool:
if getattr(cls, "_unsloth_split_exec_patch", False):
return True
orig_exec = cls.__dict__.get("_execution_device")
orig_dev = cls.__dict__.get("device")
if not isinstance(orig_exec, property) or not isinstance(orig_dev, property):
return False
def _exec(self: Any) -> Any:
home = _split_home(self)
return home if home is not None else orig_exec.fget(self)
def _dev(self: Any) -> Any:
home = _split_home(self)
return home if home is not None else orig_dev.fget(self)
cls._execution_device = property(_exec, doc = orig_exec.__doc__)
cls.device = property(_dev, doc = orig_dev.__doc__)
cls._unsloth_split_exec_patch = True
return True
def install_execution_device_patch() -> bool:
"""``_execution_device`` / ``.device`` report the primary card of a split pipeline. diffusers returns the first
non-CPU component (the VAE) or, for a modular pipeline, the first component (the text encoder), so latents landed
on the secondary. Inert without a split denoiser; covers ``from_pipe`` siblings. True when ``DiffusionPipeline``
is patched."""
bases = _pipeline_bases()
if not bases:
return False
with _EXEC_PATCH_LOCK:
patched = [_patch_execution_device(cls) for cls in bases]
return patched[0]
def companion_names(pipe: Any) -> tuple[str, ...]:
torch = _torch()
names = []
for name, module in (getattr(pipe, "components", {}) or {}).items():
if not isinstance(module, torch.nn.Module):
continue
if name in DENOISER_ATTRS or str(name).startswith(_KEEP_WITH_DENOISER_PREFIXES):
continue
names.append(str(name))
return tuple(names)
_PLACEMENTS: "weakref.WeakKeyDictionary[Any, dict]" = weakref.WeakKeyDictionary()
def place_split(
pipe: Any,
*,
primary: Any,
secondary_ordinal: int,
names: Optional[Sequence[str]] = None,
logger: Any = None,
) -> dict[str, str]:
"""Place ``names`` (default: the companions) on ``cuda:<secondary_ordinal>`` and everything else on ``primary``;
returns component -> device. Raises on failure (the caller falls back to single-card offload)."""
torch = _torch()
from .diffusion_fast_load import fast_upload
primary_dev = torch.device(primary)
if primary_dev.type == "cuda" and primary_dev.index is None:
primary_dev = torch.device("cuda", torch.cuda.current_device())
secondary_dev = torch.device("cuda", int(secondary_ordinal))
if names and tuple(names) == TEXT_ENCODERS_ONLY:
moved = tuple(n for n in companion_names(pipe) if n.startswith("text_encoder"))
else:
moved = tuple(names) if names else companion_names(pipe)
components = {
str(n): m
for n, m in (getattr(pipe, "components", {}) or {}).items()
if isinstance(m, torch.nn.Module)
}
placement: dict[str, str] = {}
order = sorted(components, key = lambda n: 0 if n in moved else 1)
for name in order:
module = components[name]
device = secondary_dev if name in moved else primary_dev
with fast_upload([module], device, logger = logger):
module.to(device)
placement[name] = str(device)
for name in moved:
if name in components:
install_boundary(components[name], primary_dev)
global _SPLIT_EVER_PLACED
_SPLIT_EVER_PLACED = True
for name, module in components.items():
if name not in moved:
module._unsloth_split_home = primary_dev
install_execution_device_patch()
info = {
"placement": placement,
"primary": str(primary_dev),
"secondary": str(secondary_dev),
"moved": list(moved),
}
try:
_PLACEMENTS[pipe] = info
except TypeError:
pass
if logger is not None:
logger.info(
"diffusion.multigpu: placed %s on %s, the rest on %s",
", ".join(moved) or "nothing",
secondary_dev,
primary_dev,
)
return placement
def undo_split(pipe: Any) -> None:
"""Remove every split placed on ``pipe``: a layer split of any denoiser (its block boundary and the planner's
record) and the component boundaries. Weights stay where they are."""
torch = _torch()
from .diffusion_layer_split import undo_layer_split as undo_layers
from .diffusion_layer_split_plan import forget_applied_layer_split
for name in DENOISER_ATTRS:
try:
undo_layers(pipe, attr = name)
except Exception: # noqa: BLE001 - best effort; the component boundaries below still come off
pass
forget_applied_layer_split(pipe)
for module in (getattr(pipe, "components", {}) or {}).values():
if isinstance(module, torch.nn.Module):
remove_boundary(module)
if "_unsloth_split_home" in module.__dict__:
del module.__dict__["_unsloth_split_home"]
try:
_PLACEMENTS.pop(pipe, None)
except TypeError:
pass
def apply_split_plan(
pipe: Any,
plan: Any,
*,
placement: str,
logger: Any = None,
) -> bool:
"""Place a split plan."""
secondary = getattr(plan, "secondary_ordinal", None)
if secondary is None:
return False
try:
place_split(
pipe,
primary = placement,
secondary_ordinal = int(secondary),
names = getattr(plan, "secondary_components", None),
logger = logger,
)
return True
except Exception as exc: # noqa: BLE001 - fall back to one card
if logger is not None:
logger.warning(
"diffusion.multigpu: split placement failed (%s); falling back to one card", exc
)
undo_split(pipe)
try:
pipe.to("cpu")
_torch().cuda.empty_cache()
except Exception: # noqa: BLE001
pass
return False
def is_split(pipe: Any) -> bool:
"""A companion split or a layer split is placed on ``pipe``."""
if pipe is None:
return False
try:
if pipe in _PLACEMENTS:
return True
except TypeError:
return False
from .diffusion_layer_split_plan import is_layer_split
return is_layer_split(pipe)
def placement_status(pipe: Any) -> Optional[dict[str, Any]]:
if pipe is None:
return None
try:
info = _PLACEMENTS.get(pipe)
cfg = _CFG_ROUTERS.get(pipe)
declined = _CFG_DECLINES.get(pipe)
except TypeError:
return None
from .diffusion_layer_split_plan import layer_split_status
layer = layer_split_status(pipe)
decision = placement_decision(pipe)
if info is None and cfg is None and declined is None and layer is None and decision is None:
return None
out: dict[str, Any] = dict(info or {})
try:
stream = _STREAM_ESTIMATES.get(pipe)
except TypeError:
stream = None
if stream is not None:
out["stream_estimate"] = dict(stream)
from .diffusion_layer_split import layer_split_stats
live = {
name: stats
for name in DENOISER_ATTRS
if (stats := layer_split_stats(getattr(pipe, name, None))) is not None
}
if layer is not None:
out["layer_split"] = dict(layer)
if live:
out["layer_split"]["live"] = live
elif (
"layer_split" in out
): # placed by the engine directly (no planner record): its own summary, live
engine = out["layer_split"]
out["layer_split"] = dict(live.get(engine.get("denoiser", "transformer")) or engine)
if cfg is not None:
out["cfg_parallel"] = cfg.describe()
elif declined is not None:
out["cfg_parallel"] = {"armed": False, "declined": declined}
if decision is not None:
if cfg is None:
cfg_rejected = declined
elif cfg.disabled_reason not in (None, "uninstalled"):
# armed, then fell back to one card at run time: say so where the placement is reported
cfg_rejected = f"disabled at run time: {cfg.disabled_reason}"
else:
cfg_rejected = None
out["decision"] = _live_decision(pipe, decision, info, layer, cfg_rejected)
return out
def _live_decision(
pipe: Any,
decision: PlacementDecision,
info: Optional[dict],
layer: Optional[dict],
cfg_declined: Optional[str],
) -> dict[str, Any]:
"""The recorded decision, corrected by what engaged: a split whose placement failed reports its one-card fallback,
and a declined CFG-parallel replica joins the rejected candidates."""
out = decision.describe()
failed = None
if decision.chosen == "companion_split" and info is None:
failed = "the companion split did not engage at placement"
elif decision.chosen == "layer_split" and not (layer or {}).get("cards"):
failed = f"the layer split did not engage ({(layer or {}).get('declined') or 'placement failed'})"
if failed is not None:
out["rejected"] = [{"kind": decision.chosen, "reason": failed}, *out["rejected"]]
if decision.fallback_resident:
out["chosen"] = "one_card"
out["summary"] = (
f"placement: one card: everything resident on cuda:{decision.primary} ({failed})"
)
else:
out["chosen"] = "offload"
out["summary"] = (
f"placement: one card with offload on cuda:{decision.primary} ({failed})"
)
if cfg_declined is not None:
out["rejected"] = [*out["rejected"], {"kind": "cfg_parallel", "reason": cfg_declined}]
return out
_COMPILE_RECORDER_LOCK = threading.Lock()
def install_compile_recorder() -> bool:
try:
import torch
except Exception: # noqa: BLE001
return False
with _COMPILE_RECORDER_LOCK:
current = torch.nn.Module.compile
if getattr(current, "_unsloth_recorder", False):
return True
def compile(self: Any, *args: Any, **kwargs: Any) -> Any: # noqa: A001 - mirrors nn.Module.compile
self.__dict__["_unsloth_compile_args"] = (args, dict(kwargs))
return current(self, *args, **kwargs)
compile._unsloth_recorder = True # type: ignore[attr-defined]
compile.__wrapped__ = current # type: ignore[attr-defined]
torch.nn.Module.compile = compile
return True
_ROOT_HOOK_DICTS = (
"_forward_pre_hooks",
"_forward_pre_hooks_with_kwargs",
"_forward_hooks",
"_forward_hooks_with_kwargs",
"_forward_hooks_always_called",
)
# Offload machinery moves weights between host and card without changing the math. A replica holds its weights
# resident, so it drops these instead of inheriting them.
_OFFLOAD_REGISTRY_HOOKS = frozenset(
{"group_offloading", "layer_execution_tracker", "lazy_prefetch_group_offloading"}
)
_OFFLOAD_MODULE_MARKERS = (
"diffusers.hooks.group_offloading",
"diffusers.hooks.hooks",
"accelerate.hooks",
"diffusion_offload_prefetch",
"diffusion_small_host",
"diffusion_pinned_arena",
)
def _origin_modules(fn: Any) -> list[str]:
out: list[str] = []
seen = 0
while fn is not None and seen < 6:
seen += 1
out.append(str(getattr(fn, "__module__", "") or ""))
out.append(str(getattr(type(fn), "__module__", "") or ""))
out.append(str(getattr(getattr(fn, "func", None), "__module__", "") or ""))
fn = (
getattr(fn, "__wrapped__", None)
or getattr(fn, "__func__", None)
or getattr(fn, "func", None)
)
return out
def _is_offload_object(obj: Any) -> bool:
return any(marker in mod for mod in _origin_modules(obj) for marker in _OFFLOAD_MODULE_MARKERS)
def _registry_kind(sub: Any) -> str:
registry = sub.__dict__.get("_diffusers_hook")
hooks = getattr(registry, "hooks", None) if registry is not None else None
if not hooks:
return "none"
return "offload" if set(hooks) <= _OFFLOAD_REGISTRY_HOOKS else "other"
def _immutable_capture(value: Any, depth: int) -> bool:
"""A closure cell a replica may share with its source. deepcopy keeps a function (and so its closure) as the same
object, and the two branches call it concurrently, so only values nobody can mutate qualify: scalars, strings,
dtypes / devices, tuples / frozensets of those, and plain functions whose own captures qualify. A list, dict, set,
tensor, bound method or arbitrary object (a counter, a cache) would be shared state."""
torch = _torch()
if depth > 3:
return False
if isinstance(
value, (str, bytes, int, float, complex, bool, type(None), torch.dtype, torch.device, type)
):
return True
if isinstance(value, (tuple, frozenset)):
return all(_immutable_capture(v, depth + 1) for v in value)
if isinstance(value, types.FunctionType):
return _portable(value, depth)
return False
def _portable(fn: Any, depth: int = 0) -> bool:
"""True when a hook / forward override can be shared by a copy of its module: nothing it captures (closure cells,
bound ``self``, partial arguments, the attributes of a captured object) is a module or a CUDA tensor, which would
keep pointing at the source card."""
torch = _torch()
if depth > 3:
return False
if isinstance(fn, torch.nn.Module) or (
isinstance(fn, torch.Tensor) and fn.device.type != "cpu"
):
return False
if isinstance(fn, (list, tuple, set)):
return all(_portable(v, depth + 1) for v in fn)
if isinstance(fn, dict):
return all(_portable(v, depth + 1) for v in fn.values())
if isinstance(
fn,
(str, bytes, int, float, bool, type(None), torch.Tensor, torch.dtype, torch.device, type),
):
return True
if isinstance(fn, (types.FunctionType, types.MethodType)):
owner = getattr(fn, "__self__", None)
if owner is not None and not _portable(owner, depth + 1):
return False
func = getattr(fn, "__func__", fn)
for cell in getattr(func, "__closure__", None) or ():
try:
value = cell.cell_contents
except ValueError:
continue
if not _immutable_capture(value, depth + 1):
return False
return True
partial_func = getattr(fn, "func", None)
if partial_func is not None and hasattr(fn, "args"):
return _portable(partial_func, depth + 1) and all(
_portable(a, depth + 1)
for a in (*fn.args, *(getattr(fn, "keywords", None) or {}).values())
)
state = getattr(fn, "__dict__", None)
if isinstance(state, dict):
return all(_portable(v, depth + 1) for v in state.values())
return True
# Foreign hooks may keep state in a module global that replica threads race on, so they decline.
_TRUSTED_HOOK_MODULES = ("core.inference.", "studio.backend.core.inference.")
def _hook_origin(fn: Any) -> str:
target = getattr(fn, "func", None) if hasattr(fn, "args") and hasattr(fn, "func") else fn
target = getattr(target, "__func__", target)
if isinstance(target, (types.FunctionType, types.BuiltinFunctionType)):
return str(getattr(target, "__module__", None) or "")
return str(getattr(type(target), "__module__", None) or "")
def _trusted_hook(fn: Any) -> bool:
return _hook_origin(fn).startswith(_TRUSTED_HOOK_MODULES) and _portable(fn)
def _hooks_present(module: Any) -> Optional[str]:
"""Name of the first submodule carrying state a replica cannot inherit faithfully, else None: a non-offload hook
registry (step caches), an accelerate hook, or a forward hook / instance forward override below the root whose
captured state points at the source (deepcopy treats functions as atoms). Offload machinery (diffusers group
offloading, Studio's prefetchers) is dropped by :func:`build_replica`, which holds every weight resident. The
ROOT's torch forward hooks run on the source around the router for every branch and the copy does not carry them,
but whatever per-call state they write must reach the replica: only Studio's own root hooks are accepted, whose
state is the ``_unsloth_*`` scalars :func:`_mirror_hook_state` copies."""
for name, sub in module.named_modules():
kind = _registry_kind(sub)
if kind == "other":
return f"{name or '<root>'}._diffusers_hook ({', '.join(sorted(sub._diffusers_hook.hooks))})"
if getattr(sub, "_hf_hook", None) is not None:
return f"{name or '<root>'}._hf_hook"
for attr in ("_forward_hooks", "_forward_pre_hooks"):
for hook in (getattr(sub, attr, None) or {}).values():
if _is_offload_object(hook):
continue
if name and not _trusted_hook(hook):
return f"{name}.{attr}"
if not name and not _hook_origin(hook).startswith(_TRUSTED_HOOK_MODULES):
return f"<root>.{attr} (a hook outside Unsloth: its per-call state cannot be mirrored)"
override = sub.__dict__.get("forward")
if override is None or isinstance(override, (types.MethodType, CfgParallelRouter)):
continue
if not name and _resident_graph(sub) is not None:
# install_cfg_parallel moves the root's graph under the router
continue
if kind == "offload" or _is_offload_object(override):
continue
if callable(override) and not _trusted_hook(override):
return f"{name or '<root>'}.forward (instance override)"
return None
def _offload_attr_keys(sub: Any) -> list[str]:
torch = _torch()
keys = []
kind = _registry_kind(sub)
for key, value in sub.__dict__.items():
if key in ("_parameters", "_buffers", "_modules") or key.startswith("_forward_"):
continue
if key == "_diffusers_hook" and kind == "offload":
keys.append(key)
elif key == "forward" and (kind == "offload" or _is_offload_object(value)):
keys.append(key)
elif not isinstance(value, (torch.Tensor, torch.nn.Module)) and _is_offload_object(value):
keys.append(key)
return keys
_DROP = object()
def _rebind_root_forward(original: Any, source: Any, replica: Any) -> None:
"""Give ``replica`` the root ``forward`` the source ran before the router took its slot: offload wrappers (math
neutral, dropped with the offload state) are peeled off, and a method bound to the source is rebound to the copy.
Anything else cannot be reproduced and raises (CFG-parallel then declines)."""
seen = 0
while original is not None and _is_offload_object(original) and seen < 8:
seen += 1
original = getattr(original, "__wrapped__", None)
if isinstance(original, types.MethodType) and original.__self__ is source:
if original.__func__ is getattr(type(source), "forward", None):
return
replica.__dict__["forward"] = types.MethodType(original.__func__, replica)
return
raise RuntimeError(
f"the denoiser's root forward override ({original!r}) cannot be reproduced on a replica"
)
def build_replica(module: Any, device: Any) -> Any:
"""A copy of ``module`` whose parameters and buffers are created directly on ``device`` (never staged on the
source card), with every recorded ``Module.compile`` re-applied to the copy. Raises when it cannot be built.
Built outside inference mode: inside it, torchao weights made outside raise "Cannot set version_counter for
inference tensor" and ones made inside silently stay on the source card."""
torch = _torch()
device = torch.device(device)
with torch.inference_mode(False), torch.no_grad():
replica = _copy_module(module, device)
stray = sorted(
{
str(t.device)
for t in (*replica.parameters(), *replica.buffers())
if t.device.type == "cuda" and t.device != device
}
)
if stray:
raise RuntimeError(f"replica weights stayed on {', '.join(stray)}, not {device}")
# The ROCm flash gate is thread-local state its root hooks set; the copy runs on its own worker thread.
from .diffusion_attention import track_rocm_flash_replica
track_rocm_flash_replica(module, replica, device)
return replica
def _copy_module(module: Any, device: Any) -> Any:
torch = _torch()
memo: dict[int, Any] = {}
keep: list[Any] = []
for _name, param in module.named_parameters(remove_duplicate = True):
if id(param) in memo:
continue
copied = param.detach().to(device)
new = copied
if isinstance(param, torch.nn.Parameter):
try:
new = torch.nn.Parameter(copied, requires_grad = param.requires_grad)
except Exception: # noqa: BLE001 - a tensor subclass that refuses the wrapper stays a plain tensor
new = copied
memo[id(param)] = new
keep.append(param)
for _name, buffer in module.named_buffers(remove_duplicate = True):
if id(buffer) in memo:
continue
memo[id(buffer)] = buffer.detach().to(device)
keep.append(buffer)
from .diffusion_qwenimage21_dense_fused import is_replica_record, rebuild_replica
for sub in module.modules():
offload_keys = set(_offload_attr_keys(sub))
for key, value in sub.__dict__.items():
if (
isinstance(value, CfgParallelRouter)
or key in offload_keys
or key in _SOURCE_ONLY_ATTRS
or is_replica_record(key, value)
):
# never traversed: offload state holds pinned host copies a deepcopy would duplicate, and a
# source-only cache holds source-card tensors
memo[id(value)] = _DROP
for attr in ("_forward_hooks", "_forward_pre_hooks"):
for hook in (getattr(sub, attr, None) or {}).values():
if _is_offload_object(hook):
memo[id(hook)] = _DROP
# Per-step precision state (the NVFP4 W4A16 protect schedule) is advanced by the source's scheduler: the copy
# must read the same controller, or the two CFG branches of one step run different GEMMs.
for value in _shared_state_values(sub):
memo[id(value)] = value
guard = module.__dict__.get("_unsloth_compile_guard")
if guard is not None:
memo[id(guard)] = _DROP
# A baked LoRA's scale pinned as a 0-d device tensor (diffusion._pin_baked_lora_scales) lives in a plain dict,
# not a buffer: the copy needs its own on ``device``.
for sub in module.modules():
scaling = sub.__dict__.get("scaling")
if isinstance(scaling, dict):
for value in scaling.values():
if isinstance(value, torch.Tensor) and id(value) not in memo:
memo[id(value)] = value.detach().to(device)
keep.append(value)
memo[id(memo)] = keep
replica = copy.deepcopy(module, memo)
router = module.__dict__.get("forward")
for sub in replica.modules():
for key in [k for k, v in sub.__dict__.items() if v is _DROP]:
del sub.__dict__[key]
for attr in _ROOT_HOOK_DICTS:
hooks = sub.__dict__.get(attr)
if hooks:
for hook_id in [k for k, v in hooks.items() if v is _DROP]:
del hooks[hook_id]
for other in _ROOT_HOOK_DICTS:
sub.__dict__.get(other, {}).pop(hook_id, None)
# Release the copies before re-fusing on the replica's card, else each replaced allocation stays alive.
memo.clear()
del keep
param = copied = new = buffer = (
None # the copy loop's last parameter, still referenced by these names
)
try:
rebuild_replica(module, replica)
except Exception: # noqa: BLE001 - the replica keeps the stock projections
pass
if isinstance(router, CfgParallelRouter) and router.had_override:
_rebind_root_forward(router.orig_forward, module, replica)
# The source's own __call__ runs its root hooks around every branch; the copy must not run them a second time.
for attr in _ROOT_HOOK_DICTS:
hooks = getattr(replica, attr, None)
if hooks is not None:
hooks.clear()
compiled_subs = 0
for (name, src), (_rname, dst) in zip(module.named_modules(), replica.named_modules()):
for key, value in list(dst.__dict__.items()):
if (
isinstance(value, torch.Tensor)
and value.device != device
and value.device.type == "cuda"
):
dst.__dict__[key] = value.to(device)
if getattr(src, "_compiled_call_impl", None) is not None:
recorded = src.__dict__.get("_unsloth_compile_args")
dst._compiled_call_impl = None
if recorded is None:
raise RuntimeError(f"compiled submodule '{name}' has no recorded compile arguments")
args, kwargs = recorded
dst.compile(*args, **kwargs)
compiled_subs += 1
elif "_compiled_call_impl" in dst.__dict__:
dst._compiled_call_impl = None
if compiled_subs:
_rewrap_compiled(module, replica)
replica._unsloth_replica_compiled = compiled_subs
replica.eval()
return replica
_SHARED_STATE_TYPES = frozenset({"NVFP4StepController"})
# Caches only the source's own hooks fill and clear: the FLUX.3 conditioning blocks reach the copy through the
# forwarded call (diffusion_cond_blocks), so a copied cache would only pin source-card tensors that no render clears.
_SOURCE_ONLY_ATTRS = frozenset({"_unsloth_cond_cache", "_unsloth_rocm_flash_replicas"})
def _shared_state_values(sub: Any) -> list:
return [value for value in sub.__dict__.values() if type(value).__name__ in _SHARED_STATE_TYPES]
def _rewrap_compiled(source: Any, replica: Any) -> None:
"""Re-apply the wrappers Studio's compile path puts around the source's compiled blocks (``dst.compile`` replaced
them): the compile guard (eager fallback on a compile-time failure, the background-compile eager gate, the compile
config on this thread) and the block-0 restride, in the order the source got them."""
if "_unsloth_compile_guard" in source.__dict__:
from .diffusion_speed import guard_compiled_blocks
guard_compiled_blocks(replica)
from . import diffusion_block_restride
if any(
diffusion_block_restride.is_wrapped(getattr(m, "_compiled_call_impl", None))
for m in source.modules()
):
diffusion_block_restride.install(replica)
_ADAPTER_ATTRS = (
"scaling",
# PEFT stores the selection here; ``active_adapters`` is a property over it (never in ``__dict__``)
"_active_adapter",
"active_adapters",
"merged_adapters",
"disable_adapters",
"_disable_adapters",
)
def weights_fingerprint(module: Any) -> int:
"""Changes whenever the denoiser's weights or adapter state do: a parameter or buffer (NVFP4 / quantised layers
keep their packed weights and scales as buffers) replaced, edited in place (fused LoRA bumps the version counter),
added or removed (PEFT injection), or an adapter's scale / activation changed. A ``.data`` swap to different values
is not seen; diffusers' LoRA paths do not do that."""
parts: list = []
for name, param in (*module.named_parameters(), *module.named_buffers()):
try:
version = param._version
except Exception: # noqa: BLE001 - inference tensors carry no version
version = -1
# Identity, not the data pointer: an encode-time offload round trip moves the same weights and must not force
# a rebuild, while a replaced parameter (new object) or an in-place edit (version) must.
parts.append((name, id(param), version))
for name, sub in module.named_modules():
for attr in _ADAPTER_ATTRS:
value = sub.__dict__.get(attr)
if isinstance(value, dict):
# A pinned scale tensor: identity + version (an in-place write bumps it), no device read per call.
value = {
k: (id(v), v._version) if hasattr(v, "_version") else v
for k, v in value.items()
}
if value is not None and not callable(value):
parts.append((name, attr, repr(value)))
return hash(tuple(parts))
_PROBE_SAMPLES = 64
def _dense_leaves(t: Any, depth: int = 0) -> Iterator[Any]:
"""The plain tensors holding ``t``'s values: ``t`` itself, or a torchao subclass's inner tensors (qdata, scales)."""
flatten = getattr(type(t), "__tensor_flatten__", None)
if flatten is None or depth > 4:
yield t
return
try:
names, _ctx = t.__tensor_flatten__()
except Exception: # noqa: BLE001 - an opaque subclass: probe the outer tensor
yield t
return
for name in names:
inner = getattr(t, name, None)
if inner is not None:
yield from _dense_leaves(inner, depth + 1)
def weights_probe(module: Any, index_cache: Optional[dict] = None) -> Optional[bytes]:
"""Sampled bytes of every inference tensor (no version counter, so invisible to :func:`weights_fingerprint`),
read back in one host sync; None when there are none. Edits missing every sampled byte go unseen."""
torch = _torch()
cache = {} if index_cache is None else index_cache
by_device: dict = {}
with torch.no_grad():
for _name, t in (*module.named_parameters(), *module.named_buffers()):
if not t.is_inference():
continue
for leaf in _dense_leaves(t):
if not isinstance(leaf, torch.Tensor) or leaf.numel() == 0 or leaf.is_meta:
continue
# bytes, not elements: a gather is not implemented for every weight dtype (fp8, packed fp4)
flat = _as_bytes(leaf)
n = flat.numel()
key = (n, flat.device)
idx = cache.get(key)
if idx is None:
# integer spacing: a float32 linspace rounds n - 1 up to n past 2**24 bytes and reads past the end
k = min(n, _PROBE_SAMPLES)
idx = torch.arange(k, device = flat.device) * (n - 1) // max(k - 1, 1)
cache[key] = idx
by_device.setdefault(flat.device, []).append(flat.index_select(0, idx))
if not by_device:
return None
return b"".join(
torch.cat(chunks).cpu().numpy().tobytes()
for _dev, chunks in sorted(by_device.items(), key = lambda kv: str(kv[0]))
)
_LATENT_KEYS = ("hidden_states", "sample", "x", "latents", "img", "latent")
def _latent_arg(args: tuple, kwargs: dict) -> Any:
torch = _torch()
for key in _LATENT_KEYS:
value = kwargs.get(key)
if isinstance(value, torch.Tensor):
return value
# Cosmos 3: ``vision_tokens=[t]``, a new list each call around the same tensor
vision = kwargs.get("vision_tokens")
if (
isinstance(vision, (list, tuple))
and len(vision) == 1
and isinstance(vision[0], torch.Tensor)
and multigpu_all_enabled()
):
return vision[0]
for value in args:
if isinstance(value, torch.Tensor):
return value
return None
_PLAIN_TYPES = (type(None), bool, int, float, str, bytes)
def _holds_opaque_tensors(obj: Any, depth: int = 0) -> bool:
"""True when ``obj`` carries tensors inside something other than a plain tuple / list / dict (a prefix-KV cache
object, a namedtuple, a value holder). The replica walker cannot move those to its card, and a mutable cache must
not be shared across the two threads, so such a branch stays on the primary."""
torch = _torch()
if isinstance(obj, (torch.Tensor, *_PLAIN_TYPES, torch.device, torch.dtype, torch.nn.Module)):
return False
if depth < 4 and isinstance(obj, tuple) and not hasattr(obj, "_fields"):
return any(_holds_opaque_tensors(v, depth + 1) for v in obj)
if depth < 4 and isinstance(obj, list):
return any(_holds_opaque_tensors(v, depth + 1) for v in obj)
if depth < 4 and isinstance(obj, dict):
return any(_holds_opaque_tensors(v, depth + 1) for v in obj.values())
if isinstance(obj, tuple):
values: Any = obj
else:
try:
values = vars(obj).values()
except TypeError:
return False
for value in values:
if isinstance(value, torch.Tensor) or (
isinstance(value, (list, tuple, dict)) and next(_iter_tensors(value), None) is not None
):
return True
if (
depth < 4
and not isinstance(value, _PLAIN_TYPES)
and _holds_opaque_tensors(value, depth + 1)
):
return True
return False
# Prefix-KV cache (Qwen-Image-2.1): step-0 prefill ("extract") runs on the primary; "cached" odd branches use a replica copy.
_KV_KWARG = "kv_cache"
_KV_MODE_KWARG = "kv_cache_mode"
class _Unmirrorable(Exception):
pass
def _walk_cache(
obj: Any,
move: Optional[Callable[[Any], Any]],
found: list,
depth: int = 0,
) -> Any:
"""Structural copy of a cache object (``move`` applied to every tensor), or with ``move`` None only the tensors
collected into ``found``. Raises :class:`_Unmirrorable` on anything but plain values, containers and ``__dict__``
value holders (a module, a slotted or foreign object), which then stays on the primary."""
torch = _torch()
if isinstance(obj, torch.Tensor):
found.append(obj)
return move(obj) if move is not None else obj
if isinstance(obj, (*_PLAIN_TYPES, torch.device, torch.dtype)):
return obj
if depth >= 4 or isinstance(obj, torch.nn.Module):
raise _Unmirrorable(type(obj).__name__)
if isinstance(obj, list):
out = [_walk_cache(v, move, found, depth + 1) for v in obj]
return out if move is not None else obj
if isinstance(obj, tuple) and not hasattr(obj, "_fields"):
out_t = tuple(_walk_cache(v, move, found, depth + 1) for v in obj)
return out_t if move is not None else obj
if isinstance(obj, dict) and type(obj) is dict:
out_d = {k: _walk_cache(v, move, found, depth + 1) for k, v in obj.items()}
return out_d if move is not None else obj
attrs = getattr(obj, "__dict__", None)
if type(attrs) is not dict or hasattr(type(obj), "__slots__"):
raise _Unmirrorable(type(obj).__name__)
copied = {k: _walk_cache(v, move, found, depth + 1) for k, v in attrs.items()}
if move is None:
return obj
cls = type(obj)
clone = cls.__new__(cls) # not object.__new__: a C base (SimpleNamespace) refuses it
clone.__dict__.update(copied)
return clone
def _cache_tensors(cache: Any) -> Optional[list]:
"""Tensors held by a mirrorable cache, in fixed order; None if not mirrorable."""
torch = _torch()
if cache is None or isinstance(cache, (torch.Tensor, list, tuple, dict, *_PLAIN_TYPES)):
return None
found: list = []
try:
_walk_cache(cache, None, found)
except _Unmirrorable:
return None
return found
def _prefix_kv_mode(kwargs: Any) -> Optional[str]:
if not isinstance(kwargs, dict) or kwargs.get(_KV_KWARG) is None:
return None
mode = kwargs.get(_KV_MODE_KWARG)
if mode not in ("extract", "cached"):
return None
return mode if _cache_tensors(kwargs[_KV_KWARG]) is not None else None
def _is_prefill(kwargs: Any) -> bool:
"""A prefill writes (or returns, FLUX.2) the cache the pipeline reads back: always on the primary."""
return isinstance(kwargs, dict) and kwargs.get(_KV_MODE_KWARG) == "extract"
def _without_prefix_kv(args: tuple, kwargs: dict) -> tuple:
if _prefix_kv_mode(kwargs) != "cached":
return args, kwargs
return args, {k: v for k, v in kwargs.items() if k != _KV_KWARG}
def _iter_tensors(obj: Any, depth: int = 0) -> Iterator[Any]:
torch = _torch()
if isinstance(obj, torch.Tensor):
yield obj
elif depth < 4 and isinstance(obj, (list, tuple)):
for v in obj:
yield from _iter_tensors(v, depth + 1)
elif depth < 4 and isinstance(obj, dict):
for v in obj.values():
yield from _iter_tensors(v, depth + 1)
def _to_primary(out: Any, primary: Any) -> Any:
"""Copy a replica result home with captures held off: the job's own hold dropped when the worker finished."""
from .diffusion_cuda_graph import outside_capture
with outside_capture():
return move_to_device(out, primary)
class _ReplicaWorker:
"""One daemon thread bound to the replica's card: set_device, grad / inference / autocast mode and the caller's
context variables (inductor overrides live in one on torch >= 2.12) are applied per job."""
def __init__(self, device: Any):
self.device = device
self._jobs: "queue.Queue[Optional[tuple]]" = queue.Queue()
self._thread = threading.Thread(
target = self._loop, name = f"unsloth-cfg-replica-{device}", daemon = True
)
# Global capture mode forbids this thread's CUDA allocations while any recording runs, so hold them out.
from .diffusion_cuda_graph import capture_hold, capture_release
capture_hold()
try:
self._thread.start()
except BaseException:
capture_release()
raise
def _loop(self) -> None:
from .diffusion_cuda_graph import capture_release, held_work
torch = _torch()
try:
with held_work():
if getattr(self.device, "type", None) == "cuda":
try:
torch.cuda.set_device(self.device)
except Exception: # noqa: BLE001
pass
except Exception: # noqa: BLE001 - refused while a capture records: each job binds its device anyway
pass
finally:
capture_release()
while True:
job = self._jobs.get()
if job is None:
return
fn, ctx, modes, done = job
job = None
try:
try:
with held_work():
try:
done["result"] = ctx.run(self._run, fn, modes)
except BaseException as exc: # noqa: BLE001 - surfaced on the caller's thread
done["error"] = exc
fn = ctx = None
except Exception as exc: # noqa: BLE001 - refused while a capture records: the job never ran
done["error"] = exc
fn = ctx = None
finally:
capture_release()
done["event"].set()
def _run(self, fn: Callable[[], Any], modes: dict) -> Any:
torch = _torch()
with contextlib.ExitStack() as stack:
if modes.get("inference"):
stack.enter_context(torch.inference_mode())
stack.enter_context(torch.set_grad_enabled(modes.get("grad", False)))
autocast_dtype = modes.get("autocast")
if autocast_dtype is not None:
stack.enter_context(torch.autocast(device_type = "cuda", dtype = autocast_dtype))
stack.enter_context(_device_context(self.device))
return fn()
def submit(self, fn: Callable[[], Any]) -> dict:
torch = _torch()
autocast = None
try:
if torch.is_autocast_enabled("cuda"):
autocast = torch.get_autocast_dtype("cuda")
except Exception: # noqa: BLE001 - older torch
pass
modes = {
"inference": torch.is_inference_mode_enabled(),
"grad": torch.is_grad_enabled(),
"autocast": autocast,
}
done: dict[str, Any] = {"event": threading.Event()}
from .diffusion_cuda_graph import capture_hold
capture_hold()
self._jobs.put((fn, contextvars.copy_context(), modes, done))
return done
@staticmethod
def wait(done: dict) -> Any:
done["event"].wait()
if "error" in done:
raise done["error"]
return done.get("result")
def call(self, fn: Callable[[], Any]) -> Any:
return self.wait(self.submit(fn))
def close(self) -> None:
self._jobs.put(None)
def _leaves(
obj: Any,
path: tuple = (),
depth: int = 0,
) -> list[tuple[tuple, Any]]:
if depth < 4 and isinstance(obj, (tuple, list)) and not hasattr(obj, "_fields"):
out: list = []
for i, v in enumerate(obj):
out.extend(_leaves(v, path + (i,), depth + 1))
return out or [(path, obj)]
if depth < 4 and isinstance(obj, dict):
out = []
for k, v in obj.items():
out.extend(_leaves(v, path + (k,), depth + 1))
return out or [(path, obj)]
return [(path, obj)]
def _rebuild(
template: Any,
values: dict,
path: tuple = (),
depth: int = 0,
) -> Any:
if (
depth < 4
and isinstance(template, (tuple, list))
and not hasattr(template, "_fields")
and len(template)
):
items = [_rebuild(v, values, path + (i,), depth + 1) for i, v in enumerate(template)]
return type(template)(items) if isinstance(template, tuple) else items
if depth < 4 and isinstance(template, dict) and template:
return type(template)(
(k, _rebuild(v, values, path + (k,), depth + 1)) for k, v in template.items()
)
return values[path]
_SMALL_TENSOR_NUMEL = 4096
def _as_bytes(t: Any) -> Any:
"""Flat uint8 view; a one-element ``t.expand(n)`` keeps stride 0 through ``contiguous()``, so copy it."""
torch = _torch()
flat = t.reshape(-1).contiguous()
if flat.numel() and flat.stride(0) != 1:
flat = flat.clone(memory_format = torch.contiguous_format)
return flat.view(torch.uint8)
def _bitwise_equal(a: Any, b: Any) -> bool:
torch = _torch()
if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):
return False
if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:
return False
if a is b:
return True
try:
return bool(torch.equal(_as_bytes(a), _as_bytes(b)))
except Exception: # noqa: BLE001 - a layout that cannot be viewed as bytes is never assumed equal
return False
def _bitwise_equal_all(pairs: Any) -> bool:
"""``all(_bitwise_equal(a, b) for a, b in pairs)`` with one host read per device instead of one per pair: each
pair's comparison stays on its device and only the combined flag is read back. Same verdict as the per-pair form
(a byte view compared elementwise is what ``torch.equal`` does on it); metadata mismatches still answer on the host
without touching the device."""
torch = _torch()
by_device: dict = {}
for a, b in pairs:
if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):
return False
if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:
return False
if a is b or a.numel() == 0:
continue
try:
if a.device.type == "cpu":
if not torch.equal(_as_bytes(a), _as_bytes(b)):
return False
continue
by_device.setdefault(a.device, []).append((_as_bytes(a) == _as_bytes(b)).all())
except Exception: # noqa: BLE001 - a layout that cannot be viewed as bytes is never assumed equal
return False
try:
return all(bool(torch.stack(flags).all().item()) for flags in by_device.values())
except Exception: # noqa: BLE001 - an unreadable verdict is never assumed equal
return False
def _unchanged_since_launch(snapshots: list) -> bool:
"""True when every (source, launch-time copy) pair still holds the same bytes. One host sync for all pairs, on
the replica's card; it waits for the primary branch (the copy is ordered after it), which the guidance combine
waits for anyway, while the replica keeps running."""
torch = _torch()
checks = []
for src, snap in snapshots:
if src.shape != snap.shape or src.dtype != snap.dtype:
return False
if src.numel() == 0:
continue
try:
now = _as_bytes(src.to(snap.device))
checks.append(
torch.equal(now, _as_bytes(snap))
if snap.device.type == "cpu"
else (now == _as_bytes(snap)).all()
)
except Exception: # noqa: BLE001 - an unverifiable tensor never accepts a prediction
return False
if not checks:
return True
host = [c for c in checks if isinstance(c, bool)]
device = [c for c in checks if not isinstance(c, bool)]
return all(host) and (not device or bool(torch.stack(device).all().item()))
def _same_value(a: Any, b: Any) -> bool:
if a is b:
return True
torch = _torch()
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor):
return False
try:
return bool(a == b) and type(a) is type(b)
except Exception: # noqa: BLE001 - incomparable objects must be the same object
return False
# Per-call state Studio's root hooks write onto denoiser submodules (``_unsloth_*`` scalars, e.g. the HunyuanVideo-1.5
# trim hook's null-mask flag). The root hooks run only on the source, around the router, so the replica's copy of that
# state is mirrored from the source before every replica call and the prediction is verified against it.
_STATE_PREFIX = "_unsloth_"
_STATE_TYPES = (bool, int, float, str, type(None))
_STATE_KEEP = frozenset({"_unsloth_replica_compiled"})
def _hook_state(pairs: list) -> tuple:
out = []
for i, (src, _dst) in enumerate(pairs):
for key, value in src.__dict__.items():
if key.startswith(_STATE_PREFIX) and isinstance(value, _STATE_TYPES):
out.append((i, key, value))
return tuple(out)
def _mirror_hook_state(pairs: list) -> tuple:
state = _hook_state(pairs)
wanted: dict = {}
for i, key, value in state:
wanted.setdefault(i, {})[key] = value
for i, (_src, dst) in enumerate(pairs):
mine = wanted.get(i, {})
for key in [
k
for k, v in dst.__dict__.items()
if k.startswith(_STATE_PREFIX)
and isinstance(v, _STATE_TYPES)
and k not in mine
and k not in _STATE_KEEP
]:
del dst.__dict__[key]
for key, value in mine.items():
if dst.__dict__.get(key, _DROP) is not value and dst.__dict__.get(key, _DROP) != value:
dst.__dict__[key] = value
return state
def _autocast_state(torch: Any) -> tuple:
try:
return (
torch.is_autocast_cache_enabled(),
*(
(dev, torch.get_autocast_dtype(dev))
if torch.is_autocast_enabled(dev)
else (dev, None)
for dev in ("cuda", "cpu")
),
)
except Exception: # noqa: BLE001 - torch without per-device autocast queries
return ()
def _call_signature(args: tuple, kwargs: dict) -> tuple:
"""What a compiled forward's guards key on, numbers by value: 0 / 1 and ``dynamic=False`` compiles stay specialised."""
torch = _torch()
sig: list = []
for path, leaf in _leaves((args, kwargs)):
if isinstance(leaf, torch.Tensor):
try:
stride = tuple(leaf.stride())
offset = leaf.storage_offset()
except Exception: # noqa: BLE001 - layouts without strides
stride = offset = None
sig.append(
(
path,
tuple(leaf.shape),
stride,
offset,
leaf.dtype,
leaf.requires_grad,
leaf.is_inference(),
)
)
elif leaf is None or isinstance(leaf, (bool, str)):
sig.append((path, leaf))
elif isinstance(leaf, (int, float)):
sig.append((path, type(leaf), leaf))
elif path and path[-1] == _KV_KWARG and (held := _cache_tensors(leaf)) is not None:
# a new prefix length (another prompt) may compile: it must not count as warm
sig.append((path, type(leaf), tuple((tuple(t.shape), t.dtype) for t in held)))
else:
sig.append((path, type(leaf)))
return (
tuple(sig),
torch.is_inference_mode_enabled(),
torch.is_grad_enabled(),
_autocast_state(torch),
)
def _shared_states(pairs: list) -> tuple:
found: dict = {}
for pair in pairs:
for sub in pair:
for value in _shared_state_values(sub):
found.setdefault(id(value), value)
return tuple(found.values())
def _shared_state_key(states: tuple) -> tuple:
key = []
for state in states:
try:
key.append(state.compile_key())
except Exception: # noqa: BLE001 - an unreadable state is never warm
key.append(object())
return tuple(key)
def _note_warm(seen: set, sig: tuple) -> None:
if len(seen) >= 256:
seen.clear()
seen.add(sig)
def _background_compile_call() -> bool:
try:
from .diffusion_bg_compile import capture_suppressed, eager_forced
except Exception: # noqa: BLE001
return False
return bool(eager_forced() or capture_suppressed())
class CfgParallelRouter:
"""Installed as the denoiser's ``forward``: routes odd branches of a step to the replica.
Once a step has shown two branches, the next step's first odd branch is predicted from the previous one (same
structure; the latent, and every tensor the previous odd call shared with its primary call, replaced by this
step's; the rest -- negative embeds, masks -- kept) and launched on the replica BEFORE the primary branch runs, so
the two cards overlap even when the primary forward blocks the host. The real call must then pass exactly the
predicted tensors (identity) and equal plain values, else the prediction is discarded and the branch runs
unpredicted.
A whole-step CUDA graph installs under the router (``inner``); split callers (Z-Image) use :meth:`run_pair`."""
_unsloth_cfg_router_layer = True
def __init__(
self,
module: Any,
*,
primary: Any,
secondary: Any,
replica_factory: Optional[Callable[[], Any]] = None,
logger: Any = None,
):
torch = _torch()
self.module_ref = weakref.ref(module)
self.orig_forward = module.forward
# FLUX.3's modular denoise step passes only kwargs named in forward's signature: expose the wrapped one.
try:
import inspect
self.__signature__ = inspect.signature(self.orig_forward)
except (TypeError, ValueError):
pass
self.inner: Any = None
self._pairing = False
self.had_override = "forward" in module.__dict__
self.primary = torch.device(primary)
self.secondary = torch.device(secondary)
self.logger = logger
self.replica: Any = None
self.replica_factory = replica_factory or (lambda: build_replica(module, self.secondary))
self.worker: Optional[_ReplicaWorker] = None
self.disabled_reason: Optional[str] = None
self.prev_latent: Any = None
self.branch = 0
self.cfg_seen = False
self.singles = 0
self.prefetched: dict[int, tuple[Any, Any]] = {}
# Tensors the odd branches of the previous step read (negative embeds, masks): copied again before the next
# step's primary branch is queued, so they are always fresh and never wait behind it.
self.carry: list[Any] = []
self.next_carry: list[Any] = []
self.stats = {
"replica_calls": 0,
"primary_calls": 0,
"steps_parallel": 0,
"replica_build_s": None,
"replica_rebuilds": 0,
"predicted_hits": 0,
"predicted_misses": 0,
"opaque_primary_calls": 0,
"pairs_parallel": 0,
"exclusion_skips": 0,
"kv_cache_copies": 0,
"kv_cache_declined": 0,
"kv_prefill_primary_calls": 0,
"replica_failures": 0,
"batch_splits": 0,
"batch_split_skip": None,
}
# the last render ran one call per step (no guidance branches): a batch can split from its first step
self.last_render_single = False
# source prefix-KV cache id -> (source cache, the tensors it held when copied, the replica's copy)
self.kv_mirrors: dict[int, tuple[Any, list, Any]] = {}
self.kv_declined: dict[int, tuple[Any, list]] = {}
self._opaque_logged = False
# Set when replica work was refused by a long recording: until that capture ends, later replica attempts go
# straight to the source instead of each waiting OUTSIDE_WAIT_S again.
self.exclusion_refused = False
self.template: Optional[tuple[Any, Any, dict]] = None
self.pending: Optional[dict] = None
self.b0_leaves: dict = {}
self.fingerprint: Optional[int] = None
self.probe: Optional[bytes] = None
self.probe_index: dict = {}
self.probed_this_render = False
self.pairs: list = []
# Shared per-step states' compile guards join every warm-call signature.
self.shared_states: tuple = ()
# A call new to either side may compile (FX tracing is process-wide up to torch 2.13): it never overlaps the other side.
self.warm_primary: set = set()
self.warm_replica: set = set()
self._pair_job: Optional[dict] = None
# why the denoiser's layers scale activations per call (a split batch would change them), read once
self._split_scaled: Optional[tuple] = None
def describe(self) -> dict[str, Any]:
return {
"secondary": str(self.secondary),
"armed": self.disabled_reason is None,
"replica_built": self.replica is not None,
"replica_compiled_submodules": getattr(self.replica, "_unsloth_replica_compiled", None),
"disabled_reason": self.disabled_reason,
**self.stats,
}
def disable(self, reason: str) -> None:
if self.disabled_reason is None:
self.disabled_reason = reason
if (
reason.startswith("replica ")
or "no longer idle" in reason
or "carries hooks" in reason
):
self.stats["replica_failures"] += 1
# the load's CFG batching lever takes over after this render (end_cfg_render), not unbatched branches
self.fallback_pending = getattr(self, "fallback", None) is not None
if self.logger is not None:
self.logger.warning(
"diffusion.multigpu: CFG-parallel disabled (%s); one card from here", reason
)
self.drop_replica()
def drop_replica(self) -> None:
if self.worker is not None:
self.worker.close()
self.worker = None
self._drop_pending()
self.replica = None
self.kv_mirrors.clear()
self.kv_declined.clear()
self.pairs = []
self.shared_states = ()
self.warm_primary.clear()
self.warm_replica.clear()
self.prefetched.clear()
self.carry = []
self.next_carry = []
self.prev_latent = None
self.template = None
self.b0_leaves = {}
def end_render(self) -> None:
self._drop_pending()
self.prefetched.clear()
self.kv_mirrors.clear()
self.kv_declined.clear()
self.carry = []
self.next_carry = []
self.prev_latent = None
self.template = None
self.b0_leaves = {}
self.last_render_single = self.singles >= 1 and not self.cfg_seen
self.branch = 0
# the next render shows its own guidance shape: no speculative branch before its first odd call
self.cfg_seen = False
self.singles = 0
self.probed_this_render = False
def _note_refusal(self, exc: BaseException) -> None:
from .diffusion_cuda_graph import OutsideWorkRefused
if isinstance(exc, OutsideWorkRefused):
self.exclusion_refused = True
def _refused_this_capture(self) -> bool:
"""Whether replica work was refused and the capture that refused it is still recording."""
if not self.exclusion_refused:
return False
from .diffusion_cuda_graph import capture_in_progress
if capture_in_progress():
self.stats["exclusion_skips"] += 1
return True
self.exclusion_refused = False
return False
def _signature(self, args: tuple, kwargs: dict) -> tuple:
"""``_call_signature`` plus guarded model state: an NVFP4 protect flip recompiles under seen args."""
return (*_call_signature(args, kwargs), _shared_state_key(self.shared_states))
def _primary(self, *args: Any, **kwargs: Any) -> Any:
inner = self.inner
return (inner if inner is not None else self.orig_forward)(*args, **kwargs)
def _primary_noting(self, sig: Optional[tuple], args: tuple, kwargs: dict) -> Any:
sig = self._signature(args, kwargs) if sig is None else sig
job = self._pair_job
if sig not in self.warm_primary and job is not None:
job["event"].wait() # may compile: not while the replica runs
out = self._primary(*args, **kwargs)
_note_warm(self.warm_primary, sig)
return out
def __call__(self, *args: Any, **kwargs: Any) -> Any:
if self._pairing:
self.stats["primary_calls"] += 1
return self._primary_noting(None, args, kwargs)
if self.disabled_reason is not None or _background_compile_call():
# Disabled: no branch state to keep. A render that must run eager while its compile is in flight, or the
# background compile thread itself replaying recorded inputs: both branches stay on the source, which
# keeps that render bit-identical to an eager one and leaves the router's per-render state untouched.
self.stats["primary_calls"] += 1
return self._primary(*args, **kwargs)
latent = _latent_arg(args, kwargs)
sig = None
if latent is not None and self.prev_latent is not None and latent is self.prev_latent:
self.branch += 1
else:
if self.branch == 0 and self.prev_latent is not None:
self.singles += 1
if self.singles >= 3:
self.cfg_seen = False
self.branch = 0
self.prev_latent = latent
self.prefetched.clear()
self.carry, self.next_carry = self.next_carry, []
if self.replica is not None and self.disabled_reason is None:
self._drop_if_stale()
self._drop_pending()
self.b0_leaves = dict(_leaves((args, kwargs)))
if self.cfg_seen and self.replica is not None and self.disabled_reason is None:
# Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after
# it would wait for it on the source stream and serialise the two branches.
self._prefetch(args, kwargs)
sig = self._signature(args, kwargs)
self._launch_prediction(latent, sig)
if (
self.branch == 0
and not self.cfg_seen
and (self.singles >= 1 or self.last_render_single)
and self.disabled_reason is None
):
out = self._batch_split(latent, args, kwargs)
if out is not None:
return out
prefill = self.branch % 2 == 1 and _is_prefill(kwargs)
if (
self.branch % 2 == 1
and self.disabled_reason is None
and not prefill
and _holds_opaque_tensors(_without_prefix_kv(args, kwargs))
):
self._drop_pending()
self.template = None
self.stats["opaque_primary_calls"] += 1
if not self._opaque_logged and self.logger is not None:
self._opaque_logged = True
self.logger.info(
"diffusion.multigpu: a guidance branch carries tensors inside an object the replica cannot copy;"
" that branch stays on %s",
self.primary,
)
elif self.branch % 2 == 1 and self.disabled_reason is None:
self.singles = 0
if self.replica is None:
self.cfg_seen = True
self._build_replica()
if self.branch == 1 and not prefill:
self._record_template(args, kwargs, latent)
elif prefill:
self._drop_pending()
self.template = None
elif self.cfg_seen and self._refused_this_capture():
# this step's prefetch or prediction was refused: the branch runs on the source, no further wait
self._drop_pending()
elif self.cfg_seen:
if self.branch == 1:
self._record_template(args, kwargs, latent)
try:
out = self._take_prediction(args, kwargs) if self.branch == 1 else None
if out is None:
out = self._run_replica(args, kwargs)
self.stats["replica_calls"] += 1
if self.branch == 1:
self.stats["steps_parallel"] += 1
return out
except _KVMirrorDeclined:
self._drop_pending()
except Exception as exc: # noqa: BLE001 - this branch is recomputed on the primary below
self._note_refusal(exc)
self.disable(f"replica call failed: {type(exc).__name__}: {exc}")
self.cfg_seen = True
if prefill:
self.stats["kv_prefill_primary_calls"] += 1
self.stats["primary_calls"] += 1
return self._primary_noting(sig, args, kwargs)
def _batch_split(self, latent: Any, args: tuple, kwargs: dict) -> Any:
"""A step without guidance branches (a distilled model, or both branches batched into one call) whose batch is
even: the first half runs on the source, the second on the replica at the same time. Only for denoiser classes
whose every batched input is known (``_DIT_BATCH_SPLIT_KWARGS`` / ``_ALL``); anything else, an odd
batch, or the kill switch keeps the call whole. None = not split (the caller runs it whole)."""
torch = _torch()
module = self.module_ref()
allowed, joins_all = _batch_split_kwargs(module)
if allowed is None or _batch_split_disabled() or torch.compiler.is_compiling():
return None
if getattr(module, "_compiled_call_impl", None) is not None:
return self._skip_split("denoiser is compiled whole")
n = latent.shape[0] if isinstance(latent, torch.Tensor) and latent.dim() >= 1 else 0
if n < 2 or n % 2:
return None
if args:
return self._skip_split("positional inputs")
extra = sorted(
k for k, v in kwargs.items() if k not in allowed and v is not None and v is not False
)
if extra:
return self._skip_split(f"unknown inputs {extra}")
if kwargs.get("joint_attention_kwargs") or (joins_all and kwargs.get("attention_kwargs")):
return self._skip_split("attention kwargs")
if self._split_scaled is None:
from .diffusion_cfg_batch import per_call_scaled
self._split_scaled = (per_call_scaled(module),)
if self._split_scaled[0] is not None:
# each half would take its own activation amax (dynamic NVFP4): not the one-card computation
return self._skip_split(f"per-call activation scaling ({self._split_scaled[0]})")
try:
(a0, k0), (a1, k1) = [
(_split_batch(args, n, h), _split_batch(kwargs, n, h)) for h in (0, 1)
]
except TypeError as exc:
return self._skip_split(f"unsplittable input ({exc})")
# inside the denoiser's own call: its root hooks already wrap the whole batch, so run_pair need not refuse them
res = self.run_pair(lambda: self._primary_noting(None, a0, k0), a1, k1, inside_call = True)
if res is None:
self._skip_split(f"pair not run ({self.disabled_reason or 'replica unavailable'})")
# run_pair forgets the step: keep this one's latent so the next call is read as a new step
self.prev_latent = latent
self.branch = 0
if res is None:
return None
first, second = res
if second is None:
second = self._primary(*a1, **k1)
if joins_all:
try:
joined = _join_outputs(first, second)
except TypeError as exc:
return self._skip_split(f"unjoinable output ({exc})")
else:
joined = None
self.stats["primary_calls"] += 1
self.stats["batch_splits"] += 1
return joined if joins_all else _join_batch(first, second)
def _skip_split(self, reason: str) -> None:
self.stats["batch_split_skip"] = reason
return None
def run_pair(
self,
primary_call: Callable[[], Any],
args: tuple,
kwargs: dict,
*,
inside_call: bool = False,
) -> Optional[tuple]:
"""Second half on the replica, ``primary_call`` on the source. ``(first, second)``; None = run both on source;
``second`` None = replica failed. ``inside_call``: called from within the denoiser's own ``__call__`` (batch
split), whose root hooks already ran around the whole call."""
if self.disabled_reason is not None or _background_compile_call():
return None
module = self.module_ref()
if module is None or _holds_opaque_tensors((args, kwargs)):
return None
if not inside_call and any(getattr(module, attr, None) for attr in _ROOT_HOOK_DICTS):
return None
self._drop_pending()
self.template = None
self.prev_latent = None
self.branch = 0
if self.replica is not None:
self._drop_if_stale()
if self.replica is None and self.disabled_reason is None:
self._build_replica()
if self.replica is None or self.worker is None or self.disabled_reason is not None:
return None
if self._refused_this_capture():
return None
self.prefetched.clear()
hook_state = _mirror_hook_state(self.pairs)
sig = self._signature(args, kwargs)
try:
r_args = self._to_replica(args)
r_kwargs = self._to_replica(kwargs)
except _KVMirrorDeclined:
return None # the prefix cache does not fit on the replica card: both halves run on the source
except Exception as exc: # noqa: BLE001 - e.g. no room left on the replica card: both halves run on the source
self.disable(f"replica input copy failed: {type(exc).__name__}: {exc}")
return None
finally:
self.next_carry = []
self.prefetched.clear()
job = self.worker.submit(_replica_call(self.replica, r_args, r_kwargs))
if sig not in self.warm_replica:
job["event"].wait() # may compile: the primary half waits
self._pairing = True
self._pair_job = job
try:
first = primary_call()
except BaseException:
with contextlib.suppress(BaseException):
_ReplicaWorker.wait(job)
raise
finally:
self._pairing = False
self._pair_job = None
try:
second = _ReplicaWorker.wait(job)
if _hook_state(self.pairs) != hook_state:
# source hook state moved during the first half: rerun
_mirror_hook_state(self.pairs)
second = self.worker.call(_replica_call(self.replica, r_args, r_kwargs))
second = _to_primary(second, self.primary)
except Exception as exc: # noqa: BLE001 - the caller recomputes the second half on the source
self.disable(f"replica call failed: {type(exc).__name__}: {exc}")
return first, None
_note_warm(self.warm_replica, sig)
self.stats["replica_calls"] += 1
self.stats["steps_parallel"] += 1
self.stats["pairs_parallel"] += 1
return first, second
def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:
torch = _torch()
b0_by_id = {id(v): p for p, v in self.b0_leaves.items() if isinstance(v, torch.Tensor)}
kinds: dict = {}
for path, leaf in _leaves((args, kwargs)):
if isinstance(leaf, torch.Tensor):
if leaf is latent:
kinds[path] = ("latent",)
elif id(leaf) in b0_by_id and self.b0_leaves.get(b0_by_id[id(leaf)]) is leaf:
kinds[path] = ("b0", b0_by_id[id(leaf)])
elif (
isinstance(self.b0_leaves.get(path), torch.Tensor)
and leaf.numel() <= _SMALL_TENSOR_NUMEL
and _bitwise_equal(leaf, self.b0_leaves[path])
):
kinds[path] = ("b0eq", path)
else:
kinds[path] = ("const", leaf)
else:
kinds[path] = ("value", leaf)
self.template = ((args, kwargs), kinds)
def _predict(self, latent: Any) -> Optional[tuple]:
if self.template is None or latent is None:
return None
structure, kinds = self.template
values: dict = {}
for path, kind in kinds.items():
if kind[0] == "latent":
values[path] = latent
elif kind[0] in ("b0", "b0eq"):
if kind[1] not in self.b0_leaves:
return None
values[path] = self.b0_leaves[kind[1]]
else:
values[path] = kind[1]
try:
return _rebuild(structure, values)
except KeyError:
return None
def _launch_prediction(self, latent: Any, sig: tuple) -> None:
if self._refused_this_capture():
return
predicted = self._predict(latent)
if predicted is None or self.worker is None:
return
p_args, p_kwargs = predicted
if _is_prefill(p_kwargs):
return
if (
sig not in self.warm_primary
or self._signature(p_args, p_kwargs) not in self.warm_replica
):
return # a side may compile: the odd branch runs unpredicted, after the primary one
try:
r_args = self._to_replica(p_args)
r_kwargs = self._to_replica(p_kwargs)
except Exception as exc: # noqa: BLE001 - no prediction this step; the branch runs unpredicted
self._note_refusal(exc)
return
torch = _torch()
# Bitwise-compared snapshots catch in-place edits; version counters miss inference tensors.
snapshots = []
for (_path, src), (_rpath, used) in zip(
_leaves((p_args, p_kwargs)), _leaves((r_args, r_kwargs))
):
if isinstance(src, torch.Tensor):
snapshots.append((src, used.clone() if used is src else used))
replica = self.replica
_structure, kinds = self.template
hook_state = _mirror_hook_state(self.pairs)
self.pending = {
"hook_state": hook_state,
"leaves": _leaves((p_args, p_kwargs)),
"by_value": {path for path, kind in kinds.items() if kind[0] == "b0eq"},
"snapshots": snapshots,
"job": self.worker.submit(_replica_call(replica, r_args, r_kwargs)),
}
def _take_prediction(self, args: tuple, kwargs: dict) -> Any:
pending, self.pending = self.pending, None
if pending is None:
return None
actual = _leaves((args, kwargs))
predicted = pending["leaves"]
by_value = pending["by_value"]
# Host-side checks first (paths, plain values), then every by-value tensor pair under one device read.
ok = (
len(actual) == len(predicted)
and all(
pa == pp and (pa in by_value or _same_value(va, vp))
for (pa, va), (pp, vp) in zip(actual, predicted)
)
and _bitwise_equal_all(
(va, vp) for (pa, va), (_pp, vp) in zip(actual, predicted) if pa in by_value
)
)
# the source's root hooks have now run for THIS call: the replica must have seen the same per-call state
ok = ok and _hook_state(self.pairs) == pending["hook_state"]
ok = ok and _unchanged_since_launch(pending["snapshots"])
try:
out = _ReplicaWorker.wait(pending["job"])
except Exception:
if ok:
raise
out = None
if not ok:
self.stats["predicted_misses"] += 1
self.prefetched.clear()
return None
self.stats["predicted_hits"] += 1
return _to_primary(out, self.primary)
def _drop_pending(self) -> None:
pending, self.pending = self.pending, None
if pending is not None:
with contextlib.suppress(Exception):
_ReplicaWorker.wait(pending["job"])
self.stats["predicted_misses"] += 1
def _drop_if_stale(self) -> None:
module = self.module_ref()
if module is None:
return
try:
current = weights_fingerprint(module)
except Exception as exc: # noqa: BLE001 - unverifiable weights: never risk a stale replica
self.disable(f"cannot fingerprint the denoiser: {exc}")
return
if current == self.fingerprint and not self.probed_this_render:
from .diffusion_cuda_graph import OutsideWorkRefused, outside_capture
try:
with outside_capture():
probe = weights_probe(module, self.probe_index)
except OutsideWorkRefused:
probe = self.probe # another pipeline is recording: re-read on the next step
except Exception as exc: # noqa: BLE001 - unverifiable weights: never risk a stale replica
self.disable(f"cannot probe the denoiser weights: {exc}")
return
else:
self.probed_this_render = True
if probe != self.probe:
current = None
if current != self.fingerprint:
if self.logger is not None:
self.logger.info(
"diffusion.multigpu: denoiser weights changed; rebuilding the CFG-parallel replica"
)
self.stats["replica_rebuilds"] += 1
self.drop_replica()
return
# A step cache (transformer_cache=auto) may be installed on the source at generate time, after the replica
# was built: the copy would then run uncached while the source's cache saw only the even branches.
hooked = _hooks_present(module)
if hooked is not None:
self.disable(f"denoiser gained hooks after the replica was built ({hooked})")
def _build_replica(self) -> None:
module = self.module_ref()
if module is None:
return
from .diffusion_cuda_graph import OutsideWorkRefused, outside_capture
t0 = time.perf_counter()
try:
hooked = _hooks_present(module)
if hooked is not None:
raise RuntimeError(f"denoiser carries hooks ({hooked})")
need_mib = sum(
t.numel() * t.element_size() for t in (*module.parameters(), *module.buffers())
) // (1 << 20)
automatic = getattr(self, "automatic_idle_only", False)
# Outside the load scope: read the card directly, before the memory read opens our context (3-7% busy).
util = _smi_util_pct(self.secondary.index) if automatic else None
free, total = self._secondary_free_total_mib()
if automatic:
if not _idle_card(
DeviceProbe(
ordinal = self.secondary.index,
name = "",
total_mib = total,
free_mib = free,
util_pct = util,
)
):
# Automatic borrowed this card because it was idle at load; something else took it since
raise RuntimeError(
f"{self.secondary} is no longer idle ({free} of {total} MiB free, {util}% busy)"
)
reserve = _replica_reserve_mib(free, total)
if free is not None and need_mib + reserve > free:
raise RuntimeError(
f"replica needs {need_mib} MiB, {free} MiB free on {self.secondary}"
)
self.fingerprint = weights_fingerprint(module)
# Copying the weights to the replica card is CUDA work another pipeline's capture prohibits.
with outside_capture(), _device_context(self.secondary):
self.probe = weights_probe(module, self.probe_index)
self.replica = self.replica_factory()
self.probed_this_render = True
self.pairs = list(zip(module.modules(), self.replica.modules()))
self.shared_states = _shared_states(self.pairs)
self.worker = _ReplicaWorker(self.secondary)
weakref.finalize(self, self.worker.close)
self.stats["replica_build_s"] = round(time.perf_counter() - t0, 3)
if self.logger is not None:
self.logger.info(
"diffusion.multigpu: CFG-parallel replica on %s built in %.2fs",
self.secondary,
time.perf_counter() - t0,
)
except Exception as exc: # noqa: BLE001
self.replica = None
# A refused build allocated nothing, and its capture is still recording: no allocator call may run now.
# Any other failure frees what it left behind, itself with captures held off.
if not isinstance(exc, OutsideWorkRefused):
with contextlib.suppress(Exception):
with outside_capture():
_torch().cuda.empty_cache()
self.disable(f"replica unavailable: {exc}")
def _prefetch(self, args: tuple, kwargs: dict) -> None:
from .diffusion_cuda_graph import outside_capture
if self._refused_this_capture():
return
try:
with outside_capture():
for tensor in (*_iter_tensors((args, kwargs)), *self.carry):
if tensor.device == self.secondary or id(tensor) in self.prefetched:
continue
self.prefetched[id(tensor)] = (tensor, tensor.to(self.secondary))
except Exception as exc: # noqa: BLE001 - only an optimisation: no prefetch, the call copies its inputs itself
self.prefetched.clear()
self._note_refusal(exc)
def _to_replica(self, obj: Any) -> Any:
torch = _torch()
def one(tensor: Any) -> Any:
if tensor is not self.prev_latent and not any(tensor is t for t in self.next_carry):
self.next_carry.append(tensor)
if tensor.device == self.secondary:
return tensor
hit = self.prefetched.get(id(tensor))
if hit is not None and hit[0] is tensor:
return hit[1]
copied = tensor.to(self.secondary)
self.prefetched[id(tensor)] = (tensor, copied)
return copied
def walk(value: Any, depth: int = 0) -> Any:
if isinstance(value, torch.Tensor):
return one(value)
if depth == 0 and isinstance(value, dict) and _prefix_kv_mode(value) == "cached":
return type(value)(
(k, self._kv_mirror(v) if k == _KV_KWARG else walk(v, depth + 1))
for k, v in value.items()
)
if depth < 4 and isinstance(value, tuple) and not hasattr(value, "_fields"):
return tuple(walk(v, depth + 1) for v in value)
if depth < 4 and isinstance(value, list):
return [walk(v, depth + 1) for v in value]
if depth < 4 and isinstance(value, dict):
return type(value)((k, walk(v, depth + 1)) for k, v in value.items())
return value
from .diffusion_cuda_graph import outside_capture
with outside_capture():
return walk(obj)
def _kv_mirror(self, cache: Any) -> Any:
"""Replica copy of a filled prefix-KV cache, reused while the source holds the same K/V tensors."""
held = _cache_tensors(cache) or []
entry = self.kv_mirrors.get(id(cache))
if (
entry is not None
and entry[0] is cache
and len(entry[1]) == len(held)
and all(a is b for a, b in zip(entry[1], held))
):
return entry[2]
declined = self.kv_declined.get(id(cache))
if (
declined is not None
and declined[0] is cache
and len(declined[1]) == len(held)
and all(a is b for a, b in zip(declined[1], held))
):
raise _KVMirrorDeclined("prefix cache does not fit on the replica card")
need = sum(t.numel() * t.element_size() for t in held)
free, total = self._secondary_free_total_mib()
if free is not None and need + (_replica_reserve_mib(free, total) << 20) > free * (1 << 20):
self.kv_declined[id(cache)] = (cache, held)
self.stats["kv_cache_declined"] += 1
if self.logger is not None:
self.logger.info(
"diffusion.multigpu: prefix KV cache (%d MiB) does not fit on %s (%d MiB free);"
" its guidance branch stays on %s this render",
need >> 20,
self.secondary,
free,
self.primary,
)
raise _KVMirrorDeclined("prefix cache does not fit on the replica card")
secondary = self.secondary
mirror = _walk_cache(cache, lambda t: t if t.device == secondary else t.to(secondary), [])
while len(self.kv_mirrors) >= 4:
self.kv_mirrors.pop(next(iter(self.kv_mirrors)))
self.kv_mirrors[id(cache)] = (cache, held, mirror)
self.stats["kv_cache_copies"] += 1
return mirror
def _secondary_free_total_mib(self) -> tuple[Optional[int], Optional[int]]:
"""(free, total) MiB on the replica card; free counts the allocator's cached-but-unused blocks, minus CUDA graph pools."""
free, total = _free_total_mib(self.secondary.index)
if free is not None:
with contextlib.suppress(Exception):
from .diffusion_memory import cuda_reusable_cache_bytes
free += cuda_reusable_cache_bytes(self.secondary) // (1 << 20)
return free, total
def _run_replica(self, args: tuple, kwargs: dict) -> Any:
# Unpredicted: nothing verified the copies made before the primary branch ran, and an input edited in place
# since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).
self.prefetched.clear()
_mirror_hook_state(self.pairs)
sig = self._signature(args, kwargs)
r_args = self._to_replica(args)
r_kwargs = self._to_replica(kwargs)
replica = self.replica
out = self.worker.call(_replica_call(replica, r_args, r_kwargs))
_note_warm(self.warm_replica, sig)
return _to_primary(out, self.primary)
class _KVMirrorDeclined(RuntimeError):
"""A prefix-KV cache too large for the replica card's free memory: the branch runs on the primary."""
def _replica_call(replica: Any, args: tuple, kwargs: dict) -> Callable[[], Any]:
return lambda: replica(*args, **kwargs)
# U-Net keyword arguments a batch split may carry; any other tensor-bearing keyword keeps the call whole on the source.
_UNET_SPLIT_KWARGS = frozenset(
{
"encoder_hidden_states",
"timestep_cond",
"added_cond_kwargs",
"cross_attention_kwargs",
"down_block_additional_residuals",
"mid_block_additional_residual",
"return_dict",
}
)
def _unet_cfg_parallel_disabled() -> bool:
return (os.environ.get(UNET_CFG_PARALLEL_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
)
def _split_batch(
value: Any,
n: int,
half: int,
depth: int = 0,
) -> Any:
"""``value`` with every tensor of leading size ``n`` cut to its ``half`` (0 = first, 1 = second) along dim 0;
other tensors and plain values are shared. Raises TypeError on an object it cannot see into."""
torch = _torch()
if isinstance(value, torch.Tensor):
if value.dim() >= 1 and value.shape[0] == n:
b = n // 2
return value[:b] if half == 0 else value[b:]
return value
if value is None or isinstance(value, _PLAIN_TYPES):
return value
if depth > 4:
raise TypeError("nested too deep")
if isinstance(value, dict):
return {k: _split_batch(v, n, half, depth + 1) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return type(value)(_split_batch(v, n, half, depth + 1) for v in value)
raise TypeError(f"cannot split {type(value).__name__}")
def _join_batch(a: Any, b: Any) -> Any:
torch = _torch()
if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
return torch.cat([a, b.to(a.device)], dim = 0)
if isinstance(a, tuple) and isinstance(b, tuple) and len(a) == len(b) and a:
return (_join_batch(a[0], b[0]),) + tuple(a[1:])
sample_a, sample_b = getattr(a, "sample", None), getattr(b, "sample", None)
if isinstance(sample_a, torch.Tensor) and isinstance(sample_b, torch.Tensor):
return type(a)(sample = torch.cat([sample_a, sample_b.to(sample_a.device)], dim = 0))
raise TypeError(f"unexpected U-Net output {type(a).__name__}")
class UnetCfgParallelRouter(CfgParallelRouter):
"""CFG-parallel for a U-Net (SDXL / SD 1.x / 2.x): diffusers sends both guidance branches through ONE call at batch
``2 * B``, so there is no second call to route. Each such call is cut in half along the batch: the first half runs
on the source card, the second on the replica at the same time (:meth:`CfgParallelRouter.run_pair`), and the two
outputs are concatenated. A U-Net treats every sample on its own, so the math is the same; the bits differ only
where the batch-B kernels round differently from batch 2B. Any even batch splits (two images without guidance
too, which then builds the replica). Odd batches, unknown inputs, a compiled-whole module and an unavailable
replica run the whole call on the source. ``UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET=0`` keeps U-Nets on one card."""
def __call__(self, *args: Any, **kwargs: Any) -> Any:
torch = _torch()
module = self.module_ref()
if (
self._pairing
or self.disabled_reason is not None
or _background_compile_call()
or torch.compiler.is_compiling()
# compiled whole after the router was installed (a deferred speed profile): its graph owns the call
or getattr(module, "_compiled_call_impl", None) is not None
):
self.stats["primary_calls"] += 1
return self._primary(*args, **kwargs)
sample = kwargs.get("sample", args[0] if args else None)
n = sample.shape[0] if isinstance(sample, torch.Tensor) and sample.dim() >= 1 else 0
halves = None
if (
n >= 2
and n % 2 == 0
and set(k for k in kwargs if k not in ("sample", "timestep")) <= _UNET_SPLIT_KWARGS
):
try:
halves = [(_split_batch(args, n, h), _split_batch(kwargs, n, h)) for h in (0, 1)]
except TypeError:
halves = None
if halves is None:
self.stats["primary_calls"] += 1
return self._primary(*args, **kwargs)
(a0, k0), (a1, k1) = halves
self.stats["primary_calls"] += 1
# the primary half waits for a cold replica (a first call may compile) and marks its signature warm
res = self.run_pair(lambda: self._primary_noting(None, a0, k0), a1, k1)
if res is None:
return self._primary(*args, **kwargs)
first, second = res
if second is None:
second = self._primary(*a1, **k1)
return _join_batch(first, second)
# DiT classes whose batch may be cut along dim 0 (every per-sample input is a dim-0 tensor; ids, when unbatched, keep
# a leading size that is the token count, never the batch). Z-Image passes per-sample lists and has its own split.
# Lumina 2: a half pads captions to its own longest, and the padding is masked out (fp order only).
_DIT_BATCH_SPLIT_KWARGS = {
"FluxTransformer2DModel": frozenset(
{
"hidden_states",
"encoder_hidden_states",
"pooled_projections",
"timestep",
"img_ids",
"txt_ids",
"guidance",
"joint_attention_kwargs",
"return_dict",
}
),
"Flux2Transformer2DModel": frozenset(
{
"hidden_states",
"encoder_hidden_states",
"timestep",
"img_ids",
"txt_ids",
"guidance",
"joint_attention_kwargs",
"return_dict",
}
),
"Lumina2Transformer2DModel": frozenset(
{
"hidden_states",
"timestep",
"encoder_hidden_states",
"encoder_attention_mask",
"return_dict",
}
),
}
# Every tensor input is per sample on dim 0 or unbatched (ids / masks); LTX-2 returns (video, audio), both joined.
# Left out: HiDream (its llama states are layers x batch, batched on dim 1), MOSS-Image (positional inputs), Anima
# (Studio's wrapper renders one image per call, so no batch reaches the denoiser), FLUX.3 /
# Qwen-Image 2.1 (a per-branch prefix KV cache object), and an attention_kwargs that is not empty.
_DIT_BATCH_SPLIT_KWARGS_ALL = {
"LTX2VideoTransformer3DModel": frozenset(
{
"hidden_states",
"audio_hidden_states",
"encoder_hidden_states",
"audio_encoder_hidden_states",
"timestep",
"audio_timestep",
"sigma",
"audio_sigma",
"encoder_attention_mask",
"audio_encoder_attention_mask",
"num_frames",
"height",
"width",
"fps",
"audio_num_frames",
"video_coords",
"audio_coords",
"isolate_modalities",
"use_cross_timestep",
"attention_kwargs",
"return_dict",
}
),
"ChromaTransformer2DModel": frozenset(
{
"hidden_states",
"encoder_hidden_states",
"timestep",
"img_ids",
"txt_ids",
"attention_mask",
"joint_attention_kwargs",
"return_dict",
}
),
"QwenImageTransformer2DModel": frozenset(
{
"hidden_states",
"encoder_hidden_states",
"encoder_hidden_states_mask",
"timestep",
"img_shapes",
"guidance",
"attention_kwargs",
"return_dict",
}
),
"HunyuanImageTransformer2DModel": frozenset(
{
"hidden_states",
"timestep",
"timestep_r",
"encoder_hidden_states",
"encoder_attention_mask",
"encoder_hidden_states_2",
"encoder_attention_mask_2",
"guidance",
"attention_kwargs",
"return_dict",
}
),
"Krea2Transformer2DModel": frozenset(
{
"hidden_states",
"encoder_hidden_states",
"timestep",
"position_ids",
"encoder_attention_mask",
"attention_kwargs",
"return_dict",
}
),
}
BATCH_SPLIT_ENV = "UNSLOTH_DIFFUSION_BATCH_SPLIT"
def _batch_split_kwargs(module: Any) -> tuple[Optional[frozenset], bool]:
"""(the inputs a batch split of ``module`` may carry, or None; whether every output tensor is joined)."""
name = type(module).__name__ if module is not None else None
allowed = _DIT_BATCH_SPLIT_KWARGS.get(name)
if allowed is not None:
return allowed, False
if name in _DIT_BATCH_SPLIT_KWARGS_ALL and multigpu_all_enabled():
return _DIT_BATCH_SPLIT_KWARGS_ALL[name], True
return None, False
def _join_outputs(
a: Any,
b: Any,
depth: int = 0,
) -> Any:
"""Join two halves' outputs on dim 0: every tensor of a tuple / list / output dict."""
torch = _torch()
if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
return torch.cat([a, b.to(a.device)], dim = 0)
if (
depth < 4
and isinstance(a, (tuple, list))
and isinstance(b, (tuple, list))
and len(a) == len(b)
):
joined = [_join_outputs(x, y, depth + 1) for x, y in zip(a, b)]
return tuple(joined) if isinstance(a, tuple) else joined
if depth < 4 and isinstance(a, dict) and isinstance(b, dict) and a.keys() == b.keys():
return type(a)(**{k: _join_outputs(a[k], b[k], depth + 1) for k in a})
if a is None and b is None:
return None
sample_a, sample_b = getattr(a, "sample", None), getattr(b, "sample", None)
if isinstance(sample_a, torch.Tensor) and isinstance(sample_b, torch.Tensor):
return type(a)(sample = torch.cat([sample_a, sample_b.to(sample_a.device)], dim = 0))
raise TypeError(f"unexpected denoiser output {type(a).__name__}")
def _batch_split_disabled() -> bool:
return (os.environ.get(BATCH_SPLIT_ENV) or "").strip().lower() in ("0", "off", "false", "no")
_CFG_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgParallelRouter]" = weakref.WeakKeyDictionary()
_CFG_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()
def _cfg_denoiser(pipe: Any) -> Any:
"""The module CFG-parallel routes: ``pipe.transformer``, else a U-Net (batch split, :class:`UnetCfgParallelRouter`)
unless ``UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET=0``."""
denoiser = getattr(pipe, "transformer", None)
if denoiser is not None:
return denoiser
unet = getattr(pipe, "unet", None)
if (
unet is not None
and type(unet).__name__ == "UNet2DConditionModel"
and not _unet_cfg_parallel_disabled()
):
if _batch_coupled_processors(unet):
return None # PAG-style processors chunk the batch themselves: a dim-0 split would mix their thirds
return unet
return None
def _batch_coupled_processors(unet: Any) -> bool:
try:
processors = getattr(unet, "attn_processors", None) or {}
return any("PAG" in type(p).__name__ for p in processors.values())
except Exception: # noqa: BLE001 - unreadable: treat as coupled
return True
def cfg_parallel_decline(
pipe: Any, *, offload_policy: str, selection: Optional[MultiGpuSelection]
) -> Optional[str]:
if selection is None:
return "one card selected"
mode = cfg_parallel_mode()
if mode == "off":
return f"{CFG_PARALLEL_ENV}=0"
from .diffusion_layer_split_plan import is_layer_split
# A replica is a whole denoiser on one card, which a denoiser cut across cards cannot be.
if is_layer_split(pipe):
return "denoiser is split across cards (layer split)"
# A replica takes a whole denoiser's worth of another card: automatic placement borrows one only when it is the
# same card model and idle at load (install_cfg_parallel keeps only those), or always under force.
if selection.automatic and mode != "force" and automatic_cfg_parallel_off():
return f"automatic GPU placement (select several cards or set {CFG_PARALLEL_ENV}=force)"
from .diffusion_bg_compile import load_time_enabled as bg_compile_enabled
if bg_compile_enabled():
# its replays call the denoiser, which would enter the router and race a render's branch state
return "background compile replays through the denoiser (UNSLOTH_DIFFUSION_BG_COMPILE)"
denoiser = _cfg_denoiser(pipe)
torch = _torch()
if not isinstance(denoiser, torch.nn.Module):
if (
isinstance(getattr(pipe, "unet", None), torch.nn.Module)
and _unet_cfg_parallel_disabled()
):
return f"U-Net CFG-parallel off ({UNET_CFG_PARALLEL_ENV}=0)"
return "no DiT denoiser (a U-Net batches its guidance branches)"
from .diffusion_layer_split import is_layer_split
if is_layer_split(denoiser):
return "denoiser is split across cards (layer split)"
# Whole-module / sequential offload page the denoiser per call; a group / streaming plan is fine only when it left
# the denoiser itself resident (encoders streamed), which shows as a denoiser without offload hooks.
if offload_policy in ("model", "sequential"):
return f"denoiser is offloaded ({offload_policy})"
hooked = _hooks_present(denoiser)
if hooked is not None:
return f"denoiser is offloaded or hooked ({offload_policy}: {hooked})"
if getattr(pipe, "transformer_2", None) is not None:
return "two-expert denoiser"
if getattr(denoiser, "_compiled_call_impl", None) is not None:
return "denoiser is compiled whole"
device = _execution_device(denoiser)
if device is None or device.type != "cuda":
return "denoiser is not on a CUDA card"
return None
def _resident_graph(module: Any) -> Any:
slot = module.__dict__.get("forward")
try:
from .diffusion_cuda_graph import GraphedCompiledCall, GraphedForward
except Exception: # noqa: BLE001
return None
if (
type(slot) is GraphedForward
and slot.placement is None
and not isinstance(slot, GraphedCompiledCall)
):
return slot
return None
def install_cfg_parallel(
pipe: Any,
*,
offload_policy: str,
logger: Any = None,
) -> Optional[str]:
"""Arm CFG-parallel on ``pipe.transformer`` (or a U-Net) when a secondary card can hold a replica. The replica is
built on the first step that shows two branches, or, for a FLUX.1 / FLUX.2 denoiser, the first step of a render
without branches whose batch is even (batch split): a distilled model rendering one image never pays for it.
Automatic placement lends one card per load (:func:`automatic_partner`). Returns the decline reason or None."""
selection = current_selection()
reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)
if reason is None:
denoiser = _cfg_denoiser(pipe)
primary = _execution_device(denoiser)
probes = probe_devices((selection.primary, *selection.secondaries))
same = [
o
for o in selection.secondaries
if probes.get(o) is not None
and probes.get(primary.index) is not None
and probes[o].name == probes[primary.index].name
]
candidates = same if cfg_parallel_mode() != "force" else list(selection.secondaries)
automatic = selection.automatic and cfg_parallel_mode() != "force"
auto_reason = None
try:
placed = ((_PLACEMENTS.get(pipe) or {}).get("placement") or {}).values()
except TypeError:
placed = ()
occupied = {str(d) for d in placed} - {str(primary)}
if automatic:
# nothing else may live on a card Automatic borrows for a replica (busy: memory in use or computing)
need = sum(
t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers())
) // (1 << 20)
pick, auto_reason = automatic_partner(
probes.get(primary.index) or DeviceProbe(primary.index, "", None, None),
# a card holding the split's components cannot take the replica: never the one Automatic lends
[
probes[o]
for o in selection.secondaries
if probes.get(o) is not None
and probes[o].free_mib is not None
and f"cuda:{o}" not in occupied
],
need,
)
candidates = [o for o in candidates if _idle_card(probes.get(o))]
if pick is not None:
candidates = [pick]
if occupied:
candidates = [o for o in candidates if f"cuda:{o}" not in occupied]
if not candidates and occupied:
reason = f"the only replica card ({', '.join(sorted(occupied))}) holds the split's components"
elif not candidates and automatic and same:
reason = (
f"automatic GPU placement: no idle card for a replica ({auto_reason}; idle = at most "
f"{AUTO_CFG_PARALLEL_IDLE_USED_MIB} MiB in use and {AUTO_CFG_PARALLEL_IDLE_UTIL_PCT:g}% busy; "
f"select several cards or set {CFG_PARALLEL_ENV}=force)"
)
elif not candidates:
reason = "secondary card is a different model (bit-identity needs the same kernels)"
else:
need_mib = sum(
t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers())
) // (1 << 20)
def usable(o: int) -> int:
free_o = probes[o].free_mib
if free_o is None:
return -1
return int(free_o) - _replica_reserve_mib(free_o, probes[o].total_mib)
best = max(candidates, key = usable)
free = probes[best].free_mib
if (
free is not None
and need_mib + _replica_reserve_mib(free, probes[best].total_mib) > free
):
reason = f"replica needs {need_mib} MiB, {free} MiB free on cuda:{best}"
else:
graph = _resident_graph(denoiser)
if graph is not None:
del denoiser.__dict__["forward"]
router_cls = (
UnetCfgParallelRouter
if denoiser is not getattr(pipe, "transformer", None)
else CfgParallelRouter
)
router = router_cls(
denoiser, primary = primary, secondary = f"cuda:{best}", logger = logger
)
router.inner = graph
router.automatic_idle_only = automatic
try:
_CFG_ROUTERS[pipe] = router
except TypeError:
if graph is not None:
denoiser.__dict__["forward"] = graph
return "pipeline cannot be tracked (not weak-referenceable)"
denoiser.forward = router
denoiser._unsloth_cfg_router = router
if logger is not None:
logger.info(
"diffusion.multigpu: CFG-parallel armed (replica on cuda:%s when guidance runs two branches or"
" an even batch splits%s)",
best,
"; U-Net batch split" if router_cls is UnetCfgParallelRouter else "",
)
return None
if selection is not None:
with contextlib.suppress(TypeError):
_CFG_DECLINES[pipe] = reason
if logger is not None:
logger.info("diffusion.multigpu: CFG-parallel not armed (%s)", reason)
return reason
def invalidate_cfg_replica(
pipe: Any,
reason: str,
logger: Any = None,
) -> bool:
"""Drop the CFG-parallel replica of ``pipe`` after its denoiser was mutated in ways the per-step staleness check
does not see (attention backend, eager patches, compile); the next guided step rebuilds it from the source. True
when a replica was dropped."""
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return False
if router is None or router.replica is None:
return False
if logger is not None:
logger.info(
"diffusion.multigpu: CFG-parallel replica dropped (%s); rebuilt on the next guided step",
reason,
)
router.stats["replica_rebuilds"] += 1
router.drop_replica()
return True
_ADAPTER_SCALE_ATTRS = ("scaling", "_active_adapter", "_disable_adapters")
def mirror_cfg_replica_adapter_scales(pipe: Any, logger: Any = None) -> bool:
"""After a LoRA weight-only change (``set_adapters`` on the same adapter set), copy each adapter layer's scales
and active-adapter state onto the CFG-parallel replica instead of dropping it. True when mirrored (the replica now
computes what the source does); False when there was nothing to mirror or the layers do not line up, in which case
the caller drops the replica as before."""
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return False
if router is None or router.replica is None:
return False
module = router.module_ref()
if module is None:
return False
src = [
(n, m) for n, m in module.named_modules() if isinstance(getattr(m, "scaling", None), dict)
]
dst = dict(
(n, m)
for n, m in router.replica.named_modules()
if isinstance(getattr(m, "scaling", None), dict)
)
if not src or len(src) != len(dst):
return False
for name, layer in src:
copy_layer = dst.get(name)
if copy_layer is None or set(copy_layer.scaling) != set(layer.scaling):
return False # a different adapter set: only a rebuild is right
for name, layer in src:
copy_layer = dst[name]
for attr in _ADAPTER_SCALE_ATTRS:
if attr in layer.__dict__:
value = layer.__dict__[attr]
copy_layer.__dict__[attr] = (
dict(value) if isinstance(value, dict) else copy.copy(value)
)
try:
# the fingerprint covers adapter scales: record the new ones as the replica's, else the next step rebuilds it
router.fingerprint = weights_fingerprint(module)
except Exception: # noqa: BLE001 - unverifiable: drop the replica as before
return False
# Python-valued scales are guards of a compiled region: both sides may recompile, so they warm one at a time again
router.warm_primary.clear()
router.warm_replica.clear()
router.stats["adapter_scale_mirrors"] = router.stats.get("adapter_scale_mirrors", 0) + 1
if logger is not None:
logger.info(
"diffusion.multigpu: CFG-parallel replica kept; LoRA scales mirrored onto it (%d layers)",
len(src),
)
return True
def _router_slot(module: Any, router: Any) -> Any:
"""``module`` when ``router`` is its forward, the outer layer (static step skip) whose inner it is, else None."""
slot = module.__dict__.get("forward")
if slot is router:
return module
if (
getattr(slot, "_unsloth_outer_forward", False) is True
and getattr(slot, "inner", None) is router
):
return slot
return None
def drop_displaced_cfg_router(
pipe: Any,
reason: str,
logger: Any = None,
) -> bool:
"""Uninstall ``pipe``'s CFG-parallel router when a later layer (the deferred speed profile's graphs) replaced the
denoiser's forward and bypasses it, so status reports CFG-parallel as declined rather than armed. True if dropped."""
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return False
if router is None:
return False
module = router.module_ref()
holder = _router_slot(module, router) if module is not None else None
if holder is module and module is not None:
return False
if holder is not None:
# The replica cannot carry a step skip (it holds the source module), so the router could never build one.
reason = "a static step skip wraps the denoiser"
uninstall_cfg_parallel(pipe)
with contextlib.suppress(TypeError):
_CFG_DECLINES[pipe] = reason
if logger is not None:
logger.info("diffusion.multigpu: CFG-parallel dropped (%s)", reason)
return True
def end_cfg_render(pipe: Any) -> None:
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return
if router is not None:
router.end_render()
if getattr(router, "fallback_pending", False):
# The replica failed at render time: hand the denoiser to the fallback (CFG batching, #16261) between
# renders, never inside one. The render that failed finished on the source.
fallback, router.fallback, router.fallback_pending = router.fallback, None, False
reason = router.disabled_reason
uninstall_cfg_parallel(pipe)
try:
result = fallback(pipe)
except Exception as exc: # noqa: BLE001 - an optional lever: one card stands
result = f"{type(exc).__name__}: {exc}"
if router.logger is not None:
router.logger.info(
"diffusion.multigpu: CFG-parallel off after a render-time replica failure (%s); CFG batching %s",
reason,
"armed" if result is None else f"not armed ({result})",
)
def set_cfg_parallel_fallback(pipe: Any, fallback: Callable[[Any], Optional[str]]) -> bool:
"""Register the lever that replaces CFG-parallel when its replica fails at render time (CFG batching): called once
with the pipeline, after ``uninstall_cfg_parallel``, between renders; returns its decline reason or None when armed.
It must not hold the pipeline itself (the router is the value of a weak-keyed map on it, so that would keep a
rolled-back load alive). True when a router was armed to carry it."""
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return False
if router is None:
return False
router.fallback = fallback
router.fallback_pending = False
return True
def release_cfg_prefix_kv(pipe: Any) -> None:
"""Drop the replica's prefix-KV copies (and their hold on the primary's tensors) after the last denoise step."""
try:
router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
except TypeError:
return
if router is not None:
router.kv_mirrors.clear()
router.kv_declined.clear()
def uninstall_cfg_parallel(pipe: Any) -> None:
try:
router = _CFG_ROUTERS.pop(pipe, None) if pipe is not None else None
except TypeError:
return
if router is None:
return
if router.disabled_reason is None:
router.disabled_reason = "uninstalled"
router.drop_replica()
module = router.module_ref()
if module is not None:
holder = _router_slot(module, router)
if holder is module:
if router.inner is not None:
module.__dict__["forward"] = router.inner
router.inner = None
elif router.had_override:
module.__dict__["forward"] = router.orig_forward
else:
del module.__dict__["forward"]
elif holder is not None:
# under a static step skip: hand the skip what the router wrapped
holder.inner = (
router.inner
if router.inner is not None
else (router.orig_forward if router.had_override else None)
)
router.inner = None
if module.__dict__.get("_unsloth_cfg_router") is router:
del module.__dict__["_unsloth_cfg_router"]