Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: split one diffusion denoiser across GPUs by block ranges (#14412)

Daniel Hanbd63cc39f62a
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 7d1c35c10a7..8bc9496a973 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -163,6 +163,7 @@ from .diffusion_memory import (
MEMORY_MODE_LOW_VRAM,
ACTIVATION_TILE,
DeviceMemory,
+ ImageActivationShortfallError,
OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_STREAMING,
@@ -282,6 +283,7 @@ from .diffusion_batched import (
from .diffusion_cache import (
FBCACHE_MIN_STEPS,
TC_AUTO,
+ TC_FBCACHE,
TC_STATIC,
apply_step_cache,
auto_static_skip_plan,
@@ -2204,6 +2206,143 @@ def _guard_vae_card(
return restore, tiled
+def _guard_layer_cards(pipe: Any, guard_target: Any, guard_kwargs: dict) -> int:
+ """The generate-time check on every other card a layer split runs blocks on. Such a card holds only blocks: no
+ text encode, no VAE decode, and one block's activations at a time, so the whole-request estimate the primary is
+ checked against does not apply. It needs what the plan reserved there at the default resolution, scaled by how
+ much this request's activations grow past that default. ``pipe`` is the LOADED pipeline (a from_pipe workflow
+ pipeline carries no split record). Raises the guard's refusal (a ValueError); returns the number of cards
+ checked. Fails open on anything unknown."""
+ from .diffusion_layer_split_plan import applied_layer_split, secondary_block_cards
+ from .diffusion_memory import _oversized_generate_override
+
+ split = applied_layer_split(pipe)
+ if split is None or guard_kwargs.get("allow_oversized") or _oversized_generate_override():
+ return 0
+ try:
+ sized = dict(
+ batch_size = guard_kwargs.get("batch_size") or 1, family = guard_kwargs.get("family")
+ )
+ needed = estimate_image_runtime_mib(
+ width = guard_kwargs.get("width"),
+ height = guard_kwargs.get("height"),
+ condition_pixels = int(guard_kwargs.get("condition_pixels") or 0),
+ **sized,
+ )
+ planned = estimate_image_runtime_mib(width = None, height = None, **{**sized, "batch_size": 1})
+ scale = max(1.0, float(needed) / float(planned)) if planned else 1.0
+ except Exception: # noqa: BLE001 - a broken estimate never blocks a generation
+ return 0
+ checked = 0
+ for ordinal in secondary_block_cards(pipe):
+ working, reserve = split.working_mib(ordinal)
+ if working + reserve <= 0:
+ continue
+ with diffusion_device_scope(int(ordinal)):
+ free = reclaimable_snapshot_device_memory(guard_target).free_mib
+ if free is None:
+ continue
+ checked += 1
+ need = int(working * scale) + reserve
+ if need > int(free):
+ raise ImageActivationShortfallError(
+ f"Generating at {guard_kwargs.get('width')}x{guard_kwargs.get('height')} needs about "
+ f"{need / 1024:.2f} GB of working memory on cuda:{int(ordinal)}, which runs part of the model split "
+ f"across GPUs, but only about {int(free) / 1024:.2f} GB is free there. Generate at a smaller "
+ "resolution, or free GPU memory on that GPU. To try anyway, turn on 'Allow oversized generations' "
+ "under Advanced on the Images page."
+ )
+ return checked
+
+
+def _layer_split_expected(
+ pipe: Any,
+ plan: Any,
+ refine: Optional[Callable[..., Any]] = None,
+) -> bool:
+ """Whether placement will cut the denoiser across cards: the multi-GPU refinement asked early, with the sizes
+ loaded so far. The speed layer runs before placement, and on a split every block is resident on some card, so it
+ is set up for that (regional compile with fullgraph, int8 kernels, graphs) and not for the one-card offload plan
+ the split replaces. A split declined later lands its one-card fallback, the same path a transient OOM re-plan
+ takes. ``refine``: the backend's own refinement (video declines some families)."""
+ if plan is None or getattr(plan, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:
+ return False
+ try:
+ preview = (refine or refine_plan_for_multigpu)(pipe, plan, None)
+ except Exception: # noqa: BLE001 - the one-card plan decides
+ return False
+ return getattr(preview, "layer_split", None) is not None
+
+
+def _companion_split_expected(
+ pipe: Any,
+ plan: Any,
+ refine: Optional[Callable[..., Any]] = None,
+) -> bool:
+ """Whether placement will keep the denoiser resident by moving other components to a second card (the multi-GPU
+ component split), previewed like :func:`_layer_split_expected`. The one-card plan offloads there, so only this
+ preview tells the quant compile rule that the denoiser ends up resident."""
+ if plan is None or getattr(plan, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:
+ return False
+ try:
+ preview = (refine or refine_plan_for_multigpu)(pipe, plan, None)
+ except Exception: # noqa: BLE001 - the one-card plan decides
+ return False
+ return (
+ getattr(preview, "secondary_ordinal", None) is not None
+ and getattr(preview, "offload_policy", None) == OFFLOAD_NONE
+ )
+
+
+def _native_quant_needs_compile(target: Any, native_scheme: Optional[str], resident: bool) -> bool:
+ """Whether a torchao-free quantised DiT must be compiled like a torchao one: the NVIDIA int8 scheme (W8A8
+ ``torch._int_mm``, chosen for an offload plan) once the DiT stays resident, on one card or split across cards,
+ where eager runs it ~3.5x slower (LTX-2 split: 0.90-0.97 vs 0.26 s/step). ROCm and Apple keep their own native
+ path and its compile policy; a streamed DiT keeps the offload path's."""
+ return (
+ native_scheme is not None
+ and bool(resident)
+ and native_offload_host(target)
+ and not native_quant_host(target)
+ )
+
+
+def _restore_step_cache_after_layer_split(
+ pipe: Any, views: Any, **cache_kwargs: Any
+) -> Optional[str]:
+ """The split takes the load-time First-Block-Cache off before it moves the blocks; once they sit on their cards the
+ cache goes back on (``diffusion_cache`` copies its residuals between the head and tail cards). Returns the mode
+ that engaged on ``pipe`` (None: it declined, the denoiser runs uncached)."""
+ engaged = None
+ for view in views:
+ mode = apply_step_cache(view, mode = TC_FBCACHE, **cache_kwargs)
+ if view is pipe:
+ engaged = mode
+ return engaged
+
+
+def _step_cache_dropped_by_layer_split(pipe: Any) -> bool:
+ """Whether the layer split placement took the load-time First-Block-Cache off ``pipe.transformer``
+ (``diffusion_layer_split._drop_step_cache``): the marker is gone, so the load's cache locals are stale."""
+ transformer = getattr(pipe, "transformer", None)
+ return transformer is not None and not getattr(transformer, "_unsloth_step_cache", None)
+
+
+def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:
+ """A layer split replaced the one-card plan whose partial residency the measured reserve priced. Left in place, a
+ conditioned request past it would give the split denoiser on-demand offload hooks onto the primary
+ (``hook_resident_denoiser``), pulling every block off its card. True when dropped."""
+ if getattr(plan, "layer_split", None) is None or not getattr(
+ pipe, "_unsloth_measured_reserve", None
+ ):
+ return False
+ try:
+ pipe._unsloth_measured_reserve = None
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
def _tile_for_verdict(
pipe: Any,
verdict: Any,
@@ -8963,12 +9102,21 @@ class DiffusionBackend:
# Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays
# bit-identical `off`.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")
- # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF).
- if (
- (transformer_quant_engaged is not None or comfy_compile)
- and native_scheme is None
- and effective_speed == SPEED_OFF
- ):
+ split_expected = _layer_split_expected(pipe, plan)
+ # A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower, losing to GGUF.
+ quant_forces_compile = (
+ transformer_quant_engaged is not None or comfy_compile
+ ) and (
+ native_scheme is None
+ or _native_quant_needs_compile(
+ target,
+ native_scheme,
+ split_expected
+ or plan_keeps_transformer_resident(plan)
+ or _companion_split_expected(pipe, plan),
+ )
+ )
+ if quant_forces_compile and effective_speed == SPEED_OFF:
logger.info(
"diffusion.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
@@ -9155,6 +9303,7 @@ class DiffusionBackend:
install_mps_fp16(getattr(pipe, "transformer", None), target, logger = logger)
self._raise_if_load_cancelled(_load_token)
+ speed_offload = plan.offload_policy != OFFLOAD_NONE and not split_expected
# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the
# first compiled forward pays the 25-58s compile once.
if (
@@ -9177,7 +9326,7 @@ class DiffusionBackend:
fam,
is_gguf = gguf_transformer,
speed_mode = effective_speed,
- offload_active = plan.offload_policy != OFFLOAD_NONE,
+ offload_active = speed_offload,
)
or (
gguf_transformer
@@ -9190,9 +9339,7 @@ class DiffusionBackend:
)
)
)
- and not fp16_unet_offloaded(
- target, pipe, offload_active = plan.offload_policy != OFFLOAD_NONE
- )
+ and not fp16_unet_offloaded(target, pipe, offload_active = speed_offload)
):
compile_ctx = compile_cache.begin(
family = fam.name,
@@ -9211,9 +9358,8 @@ class DiffusionBackend:
speed_mode = effective_speed,
fullgraph = not cache_graph_break
and not cache_may_toggle
- and plan.offload_policy == OFFLOAD_NONE,
- offload_below_hooks = plan.offload_policy != OFFLOAD_NONE
- and compile_below_hooks_enabled(),
+ and not speed_offload,
+ offload_below_hooks = speed_offload and compile_below_hooks_enabled(),
),
logger = logger,
reduction_filter = family_filters_reductions(fam),
@@ -9257,7 +9403,7 @@ class DiffusionBackend:
backend = "image",
speed_mode = effective_speed,
is_gguf = gguf_transformer,
- offload_active = plan.offload_policy != OFFLOAD_NONE,
+ offload_active = speed_offload,
audit = load_audit,
logger = logger,
apply = lambda family, log: apply_speed_optims(
@@ -9268,8 +9414,9 @@ class DiffusionBackend:
speed_mode = effective_speed,
cache_active = cache_graph_break or cache_may_toggle,
cache_engaged = cache_graph_break,
- offload_active = plan.offload_policy != OFFLOAD_NONE,
- denoiser_offloaded = not plan_keeps_transformer_resident(plan),
+ offload_active = speed_offload,
+ denoiser_offloaded = not split_expected
+ and not plan_keeps_transformer_resident(plan),
stream_int8_gemm = True,
logger = log,
),
@@ -9474,6 +9621,7 @@ class DiffusionBackend:
)
# a transient OOM during setup may have re-planned to lower residency
plan = applied_plan(pipe, plan)
+ _drop_measured_reserve_for_layer_split(pipe, plan)
identity = getattr(pipe, "_unsloth_peak_identity", None)
if isinstance(identity, tuple) and len(identity) == 3:
# a saver the plan asked for may not have engaged: key the renders by what did
@@ -9482,10 +9630,27 @@ class DiffusionBackend:
pipe._unsloth_peak_identity = (identity[0], live_vae, identity[2])
# placed from the planned saver's peak: requests size from the flat estimate instead
pipe._unsloth_peak_static = True
+ if cache_engaged == TC_FBCACHE and _step_cache_dropped_by_layer_split(pipe):
+ cache_engaged = _restore_step_cache_after_layer_split(
+ pipe,
+ (pipe,),
+ threshold = transformer_cache_threshold,
+ quant_active = cache_quant_active,
+ length_changes_ok = not cache_auto,
+ logger = logger,
+ )
+ if cache_engaged is None and getattr(plan, "layer_split", None) is not None:
+ cache_may_toggle = False
+ cache_reason = (
+ "off: the step cache declined the denoiser split across cards"
+ )
# streams the whole-resident denoiser back to the flat room while the encoders run
install_encode_release(pipe, plan, logger)
# the speed layer saw only the plan; placement may have pinned every denoiser group since
- if denoisers_pinned_resident(pipe):
+ if (
+ denoisers_pinned_resident(pipe)
+ or getattr(plan, "layer_split", None) is not None
+ ):
engage_pinned_denoisers(pipe, speed_applied, logger)
# Placement decides the graph layer: a whole-forward recording holds only while no hook moves the
# denoiser; otherwise (and for a forward that is not capture-safe) its blocks are recorded instead.
@@ -9529,8 +9694,7 @@ class DiffusionBackend:
speed_mode,
"deferred" if speed_deferred else effective_speed,
"quantized transformer requires compile"
- if (transformer_quant_engaged is not None or comfy_compile)
- and native_scheme is None
+ if quant_forces_compile
and normalize_speed_mode(speed_mode) in (None, SPEED_OFF)
else "auto: exact eager for the first two images; "
"the compile profile engages on the 3rd"
@@ -12392,6 +12556,7 @@ class DiffusionBackend:
logger = logger,
)
verdict = raise_on_image_activation_shortfall(**guard_kwargs)
+ _guard_layer_cards(state.pipe, guard_target, guard_kwargs)
restore_vae = _tile_for_verdict(
pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger
)
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index bea93715035..10af8d16478 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -288,6 +288,7 @@ class _Shared:
self.warmed: set = set()
self.pool_bytes = 0
self.static_bytes = 0
+ self.strict_device = False
_LIVE.add(self)
def capture_stream(self) -> Any:
@@ -411,6 +412,9 @@ class BlockGraph:
self.unreplayed_placements = 0
self.churned = False
self.protect: Any = None # None until resolved, then False or the NVFP4 step controller
+ self.segment: Any = None
+ self.seg_index = 0
+ self.owner: Any = None
self.placements: set = set()
self.refusals: dict = {}
self.stats = {
@@ -462,8 +466,20 @@ class BlockGraph:
if _bg_capture_suppressed() or _step_recording():
# a whole-step graph above records this block's kernels (diffusion_cuda_graph), not a nested replay
return self.compute(*args, **kwargs)
- if not self.enabled or self.bypassed or self.poisoned or self.churned or _bg_eager_forced():
+ if (
+ not self.enabled
+ or self.bypassed
+ or self.poisoned
+ or self.churned
+ or _bg_eager_forced()
+ or _step_cache_live(self.shared.root)
+ ):
return self._eager(args, kwargs)
+ segment = self.segment
+ if segment is not None:
+ out = segment.call(self.seg_index, args, kwargs)
+ if out is not _NOT_HANDLED:
+ return out
torch = _torch()
if torch.is_grad_enabled():
self.stats["refused_grad"] += 1
@@ -490,15 +506,31 @@ class BlockGraph:
# A capture never records a host read, so a replay would keep the recorded value.
self.stats["refused_host_input"] += 1
return self._eager(args, kwargs)
+ if self.shared.strict_device and (
+ torch.cuda.current_device() != self.shared.device_index
+ or any(t.device.index != self.shared.device_index for t in live)
+ ):
+ self.stats["refused_host_input"] += 1
+ return self._eager(args, kwargs)
if self.protect is None:
self.protect = self._protect_controller()
if self.protect:
# One recording per precision branch: a W4A4 replay at a W4A16 step would skip the protection.
from .diffusion_cuda_graph import protect_graph_key
placement = (placement, protect_graph_key(self.protect))
+ try:
+ extra = _root_key_extra(self.shared.root)
+ except Exception: # noqa: BLE001 - a branch flag that cannot be read: nothing safe to replay
+ self.stats["refused_object"] += 1
+ return self._eager(args, kwargs)
+ if extra is not None:
+ placement = (placement, extra)
full = (key, placement)
entry = self.cache.get(full)
if entry is None:
+ if self.owner is not None and self.owner.scope_sealed():
+ self.stats["length_eager"] = self.stats.get("length_eager", 0) + 1
+ return self._eager(args, kwargs)
if full not in self.seen:
# First call at this layout and placement: run it for real (the warm-up), record on the next one.
return self._first_sighting(full, args, kwargs)
@@ -606,6 +638,8 @@ class BlockGraph:
self.stats["recaptures"] += 1
self.cache[full] = entry
self.stats["captures"] += 1
+ if self.owner is not None:
+ self.owner.note_capture()
from .diffusion_cuda_graph import capture_device, note_capture
self._unsloth_capture_device = capture_device()
@@ -657,6 +691,424 @@ class BlockGraph:
exc.__traceback__ = None
+SEGMENT_GRAPHS_ENV = "UNSLOTH_DIFFUSION_SEGMENT_GRAPHS"
+_NOT_HANDLED = object()
+
+
+def segment_graphs_disabled() -> bool:
+ return (os.environ.get(SEGMENT_GRAPHS_ENV) or "").strip().lower() in _OFF
+
+
+class _SegRecord:
+ """One eager pass over a segment: each block's call layout, where each of its input tensors came from (an earlier
+ block's output, or one of the first block's inputs), and its output layout."""
+
+ __slots__ = ("keys", "prov", "out_specs")
+
+ def __init__(self) -> None:
+ self.keys: list = []
+ self.prov: list = []
+ self.out_specs: list = []
+
+
+class _SegEntry:
+ __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays")
+
+
+def _root_key_extra(root: Any) -> Any:
+ """The root's per-call branch state (``diffusion_cuda_graph.GRAPH_KEY_EXTRA_ATTR``, e.g. HunyuanVideo-1.5's
+ null-mask flag a pre-hook sets): part of every block and segment key, as of the whole-step graph's, so a recording
+ made on one branch never replays for the other. None without one."""
+ from .diffusion_cuda_graph import GRAPH_KEY_EXTRA_ATTR
+
+ extra = getattr(root, GRAPH_KEY_EXTRA_ATTR, None) if root is not None else None
+ return ("x", extra()) if callable(extra) else None
+
+
+def _step_cache_live(root: Any) -> bool:
+ """Whether a step cache (First-Block-Cache) is engaged on the denoiser: its hooks decide inside the forward, per
+ step and per cond / uncond context, which a recording cannot see. Belt to the caller's per-chunk ``set_bypass``,
+ as ``GraphedForward`` does: an auto cache that turns on at generate time keeps every block eager."""
+ try:
+ return bool(getattr(root, "_unsloth_step_cache", None)) if root is not None else False
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _lazy_chain(root: Any) -> bool:
+ """Whether this forward of a split ``root`` routes lazily: its class is one the split verified to pass each block's
+ outputs straight to the next block, and the call carries no residual input. Only such a chain may replay as one
+ segment, since a segment cannot see an op between blocks that rewrites a tensor in place."""
+ try:
+ from .diffusion_layer_split import layer_split_of
+ split = layer_split_of(root)
+ except Exception: # noqa: BLE001
+ return False
+ return bool(split is not None and split.lazy_now)
+
+
+def _version(t: Any) -> Any:
+ # an inference tensor carries no version counter
+ try:
+ return None if t.is_inference() else t._version
+ except Exception: # noqa: BLE001
+ return None
+
+
+class SegmentGraph:
+ """One CUDA graph over a run of blocks a layer split keeps on one card, back to back in one block list: the first
+ block's call replays the whole run, and each later block's call checks that it got exactly the tensors the run
+ handed on (the previous block's outputs, the first block's other inputs) and returns its recorded outputs. Any
+ other call (an op between blocks rewrote a tensor, an eager boundary copied it) leaves the segment to the per-block
+ path for the rest of that forward, which computes the block itself.
+
+ Lifecycle per (layout of the first block's inputs, weight placement): the first forward records the run eagerly,
+ the second warms it up once and captures it, every later one replays. All on the card's own stream and pool."""
+
+ def __init__(
+ self,
+ graphs: list,
+ shared: _Shared,
+ max_graphs: int = MAX_GRAPHS_PER_BLOCK,
+ ) -> None:
+ self.graphs = list(graphs)
+ self.shared = shared
+ self.max_graphs = int(max_graphs)
+ self.cache: "OrderedDict[tuple, _SegEntry]" = OrderedDict()
+ self.seen: "OrderedDict[tuple, _SegRecord]" = OrderedDict()
+ self.refused: set = set()
+ self.poisoned = False
+ self.enabled = True # off: every block takes its own graph's path; recordings are kept
+ self.capture_error: Optional[dict] = None
+ self.owner: Any = None
+ self._reset_forward()
+ self.stats = {
+ "segment_records": 0,
+ "segment_captures": 0,
+ "segment_replays": 0,
+ "segment_warmups": 0,
+ "segment_breaks": 0,
+ "segment_refusals": 0,
+ "segment_fallbacks": 0,
+ }
+ for i, g in enumerate(self.graphs):
+ g.segment = self
+ g.seg_index = i
+ _LIVE_BLOCKS.add(self)
+
+ def _reset_forward(self) -> None:
+ self.pos: Optional[int] = None
+ self.mode: Optional[str] = None
+ self.cur: Any = None
+ self.ext: list = []
+ self.returned: list = []
+ self.versions: list = []
+
+ def reset(self) -> None:
+ for entry in self.cache.values():
+ self._forget(entry)
+ self.cache.clear()
+ self.seen.clear()
+ self.refused.clear()
+ self._reset_forward()
+
+ def detach(self) -> None:
+ self.reset()
+ for g in self.graphs:
+ if g.segment is self:
+ g.segment = None
+
+ def _forget(self, entry: _SegEntry) -> None:
+ entry.graph = None
+ entry.outputs = None
+ self.shared.drop(getattr(entry, "slots", ()))
+
+ def _placement(self) -> Optional[tuple]:
+ out = []
+ for g in self.graphs:
+ p = g.weights.placement(self.shared.device_index)
+ if p is None:
+ return None
+ out.append(p)
+ return tuple(out)
+
+ def call(self, i: int, args: tuple, kwargs: dict) -> Any:
+ if self.poisoned or not self.enabled:
+ return _NOT_HANDLED
+ if i == 0:
+ return self._start(args, kwargs)
+ if self.pos != i:
+ return _NOT_HANDLED
+ return self._next(i, args, kwargs)
+
+ def _start(self, args: tuple, kwargs: dict) -> Any:
+ self._reset_forward()
+ torch = _torch()
+ if (
+ torch.is_grad_enabled()
+ or not _lazy_chain(self.shared.root)
+ or _step_cache_live(self.shared.root)
+ ):
+ return _NOT_HANDLED
+ live: list = []
+ try:
+ key = _walk((args, kwargs), live)
+ extra = _root_key_extra(self.shared.root)
+ except Exception: # noqa: BLE001
+ return _NOT_HANDLED
+ if _refusal(key, kwargs) or any(t.device.type != "cuda" for t in live):
+ return _NOT_HANDLED
+ if torch.cuda.current_device() != self.shared.device_index or any(
+ t.device.index != self.shared.device_index for t in live
+ ):
+ return _NOT_HANDLED
+ for g in self.graphs:
+ if g.protect is None:
+ g.protect = g._protect_controller()
+ if g.poisoned or g.churned or g.bypassed or g.protect:
+ return _NOT_HANDLED
+ placement = self._placement()
+ if placement is None:
+ return _NOT_HANDLED
+ full = (key, placement) if extra is None else (key, placement, extra)
+ if full in self.refused:
+ return _NOT_HANDLED
+ entry = self.cache.get(full)
+ if entry is None and self.owner is not None and self.owner.scope_sealed():
+ return _NOT_HANDLED # its blocks run ungraphed too (BlockGraph.owner)
+ if entry is None and full in self.seen:
+ entry = self._capture(full, live)
+ if entry is None:
+ return _NOT_HANDLED
+ if entry is not None:
+ self.cache.move_to_end(full)
+ for dst, src in zip(entry.static_in, live):
+ if dst.data_ptr() != src.data_ptr():
+ dst.copy_(src)
+ entry.graph.replay()
+ entry.replays += 1
+ self.stats["segment_replays"] += 1
+ self.mode, self.cur, self.ext = "replay", entry, live
+ return self._hand_out(0, entry)
+ rec = _SegRecord()
+ out = self.graphs[0].compute(*args, **kwargs)
+ flat: list = []
+ try:
+ spec = _flatten(out, flat)
+ except Exception: # noqa: BLE001
+ self.refused.add(full)
+ self.stats["segment_refusals"] += 1
+ return out
+ rec.keys.append(key)
+ rec.prov.append(tuple(("e", e) for e in range(len(live))))
+ rec.out_specs.append(spec)
+ self.mode, self.cur, self.ext = "record", (full, rec), live
+ self.returned = [flat]
+ self.versions = [[_version(t) for t in flat]]
+ self.pos = 1
+ self.graphs[0].stats["eager_calls"] += 1
+ return out
+
+ def _provenance(self, live: list) -> Optional[tuple]:
+ out = []
+ for t in live:
+ found = None
+ for j in range(len(self.returned) - 1, -1, -1):
+ for m, r in enumerate(self.returned[j]):
+ if t is r:
+ if _version(t) != self.versions[j][m]:
+ return None # rewritten in place since its block handed it on
+ found = ("o", j, m)
+ break
+ if found is not None:
+ break
+ if found is None:
+ for e, x in enumerate(self.ext):
+ if t is x:
+ found = ("e", e)
+ break
+ if found is None:
+ return None
+ out.append(found)
+ return tuple(out)
+
+ def _next(self, i: int, args: tuple, kwargs: dict) -> Any:
+ live: list = []
+ try:
+ key = _walk((args, kwargs), live)
+ except Exception: # noqa: BLE001
+ self._abandon()
+ return _NOT_HANDLED
+ prov = self._provenance(live)
+ last = i == len(self.graphs) - 1
+ if self.mode == "record":
+ full, rec = self.cur
+ if prov is None or _refusal(key, kwargs):
+ self.refused.add(full)
+ self.stats["segment_refusals"] += 1
+ self._abandon()
+ return _NOT_HANDLED
+ out = self.graphs[i].compute(*args, **kwargs)
+ flat: list = []
+ try:
+ spec = _flatten(out, flat)
+ except Exception: # noqa: BLE001
+ self.refused.add(full)
+ self.stats["segment_refusals"] += 1
+ self._abandon()
+ return out
+ rec.keys.append(key)
+ rec.prov.append(prov)
+ rec.out_specs.append(spec)
+ self.returned.append(flat)
+ self.versions.append([_version(t) for t in flat])
+ self.graphs[i].stats["eager_calls"] += 1
+ if last:
+ self.seen[full] = rec
+ while len(self.seen) > 4:
+ self.seen.popitem(last = False)
+ self.stats["segment_records"] += 1
+ self._reset_forward()
+ else:
+ self.pos = i + 1
+ return out
+ entry = self.cur
+ if prov is None or key != entry.record.keys[i] or prov != entry.record.prov[i]:
+ self.stats["segment_breaks"] += 1
+ self._abandon()
+ return _NOT_HANDLED
+ return self._hand_out(i, entry)
+
+ def _hand_out(self, i: int, entry: _SegEntry) -> Any:
+ tensors = entry.outputs[i]
+ last = i == len(self.graphs) - 1
+ if last:
+ tensors = [t.clone() for t in tensors]
+ self._reset_forward()
+ else:
+ if i == 0:
+ self.returned = [tensors]
+ self.versions = [[_version(t) for t in tensors]]
+ else:
+ self.returned.append(tensors)
+ self.versions.append([_version(t) for t in tensors])
+ self.pos = i + 1
+ self.graphs[i].stats["segment_hits"] = self.graphs[i].stats.get("segment_hits", 0) + 1
+ return _rebuild(entry.record.out_specs[i], tensors)
+
+ def _abandon(self) -> None:
+ self._reset_forward()
+
+ def _chain(self, rec: _SegRecord, ext: list) -> list:
+ results: list = []
+ for i, g in enumerate(self.graphs):
+ tensors = [ext[p[1]] if p[0] == "e" else results[p[1]][p[2]] for p in rec.prov[i]]
+ call_args, call_kwargs = _unwalk(rec.keys[i], iter(tensors))
+ out = g.compute(*call_args, **call_kwargs)
+ flat: list = []
+ if _flatten(out, flat) != rec.out_specs[i]:
+ raise RuntimeError("segment block output layout changed between calls")
+ results.append(flat)
+ return results
+
+ def _capture(self, full: tuple, live: list) -> Optional[_SegEntry]:
+ from .diffusion_cuda_graph import (
+ _capturing,
+ capture_device,
+ note_capture,
+ refuse_if_exhausted,
+ retire_failed_capture,
+ )
+
+ torch = _torch()
+ rec = self.seen.pop(full)
+ entry = _SegEntry()
+ slot = ("segment", id(self), full[0])
+ try:
+ entry.static_in = self.shared.statics_for(slot, live)
+ for dst, src in zip(entry.static_in, live):
+ dst.copy_(src)
+ # once, on the compute stream: a guard the static buffers trip recompiles here, never inside the capture
+ self._chain(rec, entry.static_in)
+ self.stats["segment_warmups"] += 1
+ current = torch.cuda.current_stream()
+ stream = self.shared.capture_stream()
+ stream.wait_stream(current)
+ refuse_if_exhausted(self.shared.logger)
+ graph = torch.cuda.CUDAGraph()
+ pool = self.shared.graph_pool()
+ with torch.cuda.stream(stream):
+ before = torch.cuda.memory_reserved()
+ with _capturing():
+ try:
+ graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
+ try:
+ outputs = self._chain(rec, entry.static_in)
+ finally:
+ graph.capture_end()
+ except BaseException as exc:
+ retire_failed_capture(graph, pool, exc)
+ if self.shared.pool == pool:
+ self.shared.pool = None
+ raise
+ current.wait_stream(stream)
+ self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)
+ except Exception as exc: # noqa: BLE001 - this layout stays per block for the load's life
+ self.capture_error = {"type": type(exc).__name__, "msg": str(exc)[:4000]}
+ self.refused.add(full)
+ self.stats["segment_fallbacks"] += 1
+ if self.shared.logger is not None:
+ self.shared.logger.warning(
+ "diffusion.block_graph: recording a %d-block segment failed (%s: %s); its blocks record one by one",
+ len(self.graphs),
+ type(exc).__name__,
+ exc,
+ )
+ return None
+ entry.graph = graph
+ entry.outputs = outputs
+ entry.record = rec
+ entry.slots = (slot,)
+ entry.replays = 0
+ self.shared.hold(entry.slots)
+ self.cache[full] = entry
+ self.stats["segment_captures"] += 1
+ if self.owner is not None:
+ self.owner.note_capture()
+ self._unsloth_capture_device = capture_device()
+ note_capture(self._unsloth_capture_device)
+ while len(self.cache) > self.max_graphs:
+ _, evicted = self.cache.popitem(last = False)
+ self._forget(evicted)
+ return entry
+
+
+def _segments(transformer: Any, graphs: list) -> list:
+ """Runs of at least two armed blocks a layer split keeps on one card, back to back in one block list (forward
+ order), as (graphs, shared) pairs."""
+ try:
+ from .diffusion_layer_split import find_blocks
+ except Exception: # noqa: BLE001
+ return []
+ by_block = {id(g.block): g for g in graphs}
+ runs: list = []
+ current: list = []
+ current_key = None
+ for _qual, group, block in find_blocks(transformer):
+ g = by_block.get(id(block))
+ key = (group, g.shared.device_index, id(g.shared)) if g is not None else None
+ if g is not None and key == current_key:
+ current.append(g)
+ continue
+ if len(current) > 1:
+ runs.append(current)
+ current = [g] if g is not None else []
+ current_key = key
+ if len(current) > 1:
+ runs.append(current)
+ return [(run, run[0].shared) for run in runs]
+
+
def _group_offload_hook(module: Any) -> Any:
try:
from diffusers.hooks import group_offloading as go
@@ -708,6 +1160,8 @@ class BlockGraphSet:
) -> None:
self.module = transformer
self.shared = shared
+ self.card_shared: dict = {}
+ self.segments: list = []
self.logger = logger
self.graphs: list = []
self.restores: list = []
@@ -717,6 +1171,54 @@ class BlockGraphSet:
self._prefetcher = None
self.mode = "blocks"
self.max_graphs = MAX_GRAPHS_PER_BLOCK
+ self.length_varies = False
+ self.scope: Any = None
+ self._sealed_scopes: set = set()
+ self._scope_captures: dict = {}
+ self._render_scopes: set = set()
+
+ def scope_sealed(self) -> bool:
+ return self.length_varies and self.scope in self._sealed_scopes
+
+ def note_capture(self) -> None:
+ if self.length_varies:
+ self._scope_captures[self.scope] = self._scope_captures.get(self.scope, 0) + 1
+
+ def track_scopes(self) -> None:
+ """Read each denoiser call's image scope (resolution, batch, references) before its blocks run."""
+ from .diffusion_cuda_graph import _image_scope
+
+ def _pre(module: Any, args: Any, kwargs: Any) -> None:
+ try:
+ self.scope = _image_scope(args, kwargs)
+ except Exception: # noqa: BLE001 - an unkeyable call: no scope, nothing seals
+ self.scope = None
+ self._render_scopes.add(self.scope)
+
+ self.length_varies = True
+ handle = self.module.register_forward_pre_hook(_pre, with_kwargs = True)
+ self.restores.append(handle.remove)
+
+ def shared_for(self, index: int) -> _Shared:
+ """The pool, stream and static buffers of card ``index``: the primary's own, else one per other card."""
+ if self.shared.device_index is None:
+ self.shared.device_index = index
+ if self.shared.device_index == index:
+ self.shared.strict_device = True
+ return self.shared
+ state = self.card_shared.get(index)
+ if state is None:
+ state = _Shared(index, self.logger)
+ state.root = self.shared.root
+ state.strict_device = True
+ self.card_shared[index] = state
+ return state
+
+ def cards(self) -> set:
+ return {s.device_index for s in self._all_shared()}
+
+ def _all_shared(self) -> list:
+ return [self.shared, *self.card_shared.values()]
@property
def cache(self) -> dict:
@@ -732,6 +1234,12 @@ class BlockGraphSet:
for g in self.graphs:
for k, v in g.stats.items():
total[k] = total.get(k, 0) + int(v)
+ for seg in self.segments:
+ for k, v in seg.stats.items():
+ total[k] = total.get(k, 0) + int(v)
+ total["captures"] = total.get("captures", 0) + total.get("segment_captures", 0)
+ total["warmups"] = total.get("warmups", 0) + total.get("segment_warmups", 0)
+ total["replays"] = total.get("replays", 0) + total.get("segment_replays", 0)
# The whole-forward wrapper's names, so diffusion_cuda_graph.stats aggregates both kinds alike.
total["refused_host_tensor"] = total.get("refused_host_weight", 0)
total["cap_skips"] = total.get("evictions", 0)
@@ -755,13 +1263,22 @@ class BlockGraphSet:
return self
def reset(self) -> "BlockGraphSet":
+ self._sealed_scopes.clear()
+ self._scope_captures.clear()
+ self._render_scopes.clear()
+ for seg in self.segments:
+ seg.reset()
for g in self.graphs:
g.reset()
- self.shared.release()
+ for state in self._all_shared():
+ state.release()
_release_cached()
return self
def free(self) -> "BlockGraphSet":
+ for seg in self.segments:
+ seg.detach()
+ self.segments.clear()
for g in self.graphs:
g.reset()
g.enabled = False
@@ -772,7 +1289,8 @@ class BlockGraphSet:
pass
self.restores.clear()
self.graphs.clear()
- self.shared.release()
+ for state in self._all_shared():
+ state.release()
self.enabled = False
try:
from .diffusion_offload_prefetch import module_prefetcher
@@ -801,8 +1319,18 @@ class BlockGraphSet:
getattr(getattr(self, "_prefetcher", None), "slot_bytes", 0) or 0
)
>> 20,
- "pool_mib": int(self.shared.pool_bytes >> 20),
- "static_mib": int(self.shared.static_bytes >> 20),
+ "pool_mib": int(sum(st.pool_bytes for st in self._all_shared()) >> 20),
+ "static_mib": int(sum(st.static_bytes for st in self._all_shared()) >> 20),
+ **({"cards": sorted(self.cards())} if self.card_shared else {}),
+ **(
+ {
+ "segments": len(self.segments),
+ "segment_blocks": sum(len(seg.graphs) for seg in self.segments),
+ "segment_graphs": sum(len(seg.cache) for seg in self.segments),
+ }
+ if self.segments
+ else {}
+ ),
"capture_error": None
if not self.capture_error
else {
@@ -840,7 +1368,7 @@ class BlockGraphSet:
"""Device memory this layer holds that the caching allocator cannot hand to anything else."""
pf = getattr(self, "_prefetcher", None)
slots = int(getattr(pf, "slot_bytes", 0) or 0) if pf is not None else 0
- return int(self.shared.pool_bytes + self.shared.static_bytes + slots)
+ return int(sum(st.pool_bytes + st.static_bytes for st in self._all_shared()) + slots)
def _release_cached() -> None:
@@ -869,6 +1397,21 @@ def _device_index(transformer: Any, fallback: Any = None) -> Optional[int]:
return None
+def _block_card(block: Any) -> Optional[int]:
+ """The CUDA ordinal every weight of ``block`` sits on (a layer split places each block whole), else None."""
+ index: Any = _UNSEEN
+ for t in _WeightView(block).tensors:
+ for leaf in _leaves(t):
+ dev = leaf.device
+ if dev.type != "cuda" or dev.index is None:
+ return None
+ if index is _UNSEEN:
+ index = dev.index
+ elif dev.index != index:
+ return None
+ return None if index is _UNSEEN else int(index)
+
+
def install_block_graphs(
transformer: Any,
*,
@@ -882,7 +1425,10 @@ def install_block_graphs(
blocks = repeated_blocks(transformer)
if not blocks:
return None, "no repeated blocks to record"
+ from .diffusion_layer_split import is_layer_split
+
torch = _torch()
+ split = is_layer_split(transformer)
shared = _Shared(_device_index(transformer, device), logger)
shared.root = transformer
handle = BlockGraphSet(transformer, shared, logger)
@@ -895,6 +1441,13 @@ def install_block_graphs(
continue
compiled = getattr(block, "_compiled_call_impl", None)
hook = _group_offload_hook(block)
+ block_shared = shared
+ if split:
+ index = _block_card(block)
+ if index is None:
+ refused["weights off the GPU"] = refused.get("weights off the GPU", 0) + 1
+ continue
+ block_shared = handle.shared_for(index)
if hook is not None and compiled is not None:
# compiled through its hooks (UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS=0): the compute is inside a dynamo frame
why = "compiled through its offload hooks"
@@ -919,14 +1472,14 @@ def install_block_graphs(
refused["a stacked hook chain"] = refused.get("a stacked hook chain", 0) + 1
continue
compute = target.forward
- graph = BlockGraph(block, compute, shared)
+ graph = BlockGraph(block, compute, block_shared)
target.forward = graph
def restore(ref: Any = target, fn: Any = compute) -> None:
if ref.forward is not fn:
ref.forward = fn
elif compiled is not None:
- graph = BlockGraph(block, compiled, shared)
+ graph = BlockGraph(block, compiled, block_shared)
block._compiled_call_impl = graph
def restore(
@@ -939,7 +1492,7 @@ def install_block_graphs(
else:
slot = block.__dict__.get("forward")
fwd = slot if slot is not None else type(block).forward.__get__(block)
- graph = BlockGraph(block, fwd, shared)
+ graph = BlockGraph(block, fwd, block_shared)
block.__dict__["forward"] = graph
def restore(
@@ -964,6 +1517,16 @@ def install_block_graphs(
", ".join(f"{n} block(s): {w}" for w, n in refused.items()) or "no block could be armed"
)
return None, reason
+ if split and not segment_graphs_disabled():
+ handle.segments = [
+ SegmentGraph(run, run_shared)
+ for run, run_shared in _segments(transformer, handle.graphs)
+ ]
+ handle.mode = "segments" if handle.segments else handle.mode
+ if split and _prompt_length_varies(transformer):
+ handle.track_scopes()
+ for unit in (*handle.graphs, *handle.segments):
+ unit.owner = handle
if slots and torch.cuda.is_available():
try:
from .diffusion_offload_prefetch import module_prefetcher
@@ -977,10 +1540,20 @@ def install_block_graphs(
logger.warning("diffusion.block_graph: slot ring unavailable (%s)", exc)
if logger is not None:
logger.info(
- "diffusion.block_graph: armed %d of %d %s block(s)%s%s",
+ "diffusion.block_graph: armed %d of %d %s block(s)%s%s%s",
len(handle.graphs),
len(blocks),
type(transformer).__name__,
+ (
+ f" over cards {sorted(handle.cards())}"
+ + (
+ f" as {len(handle.segments)} per-card segment graph(s)"
+ if handle.segments
+ else ""
+ )
+ )
+ if split
+ else "",
f"; streamed blocks read a {handle.slots_mib} MiB slot ring"
if handle.slots_mib
else "",
@@ -989,6 +1562,14 @@ def install_block_graphs(
return handle, "armed"
+def _prompt_length_varies(transformer: Any) -> bool:
+ try:
+ from .diffusion_cuda_graph import _length_varies
+ return _length_varies(transformer)
+ except Exception: # noqa: BLE001
+ return False
+
+
COMPILE_BELOW_HOOKS_ENV = "UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS"
diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py
index 94c1fc8daee..24c90b12229 100644
--- a/studio/backend/core/inference/diffusion_cache.py
+++ b/studio/backend/core/inference/diffusion_cache.py
@@ -432,6 +432,7 @@ def _transformer_blocks_registered(transformer: Any, logger: Any = None) -> bool
if not callable(named_children):
return True
register_unregistered_transformer_blocks(logger)
+ _register_layer_split_blocks(transformer)
blocks = [
getattr(block, "_orig_mod", block)
for name, child in named_children()
@@ -501,6 +502,107 @@ def install_fbcache_length_guard() -> bool:
return True
+def _register_layer_split_blocks(transformer: Any) -> bool:
+ """Register a layer split's block classes (same-named subclasses of the real blocks) with their base's metadata,
+ so the cache finds them. A copy, never the base's own object: ``register`` writes ``_cls`` into it. True when
+ every split block class is registered."""
+ try:
+ import dataclasses
+
+ from diffusers.hooks._helpers import TransformerBlockRegistry
+ except Exception: # noqa: BLE001 - no registry: enable_cache decides
+ return True
+ ok = True
+ for cls in {type(m) for m in transformer.modules()}:
+ base = cls.__dict__.get("_unsloth_layer_split_base")
+ if base is None:
+ continue
+ try:
+ TransformerBlockRegistry.get(cls)
+ continue
+ except Exception: # noqa: BLE001 - not registered yet
+ pass
+ try:
+ meta = TransformerBlockRegistry.get(base)
+ except Exception: # noqa: BLE001 - the base is not a cacheable block either
+ continue
+ try:
+ TransformerBlockRegistry.register(
+ model_class = cls,
+ metadata = dataclasses.replace(meta, _cls = None, _cached_parameter_indices = None),
+ )
+ except Exception: # noqa: BLE001
+ ok = False
+ return ok
+
+
+def _tensors_to(obj: Any, device: Any) -> Any:
+ if isinstance(obj, (list, tuple)):
+ return type(obj)(_tensors_to(x, device) for x in obj)
+ to = getattr(obj, "to", None)
+ return to(device) if callable(to) else obj
+
+
+def _first_device(obj: Any) -> Any:
+ if isinstance(obj, (list, tuple)):
+ for x in obj:
+ dev = _first_device(x)
+ if dev is not None:
+ return dev
+ return None
+ return getattr(obj, "device", None)
+
+
+def install_fbcache_split_guard() -> bool:
+ """Make First-Block-Cache's tail hook work when the tail block runs on another card than the head block (a layer
+ split): the head block's output is copied to the tail's card for the residual, and the residual goes back to the
+ head's card, where the head hook adds it on a cached step. Copies are exact, so the cache decides and renders as
+ on one card. Calls whose tensors share a card take the original path unchanged. Process-wide and idempotent;
+ must run before ``enable_cache`` (diffusers binds ``new_forward`` when it registers the hook). False when this
+ diffusers has no FBCache block hook."""
+ try:
+ from diffusers.hooks.first_block_cache import FBCBlockHook
+ except Exception: # noqa: BLE001 - no FBCache in this diffusers
+ return False
+ install_fbcache_mixed_stream_tail()
+ original = getattr(FBCBlockHook, "new_forward", None)
+ if original is None:
+ return False
+ if getattr(original, "_unsloth_split_guard", False):
+ return True
+
+ def new_forward(self, module, *args, **kwargs):
+ if not getattr(self, "is_tail", False):
+ return original(self, module, *args, **kwargs)
+ state = self.state_manager.get_state()
+ head = state.head_block_output
+ home = _first_device(head)
+ try:
+ here = self._metadata._get_parameter_from_args_kwargs(
+ "hidden_states", args, kwargs
+ ).device
+ except Exception: # noqa: BLE001 - the original raises the same way
+ here = None
+ if not state.should_compute or home is None or here is None or here == home:
+ return original(self, module, *args, **kwargs)
+ state.head_block_output = _tensors_to(head, here)
+ try:
+ out = original(self, module, *args, **kwargs)
+ finally:
+ state.head_block_output = head
+ residuals = state.tail_block_residuals
+ if residuals is not None:
+ state.tail_block_residuals = tuple(
+ r.to(home) if r is not None else None for r in residuals
+ )
+ return out
+
+ new_forward._unsloth_split_guard = True
+ new_forward._unsloth_mixed_stream_tail = getattr(original, "_unsloth_mixed_stream_tail", False)
+ FBCBlockHook.new_forward = new_forward
+ return True
+
+
def _split_mixed_stream_tail_residuals(tail_hidden: Any, head_output: Any) -> Optional[tuple]:
"""Tail residuals for a dual-stream head and a single-stream tail, or None when unmappable.
@@ -675,6 +777,14 @@ def apply_step_cache(
transformer = getattr(pipe, "transformer", None)
if transformer is None:
return None
+ from .diffusion_layer_split import is_layer_split
+
+ if is_layer_split(transformer) and not (
+ install_fbcache_split_guard() and _register_layer_split_blocks(transformer)
+ ):
+ # without the guard the tail hook subtracts the head block's output (primary) from its own (another card)
+ _warn(logger, mode, RuntimeError("denoiser is split across cards; running uncached"))
+ return None
# Before enable_cache, which is what raises on a block class the registry has never seen.
register_unregistered_transformer_blocks(logger)
# FLUX.2's dual-stream head + single-stream tail crashes the stock tail hook on the first step.
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 00ac17cbd71..5f96f9b226f 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -647,6 +647,31 @@ class _Entry:
)
+def _layer_split(module: Any) -> bool:
+ from .diffusion_layer_split import is_layer_split
+ return is_layer_split(module)
+
+
+def _rehome_after_layer_split(pipe: Any, logger: Any) -> None:
+ """Before any recording or first forward: the int8 / fused records the speed layer built on one card follow the
+ split's blocks to their cards (``diffusion_int8_gemm.after_layer_split``)."""
+ try:
+ from .diffusion_int8_gemm import after_layer_split
+ for module in _denoiser_modules(pipe):
+ if _layer_split(module):
+ after_layer_split(module, logger)
+ except Exception as exc: # noqa: BLE001 - the stock paths stay correct
+ _warn(logger, "layer split re-home", exc)
+
+
+def _layer_split_pipe(pipe: Any) -> bool:
+ """Any denoiser of ``pipe`` is split across cards by block ranges."""
+ try:
+ return any(_layer_split(m) for m in _denoiser_modules(pipe))
+ except Exception: # noqa: BLE001
+ return False
+
+
class GraphedForward:
"""A callable that replaces one denoiser module's ``forward`` with a CUDA-graph replay.
@@ -1054,6 +1079,7 @@ class GraphedForward:
or self.poisoned
# Belt to the caller's per-chunk bypass: one step's cond and uncond share a key.
or getattr(self.module, "_unsloth_step_cache", None)
+ or _layer_split(self.module)
):
return self._eager(args, kwargs)
@@ -1899,7 +1925,12 @@ def arm_after_placement(
handles: list = []
refusals: list = []
modes: list = []
+ from .diffusion_layer_split import is_layer_split
+
for module in _denoiser_modules(pipe):
+ if is_layer_split(module):
+ refusals.append(f"{type(module).__name__}: denoiser is split across cards")
+ continue
placement, why = offload_placement(module)
if why is not None:
refusals.append(f"{type(module).__name__}: {why}")
@@ -2027,6 +2058,11 @@ def graph_eligible(
# Lazy: ``diffusion_speed`` imports this module, so a module-level import here is a cycle.
from .diffusion_speed import _denoiser_dits, _denoiser_unet
+ from .diffusion_layer_split import is_layer_split
+
+ if any(is_layer_split(m) for m in _denoiser_modules(pipe)):
+ return False, "denoiser is split across cards"
+
# A whole-compiled U-Net serves from ``_compiled_call_impl``: GraphedCompiledCall captures there.
if not _denoiser_dits(pipe) and _denoiser_unet(pipe) is None:
return False, "no denoiser transformer"
@@ -2335,6 +2371,11 @@ def arm_block_graphs(
whole = [h for h in handles if isinstance(h, GraphedForward)]
prior = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")
step = [h for h in whole if h.placement is not None or h.plan is not None]
+ split = _layer_split_pipe(pipe)
+ if split:
+ # a whole-step capture records one card; a split's blocks record per block on their own card
+ step = []
+ _rehome_after_layer_split(pipe, logger)
if whole and len(step) == len(whole):
# The whole step stays primary; where per-block graphs are the default they arm for the calls it leaves eager.
pipe._unsloth_cuda_graph_mode = "step"
@@ -2344,13 +2385,13 @@ def arm_block_graphs(
if why is None:
_attach_block_fallback(pipe, step, target, hooked, pinned, logger)
return handles
- if not hooked and (whole or "not capture-safe" not in prior):
+ if not split and not hooked and (whole or "not capture-safe" not in prior):
# Nothing moves the denoiser: the speed layer's decision stands, except a forward that is not capture-safe,
# whose blocks still record.
return handles
- why = _block_basics(
- target, family, cache_engaged, speed_mode, family_default
- ) or prompt_length_reason(pipe, family)
+ why = _block_basics(target, family, cache_engaged, speed_mode, family_default) or (
+ None if split else prompt_length_reason(pipe, family)
+ )
if whole:
uninstall_all(whole, logger = logger)
pipe._unsloth_cuda_graph_mode = None
@@ -2414,6 +2455,8 @@ def arm_block_graphs(
if hooked and pinned
else "offloaded denoiser"
if hooked
+ else "split denoiser (each card's blocks on that card)"
+ if split
else "denoiser"
)
slots = sum(int(getattr(h, "slots_mib", 0) or 0) for h in armed)
diff --git a/studio/backend/core/inference/diffusion_dynamic_text.py b/studio/backend/core/inference/diffusion_dynamic_text.py
index 8f913cd52d4..bbf8541c470 100644
--- a/studio/backend/core/inference/diffusion_dynamic_text.py
+++ b/studio/backend/core/inference/diffusion_dynamic_text.py
@@ -129,6 +129,13 @@ _FAMILY_SOURCES: dict[str, tuple[str, ...]] = {
"MiniMaxH3Transformer3DModel": _MINIMAX_H3_SOURCES,
}
+_ARMED_WHEN_DYNAMIC: frozenset[str] = frozenset({"QwenImageTransformer2DModel"})
+
+
+def _armed_when_dynamic(transformer: Any) -> bool:
+ return type(transformer).__name__ in _ARMED_WHEN_DYNAMIC
+
+
# Unbacked: a backed symbol specialises size 1, and H3's temb has 1 row on step 1, 2 after (a second compile).
_FAMILY_UNBACKED: dict[str, tuple[str, ...]] = {
"MiniMaxH3Transformer3DModel": ("L['temb']",),
@@ -182,11 +189,11 @@ def sources_for(transformer: Any) -> tuple[str, ...]:
def fingerprint(transformer: Any, dynamic: Any) -> Optional[str]:
- if dynamic is True:
+ if dynamic is True and not _armed_when_dynamic(transformer):
# dynamic=True already makes every dim dynamic; only the unbacked sources are armed (see install).
unbacked = unbacked_sources_for(transformer)
return "unbacked:" + ",".join(unbacked) if unbacked else None
- if dynamic is not None:
+ if dynamic is not None and dynamic is not True:
return None
sources = sources_for(transformer)
if not sources or not supported():
@@ -273,7 +280,8 @@ def install(
*,
dynamic: Any = None,
) -> bool:
- """Arm sources for a ``dynamic`` compile: None arms dynamic + unbacked, True only unbacked, False nothing."""
+ """Arm sources for a ``dynamic`` compile: None arms dynamic + unbacked, True only unbacked (dynamic + unbacked
+ for :data:`_ARMED_WHEN_DYNAMIC`), False nothing."""
if dynamic is False:
return False
if getattr(transformer, "_unsloth_dynamic_text", None) is not None:
@@ -286,7 +294,9 @@ def install(
if logger is not None:
logger.warning("diffusion.dynamic_text: slice identity fix failed: %s", exc)
unbacked = unbacked_sources_for(transformer)
- sources = sources_for(transformer) if dynamic is None else ()
+ sources = (
+ sources_for(transformer) if dynamic is None or _armed_when_dynamic(transformer) else ()
+ )
cfg = _compiler_config()
if not (sources or unbacked) or cfg is None:
return False
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index cd6672dfc79..f04e4eee464 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -260,6 +260,8 @@ class DiffusionFamily:
# Fresh servers rendered 2-4 variants of one seed here (FLUX.1, Z-Image, Qwen-Image; the edit siblings share their DiT):
# near-tied norm-reduction configs, benchmarked per process, sum in different orders. B200 measured deterministic.
_REDUCTION_RACE_ARCHS: tuple[tuple[int, int], ...] = ((8, 0), (8, 9), (12, 0))
+# Qwen-Image races on B200 too (torch 2.14): fresh processes rendered one seed 26 dB apart; with the filter 6/6 matched bitwise.
+_QWEN_REDUCTION_RACE_ARCHS: tuple[tuple[int, int], ...] = _REDUCTION_RACE_ARCHS + ((10, 0),)
# --- Shared FamilyProfile data (see family_profile.FamilyProfile) ---
@@ -611,6 +613,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
label = "FLUX.2-dev",
profile = FamilyProfile(te_int8_skip = (3, 0), preview_factors = "flux2"),
pipeline_class = "Flux2Pipeline",
+ # cudnn.benchmark's per-process VAE conv pick rendered 5 variants of one seed over 7 fresh B200 processes.
+ cudnn_benchmark = False,
uses_negative_prompt = False,
transformer_class = "Flux2Transformer2DModel",
base_repo = "black-forest-labs/FLUX.2-dev",
@@ -684,7 +688,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
te_int8_skip = _QWENIMAGE_TE_INT8_SKIP,
preview_factors = "qwen_image",
),
- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,
+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,
comfy_flow_shift = 3.1, # ComfyUI ModelSamplingAuraFlow 3.1 (Qwen-Image-Edit 2511 template)
comfy_flow_shift_variants = (
("qwen-image-edit-2511", 3.1),
@@ -757,7 +761,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
scheme_deny = _QWENIMAGE_SCHEME_DENY,
te_int8_skip = _QWENIMAGE_TE_INT8_SKIP,
),
- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,
+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,
# ComfyUI's Image to Layers template: ModelSamplingAuraFlow 1, 2 layers, the input scaled to 640.
comfy_flow_shift = 1.0,
layer_count = 2,
@@ -805,7 +809,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
preview_factors = "qwen_image",
auto_static_skip = (("qwen/qwen-image", (("default", 2), ("max", 3), ("steps", 20))),),
),
- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,
+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,
comfy_flow_shift = 3.1, # ComfyUI ModelSamplingAuraFlow 3.1 (Qwen-Image templates)
pipeline_class = "QwenImagePipeline",
transformer_class = "QwenImageTransformer2DModel",
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index aa18f95d337..908c14b2609 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -1334,7 +1334,10 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->
ok = ok and hidden_states.dtype == torch.bfloat16
if ok:
fused_in, parts, down_name, n, gate_col, value_col, in_group, down_group = rec
- ok = all(_get(self, nm) is mod for nm, mod in parts)
+ ok = (
+ all(_get(self, nm) is mod for nm, mod in parts)
+ and fused_in.weight.device == hidden_states.device
+ )
if not ok:
return type(self).forward(self, hidden_states, *args, **kwargs)
down = _get(self, down_name)
@@ -1500,6 +1503,18 @@ def resident_cuda_device(module: Any) -> Any:
return dev if dev.type == "cuda" else None
+def resident_cuda_cards(module: Any) -> tuple:
+ """The CUDA ordinals ``module``'s weights sit on: its one resident card, or every card of a layer split (each
+ block on one card). Empty when any weight is off the GPU, or on ROCm."""
+ dev = resident_cuda_device(module)
+ if dev is not None:
+ import torch
+ return (dev.index if dev.index is not None else torch.cuda.current_device(),)
+ from .diffusion_int8_gemm import _split_cards
+
+ return _split_cards(module)
+
+
def run_on_first_call(module: Any, key: str, fn: Any) -> None:
"""Run ``fn(module)`` once, eagerly, right before ``module``'s first forward (after load-time placement moved
the weights, before the regional compile traces the blocks and before any CUDA-graph capture). Idempotent per key."""
@@ -1599,8 +1614,12 @@ def _has_eligible(transformer: Any) -> bool:
def _finalize(transformer: Any, logger: Any = None) -> int:
global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES, _INTMM_MODULE
dev = resident_cuda_device(transformer)
+ cards: tuple = ()
if dev is None:
- return 0
+ from .diffusion_int8_gemm import _split_cards
+ cards = _split_cards(transformer)
+ if not cards:
+ return 0
# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.
if not _has_eligible(transformer):
return 0
@@ -1612,23 +1631,33 @@ def _finalize(transformer: Any, logger: Any = None) -> int:
_INTMM_MODULE = intmm
import torch
- if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):
- return 0
+ if cards:
+ usable = {index for index in cards if _device_ok(index)}
+ if not usable:
+ return 0
+ indices = sorted(usable)
+ else:
+ if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):
+ return 0
+ indices = [dev.index if dev.index is not None else torch.cuda.current_device()]
count = 0
rearm = False
- index = dev.index if dev.index is not None else torch.cuda.current_device()
- actq = _act_quant_op() if _act_quant_device_ok(index) else None
+ actq_ok = {index: _act_quant_device_ok(index) for index in indices}
+ actq = _act_quant_op() if any(actq_ok.values()) else None
with _LOCK:
_OP_HANDLE = _op()
- if actq is not None:
- _ACTQ_HANDLE = actq
- _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
- else:
- _ACTQ_DEVICES = _ACTQ_DEVICES - {index}
+ for index, ok in actq_ok.items():
+ if ok and actq is not None:
+ _ACTQ_HANDLE = actq
+ _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
+ else:
+ _ACTQ_DEVICES = _ACTQ_DEVICES - {index}
for _name, module in transformer.named_modules():
if _MARK in module.__dict__:
count += 1
continue
+ if cards and _module_card(module) not in usable:
+ continue # its card failed the kernel probe (or holds no weights of it): stock path
if _ff_eligible(module):
fn = _ff_forward
elif _prepare_swiglu(module):
@@ -1668,6 +1697,102 @@ def _finalize(transformer: Any, logger: Any = None) -> int:
return count
+def _module_card(module: Any) -> Optional[int]:
+ for t in module.parameters():
+ dev = t.device
+ return dev.index if dev.type == "cuda" else None
+ return None
+
+
+_OFF_TREE_RECORDS = (
+ (_SWIGLU_ATTR, "swiglu"),
+ ("_unsloth_fused_qkv", "qkv"),
+ ("_unsloth_q21_fused_qkv", "qkv"),
+)
+
+
+def relocate_after_layer_split(transformer: Any, logger: Any = None) -> dict:
+ """A layer split moved blocks to other cards after their fused records were built on one card. Each off-tree
+ fused Linear (which shared its parts' storage there) is rebuilt on its parts' new card, freeing the stale copy, or
+ its record is dropped (the module then runs its stock path). Regions on a card whose fused kernels fail the probe
+ get their stock forward back. Returns counts; idempotent. Must run before the first forward (the regional compile
+ traces what it finds)."""
+ from .diffusion_zimage_fused import _fuse_linears, _share_storage
+
+ out = {"rebuilt": 0, "dropped": 0, "restored": 0}
+ global _ACTQ_DEVICES
+ cards = {}
+ with _LOCK:
+ for module in transformer.modules():
+ for attr, kind in _OFF_TREE_RECORDS:
+ rec = module.__dict__.get(attr)
+ if rec is None:
+ continue
+ if kind == "swiglu":
+ parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None
+ else:
+ parts = list(rec[1:4])
+ if parts is None:
+ continue # the in-projection is the module's own Linear: it moved with the block
+ fused = rec[0]
+ here = parts[0].weight.device
+ if fused.weight.device == here:
+ continue
+ rebuilt = _fuse_linears(parts)
+ if rebuilt is not None and _share_storage(rebuilt, parts):
+ module.__dict__[attr] = (rebuilt, *rec[1:])
+ out["rebuilt"] += 1
+ else:
+ module.__dict__.pop(attr, None)
+ out["dropped"] += 1
+ if _MARK in module.__dict__:
+ index = _module_card(module)
+ if index is not None:
+ cards.setdefault(index, []).append(module)
+ for index, modules in cards.items():
+ if _device_ok(index):
+ if _act_quant_device_ok(index) and _ACTQ_HANDLE is not None:
+ _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
+ continue
+ for module in modules:
+ _restore_module(module)
+ out["restored"] += 1
+ if logger is not None and any(out.values()):
+ logger.info(
+ "diffusion.int8_fused: layer split re-homed fused records (%d rebuilt, %d dropped, %d regions stock)",
+ out["rebuilt"],
+ out["dropped"],
+ out["restored"],
+ )
+ return out
+
+
+def drop_stranded_fused_records(transformer: Any) -> int:
+ """Drop fused records whose parts left their card after a failed split (``pipe.to("cpu")`` cannot free them).
+ Returns how many were dropped."""
+ dropped = 0
+ with _LOCK:
+ for module in transformer.modules():
+ for attr, kind in _OFF_TREE_RECORDS:
+ rec = module.__dict__.get(attr)
+ if rec is None:
+ continue
+ if kind == "swiglu":
+ parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None
+ else:
+ parts = list(rec[1:4])
+ if parts is None:
+ continue # the in-projection is the module's own Linear: it moved with the block
+ try:
+ stranded = any(p.weight.device != rec[0].weight.device for p in parts)
+ except Exception: # noqa: BLE001 - an unreadable record cannot be trusted either
+ stranded = True
+ if stranded:
+ module.__dict__.pop(attr, None)
+ dropped += 1
+ return dropped
+
+
def _hooked_forward_slot(module: Any) -> Optional[tuple]:
"""(fn_ref, attribute) holding the module's own forward in the innermost ``HookFunctionReference``, else None."""
registry = getattr(module, "_diffusers_hook", None)
@@ -1735,21 +1860,25 @@ def uninstall(transformer: Any = None) -> None:
cancel_first_call(transformer, "int8_fused")
with _LOCK:
for module in transformer.modules():
- prev = module.__dict__.pop(_MARK, None)
- if prev is None:
- continue
- slot = _hooked_forward_slot(module)
- if slot is not None and _is_fused_forward(getattr(*slot), module):
- # Swapped under a hook, or a cache engaged after the swap: restore the hook's inner, keep its wrapper.
- setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))
- elif _is_fused_forward(module.__dict__.get("forward"), module):
- if prev is _NO_PREV:
- module.__dict__.pop("forward", None)
- else:
- module.forward = prev
- _disarm_fused_inner(module, prev)
- module.__dict__.pop("_unsloth_i8_addcmul", None)
- module.__dict__.pop(_SWIGLU_ATTR, None)
+ _restore_module(module)
+
+
+def _restore_module(module: Any) -> None:
+ """One region's stock forward back (caller holds ``_LOCK`` or owns the module)."""
+ prev = module.__dict__.pop(_MARK, None)
+ if prev is None:
+ return
+ slot = _hooked_forward_slot(module)
+ if slot is not None and _is_fused_forward(getattr(*slot), module):
+ setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))
+ elif _is_fused_forward(module.__dict__.get("forward"), module):
+ if prev is _NO_PREV:
+ module.__dict__.pop("forward", None)
+ else:
+ module.forward = prev
+ _disarm_fused_inner(module, prev)
+ module.__dict__.pop("_unsloth_i8_addcmul", None)
+ module.__dict__.pop(_SWIGLU_ATTR, None)
def is_installed(module: Any) -> bool:
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index d60673ff69c..484ce6d2703 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -767,7 +767,6 @@ def _swap(
logger: Any = None,
device: Any = None,
) -> int:
- global _OP_HANDLE
from .diffusion_int8_fused import resident_cuda_device
import torch
@@ -785,12 +784,50 @@ def _swap(
return 0
else:
dev = resident_cuda_device(transformer)
+ if dev is None:
+ cards = _split_cards(transformer)
+ if cards:
+ count = sum(_swap_on(transformer, logger, index) for index in cards)
+ return count
if dev is None:
return 0
index = dev.index if dev.index is not None else torch.cuda.current_device()
+ return _swap_on(transformer, logger, index)
+
+
+def _split_cards(transformer: Any) -> tuple:
+ """The CUDA ordinals of a layer-split denoiser whose every weight sits on a card (NVIDIA), else ()."""
+ try:
+ import torch
+
+ from .diffusion_layer_split import is_layer_split
+
+ if getattr(torch.version, "hip", None) or not is_layer_split(transformer):
+ return ()
+ devices = {p.device for p in transformer.parameters()}
+ except Exception: # noqa: BLE001
+ return ()
+ if not devices or any(d.type != "cuda" or d.index is None for d in devices):
+ return ()
+ return tuple(sorted(d.index for d in devices))
+
+
+def _weight_card(module: Any) -> Optional[int]:
+ weight = getattr(module, "weight", None)
+ dev = getattr(weight, "device", None)
+ return dev.index if dev is not None and dev.type == "cuda" else None
+
+
+def _swap_on(transformer: Any, logger: Any, index: int) -> int:
+ """Swap the eligible int8 Linears of ``transformer`` against card ``index``'s probed kernels. On a layer split
+ (several cards) only the Linears whose weight sits on ``index``."""
+ global _OP_HANDLE
+ import torch
+
+ multi = len(_split_cards(transformer)) > 1
recs = [(m, _eligible(m)) for m in transformer.modules()]
- recs = [(m, r) for m, r in recs if r is not None]
+ recs = [(m, r) for m, r in recs if r is not None and (not multi or _weight_card(m) == index)]
if not recs:
return 0
from .diffusion_convrot_quant import rotquant_device_config
@@ -812,6 +849,14 @@ def _swap(
return 0
for module, rec in recs:
if _MARK in module.__dict__:
+ if multi:
+ module.__dict__[_REC] = (
+ rec[0],
+ rec[1],
+ bool(rotq and rec[1] is not None),
+ rec[3],
+ gemm,
+ )
count += 1
continue
# kind, rotation group, fused rotation + act quant on, the Parameter (no payload alias), fused GEMM on
@@ -836,6 +881,45 @@ def _swap(
return count
+def after_layer_split(transformer: Any, logger: Any = None) -> dict:
+ """Run once a layer split placed ``transformer``'s blocks, before its first forward: re-home the fused int8 / QKV
+ records built while it sat on one card (``diffusion_int8_fused.relocate_after_layer_split``), and probe each card's
+ GEMM tiles so the int8 Linears swapped before the split run their own card's kernel; each RoPE patch's probe map
+ covers every card. Idempotent; a no-op on a denoiser that is not split."""
+ out: dict = {"cards": (), "gemm": 0}
+ cards = _split_cards(transformer)
+ if len(cards) < 2:
+ return out
+ out["cards"] = cards
+ try:
+ from .diffusion_int8_fused import relocate_after_layer_split
+ out.update(relocate_after_layer_split(transformer, logger))
+ except Exception as exc: # noqa: BLE001 - the stock paths stay correct
+ if logger is not None:
+ logger.warning(
+ "diffusion.int8_gemm: fused record re-home after the layer split failed (%s)", exc
+ )
+ try:
+ from .diffusion_qwenimage21_rope import probe_split_cards
+ out["rope_cards"] = probe_split_cards(cards)
+ except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE
+ pass
+ try:
+ from .diffusion_zimage_fused import probe_split_cards as zimage_probe
+ out["zimage_rope_cards"] = zimage_probe(cards)
+ except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE
+ pass
+ if int8_gemm_mode() != "off" and any(_MARK in m.__dict__ for m in transformer.modules()):
+ try:
+ out["gemm"] = sum(_swap_on(transformer, logger, index) for index in cards)
+ except Exception as exc: # noqa: BLE001
+ if logger is not None:
+ logger.warning(
+ "diffusion.int8_gemm: per-card swap after the layer split failed (%s)", exc
+ )
+ return out
+
+
def uninstall(transformer: Any = None) -> None:
if transformer is None:
return
diff --git a/studio/backend/core/inference/diffusion_layer_split.py b/studio/backend/core/inference/diffusion_layer_split.py
new file mode 100644
index 00000000000..767b99565e0
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_layer_split.py
@@ -0,0 +1,898 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""One denoiser split across several CUDA cards by contiguous ranges of its repeated blocks (layer split).
+
+For a DiT that does not fit one selected card: each repeated block lives whole on one card, everything else of the
+denoiser (embedders, modulation, final norm and projection) stays on the primary. Blocks run one after another, so
+this buys memory, not speed; the weights never move after placement and only activations cross a card boundary.
+
+The boundary is a same-named subclass of each block's class whose ``__call__`` moves tensor arguments to the block's
+card before ``nn.Module.__call__`` runs. It runs outside ``_compiled_call_impl``, so a regional compile only ever sees
+tensors on the block's own card (no graph break; the shared block code compiles once per card). Tensors computed once
+per forward on the primary (rotary tables, modulation, masks) are copied once per card per forward, not per block.
+
+Outputs: in lazy mode a block's outputs move only where the next block of its list sits on another card, and home at
+the end of the list. That is only correct when the denoiser's loop does no tensor op of its own between blocks, so it
+is limited to the classes below and to calls without a residual input (ControlNet, VACE hints); every other call is
+eager: outputs return to the primary after every off-primary block. Both are exact copies, so the image matches a
+one-card run bit for bit.
+
+Every route reads where the block's weights and the denoiser's own weights are NOW, per call, so a later ``.to()`` of
+the pipeline or of the denoiser (eviction to the CPU, a move onto one card) keeps the forward on one device, as the
+component boundary does.
+
+Kill switch ``UNSLOTH_DIFFUSION_LAYER_SPLIT=0`` (never split a denoiser; the planner offloads as before); ``force``
+splits across the selected cards even when one card would hold the denoiser (for proving bit-identity).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import inspect
+import os
+import threading
+import weakref
+from typing import Any, Mapping, Optional, Sequence
+
+LAYER_SPLIT_ENV = "UNSLOTH_DIFFUSION_LAYER_SPLIT"
+
+# Denoisers whose forward passes block outputs straight to the next block (checked against diffusers' loops).
+LAZY_CLASSES: frozenset[str] = frozenset(
+ {
+ "Flux2Transformer2DModel",
+ "Flux3Transformer2DModel",
+ "FluxTransformer2DModel",
+ "HunyuanImageTransformer2DModel",
+ "QwenImageTransformer2DModel",
+ "QwenImage21Transformer2DModel",
+ "ZImageTransformer2DModel",
+ "WanTransformer3DModel",
+ "LTX2VideoTransformer3DModel",
+ # HiDream's llama states / cats follow hidden_states.device (bitwise vs one card, test_diffusion_layer_split_cuda.py).
+ "HiDreamImageTransformer2DModel",
+ "HunyuanVideo15Transformer3DModel",
+ "Ideogram4Transformer2DModel",
+ "Krea2Transformer2DModel",
+ "Lumina2Transformer2DModel",
+ "MiniMaxH3Transformer3DModel",
+ }
+)
+# Block lists run in another order than registered (forward order, so each card is entered once).
+FORWARD_ORDER: dict[str, tuple[str, ...]] = {
+ "Flux3Transformer2DModel": (
+ "txt_mode_blocks",
+ "mode_blocks.image",
+ "mode_blocks.image_cond",
+ "single_transformer_blocks",
+ ),
+}
+# A non-None keyword whose name contains one of these feeds a root-level add between blocks: eager for that call.
+_RESIDUAL_KWARG_PARTS: tuple[str, ...] = (
+ "controlnet",
+ "control_hidden",
+ "residual",
+ "hint",
+ "vace",
+)
+
+_STEP_CACHE_HOOKS: frozenset[str] = frozenset({"fbc_leader_block_hook", "fbc_block_hook"})
+
+_SPLIT_ATTR = "_unsloth_layer_split"
+_SUBCLASSES: dict[type, type] = {}
+_SUBCLASS_LOCK = threading.Lock()
+
+
+class LayerSplitError(RuntimeError):
+ """The denoiser cannot be split (no repeated blocks, hooks that own its placement, or the budgets are too small).
+ The caller falls back to its one-card plan."""
+
+
+def layer_split_mode() -> str:
+ """``auto`` (default: split only a denoiser that fits no single selected card), ``off``, or ``force``.
+ ``UNSLOTH_DIFFUSION_MULTIGPU=0`` turns it off too."""
+ from .diffusion_multigpu import multigpu_mode
+
+ if multigpu_mode() == "off":
+ return "off"
+ raw = (os.environ.get(LAYER_SPLIT_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw == "force":
+ return "force"
+ return "auto"
+
+
+def _torch():
+ import torch
+ return torch
+
+
+def _leaves(t: Any) -> list:
+ from .diffusion_block_graph import _leaves as leaves
+ return leaves(t)
+
+
+def module_bytes(module: Any) -> int:
+ """Bytes the module's weights take on a card: tensor subclasses (torchao) count their inner tensors."""
+ seen: set = set()
+ total = 0
+ for t in (*module.parameters(), *module.buffers()):
+ if id(t) in seen:
+ continue
+ seen.add(id(t))
+ for leaf in _leaves(t):
+ try:
+ total += leaf.numel() * leaf.element_size()
+ except Exception: # noqa: BLE001 - opaque leaf: count the wrapper
+ total += t.numel() * t.element_size()
+ break
+ return total
+
+
+def _block_list_name(qual: str) -> str:
+ """The block list a repeated block belongs to: the module path before its last list index. A list entry that
+ wraps the block (HiDream's ``double_stream_blocks.3.block``) belongs to ``double_stream_blocks``, not to a group
+ of its own, so consecutive entries route lazily and record as one segment."""
+ parts = qual.split(".")
+ for i in range(len(parts) - 1, -1, -1):
+ if parts[i].isdigit():
+ return ".".join(parts[:i])
+ return qual.rpartition(".")[0]
+
+
+def find_blocks(transformer: Any) -> list[tuple[str, str, Any]]:
+ """(qualified name, list name, block) for every repeated block, in forward order (registration order unless
+ :data:`FORWARD_ORDER` names the lists). Nested repeated blocks are part of their outer block."""
+ names: tuple[str, ...] = ()
+ try:
+ from .diffusion_regional_compile import ensure_repeated_blocks
+ names = tuple(ensure_repeated_blocks(transformer) or ())
+ except Exception: # noqa: BLE001 - fall back to what the class declares
+ pass
+ if not names:
+ names = tuple(getattr(transformer, "_repeated_blocks", None) or ())
+ if not names:
+ names = tuple(getattr(transformer, "_no_split_modules", None) or ())
+ wanted = set(names)
+ out: list[tuple[str, str, Any]] = []
+ inside: Optional[str] = None
+ for qual, module in transformer.named_modules():
+ if module is transformer:
+ continue
+ if inside is not None and qual.startswith(inside + "."):
+ continue
+ inside = None
+ base = getattr(type(module), "_unsloth_layer_split_base", type(module))
+ if base.__name__ in wanted:
+ out.append((qual, _block_list_name(qual), module))
+ inside = qual
+ order = FORWARD_ORDER.get(type(transformer).__name__)
+ if order:
+ rank = {g: i for i, g in enumerate(order)}
+ out.sort(
+ key = lambda item: rank.get(item[1], len(rank))
+ ) # stable: keeps each list's own order
+ return out
+
+
+def _module_device(module: Any) -> Any:
+ for t in module.parameters():
+ return t.device
+ for t in module.buffers():
+ return t.device
+ return None
+
+
+def _flex_processor(transformer: Any) -> Optional[str]:
+ try:
+ processors = getattr(transformer, "attn_processors", None) or {}
+ except Exception: # noqa: BLE001 - no processor registry: nothing flex to find
+ return None
+ for processor in processors.values():
+ name = type(processor).__name__
+ if "Flex" in name:
+ return name
+ return None
+
+
+def _diffusers_hook_names(module: Any) -> set:
+ registry = getattr(module, "_diffusers_hook", None)
+ return set(getattr(registry, "hooks", None) or {}) if registry is not None else set()
+
+
+def step_cache_hooked(transformer: Any) -> bool:
+ """Whether Studio's First-Block-Cache (``diffusion_cache``) is the ONLY diffusers hook on ``transformer``. It keeps
+ its residuals on one card, so a split removes it first (:func:`_drop_step_cache`) instead of refusing."""
+ try:
+ names = set().union(*(_diffusers_hook_names(m) for m in transformer.modules()))
+ except Exception: # noqa: BLE001 - not a torch module
+ return False
+ return bool(names) and names <= _STEP_CACHE_HOOKS
+
+
+def layer_split_refusal(transformer: Any, *, step_cache_ok: bool = False) -> Optional[str]:
+ """Why this denoiser cannot be split, else None. ``step_cache_ok``: Studio's own First-Block-Cache hooks do not
+ refuse (placement removes them)."""
+ torch = _torch()
+ if not isinstance(transformer, torch.nn.Module):
+ return "no denoiser module"
+ if getattr(transformer, "_compiled_call_impl", None) is not None:
+ return "denoiser is compiled whole"
+ cache_only = step_cache_ok and step_cache_hooked(transformer)
+ for name, sub in transformer.named_modules():
+ if getattr(sub, "_hf_hook", None) is not None:
+ return f"{name or '<root>'} carries an accelerate hook"
+ hooks = _diffusers_hook_names(sub)
+ if hooks and not cache_only:
+ return f"{name or '<root>'} carries diffusers hooks ({', '.join(sorted(hooks))})"
+ for t in (*transformer.parameters(), *transformer.buffers()):
+ if t.device.type == "meta":
+ return "denoiser has meta weights"
+ break
+ flex = _flex_processor(transformer)
+ if flex is not None:
+ # its BlockMask's mask_mod closes over tensors built on the primary, which no boundary copy reaches
+ return f"{flex} drives flex attention through a BlockMask built on the primary"
+ if len(find_blocks(transformer)) < 2:
+ return "denoiser has fewer than two repeated blocks"
+ return None
+
+
+class LayerRange:
+ device: str
+ blocks: list[str]
+ mib: int
+
+ def describe(self) -> dict:
+ return {
+ "device": self.device,
+ "count": len(self.blocks),
+ "first": self.blocks[0] if self.blocks else None,
+ "last": self.blocks[-1] if self.blocks else None,
+ "mib": self.mib,
+ }
+
+
+class LayerSplitPlan:
+ primary: str
+ rest_mib: int
+ ranges: list[LayerRange]
+ assignment: dict[str, str] # block qualified name -> device
+
+ def per_card_mib(self) -> dict[str, int]:
+ out: dict[str, int] = {self.primary: self.rest_mib}
+ for r in self.ranges:
+ out[r.device] = out.get(r.device, 0) + r.mib
+ return out
+
+
+def _cuda(ordinal_or_device: Any) -> Any:
+ torch = _torch()
+ if isinstance(ordinal_or_device, int):
+ return torch.device("cuda", ordinal_or_device)
+ dev = torch.device(ordinal_or_device)
+ if dev.type == "cuda" and dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ return dev
+
+
+def plan_layer_split(
+ transformer: Any,
+ *,
+ primary: Any,
+ cards: Sequence[Any],
+ budgets_mib: Mapping[Any, int],
+ strategy: str = "fill",
+) -> LayerSplitPlan:
+ """Contiguous block ranges over ``cards`` (primary first; added if missing), in forward order.
+
+ ``budgets_mib``: denoiser weights each card may take (keys: ordinal or device string); the primary's budget also
+ pays for the non-block modules. ``fill`` packs each card up to its budget before the next (fewest cards, one
+ crossing per used card); ``balance`` spreads the blocks in proportion to the budgets. Raises
+ :class:`LayerSplitError` when the blocks do not fit."""
+ primary_dev = _cuda(primary)
+ order = [primary_dev]
+ for c in cards:
+ dev = _cuda(c)
+ if dev not in order:
+ order.append(dev)
+ if len(order) < 2:
+ raise LayerSplitError("a layer split needs at least two cards")
+
+ def _budget(dev: Any) -> int:
+ for key in (dev.index, str(dev), dev):
+ if key in budgets_mib:
+ return int(budgets_mib[key])
+ raise LayerSplitError(f"no budget for {dev}")
+
+ mib = 1 << 20
+ blocks = find_blocks(transformer)
+ if len(blocks) < 2:
+ raise LayerSplitError("denoiser has fewer than two repeated blocks")
+ sizes = [module_bytes(b) for _, _, b in blocks]
+ rest = max(0, module_bytes(transformer) - sum(sizes))
+ budgets = [_budget(d) * mib for d in order]
+ budgets[0] -= rest
+ if budgets[0] < 0:
+ raise LayerSplitError(
+ f"the primary's budget ({_budget(order[0])} MiB) does not cover the non-block modules "
+ f"({rest // mib} MiB)"
+ )
+ total = sum(sizes)
+ if total > sum(max(0, b) for b in budgets):
+ raise LayerSplitError(
+ f"blocks need {total // mib} MiB, the cards' budgets give {sum(max(0, b) for b in budgets) // mib} MiB"
+ )
+ if strategy == "balance":
+ weight = sum(max(0, b) for b in budgets)
+ targets = [min(b, total * max(0, b) / weight) for b in budgets]
+ elif strategy == "fill":
+ targets = list(budgets)
+ else:
+ raise LayerSplitError(f"unknown strategy {strategy!r}")
+
+ assignment: dict[str, str] = {}
+ ranges: list[LayerRange] = []
+ card = 0
+ used = 0
+ current: list[str] = []
+ current_bytes = 0
+
+ def _close() -> None:
+ if current:
+ ranges.append(LayerRange(str(order[card]), list(current), current_bytes // mib))
+
+ for (qual, _, _), size in zip(blocks, sizes):
+ while card < len(order) - 1 and (
+ used + size > budgets[card] or (strategy == "balance" and used >= targets[card])
+ ):
+ _close()
+ current, current_bytes, used = [], 0, 0
+ card += 1
+ if used + size > budgets[card]:
+ raise LayerSplitError(
+ f"block {qual} ({size // mib} MiB) does not fit any remaining card"
+ )
+ assignment[qual] = str(order[card])
+ current.append(qual)
+ current_bytes += size
+ used += size
+ _close()
+ return LayerSplitPlan(str(primary_dev), rest // mib, ranges, assignment)
+
+
+def assigned_plan(
+ transformer: Any, *, primary: Any, assignment: Mapping[str, Any]
+) -> LayerSplitPlan:
+ """A plan from an explicit block -> card map (qualified names from :func:`find_blocks`; card = ordinal or device).
+ Every block must be assigned. Raises :class:`LayerSplitError` on an unknown or missing block."""
+ primary_dev = _cuda(primary)
+ mib = 1 << 20
+ blocks = find_blocks(transformer)
+ if len(blocks) < 2:
+ raise LayerSplitError("denoiser has fewer than two repeated blocks")
+ known = {qual for qual, _, _ in blocks}
+ unknown = sorted(set(assignment) - known)
+ if unknown:
+ raise LayerSplitError(f"assignment names unknown blocks ({', '.join(unknown[:3])})")
+ sizes = [module_bytes(b) for _, _, b in blocks]
+ rest = max(0, module_bytes(transformer) - sum(sizes))
+ out: dict[str, str] = {}
+ ranges: list[LayerRange] = []
+ for (qual, _, _), size in zip(blocks, sizes):
+ if qual not in assignment:
+ raise LayerSplitError(f"block {qual} has no card")
+ dev = str(_cuda(assignment[qual]))
+ out[qual] = dev
+ if ranges and ranges[-1].device == dev:
+ ranges[-1].blocks.append(qual)
+ ranges[-1].mib += size // mib
+ else:
+ ranges.append(LayerRange(dev, [qual], size // mib))
+ if len({r.device for r in ranges} | {str(primary_dev)}) < 2:
+ raise LayerSplitError("assignment keeps every block on the primary")
+ return LayerSplitPlan(str(primary_dev), rest // mib, ranges, out)
+
+
+def cross_copy(t: Any, device: Any) -> Any:
+ """Every input copy onto a block's card goes through here (the memo and the per-call hand-off): the one place to
+ redirect, e.g. into a per-card graph's static buffer."""
+ return t.to(device)
+
+
+def cross_outputs(out: Any, device: Any) -> Any:
+ """Every block output sent to another card (the next block's, or home) goes through here."""
+ from .diffusion_multigpu import move_to_device
+ return move_to_device(out, device, objects = False)
+
+
+class _Memo:
+ """Copies of one forward's tensors per card, keyed by the source's id and checked against a weak reference, its
+ data pointer and its version, so a freed or rewritten source never serves a stale copy. Inference tensors (every
+ render) keep no version, so the memo is only trusted across blocks in lazy mode, where the denoiser does no tensor
+ op of its own between blocks; an eager call clears it after every block (``_split_call``)."""
+
+ def __init__(self) -> None:
+ self.entries: dict[int, tuple[Any, int, Any, dict]] = {}
+ self.copies = 0
+
+ def clear(self) -> None:
+ self.entries.clear()
+
+ @staticmethod
+ def _version(t: Any) -> Any:
+ try:
+ return t._version
+ except Exception: # noqa: BLE001 - inference tensors keep no version counter
+ return None
+
+ def get(self, t: Any, device: Any) -> Any:
+ torch = _torch()
+ if type(t) is not torch.Tensor and not isinstance(t, torch.nn.Parameter):
+ self.copies += 1
+ return cross_copy(t, device) # tensor subclasses: no stable data pointer to key on
+ key = id(t)
+ hit = self.entries.get(key)
+ if hit is not None:
+ ref, ptr, version, copies = hit
+ if ref() is t and ptr == t.data_ptr() and version == self._version(t):
+ moved = copies.get(device)
+ if moved is None:
+ self.copies += 1
+ moved = copies[device] = cross_copy(t, device)
+ return moved
+ self.copies += 1
+ moved = cross_copy(t, device)
+ entries = self.entries
+
+ def _drop(_ref: Any, key: int = key) -> None:
+ current = entries.get(key)
+ if current is not None and current[0] is _ref:
+ entries.pop(key, None)
+
+ try:
+ ref = weakref.ref(t, _drop)
+ except TypeError:
+ return moved
+ entries[key] = (ref, t.data_ptr(), self._version(t), {device: moved})
+ return moved
+
+
+def _probe(module: Any) -> Optional[tuple]:
+ """(weak module, kind, name) of one tensor ``module`` itself or a descendant holds, read per call for its current
+ device; None when it holds none. ``skip``: ids of submodules (the blocks) whose tensors are not the module's."""
+ return _probe_outside(module, frozenset())
+
+
+def _probe_outside(module: Any, skip: frozenset) -> Optional[tuple]:
+ stack = [module]
+ while stack:
+ current = stack.pop()
+ if id(current) in skip:
+ continue
+ for kind in ("_parameters", "_buffers"):
+ for name, tensor in getattr(current, kind, {}).items():
+ if tensor is not None:
+ return (weakref.ref(current), kind, name)
+ stack.extend(reversed(list(current.children())))
+ return None
+
+
+def _probe_device(probe: Optional[tuple], default: Any) -> Any:
+ if probe is None:
+ return default
+ owner = probe[0]()
+ tensor = getattr(owner, probe[1], {}).get(probe[2]) if owner is not None else None
+ return default if tensor is None else tensor.device
+
+
+class _BlockState:
+ device: Any # the card placement put the block on (used only when the block holds no tensor)
+ next_block: (
+ Any # weak reference to the next block of the same list, or None at the end of the list
+ )
+ split: "_Split"
+ probe: Optional[tuple] = None
+
+ def current(self) -> Any:
+ return _probe_device(self.probe, self.device)
+
+
+class _Split:
+ def __init__(self, transformer: Any, home: Any, lazy_class: bool) -> None:
+ self.transformer = weakref.ref(transformer)
+ self.home = home
+ self.lazy_class = lazy_class
+ self.lazy_now = False
+ self.memo = _Memo()
+ self.handles: list = []
+ # weak: blocks hold their state, which holds this split; a strong list would make every split a cycle
+ self.blocks: list[tuple[Any, type]] = []
+ self.owns_home = False
+ self.home_probe: Optional[tuple] = None
+ self.calls = 0
+ self.crossings = 0
+ self.lazy_calls = 0
+ self.eager_calls = 0
+
+ def current_home(self) -> Any:
+ """Where the denoiser's own (non-block) weights are now: the placement's primary until something moves them."""
+ return _probe_device(self.home_probe, self.home)
+
+ def move_in(
+ self,
+ obj: Any,
+ device: Any,
+ home: Any = None,
+ ) -> Any:
+ from .diffusion_multigpu import move_to_device
+
+ home = self.home if home is None else home
+ memo = self.memo
+
+ def _tensor(t: Any) -> Any:
+ self.crossings += 1
+ if t.device == home:
+ return memo.get(t, device)
+ memo.copies += 1
+ return cross_copy(t, device)
+
+ return move_to_device(obj, device, tensor_fn = _tensor, objects = False)
+
+
+def _split_call(self: Any, *args: Any, **kwargs: Any) -> Any:
+ state: Optional[_BlockState] = self.__dict__.get(_SPLIT_ATTR)
+ base_call = type(self)._unsloth_layer_split_base.__call__
+ if state is None:
+ return base_call(self, *args, **kwargs)
+ from .diffusion_multigpu import _device_context
+
+ split = state.split
+ device = state.current()
+ home = split.current_home()
+ args, kwargs = split.move_in((args, kwargs), device, home)
+ with _device_context(device):
+ out = base_call(self, *args, **kwargs)
+ target = home if device != home else None
+ if split.lazy_now:
+ nxt = state.next_block() if state.next_block is not None else None
+ nxt_state = nxt.__dict__.get(_SPLIT_ATTR) if nxt is not None else None
+ if nxt_state is not None:
+ nxt_device = nxt_state.current()
+ target = nxt_device if nxt_device != device else None
+ else:
+ # eager: an in-place write between blocks is invisible (inference tensors carry no version), so no copy outlives its block
+ split.memo.clear()
+ if target is not None:
+ out = cross_outputs(out, target)
+ return out
+
+
+def _split_class(cls: type) -> type:
+ base = getattr(cls, "_unsloth_layer_split_base", None)
+ if base is not None:
+ return cls
+ with _SUBCLASS_LOCK:
+ sub = _SUBCLASSES.get(cls)
+ if sub is None:
+ # Same name and module: class-name checks (``_repeated_blocks``, block graph / cache lookups) still match.
+ sub = type(
+ cls.__name__,
+ (cls,),
+ {
+ "__call__": _split_call,
+ "__module__": cls.__module__,
+ "__qualname__": cls.__qualname__,
+ "_unsloth_layer_split_base": cls,
+ },
+ )
+ _SUBCLASSES[cls] = sub
+ return sub
+
+
+_POSITIONAL_NAMES: "weakref.WeakKeyDictionary[type, tuple]" = weakref.WeakKeyDictionary()
+
+
+def _positional_names(module: Any) -> tuple:
+ """Names of ``module.forward``'s positional parameters in order, cached per class."""
+ cls = type(module)
+ names = _POSITIONAL_NAMES.get(cls)
+ if names is None:
+ try:
+ params = inspect.signature(cls.forward).parameters.values()
+ names = tuple(
+ p.name
+ for p in params
+ if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.name != "self"
+ )
+ except (TypeError, ValueError):
+ names = ()
+ _POSITIONAL_NAMES[cls] = names
+ return names
+
+
+def _is_residual_name(name: Any) -> bool:
+ lowered = str(name).lower()
+ return any(part in lowered for part in _RESIDUAL_KWARG_PARTS)
+
+
+def _residual_call(
+ kwargs: Mapping[str, Any],
+ args: Sequence[Any] = (),
+ module: Any = None,
+) -> bool:
+ """The call feeds a residual the denoiser adds between blocks (ControlNet / VACE hints), by keyword or by position."""
+ for key, value in kwargs.items():
+ if value is not None and _is_residual_name(key):
+ return True
+ if args and module is not None:
+ names = _positional_names(module)
+ for name, value in zip(names, args):
+ if value is not None and _is_residual_name(name):
+ return True
+ if len(args) > len(names) and any(_is_residual_name(n) for n in names):
+ return True # *args past the named slots: cannot tell, so stay exact
+ return False
+
+
+def _move_rest(transformer: Any, block_ids: set, device: Any, logger: Any) -> None:
+ """Every parameter and buffer of ``transformer`` outside the blocks to ``device``."""
+ from .diffusion_fast_load import fast_upload
+
+ owners = []
+ stack = [transformer]
+ while stack:
+ module = stack.pop()
+ if id(module) in block_ids:
+ continue
+ owners.append(module)
+ stack.extend(module.children())
+ with fast_upload(owners, device, logger = logger):
+ for module in owners:
+ module._apply(lambda t: t.to(device), recurse = False)
+
+
+def _move_weights(
+ transformer: Any, plan: LayerSplitPlan, blocks: list, home: Any, logger: Any
+) -> None:
+ """Each block once to its card (blocks first, so the primary only ever holds its own share), then the rest."""
+ torch = _torch()
+ from .diffusion_fast_load import fast_upload
+
+ by_device: dict[str, list] = {}
+ for qual, _, block in blocks:
+ by_device.setdefault(plan.assignment[qual], []).append(block)
+ for dev_name, mods in by_device.items():
+ dev = torch.device(dev_name)
+ with fast_upload(mods, dev, logger = logger):
+ for block in mods:
+ block.to(dev)
+ _move_rest(transformer, {id(b) for _, _, b in blocks}, home, logger)
+
+
+def _install_boundary(transformer: Any, plan: LayerSplitPlan, blocks: list, split: _Split) -> None:
+ torch = _torch()
+ for i, (qual, group, block) in enumerate(blocks):
+ dev = torch.device(plan.assignment[qual])
+ nxt = blocks[i + 1] if i + 1 < len(blocks) else None
+ nxt_ref = weakref.ref(nxt[2]) if nxt is not None and nxt[1] == group else None
+ original = type(block)
+ block.__dict__[_SPLIT_ATTR] = _BlockState(dev, nxt_ref, split, _probe(block))
+ block.__class__ = _split_class(original)
+ split.blocks.append(
+ (weakref.ref(block), getattr(original, "_unsloth_layer_split_base", original))
+ )
+ split.home_probe = _probe_outside(transformer, frozenset(id(b) for _, _, b in blocks))
+
+ def _pre(module: Any, args: Any, kwargs: Any) -> None:
+ split.memo.clear()
+ split.calls += 1
+ split.lazy_now = split.lazy_class and not _residual_call(kwargs, args, module)
+ if split.lazy_now:
+ split.lazy_calls += 1
+ else:
+ split.eager_calls += 1
+
+ def _post(module: Any, args: Any, kwargs: Any, output: Any) -> None:
+ split.memo.clear()
+ split.lazy_now = False
+
+ split.handles.append(transformer.register_forward_pre_hook(_pre, with_kwargs = True))
+ split.handles.append(
+ transformer.register_forward_hook(_post, with_kwargs = True, always_call = True)
+ )
+ transformer.__dict__[_SPLIT_ATTR + "_root"] = split
+ split.owns_home = "_unsloth_split_home" not in transformer.__dict__
+ transformer._unsloth_split_home = split.home
+
+
+def layer_split_of(module: Any) -> Optional[_Split]:
+ if module is None:
+ return None
+ return getattr(module, "__dict__", {}).get(_SPLIT_ATTR + "_root")
+
+
+def is_layer_split(module: Any) -> bool:
+ return layer_split_of(module) is not None
+
+
+def _drop_step_cache(pipe: Any, attr: str, transformer: Any, logger: Any) -> bool:
+ """Take Studio's First-Block-Cache off ``transformer`` before a split (it keeps residuals on one card and its
+ hooks own the blocks' forward). The load re-reads ``_unsloth_step_cache`` afterwards, so status reports it off.
+ Raises :class:`LayerSplitError` when the hooks cannot be verified gone. True when a cache was removed."""
+ if not step_cache_hooked(transformer):
+ return False
+ from . import diffusion_cache
+
+ view = pipe if attr == "transformer" else type("_View", (), {"transformer": transformer})()
+ diffusion_cache.maybe_toggle_step_cache(view, steps = 0, logger = logger)
+ if step_cache_hooked(transformer) or getattr(transformer, "_unsloth_step_cache", None):
+ raise LayerSplitError("the step cache on the denoiser could not be removed")
+ if logger is not None:
+ logger.info(
+ "diffusion.layer_split: step cache removed from %s (it keeps residuals on one card)",
+ attr,
+ )
+ return True
+
+
+def place_layer_split(
+ pipe: Any,
+ *,
+ primary: Any,
+ cards: Sequence[Any] = (),
+ budgets_mib: Optional[Mapping[Any, int]] = None,
+ strategy: str = "fill",
+ attr: str = "transformer",
+ assignment: Optional[Mapping[str, Any]] = None,
+ logger: Any = None,
+) -> dict[str, Any]:
+ """Split ``pipe.<attr>`` across ``cards`` (see :func:`plan_layer_split`), or by an explicit block -> card
+ ``assignment`` (see :func:`assigned_plan`), and place it. Other components are left where they are. Returns the placement summary also shown by ``placement_status``. Raises :class:`LayerSplitError`
+ (nothing moved) when the split is refused or does not fit; on a failure while moving, the boundary is removed and
+ the error re-raised, and the caller re-places the pipeline."""
+ torch = _torch()
+ from . import diffusion_multigpu as mg
+
+ if layer_split_mode() == "off":
+ raise LayerSplitError(f"{LAYER_SPLIT_ENV}=0 or {mg.MULTIGPU_ENV}=0")
+ transformer = getattr(pipe, attr, None)
+ why = layer_split_refusal(transformer, step_cache_ok = True)
+ if why is not None:
+ raise LayerSplitError(why)
+ if is_layer_split(transformer):
+ undo_layer_split(pipe, attr = attr)
+ if assignment is not None:
+ assigned_plan(
+ transformer, primary = primary, assignment = assignment
+ ) # validate before touching the cache
+ else:
+ plan_layer_split(
+ transformer, primary = primary, cards = cards, budgets_mib = budgets_mib, strategy = strategy
+ )
+ _drop_step_cache(pipe, attr, transformer, logger)
+ if assignment is not None:
+ plan = assigned_plan(transformer, primary = primary, assignment = assignment)
+ strategy = "assigned"
+ else:
+ plan = plan_layer_split(
+ transformer, primary = primary, cards = cards, budgets_mib = budgets_mib, strategy = strategy
+ )
+ home = torch.device(plan.primary)
+ blocks = find_blocks(transformer)
+ split = _Split(transformer, home, type(transformer).__name__ in LAZY_CLASSES)
+ try:
+ _move_weights(transformer, plan, blocks, home, logger)
+ _install_boundary(transformer, plan, blocks, split)
+ mg._SPLIT_EVER_PLACED = True
+ mg.install_execution_device_patch()
+ except Exception:
+ _remove_boundary(transformer, split)
+ raise
+
+ info = {
+ "primary": plan.primary,
+ "strategy": strategy,
+ "denoiser": attr,
+ "rest_mib": plan.rest_mib,
+ "ranges": [r.describe() for r in plan.ranges],
+ "per_card_mib": plan.per_card_mib(),
+ "lazy": split.lazy_class,
+ }
+ split.info = info # type: ignore[attr-defined]
+ try:
+ record = mg._PLACEMENTS.get(pipe)
+ if record is None:
+ record = {
+ "placement": {
+ str(n): str(_module_device(m))
+ for n, m in (getattr(pipe, "components", {}) or {}).items()
+ if isinstance(m, torch.nn.Module) and m is not transformer
+ },
+ "primary": plan.primary,
+ }
+ mg._PLACEMENTS[pipe] = record
+ record.setdefault("placement", {})[attr] = "+".join(r.device for r in plan.ranges)
+ record["layer_split"] = info
+ except TypeError: # not weak-referenceable
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.layer_split: %s split %s",
+ type(transformer).__name__,
+ ", ".join(f"{r.device}: {len(r.blocks)} blocks {r.mib} MiB" for r in plan.ranges),
+ )
+ return info
+
+
+def _remove_boundary(transformer: Any, split: _Split) -> None:
+ for handle in split.handles:
+ try:
+ handle.remove()
+ except Exception: # noqa: BLE001
+ pass
+ split.handles.clear()
+ for ref, base in split.blocks:
+ block = ref()
+ if block is None:
+ continue
+ block.__dict__.pop(_SPLIT_ATTR, None)
+ if getattr(type(block), "_unsloth_layer_split_base", None) is base:
+ block.__class__ = base
+ split.blocks.clear()
+ split.memo.clear()
+ if transformer is not None:
+ transformer.__dict__.pop(_SPLIT_ATTR + "_root", None)
+ if split.owns_home:
+ transformer.__dict__.pop("_unsloth_split_home", None)
+
+
+def undo_layer_split(
+ pipe: Any,
+ *,
+ attr: str = "transformer",
+ device: Any = None,
+) -> bool:
+ """Remove the boundary (weights stay where they are unless ``device`` is given, then the denoiser moves there).
+ True when a split was removed."""
+ from . import diffusion_multigpu as mg
+
+ transformer = getattr(pipe, attr, None)
+ split = layer_split_of(transformer)
+ if split is None:
+ return False
+ _remove_boundary(transformer, split)
+ try:
+ record = mg._PLACEMENTS.get(pipe)
+ except TypeError:
+ record = None
+ if record is not None:
+ record.pop("layer_split", None)
+ if not record.get("moved"): # no component split under it: the record was ours
+ mg._PLACEMENTS.pop(pipe, None)
+ else:
+ record.get("placement", {}).pop(attr, None)
+ if device is not None:
+ transformer.to(device)
+ return True
+
+
+def layer_split_stats(module: Any) -> Optional[dict]:
+ split = layer_split_of(module)
+ if split is None:
+ return None
+ out = dict(getattr(split, "info", {}) or {})
+ out.update(
+ {
+ "calls": split.calls,
+ "lazy_calls": split.lazy_calls,
+ "eager_calls": split.eager_calls,
+ "input_handoffs": split.crossings,
+ "input_copies": split.memo.copies,
+ }
+ )
+ return out
diff --git a/studio/backend/core/inference/diffusion_layer_split_plan.py b/studio/backend/core/inference/diffusion_layer_split_plan.py
new file mode 100644
index 00000000000..3a8656a119a
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_layer_split_plan.py
@@ -0,0 +1,854 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Layer split planner: a denoiser too big for one card, cut into contiguous block ranges across the selected cards.
+
+Order of preference on a multi-card selection: companion split (diffusion_multigpu), then a layer split, then the
+one-card offload plan. A layer split is only considered when the one-card plan offloads the DENOISER itself and the
+companion assignment found nothing; it is priced with the same per-card reserves as the companion split. The VAE and
+every non-text-encoder companion stay on the primary (the decode and its tile verdict stay where today's guard reads
+them); the text encoders may move to any used card. One card, Keep multiple models loaded, ROCm and the kill switches
+keep today's plan unchanged.
+
+The engine (block placement + device-crossing hooks) is a separate piece; this module only defines the interface it
+must provide (``split_blocks``, ``engine_available``, ``place_layer_split``, ``undo_layer_split``) with stubs that keep
+production behaviour unchanged until it lands.
+
+``UNSLOTH_DIFFUSION_LAYER_SPLIT=0`` turns only this off; ``force`` also cuts a denoiser one card would hold resident
+(for proving the engine on cards that are too big to need it).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import sys
+import weakref
+from typing import Any, Callable, Mapping, Optional, Sequence
+
+LAYER_SPLIT_ENV = "UNSLOTH_DIFFUSION_LAYER_SPLIT"
+_TE_ROLE = "text_encoder"
+_DIT_ROLE = "dit"
+_DENOISER_OFFLOAD_POLICIES = ("model", "sequential", "streaming")
+# A block-only secondary holds one block's working set (B200 1024px: 0.6 GiB klein-4B, 1.2 GiB FLUX.2-dev):
+# priced as twice the largest block, at least 1 GiB, never above the plan's denoise headroom.
+_WORKING_SET_BLOCKS = 2
+_WORKING_SET_MIN_MIB = 1024
+_TE_ONLY_HEADROOM_MIB = (
+ 1024 # diffusion_multigpu.TEXT_ENCODER_HEADROOM_MIB: an encoder-only phase on a card
+)
+
+
+def secondary_reserve_mib() -> int:
+ """ComfyUI's per-card floor (``minimum_inference_memory``: 0.8 GiB + EXTRA_RESERVED_VRAM, 400 MiB on Linux, 600 MiB
+ on Windows plus 100 MiB on a 16 GB+ card, taken here unconditionally) for a card that only runs blocks. The primary
+ keeps Studio's base overhead when that is larger."""
+ return 800 + (700 if sys.platform == "win32" else 400)
+
+
+def block_working_set_mib(
+ denoisers: Mapping[str, "DenoiserLayout"], denoise_headroom_mib: int
+) -> int:
+ largest = max((max(d.block_mib) for d in denoisers.values() if d.block_mib), default = 0)
+ estimate = max(_WORKING_SET_MIN_MIB, _WORKING_SET_BLOCKS * int(largest))
+ return min(estimate, int(denoise_headroom_mib)) if denoise_headroom_mib > 0 else estimate
+
+
+def layer_split_mode() -> str:
+ """``off`` / ``auto`` (default) / ``force``; off whenever the whole multi-GPU lever is off."""
+ from .diffusion_layer_split import layer_split_mode as engine_mode
+ return engine_mode()
+
+
[email protected](frozen = True)
+class BlockRange:
+ denoiser: str
+ start: int
+ stop: int
+ card: int
+ mib: int
+
+
[email protected](frozen = True)
+class LayerSplit:
+ primary: int
+ cards: tuple[int, ...]
+ ranges: tuple[BlockRange, ...]
+ text_encoders_card: Optional[int]
+ card_required_mib: tuple[tuple[int, int], ...]
+ card_limit_mib: tuple[tuple[int, int], ...]
+ card_weights_mib: tuple[tuple[int, int], ...] = ()
+ reason: str = ""
+ text_encoders_streamed: bool = False
+ card_reserve_mib: tuple[tuple[int, int], ...] = ()
+
+ def working_mib(self, ordinal: int) -> tuple[int, int]:
+ """(activation working set, fixed reserve) the plan held on card ``ordinal`` beside its weights, at the default
+ request. A request larger than that scales the first only."""
+ required = dict(self.card_required_mib).get(ordinal, 0)
+ weights = dict(self.card_weights_mib).get(ordinal, 0)
+ reserve = dict(self.card_reserve_mib).get(ordinal, 0)
+ return max(0, int(required) - int(weights) - int(reserve)), int(reserve)
+
+ @property
+ def boundaries(self) -> int:
+ """Device crossings per pass of every denoiser, the return of the output to the primary included."""
+ out = 0
+ for name in dict.fromkeys(r.denoiser for r in self.ranges):
+ path = [
+ self.primary,
+ *(r.card for r in self.ranges if r.denoiser == name),
+ self.primary,
+ ]
+ out += sum(1 for a, b in zip(path, path[1:]) if a != b)
+ return out
+
+ def card_of(self, denoiser: str, index: int) -> Optional[int]:
+ for r in self.ranges:
+ if r.denoiser == denoiser and r.start <= index < r.stop:
+ return r.card
+ return None
+
+ def as_public_dict(self) -> dict[str, Any]:
+ return {
+ "primary": self.primary,
+ "cards": list(self.cards),
+ "ranges": [dataclasses.asdict(r) for r in self.ranges],
+ "text_encoders_card": self.text_encoders_card,
+ "text_encoders_streamed": self.text_encoders_streamed,
+ "card_required_mib": {str(o): m for o, m in self.card_required_mib},
+ "card_limit_mib": {str(o): m for o, m in self.card_limit_mib},
+ "card_weights_mib": {str(o): m for o, m in self.card_weights_mib},
+ "card_reserve_mib": {str(o): m for o, m in self.card_reserve_mib},
+ "boundaries": self.boundaries,
+ "reason": self.reason,
+ }
+
+
[email protected](frozen = True)
+class DenoiserLayout:
+ """Storage MiB of one denoiser: ``stem`` (everything outside the sequential blocks) and each block in order."""
+
+ stem_mib: int
+ block_mib: tuple[int, ...]
+
+
+def _partition(blocks: Sequence[int], capacities: Sequence[int]) -> Optional[list[int]]:
+ """Cut points (len(capacities) + 1, first 0, last len(blocks)) of a contiguous partition with part i <= capacity i,
+ proportional to the capacities when that fits, else greedy first-fit. None when nothing fits."""
+ n = len(blocks)
+ total = sum(blocks)
+ cap_sum = sum(max(0, c) for c in capacities)
+ if n == 0:
+ return [0] * (len(capacities) + 1)
+ if cap_sum < total:
+ return None
+
+ def check(cuts: list[int]) -> bool:
+ return all(
+ sum(blocks[cuts[i] : cuts[i + 1]]) <= capacities[i] for i in range(len(capacities))
+ )
+
+ cuts = [0]
+ acc = 0
+ prefix = [0]
+ for b in blocks:
+ prefix.append(prefix[-1] + b)
+ for i, cap in enumerate(capacities[:-1]):
+ acc += max(0, cap)
+ target = total * acc / cap_sum
+ best = min(range(cuts[-1], n + 1), key = lambda j: (abs(prefix[j] - target), j))
+ cuts.append(best)
+ cuts.append(n)
+ if check(cuts):
+ return cuts
+ cuts = [0]
+ for cap in capacities:
+ j = cuts[-1]
+ used = 0
+ while j < n and used + blocks[j] <= cap:
+ used += blocks[j]
+ j += 1
+ cuts.append(j)
+ if cuts[-1] != n:
+ return None
+ return cuts
+
+
+_SEARCH_BUDGET = 200_000
+
+
+def _search_cuts(
+ layouts: Sequence[Sequence[int]], capacities: Sequence[int]
+) -> Optional[list[list[int]]]:
+ """Cut points for several denoisers sharing the cards, by backtracking: each denoiser contiguous over the cards in
+ order, at least one keeping a block off the primary (a denoiser wholly on the primary is placed there unsplit),
+ each card's parts within its capacity. The fallback when cutting each denoiser in turn by proportion leaves a later one no fit.
+ None when nothing fits within the search budget."""
+ k = len(capacities)
+ prefixes = []
+ for blocks in layouts:
+ prefix = [0]
+ for b in blocks:
+ prefix.append(prefix[-1] + int(b))
+ prefixes.append(prefix)
+ visited = [0]
+
+ def options(d: int, remaining: list[int]):
+ prefix = prefixes[d]
+ n = len(prefix) - 1
+
+ def rec(i: int, start: int, cuts: list[int]):
+ if i == k - 1:
+ if prefix[n] - prefix[start] <= remaining[i]:
+ yield [*cuts, n]
+ return
+ for j in range(start, n + 1):
+ if prefix[j] - prefix[start] > remaining[i]:
+ break
+ yield from rec(i + 1, j, [*cuts, j])
+
+ yield from rec(0, 0, [0])
+
+ def dfs(d: int, remaining: list[int], split: bool) -> Optional[list[list[int]]]:
+ if d == len(layouts):
+ return [] if split else None
+ prefix = prefixes[d]
+ n = len(prefix) - 1
+ for cuts in options(d, remaining):
+ visited[0] += 1
+ if visited[0] > _SEARCH_BUDGET:
+ return None
+ left = [remaining[i] - (prefix[cuts[i + 1]] - prefix[cuts[i]]) for i in range(k)]
+ rest = dfs(d + 1, left, split or cuts[1] < n)
+ if rest is not None:
+ return [cuts, *rest]
+ return None
+
+ return dfs(0, [int(c) for c in capacities], False)
+
+
+def plan_layer_split(
+ *,
+ denoisers: Mapping[str, DenoiserLayout],
+ companions: Mapping[str, tuple[int, str]],
+ primary: int,
+ card_limits: Mapping[int, int],
+ headroom_mib: int,
+ overhead_mib: int,
+ phase_headroom_mib: Optional[tuple[int, int, int]] = None,
+ secondary_overhead_mib: Optional[int] = None,
+ working_set_mib: Optional[int] = None,
+) -> tuple[Optional[LayerSplit], str]:
+ """Pure decision: ``(LayerSplit, reason)`` or ``(None, decline reason)``.
+
+ ``card_limits`` maps every usable card (the primary included) to its resident limit. Fewest cards first: the
+ primary, then the secondaries by limit (largest first). ``phase_headroom_mib`` = ``(encode, denoise, decode)``.
+
+ The primary holds the stems, the VAE, the latents and the activation peak: it is charged the plan's full headroom
+ and ``max(overhead_mib, secondary reserve)``. A secondary runs blocks only: one block's working set
+ (``working_set_mib``, default :func:`block_working_set_mib`) plus ``secondary_overhead_mib`` (default
+ :func:`secondary_reserve_mib`), and the encode headroom when the text encoders live there."""
+ if primary not in card_limits:
+ return None, "primary card limit unknown"
+ if not denoisers or any(not d.block_mib for d in denoisers.values()):
+ return None, "denoiser blocks unknown"
+ secondaries = sorted(
+ (o for o in card_limits if o != primary), key = lambda o: (-card_limits[o], o)
+ )
+ if not secondaries:
+ return None, "no secondary card"
+ stems = sum(d.stem_mib for d in denoisers.values())
+ te_mib = sum(m for m, role in companions.values() if role == _TE_ROLE)
+ other_mib = sum(m for m, role in companions.values() if role not in (_TE_ROLE, _DIT_ROLE))
+ if phase_headroom_mib is not None:
+ encode, denoise, decode = (int(x or 0) for x in phase_headroom_mib)
+ else:
+ denoise = decode = int(headroom_mib)
+ encode = _TE_ONLY_HEADROOM_MIB
+ sec_reserve = (
+ secondary_reserve_mib() if secondary_overhead_mib is None else int(secondary_overhead_mib)
+ )
+ pri_reserve = max(int(overhead_mib), sec_reserve)
+ working = (
+ block_working_set_mib(denoisers, denoise)
+ if working_set_mib is None
+ else int(working_set_mib)
+ )
+ reserve = {primary: pri_reserve, **{o: sec_reserve for o in secondaries}}
+ blocks_total = sum(sum(d.block_mib) for d in denoisers.values())
+ need_total = blocks_total + stems + te_mib + other_mib
+ best_shortfall: Optional[int] = None
+ for k in range(2, len(secondaries) + 2):
+ cards = [primary, *secondaries[: k - 1]]
+ # TE home: last card first, primary last; streaming the TE from host (one paged encode per prompt) is the last resort
+ te_homes = (
+ [(o, False) for o in reversed(cards)] + [(None, True)] if te_mib else [(None, False)]
+ )
+ for te_home, te_streamed in te_homes:
+ fixed = {o: 0 for o in cards}
+ head = {o: working for o in cards}
+ fixed[primary] += stems + other_mib
+ head[primary] = max(denoise, decode)
+ if te_home is not None:
+ fixed[te_home] += te_mib
+ head[te_home] = max(head[te_home], encode)
+ elif te_streamed:
+ head[primary] = max(head[primary], encode)
+ caps = [card_limits[o] - fixed[o] - head[o] - reserve[o] for o in cards]
+ if any(c < 0 for c in caps):
+ continue
+ ranges: list[BlockRange] = []
+ remaining = list(caps)
+ ok = True
+ names = list(denoisers)
+ for idx, name in enumerate(names):
+ layout = denoisers[name]
+ left = sum(sum(denoisers[n].block_mib) for n in names[idx:])
+ share = sum(layout.block_mib) / left if left else 1.0
+ caps_d = [int(r * share) if idx < len(names) - 1 else r for r in remaining]
+ cuts = _partition(layout.block_mib, caps_d) or _partition(
+ layout.block_mib, remaining
+ )
+ n = len(layout.block_mib)
+ if cuts is not None and n > 1 and cuts[1] == n:
+ # the cut rounded every block onto the primary, which the engine refuses: move its last block to the furthest card with room
+ room = [
+ c
+ for c in range(len(cards) - 1, 0, -1)
+ if layout.block_mib[-1] <= remaining[c]
+ ]
+ cuts = (
+ [0, *([n - 1] * room[0]), *([n] * (len(cards) - room[0]))] if room else None
+ )
+ if cuts is None:
+ ok = False
+ break
+ for i, card in enumerate(cards):
+ lo, hi = cuts[i], cuts[i + 1]
+ if hi > lo:
+ mib = sum(layout.block_mib[lo:hi])
+ ranges.append(BlockRange(name, lo, hi, card, mib))
+ remaining[i] -= mib
+ if not ok and len(names) > 1:
+ # cut in turn, an earlier denoiser can take the room a later one needed: search the cuts jointly
+ joint = _search_cuts([denoisers[n].block_mib for n in names], caps)
+ if joint is not None:
+ ok, ranges, remaining = True, [], list(caps)
+ for name, cuts in zip(names, joint):
+ blocks = denoisers[name].block_mib
+ for i, card in enumerate(cards):
+ lo, hi = cuts[i], cuts[i + 1]
+ if hi > lo:
+ mib = sum(blocks[lo:hi])
+ ranges.append(BlockRange(name, lo, hi, card, mib))
+ remaining[i] -= mib
+ if not ok:
+ continue
+ used = {r.card for r in ranges} | {primary}
+ if te_home is not None:
+ used.add(te_home)
+ if len(used) < 2:
+ continue # everything landed on the primary: no split needed (and the caller only asks when it is)
+ required = {
+ o: card_limits[o] - caps[i] + (caps[i] - remaining[i])
+ for i, o in enumerate(cards)
+ if o in used
+ }
+ split = LayerSplit(
+ primary = primary,
+ cards = tuple(o for o in cards if o in used),
+ ranges = tuple(ranges),
+ text_encoders_card = te_home,
+ text_encoders_streamed = te_streamed,
+ card_required_mib = tuple(sorted(required.items())),
+ card_limit_mib = tuple(sorted((o, int(card_limits[o])) for o in used)),
+ card_weights_mib = tuple(
+ sorted(
+ (o, fixed[o] + caps[i] - remaining[i])
+ for i, o in enumerate(cards)
+ if o in used
+ )
+ ),
+ card_reserve_mib = tuple(sorted((o, int(reserve[o])) for o in used)),
+ )
+ per_card = ", ".join(
+ f"cuda:{o} {required[o]}/{card_limits[o]} MiB" for o in split.cards
+ )
+ reason = f"multi-GPU layer split: {', '.join(names)} cut over {len(split.cards)} cards ({per_card})"
+ return dataclasses.replace(split, reason = reason), reason
+ available = (
+ sum(card_limits[o] - reserve[o] for o in cards)
+ - max(denoise, decode)
+ - (k - 1) * working
+ )
+ shortfall = need_total - available
+ best_shortfall = shortfall if best_shortfall is None else min(best_shortfall, shortfall)
+ return None, (
+ f"layer split does not fit the selected cards ({need_total} MiB of weights, "
+ f"short by about {max(0, best_shortfall or 0)} MiB after the per-card reserves)"
+ )
+
+
+_BLOCK_LIST_ATTRS: tuple[str, ...] = (
+ "transformer_blocks",
+ "single_transformer_blocks",
+ "blocks",
+ "layers",
+)
+
+
+def split_blocks(module: Any) -> Optional[list[tuple[str, Any]]]:
+ """The denoiser's repeated blocks in forward order as ``(qualified name, module)`` (the engine's own list, so a
+ plan's indices name the blocks it places); None when the engine cannot cut this denoiser."""
+ from .diffusion_layer_split import find_blocks, layer_split_refusal
+ try:
+ if layer_split_refusal(module, step_cache_ok = True) is not None:
+ return None
+ return [(qual, block) for qual, _, block in find_blocks(module)] or None
+ except Exception: # noqa: BLE001 - no torch, or a module the engine cannot walk
+ return None
+
+
+def engine_available() -> bool:
+ """Whether the layer split engine can run here (CUDA torch)."""
+ try:
+ import torch
+ return bool(torch.cuda.is_available())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _denoisers(split: LayerSplit) -> list[str]:
+ return list(dict.fromkeys(r.denoiser for r in split.ranges))
+
+
+def place_layer_split(
+ pipe: Any,
+ *,
+ primary: str,
+ split: LayerSplit,
+ logger: Any = None,
+) -> dict:
+ """Place every block range on its card, the stems and non-TE companions on ``primary``, the text encoders on
+ ``split.text_encoders_card``, and install the device-crossing hooks. Raises on failure."""
+ import torch
+
+ from . import diffusion_multigpu as mg
+ from .diffusion_fast_load import fast_upload
+ from .diffusion_layer_split import LayerSplitError
+ from .diffusion_layer_split import place_layer_split as engine_place
+
+ primary_dev = torch.device(primary)
+ if primary_dev.type == "cuda" and primary_dev.index is None:
+ primary_dev = torch.device("cuda", torch.cuda.current_device())
+ names = _denoisers(split)
+ te_card = split.text_encoders_card
+ te_dev = torch.device("cuda", int(te_card)) if te_card is not None else primary_dev
+ components = {
+ str(n): m
+ for n, m in (getattr(pipe, "components", {}) or {}).items()
+ if isinstance(m, torch.nn.Module) and str(n) not in names
+ }
+ streamed = (
+ [n for n in components if n.startswith("text_encoder")]
+ if split.text_encoders_streamed
+ else []
+ )
+ moved = [
+ n
+ for n in components
+ if n.startswith("text_encoder") and te_dev != primary_dev and n not in streamed
+ ]
+ placement: dict[str, str] = {}
+ for name, module in components.items():
+ if name in streamed:
+ continue
+ device = te_dev if name in moved else primary_dev
+ with fast_upload([module], device, logger = logger):
+ module.to(device)
+ placement[name] = str(device)
+ if streamed:
+ from .diffusion_memory import stream_text_encoders_onto
+ for name in stream_text_encoders_onto(pipe, primary_dev, logger):
+ placement[name] = f"cpu (streamed to {primary_dev})"
+ for name in moved:
+ mg.install_boundary(components[name], primary_dev)
+ if moved or streamed:
+ for name, module in components.items():
+ if name not in moved and name not in streamed:
+ module._unsloth_split_home = primary_dev
+ with contextlib.suppress(TypeError):
+ mg._PLACEMENTS[pipe] = {
+ "placement": placement,
+ "primary": str(primary_dev),
+ "secondary": str(te_dev) if moved else None,
+ "moved": moved,
+ "streamed": streamed,
+ }
+ out: dict[str, Any] = {}
+ for name in names:
+ module = getattr(pipe, name, None)
+ blocks = split_blocks(module)
+ if not blocks:
+ raise LayerSplitError(f"{name}: no blocks to split")
+ assignment: dict[str, int] = {}
+ for r in split.ranges:
+ if r.denoiser == name:
+ for qual, _ in blocks[r.start : r.stop]:
+ assignment[qual] = int(r.card)
+ if assignment and all(card == primary_dev.index for card in assignment.values()):
+ with fast_upload([module], primary_dev, logger = logger):
+ module.to(primary_dev)
+ out[name] = {"primary": str(primary_dev), "denoiser": name, "whole_on_primary": True}
+ continue
+ out[name] = engine_place(
+ pipe, primary = primary_dev, attr = name, assignment = assignment, logger = logger
+ )
+ return out
+
+
+def undo_layer_split(pipe: Any) -> None:
+ """Remove the crossing hooks a (partial) ``place_layer_split`` installed. Never raises."""
+ from . import diffusion_multigpu as mg
+ from .diffusion_layer_split import undo_layer_split as engine_undo
+
+ try:
+ record = dict(
+ mg._PLACEMENTS.get(pipe) or {}
+ ) # read first: the engine's undo drops a record it owns
+ except TypeError:
+ record = {}
+ for name in mg.DENOISER_ATTRS:
+ with contextlib.suppress(Exception):
+ engine_undo(pipe, attr = name)
+ with contextlib.suppress(Exception):
+ if record.get("streamed"):
+ from .diffusion_memory import _remove_group_offload_hooks
+ for name in record["streamed"]:
+ module = getattr(pipe, name, None)
+ if module is not None:
+ _remove_group_offload_hooks(module)
+ with contextlib.suppress(Exception):
+ mg.undo_split(pipe)
+
+
+_APPLIED: "weakref.WeakKeyDictionary[Any, LayerSplit]" = weakref.WeakKeyDictionary()
+_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()
+
+
+def _record_decline(pipe: Any, reason: str, logger: Any) -> None:
+ with contextlib.suppress(TypeError):
+ _DECLINES[pipe] = reason
+ if logger is not None:
+ logger.info("diffusion.multigpu: layer split not used (%s)", reason)
+
+
+def _is_rocm() -> bool:
+ try:
+ import torch
+ return bool(getattr(torch.version, "hip", None))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def plan_offloads_denoiser(plan: Any) -> bool:
+ """The one-card plan pages the denoiser's weights per call (what a layer split exists to remove)."""
+ policy = getattr(plan, "offload_policy", "none")
+ if policy in _DENOISER_OFFLOAD_POLICIES:
+ return True
+ return policy == "group" and bool(getattr(plan, "stream_transformer", True))
+
+
+def _denoiser_layouts(pipe: Any) -> Optional[dict[str, DenoiserLayout]]:
+ from .diffusion_memory import _storage_nbytes
+ from .diffusion_multigpu import DENOISER_ATTRS
+
+ mib = 1 << 20
+ out: dict[str, DenoiserLayout] = {}
+ for name in DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ if module is None:
+ continue
+ blocks = split_blocks(module)
+ if not blocks:
+ return None
+ seen: set[int] = set()
+
+ def size(m: Any) -> int:
+ nbytes = 0
+ for t in (*m.parameters(), *m.buffers()):
+ if id(t) in seen:
+ continue
+ seen.add(id(t))
+ nbytes += sum(_storage_nbytes(t))
+ return nbytes
+
+ block_bytes = [size(b) for _, b in blocks]
+ stem_bytes = size(module) # what is left after the blocks claimed theirs
+ out[name] = DenoiserLayout(
+ stem_mib = -(-stem_bytes // mib), block_mib = tuple(-(-b // mib) for b in block_bytes)
+ )
+ return out or None
+
+
+def _layout_refusal(pipe: Any) -> str:
+ """The engine's reason the first unsplittable denoiser of ``pipe`` cannot be cut (for the decline status)."""
+ from .diffusion_layer_split import layer_split_refusal
+ from .diffusion_multigpu import DENOISER_ATTRS
+
+ for name in DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ if module is None:
+ continue
+ try:
+ why = layer_split_refusal(module, step_cache_ok = True)
+ except Exception as exc: # noqa: BLE001
+ why = f"{type(exc).__name__}: {exc}"
+ if why is not None:
+ return f"{name}: {why}"
+ return "no sequential block list the engine can cut"
+
+
+# Backends gated on the primary's arch only (flash3, flash4, Sage, FA2): split only onto identical cards.
+_PRIMARY_ARCH_BACKENDS = frozenset({"_flash_3_hub", "flash_4_hub", "sage", "sage_hub", "flash"})
+
+
+def _backend_cards(
+ pipe: Any, primary: int, secondaries: Sequence[int]
+) -> tuple[tuple[int, ...], Optional[str]]:
+ """``(secondaries the engaged attention backend runs on, that backend)``; a card of unknown capability is kept."""
+ from . import diffusion_multigpu as mg
+
+ backend = None
+ with contextlib.suppress(Exception):
+ from .diffusion_attention import ATTENTION_BACKEND_ATTR
+ for name in mg.DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ tag = getattr(module, ATTENTION_BACKEND_ATTR, None) if module is not None else None
+ if tag in _PRIMARY_ARCH_BACKENDS:
+ backend = tag
+ break
+ cards = tuple(int(o) for o in secondaries)
+ if backend is None:
+ return cards, None
+ need = mg._capability(primary)
+ if need is None:
+ return cards, backend
+ return tuple(o for o in cards if (mg._capability(o) or need) == need), backend
+
+
+def refine_plan_for_layer_split(
+ pipe: Any,
+ plan: Any,
+ *,
+ selection: Any,
+ sizes: Mapping[str, tuple[int, str]],
+ primary_limit_mib: int,
+ secondary_limit_mib: Callable[[int], Optional[int]],
+ headroom_mib: int,
+ overhead_mib: int,
+ logger: Any = None,
+) -> Any:
+ """Called by ``refine_plan_for_multigpu`` once the companion assignment found nothing (or under ``force``). The
+ plan unchanged unless a layer split fits; then a resident plan carrying ``layer_split`` and the one-card plan as
+ ``fallback_plan``. The caller has already checked the selection, mode, explicit offload and CUDA gates."""
+ mode = layer_split_mode()
+ if mode == "off" or not engine_available():
+ return plan
+ if not plan_offloads_denoiser(plan) and mode != "force":
+ return plan
+ if _is_rocm():
+ _record_decline(pipe, "ROCm (layer split is CUDA only)", logger)
+ return plan
+ layouts = _denoiser_layouts(pipe)
+ if layouts is None:
+ _record_decline(pipe, f"denoiser cannot be split ({_layout_refusal(pipe)})", logger)
+ return plan
+ limits: dict[int, int] = {int(selection.primary): int(primary_limit_mib)}
+ secondaries, backend = _backend_cards(pipe, int(selection.primary), selection.secondaries)
+ if not secondaries:
+ _record_decline(
+ pipe,
+ f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",
+ logger,
+ )
+ return plan
+ for o in secondaries:
+ limit = secondary_limit_mib(int(o))
+ if limit is not None:
+ limits[int(o)] = int(limit)
+ companions = {n: v for n, v in sizes.items() if v[1] != _DIT_ROLE}
+ split, reason = plan_layer_split(
+ denoisers = layouts,
+ companions = companions,
+ primary = int(selection.primary),
+ card_limits = limits,
+ headroom_mib = int(headroom_mib),
+ overhead_mib = int(overhead_mib),
+ )
+ if split is None:
+ _record_decline(pipe, reason, logger)
+ return plan
+ reason = f"{reason}; single card would use '{plan.offload_policy}'"
+ if logger is not None:
+ logger.info("diffusion.multigpu: %s", reason)
+ with contextlib.suppress(TypeError):
+ _DECLINES.pop(pipe, None)
+ return dataclasses.replace(
+ plan,
+ offload_policy = "none",
+ vae_tiling = plan.vae_tiling,
+ vae_slicing = plan.vae_slicing,
+ stream_text_encoders = False,
+ stream_transformer = True,
+ resident_transformer_mib = None,
+ resident_text_encoder_mib = None,
+ reasons = tuple(plan.reasons) + (reason,),
+ layer_split = dataclasses.replace(split, reason = reason),
+ fallback_plan = plan,
+ )
+
+
+def apply_layer_split_plan(
+ pipe: Any,
+ plan: Any,
+ *,
+ placement: str,
+ logger: Any = None,
+) -> bool:
+ """Place a layer split plan. True when placed; on any failure the split is undone, the pipe returns to the host and
+ False is returned (the caller then applies ``fallback_plan``)."""
+ split = getattr(plan, "layer_split", None)
+ if split is None:
+ return False
+ try:
+ place_layer_split(pipe, primary = placement, split = split, logger = logger)
+ except Exception as exc: # noqa: BLE001 - fall back to one card
+ if logger is not None:
+ logger.warning(
+ "diffusion.multigpu: layer split placement failed (%s); falling back to one card",
+ exc,
+ )
+ with contextlib.suppress(Exception):
+ undo_layer_split(pipe)
+ forget_applied_layer_split(pipe)
+ with contextlib.suppress(Exception):
+ pipe.to("cpu")
+ _drop_stranded_fused_records(pipe)
+ with contextlib.suppress(Exception):
+ import torch
+ torch.cuda.empty_cache()
+ _record_decline(pipe, f"placement failed ({exc})", None)
+ return False
+ with contextlib.suppress(TypeError):
+ _APPLIED[pipe] = split
+ for name in _denoisers(split):
+ module = getattr(pipe, name, None)
+ if module is not None:
+ module.__dict__[_RECORD_ATTR] = split
+ return True
+
+
+_RECORD_ATTR = "_unsloth_layer_split_record"
+
+
+def _drop_stranded_fused_records(pipe: Any) -> None:
+ """Drop fused records a failed placement left on a card. Never raises."""
+ from .diffusion_multigpu import DENOISER_ATTRS
+ for name in DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ if module is None:
+ continue
+ with contextlib.suppress(Exception):
+ from .diffusion_int8_fused import drop_stranded_fused_records
+ drop_stranded_fused_records(module)
+
+
+def applied_layer_split(pipe: Any) -> Optional[LayerSplit]:
+ """The split placed on ``pipe``, or on a denoiser it shares with the pipeline that placed it."""
+ if pipe is None:
+ return None
+ try:
+ split = _APPLIED.get(pipe)
+ except TypeError:
+ split = None
+ if split is not None:
+ return split
+ from .diffusion_layer_split import is_layer_split as engine_split
+ from .diffusion_multigpu import DENOISER_ATTRS
+
+ for name in DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ record = getattr(module, "__dict__", {}).get(_RECORD_ATTR) if module is not None else None
+ if record is not None and engine_split(module):
+ return record
+ return None
+
+
+def forget_applied_layer_split(pipe: Any) -> None:
+ """Teardown: the placed split's record goes (the pipeline's and its denoisers'), a decline reason stays."""
+ from .diffusion_multigpu import DENOISER_ATTRS
+
+ with contextlib.suppress(TypeError):
+ _APPLIED.pop(pipe, None)
+ for name in DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ if module is not None:
+ getattr(module, "__dict__", {}).pop(_RECORD_ATTR, None)
+
+
+def is_layer_split(pipe: Any) -> bool:
+ return applied_layer_split(pipe) is not None
+
+
+def forget_layer_split(pipe: Any) -> None:
+ """Unload / re-place: drop the record (the engine's ``undo_layer_split`` removes the hooks)."""
+ forget_applied_layer_split(pipe)
+ with contextlib.suppress(TypeError):
+ _DECLINES.pop(pipe, None)
+
+
+def layer_split_status(pipe: Any) -> Optional[dict[str, Any]]:
+ """``status()["multi_gpu"]["layer_split"]``: the applied split, ``{"declined": reason}`` after an attempt that did
+ not engage, or None when none was considered."""
+ split = applied_layer_split(pipe)
+ if split is not None:
+ return split.as_public_dict()
+ try:
+ declined = _DECLINES.get(pipe) if pipe is not None else None
+ except TypeError:
+ declined = None
+ return {"declined": declined} if declined is not None else None
+
+
+def layer_split_card_floors(pipe: Any) -> Optional[tuple]:
+ """``vram_split_cards`` for the video guard: ``(ordinal, resident MiB, decodes, home, denoises, block headroom
+ MiB, reserve MiB)`` per card. The VAE and the stems stay on the primary, and every card runs blocks. The last two
+ fields are what the plan held on a card that holds only blocks: one block's working set at the family's default
+ request (it scales with the request) and the fixed per-card reserve (it does not); None on the primary, which the
+ whole-request estimate prices."""
+ split = applied_layer_split(pipe)
+ if split is None:
+ return None
+ weights = dict(split.card_weights_mib)
+ return tuple(
+ (
+ o,
+ int(weights.get(o, 0)),
+ o == split.primary,
+ o == split.primary,
+ True,
+ None if o == split.primary else split.working_mib(o)[0],
+ None if o == split.primary else split.working_mib(o)[1],
+ )
+ for o in split.cards
+ )
+
+
+def secondary_block_cards(pipe: Any) -> tuple[int, ...]:
+ """Cards other than the primary that run blocks (each needs its own generate-time activation check)."""
+ split = applied_layer_split(pipe)
+ if split is None:
+ return ()
+ return tuple(sorted({r.card for r in split.ranges if r.card != split.primary}))
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 54d092e751a..82b777cfad1 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -621,6 +621,7 @@ class MemoryPlan:
# The single-card plan a split replaced, applied on the primary if the split placement fails (a torchao denoiser
# cannot take the whole-module offload a blind fallback would choose).
fallback_plan: Optional["MemoryPlan"] = field(default = None, compare = False, repr = False)
+ layer_split: Optional[Any] = None
@property
def engages_offload(self) -> bool:
@@ -643,6 +644,11 @@ class MemoryPlan:
"secondary_components": (
list(self.secondary_components) if self.secondary_components is not None else None
),
+ **(
+ {"layer_split": self.layer_split.as_public_dict()}
+ if self.layer_split is not None
+ else {}
+ ),
}
@@ -3513,6 +3519,12 @@ def resident_group_mib(pipe: Any) -> int:
return total >> 20
+def _any_layer_split(pipe: Any) -> bool:
+ from .diffusion_layer_split import is_layer_split
+ from .diffusion_multigpu import DENOISER_ATTRS
+ return any(is_layer_split(getattr(pipe, name, None)) for name in DENOISER_ATTRS)
+
+
def hook_resident_denoiser(
pipe: Any,
device: Any,
@@ -3530,6 +3542,7 @@ def hook_resident_denoiser(
transformer is None
or _offload_groups(transformer)
or not _pipe_denoisers_hold_torchao(pipe)
+ or _any_layer_split(pipe)
):
return False
import torch
@@ -3976,11 +3989,21 @@ def apply_memory_plan(
)
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
- elif getattr(plan, "secondary_ordinal", None) is not None:
- from .diffusion_multigpu import apply_split_plan
- if not apply_split_plan(pipe, plan, placement = placement, logger = logger):
+ elif (
+ getattr(plan, "secondary_ordinal", None) is not None
+ or getattr(plan, "layer_split", None) is not None
+ ):
+ if getattr(plan, "layer_split", None) is not None:
+ from .diffusion_layer_split_plan import apply_layer_split_plan as _apply_split
+ else:
+ from .diffusion_multigpu import apply_split_plan as _apply_split
+ if not _apply_split(pipe, plan, placement = placement, logger = logger):
fallback = getattr(plan, "fallback_plan", None)
- if fallback is not None and getattr(fallback, "secondary_ordinal", None) is None:
+ if (
+ fallback is not None
+ and getattr(fallback, "secondary_ordinal", None) is None
+ and getattr(fallback, "layer_split", None) is None
+ ):
# the policy one card would have used (group offload keeps a torchao denoiser loadable)
result = apply_memory_plan(
pipe, fallback, device = device, placement_device = placement_device, logger = logger
@@ -4001,6 +4024,7 @@ def apply_memory_plan(
secondary_ordinal = None,
secondary_components = None,
fallback_plan = None,
+ layer_split = None,
)
except Exception: # noqa: BLE001
pass
@@ -5033,6 +5057,11 @@ def _apply_group_offload(
return False
if not stream_transformer and not stream_text_encoders:
return False
+ if _any_layer_split(pipe):
+ from .diffusion_layer_split_plan import undo_layer_split
+ undo_layer_split(pipe)
+ if logger is not None:
+ logger.info("diffusion.memory: removed the layer split before applying group offload")
installed = 0
# every component's torch forward hooks before this call: an OOM after hooks exist unwinds to this state
hook_snapshots: list = []
@@ -5257,6 +5286,62 @@ def _apply_group_offload(
return False
+def stream_text_encoders_onto(
+ pipe: Any,
+ device: Any,
+ logger: Any = None,
+) -> list[str]:
+ """Leaf-level group offload of every ``text_encoder*`` onto ``device`` and nothing else: the denoisers and the other
+ components are not touched (a layer split places them; ``_apply_group_offload`` would move every unstreamed
+ component onto one card). A text encoder runs once, before step 0, so it pages in for that encode. Returns the
+ streamed names; raises (with this call's hooks removed) when an encoder refuses, and the caller falls back."""
+ import inspect
+
+ import torch
+ from diffusers.hooks import apply_group_offloading
+
+ install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
+ onload = torch.device(device)
+ use_stream = onload.type == "cuda"
+ kwargs: dict[str, Any] = {
+ "onload_device": onload,
+ "offload_device": torch.device("cpu"),
+ "offload_type": "leaf_level",
+ "use_stream": use_stream,
+ }
+ params = inspect.signature(apply_group_offloading).parameters
+ if use_stream:
+ if "non_blocking" in params:
+ kwargs["non_blocking"] = True
+ if "record_stream" in params:
+ kwargs["record_stream"] = True
+ encoders = {
+ str(n): m
+ for n, m in (getattr(pipe, "components", {}) or {}).items()
+ if str(n).startswith("text_encoder") and isinstance(m, torch.nn.Module)
+ }
+ if "low_cpu_mem_usage" in params:
+ pin = _streamed_pin_plan(0, sum(_module_host_mib(m) for m in encoders.values()), logger)
+ kwargs["low_cpu_mem_usage"] = not pin[1]
+ done: list[str] = []
+ try:
+ for name, module in encoders.items():
+ module.to("cpu")
+ done.append(name)
+ apply_group_offloading(module, **kwargs)
+ _pin_vision_embedding_device(module)
+ except Exception:
+ for name in done:
+ _remove_group_offload_hooks(encoders[name])
+ raise
+ if logger is not None and done:
+ logger.info(
+ "diffusion.memory: streaming %s onto %s for the encode", ", ".join(done), onload
+ )
+ return done
+
+
# Generate-time activation guard. The load-time plan cannot know the output resolution: a model is loaded once and
# then generates at whatever size the sliders say, so ``_plan_memory`` budgets the 1024x1024 default. That is the
# right call for PLACEMENT, but it means a request for a much larger frame is never checked against anything, so this
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index 1f5d12dcfa9..fdda8d74802 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -9,6 +9,9 @@ Three levers, all at a component or denoiser-call boundary, never inside a model
other companions sit on a secondary card, when that removes offload on the primary. Inputs move to a module's own
card at its entry points and outputs come back to the primary, so the pipeline still sees one device. Copies are
exact, so the output is bit-identical to a single-card resident run.
+* Layer split (``diffusion_layer_split``): a DiT that fits no single selected card is split by contiguous ranges of
+ its repeated blocks, the rest of it on the primary. Activations cross at the range boundaries only; outputs are
+ bit-identical to a one-card run.
* Second denoiser (after load): a two-DiT pipeline (an MoE ``transformer_2``) keeps one expert per card when both
experts do not fit one card.
* CFG-parallel (generate): when guidance runs separate cond / uncond denoiser calls on the same latents, every odd
@@ -17,6 +20,8 @@ Three levers, all at a component or denoiser-call boundary, never inside a model
step stay on the primary. Branches are recognised from the call boundary alone (same latent tensor object in
consecutive calls), so true CFG, rescale and multi-branch guidance work unchanged and a guidance-distilled model
(one call per step) never builds a replica.
+* Layer split (after load, diffusion_layer_split_plan): when one card would offload the denoiser itself and no
+ component assignment fits, the denoiser's blocks are cut across the cards. Excludes CFG-parallel.
Kill switches: ``UNSLOTH_DIFFUSION_MULTIGPU=0`` (everything here; one card as before), ``UNSLOTH_DIFFUSION_CFG_PARALLEL=0``
(only the replica). ``UNSLOTH_DIFFUSION_MULTIGPU=force`` splits the companions even when one card would hold
@@ -425,13 +430,20 @@ def refine_plan_for_multigpu(
"""Post-load: an offloading plan becomes a resident split when the LOADED components can be shared by the cards.
No-op without a selection, for explicit offload modes, a plan already resident (unless forced) or split, unified memory
- or an unknown budget. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""
+ or an unknown budget. When no component assignment fits, a layer split (diffusion_layer_split_plan) is tried before
+ keeping the offload plan. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""
selection = current_selection()
if selection is None or plan is None:
return plan
- if getattr(plan, "secondary_ordinal", None) is not None:
+ if (
+ getattr(plan, "secondary_ordinal", None) is not None
+ or getattr(plan, "layer_split", None) is not None
+ ):
return plan
- if plan.offload_policy == "none" and multigpu_mode() != "force":
+ from .diffusion_layer_split_plan import layer_split_mode, refine_plan_for_layer_split
+
+ layer_force = layer_split_mode() == "force"
+ if plan.offload_policy == "none" and multigpu_mode() != "force" and not layer_force:
return plan
if getattr(plan, "requested_mode", None) not in ("auto", "fast"):
return plan
@@ -469,6 +481,22 @@ def refine_plan_for_multigpu(
or 0
)
overhead = int(plan.estimates.get("base_overhead_mib") or 0)
+
+ def layer_split_or_plan() -> Any:
+ return refine_plan_for_layer_split(
+ pipe,
+ plan,
+ selection = selection,
+ sizes = sizes,
+ primary_limit_mib = int(primary_limit),
+ secondary_limit_mib = lambda o: secondary_resident_limit_mib(o, mode),
+ headroom_mib = headroom,
+ overhead_mib = overhead,
+ logger = logger,
+ )
+
+ if layer_force:
+ return layer_split_or_plan()
moved = choose_component_assignment(
sizes,
headroom_mib = headroom,
@@ -477,7 +505,7 @@ def refine_plan_for_multigpu(
secondary_limit_mib = int(sec_limit),
)
if moved is None:
- return plan
+ return layer_split_or_plan()
reason = (
f"multi-GPU (loaded sizes): {', '.join(moved)} on cuda:{secondary}, the rest resident on "
f"cuda:{selection.primary}; single card would use '{plan.offload_policy}'"
@@ -538,10 +566,18 @@ def move_to_device(
obj: Any,
device: Any,
_depth: int = 0,
+ *,
+ tensor_fn: Optional[Callable[[Any], Any]] = None,
+ objects: bool = True,
) -> Any:
+ """Tensors in ``obj`` (containers, dataclasses, ModelOutputs, plain attribute holders) on ``device``.
+ ``tensor_fn(t)`` replaces ``t.to(device)`` for a tensor that is elsewhere. ``objects=False`` leaves dataclasses
+ and attribute holders as they are (a per-layer KV cache the callee writes into must keep its identity)."""
torch = _torch()
if isinstance(obj, torch.Tensor):
- return obj if obj.device == device else obj.to(device)
+ if obj.device == device:
+ return obj
+ return obj.to(device) if tensor_fn is None else tensor_fn(obj)
if (
_depth > 6
or obj is None
@@ -553,28 +589,39 @@ def move_to_device(
try:
out = copy.copy(obj)
for key, value in obj.items():
- moved = move_to_device(value, device, _depth + 1)
+ moved = move_to_device(
+ value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects
+ )
if moved is not value:
out[key] = moved
return out
except Exception: # noqa: BLE001
- return type(obj)((k, move_to_device(v, device, _depth + 1)) for k, v in obj.items())
+ return type(obj)(
+ (k, move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects))
+ for k, v in obj.items()
+ )
if isinstance(obj, tuple):
- moved = [move_to_device(v, device, _depth + 1) for v in obj]
+ moved = [
+ move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj
+ ]
if all(a is b for a, b in zip(moved, obj)):
return obj
if hasattr(obj, "_fields"):
return type(obj)(*moved)
return type(obj)(moved) if type(obj) is not tuple else tuple(moved)
if isinstance(obj, list):
- return [move_to_device(v, device, _depth + 1) for v in obj]
- if isinstance(obj, torch.nn.Module):
+ return [
+ move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj
+ ]
+ if type(obj).__name__ == "BlockMask" and callable(getattr(obj, "to", None)):
+ return obj.to(device)
+ if isinstance(obj, torch.nn.Module) or not objects:
return obj
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
changes = {}
for f in dataclasses.fields(obj):
value = getattr(obj, f.name, None)
- moved = move_to_device(value, device, _depth + 1)
+ moved = move_to_device(value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects)
if moved is not value:
changes[f.name] = moved
if not changes:
@@ -601,7 +648,7 @@ def move_to_device(
out = copy.copy(obj)
for k, v in state.items():
if isinstance(v, torch.Tensor) and v.device != device:
- setattr(out, k, v.to(device))
+ setattr(out, k, v.to(device) if tensor_fn is None else tensor_fn(v))
return out
return obj
@@ -687,6 +734,12 @@ def _split_home(pipe: Any) -> Any:
return None
if not isinstance(components, dict):
return None
+ from .diffusion_layer_split import layer_split_of
+
+ for module in components.values():
+ layered = layer_split_of(module)
+ if layered is not None:
+ return layered.current_home()
for module in components.values():
home = getattr(module, "_unsloth_split_home", None)
if home is not None:
@@ -803,7 +856,18 @@ def place_split(
def undo_split(pipe: Any) -> None:
+ """Remove every split placed on ``pipe``: a layer split of any denoiser (its block boundary and the planner's
+ record) and the component boundaries. Weights stay where they are."""
torch = _torch()
+ from .diffusion_layer_split import undo_layer_split as undo_layers
+ from .diffusion_layer_split_plan import forget_applied_layer_split
+
+ for name in DENOISER_ATTRS:
+ try:
+ undo_layers(pipe, attr = name)
+ except Exception: # noqa: BLE001 - best effort; the component boundaries below still come off
+ pass
+ forget_applied_layer_split(pipe)
for module in (getattr(pipe, "components", {}) or {}).values():
if isinstance(module, torch.nn.Module):
remove_boundary(module)
@@ -850,12 +914,17 @@ def apply_split_plan(
def is_split(pipe: Any) -> bool:
+ """A companion split or a layer split is placed on ``pipe``."""
if pipe is None:
return False
try:
- return pipe in _PLACEMENTS
+ if pipe in _PLACEMENTS:
+ return True
except TypeError:
return False
+ from .diffusion_layer_split_plan import is_layer_split
+
+ return is_layer_split(pipe)
def placement_status(pipe: Any) -> Optional[dict[str, Any]]:
@@ -867,9 +936,28 @@ def placement_status(pipe: Any) -> Optional[dict[str, Any]]:
declined = _CFG_DECLINES.get(pipe)
except TypeError:
return None
- if info is None and cfg is None and declined is None:
+ from .diffusion_layer_split_plan import layer_split_status
+
+ layer = layer_split_status(pipe)
+ if info is None and cfg is None and declined is None and layer is None:
return None
out: dict[str, Any] = dict(info or {})
+ from .diffusion_layer_split import layer_split_stats
+
+ live = {
+ name: stats
+ for name in DENOISER_ATTRS
+ if (stats := layer_split_stats(getattr(pipe, name, None))) is not None
+ }
+ if layer is not None:
+ out["layer_split"] = dict(layer)
+ if live:
+ out["layer_split"]["live"] = live
+ elif (
+ "layer_split" in out
+ ): # placed by the engine directly (no planner record): its own summary, live
+ engine = out["layer_split"]
+ out["layer_split"] = dict(live.get(engine.get("denoiser", "transformer")) or engine)
if cfg is not None:
out["cfg_parallel"] = cfg.describe()
elif declined is not None:
@@ -1919,6 +2007,11 @@ def cfg_parallel_decline(
mode = cfg_parallel_mode()
if mode == "off":
return f"{CFG_PARALLEL_ENV}=0"
+ from .diffusion_layer_split_plan import is_layer_split
+
+ # A replica is a whole denoiser on one card, which a denoiser cut across cards cannot be.
+ if is_layer_split(pipe):
+ return "denoiser is split across cards (layer split)"
# A replica takes a whole denoiser's worth of another card; automatic placement only borrows one to avoid offload.
if selection.automatic and mode != "force":
return f"automatic GPU placement (select several cards or set {CFG_PARALLEL_ENV}=force)"
@@ -1931,6 +2024,10 @@ def cfg_parallel_decline(
torch = _torch()
if not isinstance(denoiser, torch.nn.Module):
return "no DiT denoiser (a U-Net batches its guidance branches)"
+ from .diffusion_layer_split import is_layer_split
+
+ if is_layer_split(denoiser):
+ return "denoiser is split across cards (layer split)"
# Whole-module / sequential offload page the denoiser per call; a group / streaming plan is fine only when it left
# the denoiser itself resident (encoders streamed), which shows as a denoiser without offload hooks.
if offload_policy in ("model", "sequential"):
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
index 8b9d6917d04..b75e64047c6 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
@@ -91,6 +91,8 @@ def _qkv_intact(attn: Any) -> Any:
lin, q, k, v = rec
if attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:
return None
+ if lin.weight.device != q.weight.device:
+ return None # a layer split moved the parts after the fusion; the stale fused copy is on another card
return lin
@@ -191,9 +193,9 @@ def install(
return result
if offload_active or type(transformer).__name__ != "QwenImage21Transformer2DModel":
return result
- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call
- if resident_cuda_device(transformer) is not None:
+ if resident_cuda_cards(transformer):
return install_modules(transformer, logger)
run_on_first_call(transformer, "q21_convrot_fused", lambda t: install_modules(t, logger))
return {"fused_qkv": None, "out": None}
@@ -203,10 +205,12 @@ def install_modules(root: Any, logger: Any = None) -> dict:
result = {"fused_qkv": 0, "out": 0}
if q21_convrot_fused_disabled():
return result
- from .diffusion_int8_fused import resident_cuda_device
+ import torch
+
+ from .diffusion_int8_fused import resident_cuda_cards
- dev = resident_cuda_device(root)
- if dev is None or not arch_ok(dev):
+ cards = resident_cuda_cards(root) # one card, or each card of a layer split
+ if not cards or not all(arch_ok(torch.device("cuda", index)) for index in cards):
return result
from .diffusion_zimage_fused import _fuse_linears, _share_storage
@@ -217,6 +221,8 @@ def install_modules(root: Any, logger: Any = None) -> dict:
parts = [getattr(module, n, None) for n in ("to_q", "to_k", "to_v")]
if any(p is None for p in parts) or not _rotated_group(parts):
continue
+ if len({p.weight.device for p in parts}) != 1:
+ continue
todo.append((module, parts))
if not todo or not _patch_prepare(logger):
return result
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_rope.py b/studio/backend/core/inference/diffusion_qwenimage21_rope.py
index 982e7cdcdcf..4089b786062 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_rope.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_rope.py
@@ -229,6 +229,23 @@ def install(logger: Any = None, device: Any = None) -> bool:
return True
+def probe_split_cards(cards: Any) -> tuple:
+ """A layer split runs its blocks on several cards, and the real-arithmetic wrapper reads each card's own probe
+ (an unprobed card keeps the complex form). Probes every card not yet probed; nothing when no install probed a
+ card (the patch is not in use). Returns the cards whose probe passed."""
+ if real_rope_disabled() or not _FUSION:
+ return ()
+ import torch
+
+ for index in cards:
+ if index not in _FUSION:
+ try:
+ _FUSION[index] = probe_fusion(torch.device("cuda", int(index)))
+ except Exception: # noqa: BLE001 - that card keeps the complex form
+ _FUSION[index] = None
+ return tuple(index for index in cards if _FUSION.get(index) is not None)
+
+
def uninstall() -> None:
with _LOCK:
for mod, stock in list(_INSTALLED.items()):
diff --git a/studio/backend/core/inference/diffusion_qwenimage_rope.py b/studio/backend/core/inference/diffusion_qwenimage_rope.py
index 93473b4c7c0..4a97058cb33 100644
--- a/studio/backend/core/inference/diffusion_qwenimage_rope.py
+++ b/studio/backend/core/inference/diffusion_qwenimage_rope.py
@@ -145,19 +145,14 @@ def install(transformer: Any, logger: Any = None) -> bool:
return False
if type(transformer).__name__ != "QwenImageTransformer2DModel":
return False
- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call
def _finish(t: Any) -> bool:
- dev = resident_cuda_device(t)
- if dev is None:
- return False
- import torch
-
- return _patch_table(
- dev.index if dev.index is not None else torch.cuda.current_device(), logger
- )
+ cards = resident_cuda_cards(t)
+ passed = [index for index in cards if _probe(index)]
+ return bool(passed) and _patch_table(passed[0], logger)
- if resident_cuda_device(transformer) is not None:
+ if resident_cuda_cards(transformer):
return _finish(transformer)
run_on_first_call(transformer, "qwenimage_rope", _finish)
return True
diff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.py
index 2b84b0b04e5..ea283ee7194 100644
--- a/studio/backend/core/inference/diffusion_zimage_fused.py
+++ b/studio/backend/core/inference/diffusion_zimage_fused.py
@@ -73,6 +73,8 @@ def _qkv_intact(attn: Any) -> Any:
lin, q, k, v = rec
if attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:
return None
+ if lin.weight.device != q.weight.device:
+ return None # a layer split moved the parts after the fusion; the stale fused copy is on another card
return lin
@@ -274,9 +276,9 @@ def install(
return {"real_rope": False, "fused_qkv": 0}
if not _patch_class(logger):
return {"real_rope": False, "fused_qkv": 0}
- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call
- if resident_cuda_device(transformer) is not None:
+ if resident_cuda_cards(transformer):
return install_modules(transformer, logger, fuse_qkv = not offload_active)
run_on_first_call(
transformer,
@@ -320,15 +322,18 @@ def install_modules(
import torch
- from .diffusion_int8_fused import resident_cuda_device
+ from .diffusion_int8_fused import resident_cuda_cards
cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
- dev = resident_cuda_device(root)
- if dev is None:
+ cards = resident_cuda_cards(
+ root
+ ) # one card, or each card of a layer split (the processor reads its query's card)
+ if not cards:
return result
with _LOCK:
- index = dev.index if dev.index is not None else torch.cuda.current_device()
- if index not in _FUSION:
+ for index in cards:
+ if index in _FUSION:
+ continue
try:
from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion
_FUSION[index] = (
@@ -336,7 +341,7 @@ def install_modules(
)
except Exception: # noqa: BLE001
_FUSION[index] = None
- result["real_rope"] = _FUSION[index] is not None
+ result["real_rope"] = all(_FUSION[index] is not None for index in cards)
for _name, module in root.named_modules():
if not fuse_qkv or type(getattr(module, "processor", None)) is not cls:
continue
@@ -360,6 +365,28 @@ def install_modules(
return result
+def probe_split_cards(cards: Any) -> tuple:
+ """Probe the split's other cards (an unprobed card takes the complex RoPE the regional compile cannot lower).
+ Returns the cards whose probe passed."""
+ if zimage_fused_disabled() or not _FUSION:
+ return ()
+ import torch
+
+ with _LOCK:
+ for index in cards:
+ index = int(index)
+ if index in _FUSION:
+ continue
+ try:
+ from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion
+ _FUSION[index] = (
+ probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None
+ )
+ except Exception: # noqa: BLE001 - that card keeps the complex form
+ _FUSION[index] = None
+ return tuple(index for index in cards if _FUSION.get(int(index)) is not None)
+
+
def uninstall(transformer: Any = None) -> None:
"""Restore the stock processor; drop the fused Linears (the per-row views stay valid) and pending install."""
if transformer is not None:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index adac540845e..3bf8ce46be8 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -73,6 +73,7 @@ from .video_moe_pair import moe_expert_of, moe_partner_filename, moe_pick_pairs
from .diffusion_cache import (
FBCACHE_MIN_STEPS,
TC_AUTO,
+ TC_FBCACHE,
TC_STATIC,
apply_step_cache,
auto_static_skip_entry,
@@ -166,6 +167,7 @@ from .diffusion_speed import (
SPEED_OFF,
apply_speed_optims,
arm_graphs_after_placement,
+ engage_pinned_denoisers,
rocm_probe_load_scoped,
resolve_speed_mode,
restore_backend_flags,
@@ -205,9 +207,18 @@ from .diffusion_transformer_quant import (
stored_denoiser_precision,
transformer_is_quantised,
)
-from .diffusion import _family_override_resolved, _memory_request_forces_offload
+from .diffusion import (
+ _family_override_resolved,
+ _companion_split_expected,
+ _layer_split_expected,
+ _memory_request_forces_offload,
+ _native_quant_needs_compile,
+ _restore_step_cache_after_layer_split,
+ _step_cache_dropped_by_layer_split,
+)
from .diffusion_native_quant import native_quant_reason
from .diffusion_batched import is_oom_error
+from .diffusion_block_graph import compile_pipe_below_offload_hooks
from .diffusion_precision import (
effective_te_quant,
normalize_te_quant,
@@ -3436,6 +3447,11 @@ def _video_split_card_floors(pipe: Any) -> Optional[tuple]:
"""``((ordinal, resident MiB, runs the decode), ...)`` for a pipeline split across cards, else None. A split plan
reports no offload policy, so the offload floor is None and the request guard would be skipped; this records what
each card holds so the guard can run per card."""
+ from .diffusion_layer_split_plan import layer_split_card_floors
+
+ layered = layer_split_card_floors(pipe)
+ if layered is not None:
+ return layered
status = multigpu_placement_status(pipe) if multigpu_is_split(pipe) else None
placement = (status or {}).get("placement") or {}
if not placement:
@@ -3491,16 +3507,39 @@ def video_split_shortfall_message(
width: Optional[int] = None,
height: Optional[int] = None,
frames: Optional[int] = None,
+ planned_shape: Optional[tuple] = None,
) -> Optional[str]:
"""The offload guard run on every card of a split: each card's resident weights plus the base headroom, the
decoded-clip share on the card that runs the VAE decode, and on the denoiser's card, when the decode ran
- elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. The first refusal,
- else None."""
+ elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. A layer split's
+ block-only card (a sixth field: the headroom its plan reserved at ``planned_shape`` = (width, height, frames))
+ is charged that headroom, scaled by how many more tokens this request carries, plus its fixed reserve (a seventh
+ field, unscaled). The first refusal, else None."""
for card in cards:
ordinal, floor_mib, decodes = card[:3]
home = len(card) > 3 and bool(card[3])
denoises = bool(card[4]) if len(card) > 4 else home
shaped = bool(width and height and frames)
+ block_headroom = card[5] if len(card) > 5 else None
+ block_reserve = int(card[6] or 0) if len(card) > 6 else 0
+ if block_headroom is not None and not home and not decodes:
+ scale = 1.0
+ if shaped and planned_shape and all(planned_shape):
+ planned = int(planned_shape[0]) * int(planned_shape[1]) * int(planned_shape[2])
+ scale = max(1.0, int(width) * int(height) * int(frames) / float(planned))
+ headroom = int(int(block_headroom) * scale) + block_reserve
+ required = int(floor_mib) + headroom
+ available = int(available_mib(int(ordinal)))
+ if available < required:
+ shape = f" for {width}x{height} at {frames} frames" if shaped else ""
+ return (
+ f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "
+ f"placement (split across GPUs, cuda:{int(ordinal)}: {int(floor_mib) / 1024:.1f} GiB of model "
+ f"blocks plus {headroom / 1024:.1f} GiB for their activations); {available / 1024:.1f} GiB is "
+ "available. Free VRAM held by other processes, generate a shorter or smaller clip, or set the "
+ "DiT precision lower."
+ )
+ continue
frames_mib = (
-(-int(width) * int(height) * int(frames) * 3 * 4 // (1 << 20)) if shaped else 0
)
@@ -9125,13 +9164,20 @@ class VideoBackend:
effective_speed = resolve_speed_mode(
speed_mode, is_gguf = kind == "gguf", dense_default = SPEED_DEFAULT
)
- # A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant
- # engaged but speed was off.
- if (
- transformer_quant_engaged is not None
- and native_scheme is None
- and effective_speed == SPEED_OFF
- ):
+ video_refine = lambda p, pl, lg: _video_refine_for_multigpu(fam, p, pl, lg) # noqa: E731
+ split_expected = _layer_split_expected(pipe, plan, video_refine)
+ # A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower.
+ quant_forces_compile = transformer_quant_engaged is not None and (
+ native_scheme is None
+ or _native_quant_needs_compile(
+ target,
+ native_scheme,
+ split_expected
+ or plan.offload_policy == "none"
+ or _companion_split_expected(pipe, plan, video_refine),
+ )
+ )
+ if quant_forces_compile and effective_speed == SPEED_OFF:
logger.info(
"video.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
@@ -9278,6 +9324,7 @@ class VideoBackend:
from .video_hv15_vae import install_vectorised_causal_mask
hv15_mask_engaged = install_vectorised_causal_mask(pipe, logger = logger) > 0
speed_optims: tuple = ()
+ speed_offload = plan.offload_policy != "none" and not split_expected
for view in views:
# Both helpers act on ``view.transformer``; call once per view (engaged values match, so record the first).
# is_gguf needs kind==gguf AND no quant engaged.
@@ -9305,7 +9352,7 @@ class VideoBackend:
backend = "video",
speed_mode = effective_speed,
is_gguf = gguf_transformer,
- offload_active = plan.offload_policy != "none",
+ offload_active = speed_offload,
audit = load_audit,
apply = lambda family, log, view = view: apply_speed_optims(
view,
@@ -9316,7 +9363,7 @@ class VideoBackend:
# An auto cache that could still engage also drops fullgraph (FBCache under a fullgraph-compiled
# DiT crashes)
cache_active = cache_graph_break or cache_may_toggle,
- offload_active = plan.offload_policy != "none",
+ offload_active = speed_offload,
cuda_graph_default = False,
logger = log,
),
@@ -9373,7 +9420,21 @@ class VideoBackend:
# a transient OOM during setup may have re-planned to lower residency
planned = plan
plan = applied_plan(pipe, plan)
- if getattr(planned, "secondary_ordinal", None) is not None and plan is not planned:
+ if cache_engaged == TC_FBCACHE and _step_cache_dropped_by_layer_split(pipe):
+ cache_engaged = _restore_step_cache_after_layer_split(
+ pipe,
+ views,
+ threshold = transformer_cache_threshold,
+ quant_active = cache_quant_active,
+ logger = logger,
+ )
+ if cache_engaged is None and getattr(plan, "layer_split", None) is not None:
+ cache_may_toggle = False
+ cache_reason = "off: the step cache declined the denoiser split across cards"
+ if (
+ getattr(planned, "secondary_ordinal", None) is not None
+ or getattr(planned, "layer_split", None) is not None
+ ) and plan is not planned:
vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)
if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:
@@ -9428,6 +9489,9 @@ class VideoBackend:
speed_optims += ("vae_untiled_when_fits",)
# Wan's decode also grows within a single tile, which tiling alone cannot bound.
install_decoder_sync(pipe, target, logger = logger)
+ if not speed_offload and _video_denoiser_hooked(pipe):
+ # compiled for a resident denoiser but it landed hooked: move the compile below the offload hooks (as the image backend)
+ compile_pipe_below_offload_hooks(pipe, logger)
# A family that opts in to graphs arms them against the placement that landed (offload hooks included).
graph_applied = arm_graphs_after_placement(
pipe, {"cuda_graph": "cuda_graph" in speed_optims}, logger
@@ -9436,6 +9500,27 @@ class VideoBackend:
speed_optims += ("cuda_graph",)
elif not graph_applied.get("cuda_graph"):
speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")
+ if getattr(plan, "layer_split", None) is not None:
+ split_applied = {k: True for k in speed_optims}
+ engage_pinned_denoisers(pipe, split_applied, logger)
+ from . import diffusion_cuda_graph
+
+ diffusion_cuda_graph.arm_block_graphs(
+ pipe,
+ split_applied,
+ target = target,
+ family = fam,
+ hooked = _video_denoiser_hooked(pipe),
+ cache_engaged = bool(cache_graph_break),
+ speed_mode = effective_speed,
+ family_default = False,
+ logger = logger,
+ )
+ for key in ("int8_gemm", "cuda_graph"):
+ if split_applied.get(key) and key not in speed_optims:
+ speed_optims += (key,)
+ elif not split_applied.get(key):
+ speed_optims = tuple(o for o in speed_optims if o != key)
# Last, so the bf16 entry wraps whatever decode path the steps above installed. Not on SPEED_OFF.
if getattr(fam, "vae_force_fp32", False) and effective_speed != SPEED_OFF:
vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)
@@ -9458,7 +9543,7 @@ class VideoBackend:
speed_mode,
effective_speed,
"quantized transformer requires compile"
- if transformer_quant_engaged is not None and native_scheme is None
+ if quant_forces_compile
else "clip denoises amortise the one-time compile within a single run"
if speed_mode is None
else "requested",
@@ -11685,6 +11770,13 @@ class VideoBackend:
width = width,
height = height,
frames = frames,
+ planned_shape = (
+ fam.resolution_presets[0][0],
+ fam.resolution_presets[0][1],
+ fam.default_num_frames,
+ )
+ if getattr(fam, "resolution_presets", None)
+ else None,
)
if shortfall is not None:
raise RuntimeError(shortfall)
@@ -12169,6 +12261,13 @@ class VideoBackend:
+ ("reaches" if toggled else "is below")
+ f" {FBCACHE_MIN_STEPS}"
)
+ # A step cache keys residuals on cond / uncond context, invisible to a graph key: graphs armed with the cache off run eager.
+ graph_handles = tuple(getattr(state.pipe, "_unsloth_cuda_graphs", ()) or ())
+ if graph_handles:
+ from . import diffusion_cuda_graph
+ diffusion_cuda_graph.set_bypass(
+ graph_handles, cache_breaks_graph(state.transformer_cache)
+ )
if static_skip:
# Auto skip was measured on text-to-video only.
conditioned = bool(first_pil is not None or last_pil is not None or references)
diff --git a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json b/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
index 0dc592cb74b..b42b00ac69d 100644
--- a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
+++ b/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
@@ -65,30 +65,30 @@
"image|flux.1|off|float16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
"image|flux.1|off|float16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
"image|flux.1|off|float16|offload=1|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|bfloat16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|bfloat16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|bfloat16|offload=1|gguf=1": {"applied":[["compiled_dequant","cudnn_benchmark","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|float16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|float16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|default|float16|offload=1|gguf=1": {"applied":[["compiled_dequant","cudnn_benchmark"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|bfloat16|offload=0|gguf=0": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|bfloat16|offload=0|gguf=1": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|bfloat16|offload=1|gguf=0": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|bfloat16|offload=1|gguf=1": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|float16|offload=0|gguf=0": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|float16|offload=0|gguf=1": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|float16|offload=1|gguf=0": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|eager|float16|offload=1|gguf=1": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|bfloat16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|bfloat16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|bfloat16|offload=1|gguf=1": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|float16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|float16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
- "image|flux.2-dev|max|float16|offload=1|gguf=1": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|bfloat16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|bfloat16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|bfloat16|offload=1|gguf=1": {"applied":[["compiled_dequant","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|float16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|float16|offload=1|gguf=0": {"applied":[["compiled"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|default|float16|offload=1|gguf=1": {"applied":[["compiled_dequant"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|bfloat16|offload=0|gguf=0": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|bfloat16|offload=0|gguf=1": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|bfloat16|offload=1|gguf=0": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|bfloat16|offload=1|gguf=1": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|float16|offload=0|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|float16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|float16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|eager|float16|offload=1|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|bfloat16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|bfloat16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|bfloat16|offload=1|gguf=1": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|float16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|float16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
+ "image|flux.2-dev|max|float16|offload=1|gguf=1": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},
"image|flux.2-dev|off|bfloat16|offload=0|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
"image|flux.2-dev|off|bfloat16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
"image|flux.2-dev|off|bfloat16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index 3daa6adeae7..1280e2062d8 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -306,7 +306,7 @@
]
},
"flux.2-dev": {
- "cudnn_benchmark": true,
+ "cudnn_benchmark": false,
"filter_reduction_configs": false,
"filter_reduction_configs_archs": []
},
@@ -390,6 +390,10 @@
[
12,
0
+ ],
+ [
+ 10,
+ 0
]
]
},
@@ -413,6 +417,10 @@
[
12,
0
+ ],
+ [
+ 10,
+ 0
]
]
},
@@ -431,6 +439,10 @@
[
12,
0
+ ],
+ [
+ 10,
+ 0
]
]
},
diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.py
index db0426f7806..31d366f8225 100644
--- a/studio/backend/tests/test_diffusion_block_graph.py
+++ b/studio/backend/tests/test_diffusion_block_graph.py
@@ -172,6 +172,56 @@ def test_a_host_tensor_input_is_never_recorded():
handle.free()
+def test_a_block_graph_keys_on_the_roots_branch_flag():
+ _cuda()
+ block = Block().cuda()
+ root = types.SimpleNamespace(masked = False)
+ root._unsloth_graph_key_extra = lambda: root.masked
+
+ def compute(x):
+ y = block.lin(x)
+ return y * 0.5 if root.masked else y
+
+ shared = bg._Shared(torch.cuda.current_device(), None)
+ shared.root = root
+ graph = bg.BlockGraph(block, compute, shared)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.inference_mode():
+ for _ in range(3):
+ assert torch.equal(graph(x), compute(x))
+ assert graph.stats["captures"] == 1
+ root.masked = True
+ for _ in range(3):
+ assert torch.equal(graph(x), compute(x))
+ assert graph.stats["captures"] == 2
+
+
+def test_a_block_graph_runs_eager_while_a_step_cache_is_on():
+ _cuda()
+ block = Block().cuda()
+ root = types.SimpleNamespace(_unsloth_step_cache = None)
+
+ def compute(x):
+ y = block.lin(x)
+ return (
+ y * 0.5 if root._unsloth_step_cache else y
+ ) # what the cache hooks change inside the forward
+
+ shared = bg._Shared(torch.cuda.current_device(), None)
+ shared.root = root
+ graph = bg.BlockGraph(block, compute, shared)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.inference_mode():
+ for _ in range(3):
+ assert torch.equal(graph(x), compute(x))
+ assert graph.stats["captures"] == 1
+ replays = graph.stats["replays"]
+ root._unsloth_step_cache = "[email protected]"
+ for _ in range(3):
+ assert torch.equal(graph(x), compute(x))
+ assert graph.stats["captures"] == 1 and graph.stats["replays"] == replays
+
+
def test_weight_placement_is_none_while_a_weight_is_on_the_host():
view = bg._WeightView(Block())
assert view.placement(None) is None
diff --git a/studio/backend/tests/test_diffusion_layer_split.py b/studio/backend/tests/test_diffusion_layer_split.py
new file mode 100644
index 00000000000..9158c364036
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split.py
@@ -0,0 +1,711 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Layer split of one denoiser across cards (core/inference/diffusion_layer_split.py), CPU only.
+
+The planner runs on real module sizes. The boundary is exercised with the blocks assigned to ``cuda:0`` / ``cuda:1``
+while their weights stay on the CPU: the cross-card copy is replaced by a recorder, which checks where every block's
+inputs and outputs would go without a second card."""
+
+from __future__ import annotations
+
+import contextlib
+import gc
+
+import pytest
+import torch
+from torch import nn
+
+from core.inference import diffusion_layer_split as ls
+from core.inference import diffusion_multigpu as mg
+
+
+class FakeBlock(nn.Module):
+ def __init__(self, dim: int = 8):
+ super().__init__()
+ self.lin = nn.Linear(dim, dim)
+
+ def forward(
+ self,
+ hidden_states,
+ temb,
+ rotary = None,
+ ):
+ out = self.lin(hidden_states) * temb
+ if rotary is not None:
+ out = out + rotary[0]
+ return out
+
+
+class FakeSingle(nn.Module):
+ def __init__(self, dim: int = 8):
+ super().__init__()
+ self.lin = nn.Linear(dim, dim)
+ self.inner = FakeBlock(dim) # a repeated class nested in a block stays with its block
+
+ def forward(self, hidden_states, temb):
+ return self.inner(self.lin(hidden_states), temb)
+
+
+class FakeDiT(nn.Module):
+ _repeated_blocks = ["FakeBlock", "FakeSingle"]
+
+ def __init__(
+ self,
+ dim: int = 8,
+ double: int = 3,
+ single: int = 3,
+ ):
+ super().__init__()
+ self.x_embedder = nn.Linear(dim, dim)
+ self.transformer_blocks = nn.ModuleList(FakeBlock(dim) for _ in range(double))
+ self.single_transformer_blocks = nn.ModuleList(FakeSingle(dim) for _ in range(single))
+ self.proj_out = nn.Linear(dim, dim)
+
+ def forward(
+ self,
+ hidden_states,
+ controlnet_block_samples = None,
+ ):
+ h = self.x_embedder(hidden_states)
+ temb = torch.ones_like(h[:1])
+ rotary = (torch.zeros_like(h), torch.zeros_like(h))
+ for i, block in enumerate(self.transformer_blocks):
+ h = block(h, temb, rotary)
+ if controlnet_block_samples is not None:
+ h = h + controlnet_block_samples[i]
+ for block in self.single_transformer_blocks:
+ h = block(h, temb)
+ return self.proj_out(h)
+
+
+LazyDiT = type("Flux2Transformer2DModel", (FakeDiT,), {})
+
+
+class Pipe:
+ def __init__(self, transformer):
+ self.transformer = transformer
+ self.vae = nn.Linear(2, 2)
+
+ @property
+ def components(self):
+ return {"transformer": self.transformer, "vae": self.vae}
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(ls.LAYER_SPLIT_ENV, raising = False)
+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)
+ yield
+
+
+def _mib(module) -> int:
+ return ls.module_bytes(module) / (1 << 20)
+
+
+def test_mode_parsing(monkeypatch):
+ assert ls.layer_split_mode() == "auto"
+ for raw in ("0", "off", "False"):
+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, raw)
+ assert ls.layer_split_mode() == "off"
+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, "force")
+ assert ls.layer_split_mode() == "force"
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
+ assert ls.layer_split_mode() == "off"
+
+
+def test_find_blocks_in_registration_order_and_skips_nested():
+ dit = FakeDiT()
+ found = ls.find_blocks(dit)
+ assert [q for q, _, _ in found] == [
+ "transformer_blocks.0",
+ "transformer_blocks.1",
+ "transformer_blocks.2",
+ "single_transformer_blocks.0",
+ "single_transformer_blocks.1",
+ "single_transformer_blocks.2",
+ ]
+ assert {g for _, g, _ in found} == {"transformer_blocks", "single_transformer_blocks"}
+
+
+def test_find_blocks_follows_the_forward_order_of_a_listed_class(monkeypatch):
+ class ModeDiT(nn.Module):
+ _repeated_blocks = ["FakeBlock"]
+
+ def __init__(self):
+ super().__init__()
+ self.mode_blocks = nn.ModuleDict(
+ {"image": nn.ModuleList(FakeBlock() for _ in range(2))}
+ )
+ self.txt_mode_blocks = nn.ModuleList(FakeBlock() for _ in range(2))
+
+ monkeypatch.setitem(ls.FORWARD_ORDER, "ModeDiT", ("txt_mode_blocks", "mode_blocks.image"))
+ assert [q for q, _, _ in ls.find_blocks(ModeDiT())] == [
+ "txt_mode_blocks.0",
+ "txt_mode_blocks.1",
+ "mode_blocks.image.0",
+ "mode_blocks.image.1",
+ ]
+
+
+def test_flux3_forward_order_names_its_real_lists():
+ try:
+ from diffusers.models.transformers import transformer_flux3
+ except Exception: # noqa: BLE001 - this diffusers has no FLUX.3
+ pytest.skip("diffusers without FLUX.3")
+ import inspect
+
+ src = inspect.getsource(transformer_flux3)
+ for group in ls.FORWARD_ORDER["Flux3Transformer2DModel"]:
+ head = group.split(".")[0]
+ assert f"self.{head}" in src
+
+
+def test_a_wrapped_list_entry_groups_with_its_list():
+ assert ls._block_list_name("double_stream_blocks.3.block") == "double_stream_blocks"
+ assert ls._block_list_name("transformer_blocks.7") == "transformer_blocks"
+ assert ls._block_list_name("mode_blocks.image.2") == "mode_blocks.image"
+ assert ls._block_list_name("stages.1.blocks.0") == "stages.1.blocks"
+
+ class Wrap(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.block = FakeBlock(8)
+
+ class Wrapped(nn.Module):
+ _repeated_blocks = ["FakeBlock"]
+
+ def __init__(self):
+ super().__init__()
+ self.double = nn.ModuleList([Wrap() for _ in range(3)])
+ self.single = nn.ModuleList([Wrap() for _ in range(2)])
+
+ groups = [g for _q, g, _b in ls.find_blocks(Wrapped())]
+ assert groups == ["double"] * 3 + ["single"] * 2
+
+
+def test_module_bytes_counts_parameters_and_buffers_once():
+ lin = nn.Linear(4, 4)
+ lin.register_buffer("scale", torch.ones(3, dtype = torch.float16))
+ lin.alias = lin.weight # the same tensor twice
+ assert ls.module_bytes(lin) == (16 + 4) * 4 + 3 * 2
+
+
+def test_plan_fill_packs_the_primary_first():
+ dit = FakeDiT(dim = 1024, double = 4, single = 4)
+ block = _mib(dit.transformer_blocks[0])
+ single = _mib(dit.single_transformer_blocks[0])
+ rest = _mib(dit) - 4 * block - 4 * single
+ budget0 = int(rest + 3 * block + 0.5 * block + 1)
+ plan = ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: budget0, 1: 100})
+ assert plan.primary == "cuda:0"
+ assert [r.device for r in plan.ranges] == ["cuda:0", "cuda:1"]
+ assert plan.ranges[0].blocks == [f"transformer_blocks.{i}" for i in range(3)]
+ assert plan.ranges[1].blocks[0] == "transformer_blocks.3"
+ assert len(plan.ranges[1].blocks) == 5
+ assert set(plan.assignment.values()) == {"cuda:0", "cuda:1"}
+ plan = ls.plan_layer_split(
+ dit, primary = "cuda:0", cards = ["cuda:1"], budgets_mib = {"cuda:0": 100, "cuda:1": 100}
+ )
+ assert [r.device for r in plan.ranges] == ["cuda:0"]
+
+
+def test_plan_balance_spreads_by_budget():
+ dit = FakeDiT(dim = 1024, double = 4, single = 4)
+ plan = ls.plan_layer_split(
+ dit, primary = 0, cards = [1], budgets_mib = {0: 100, 1: 100}, strategy = "balance"
+ )
+ counts = [len(r.blocks) for r in plan.ranges]
+ assert counts and abs(counts[0] - counts[1]) <= 2 and sum(counts) == 8
+
+
+def test_plan_refusals():
+ dit = FakeDiT(dim = 1024, double = 4, single = 4)
+ with pytest.raises(ls.LayerSplitError, match = "two cards"):
+ ls.plan_layer_split(dit, primary = 0, cards = [0], budgets_mib = {0: 100})
+ with pytest.raises(ls.LayerSplitError, match = "no budget"):
+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 100})
+ with pytest.raises(ls.LayerSplitError, match = "non-block"):
+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 0, 1: 100})
+ with pytest.raises(ls.LayerSplitError, match = "blocks need"):
+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 10, 1: 1})
+ with pytest.raises(ls.LayerSplitError, match = "strategy"):
+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 100, 1: 100}, strategy = "x")
+
+
+def test_refusal_reasons():
+ assert ls.layer_split_refusal(None) == "no denoiser module"
+ dit = FakeDiT()
+ assert ls.layer_split_refusal(dit) is None
+ dit.transformer_blocks[0]._hf_hook = object()
+ assert "accelerate" in ls.layer_split_refusal(dit)
+ dit = FakeDiT()
+ dit.compile()
+ assert ls.layer_split_refusal(dit) == "denoiser is compiled whole"
+ assert "fewer than two" in ls.layer_split_refusal(FakeDiT(double = 1, single = 0))
+
+
+def test_off_switch_refuses(monkeypatch):
+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, "0")
+ with pytest.raises(ls.LayerSplitError, match = ls.LAYER_SPLIT_ENV):
+ ls.place_layer_split(Pipe(FakeDiT()), primary = 0, cards = [1], budgets_mib = {0: 9, 1: 9})
+
+
+class _Recorder:
+ """Stands in for the cross-card copies: records (what, device) and leaves the CPU tensors where they are."""
+
+ def __init__(
+ self,
+ monkeypatch,
+ simulate_cards = True,
+ ):
+ self.out_moves: list = []
+ self.in_moves: list = []
+ real = mg.move_to_device
+
+ def fake_move(
+ obj,
+ device,
+ _depth = 0,
+ *,
+ tensor_fn = None,
+ objects = True,
+ ):
+ assert (
+ objects is False
+ ) # a split never copies attribute holders (KV caches keep their identity)
+ if tensor_fn is None:
+ self.out_moves.append(str(device))
+ else:
+
+ def counted(t):
+ self.in_moves.append(str(device))
+ return t
+
+ return real(obj, torch.device("meta"), _depth, tensor_fn = counted, objects = False)
+ return obj
+
+ monkeypatch.setattr(mg, "move_to_device", fake_move)
+ monkeypatch.setattr(mg, "_device_context", lambda device: contextlib.nullcontext())
+ if simulate_cards:
+ monkeypatch.setattr(ls, "_probe_device", lambda probe, default: default)
+
+
+def _install(dit, budgets):
+ plan = ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = budgets)
+ split = ls._Split(dit, torch.device("cuda", 0), type(dit).__name__ in ls.LAZY_CLASSES)
+ ls._install_boundary(dit, plan, ls.find_blocks(dit), split)
+ return plan, split
+
+
+def _two_card_budget(dit):
+ block = _mib(dit.transformer_blocks[0])
+ rest = _mib(dit) - sum(_mib(b) for _, _, b in ls.find_blocks(dit))
+ return {0: int(rest + 2 * block + 0.5 * block + 1), 1: 100}
+
+
+def test_lazy_routing_moves_only_at_card_and_list_boundaries(monkeypatch):
+ torch.manual_seed(0)
+ dit = LazyDiT(dim = 1024, double = 4, single = 3)
+ x = torch.randn(2, 1024)
+ with torch.no_grad():
+ reference = dit(x)
+ plan, split = _install(dit, _two_card_budget(dit))
+ assert plan.ranges[0].blocks == ["transformer_blocks.0", "transformer_blocks.1"]
+ rec = _Recorder(monkeypatch)
+ with torch.no_grad():
+ out = dit(x)
+ assert torch.equal(out, reference)
+ assert rec.out_moves == ["cuda:1", "cuda:0", "cuda:0"]
+ assert split.lazy_calls == 1 and split.eager_calls == 0
+ assert not split.lazy_now # reset after the forward
+
+
+def test_eager_routing_with_a_residual_kwarg(monkeypatch):
+ torch.manual_seed(0)
+ dit = LazyDiT(dim = 1024, double = 4, single = 3)
+ x = torch.randn(2, 1024)
+ residual = [torch.randn(2, 1024) for _ in range(4)]
+ with torch.no_grad():
+ reference = dit(x, controlnet_block_samples = residual)
+ _, split = _install(dit, _two_card_budget(dit))
+ rec = _Recorder(monkeypatch)
+ with torch.no_grad():
+ out = dit(x, controlnet_block_samples = residual)
+ assert torch.equal(out, reference)
+ assert rec.out_moves == ["cuda:0"] * 5
+ assert split.eager_calls == 1
+
+
+def test_unlisted_class_is_always_eager(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ _, split = _install(dit, _two_card_budget(dit))
+ rec = _Recorder(monkeypatch)
+ with torch.no_grad():
+ dit(torch.randn(2, 1024))
+ assert rec.out_moves == ["cuda:0"] * 5
+ assert split.lazy_calls == 0
+
+
+def test_class_swap_keeps_names_and_undo_restores(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ budgets = _two_card_budget(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ info = ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = budgets)
+ assert [r["count"] for r in info["ranges"]] == [2, 5]
+ block = dit.transformer_blocks[3]
+ assert type(block).__name__ == "FakeBlock" and type(block) is not FakeBlock
+ assert isinstance(block, FakeBlock)
+ from core.inference.diffusion_block_graph import repeated_blocks
+
+ assert len(repeated_blocks(dit)) == len(
+ list(m for m in dit.modules() if isinstance(m, (FakeBlock, FakeSingle)))
+ )
+ assert ls.is_layer_split(dit) and mg.is_split(pipe)
+ assert dit._unsloth_split_home == torch.device("cuda", 0)
+ status = mg.placement_status(pipe)
+ assert status["placement"]["transformer"] == "cuda:0+cuda:1"
+ assert status["layer_split"]["per_card_mib"].keys() == {"cuda:0", "cuda:1"}
+ assert status["layer_split"]["calls"] == 0
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = budgets)
+ assert type(block).__mro__[1] is FakeBlock
+
+ assert ls.undo_layer_split(pipe)
+ assert all(type(b) in (FakeBlock, FakeSingle) for _, _, b in ls.find_blocks(dit))
+ assert not dit._forward_pre_hooks and not dit._forward_hooks
+ assert "_unsloth_split_home" not in dit.__dict__
+ assert not ls.is_layer_split(dit) and not mg.is_split(pipe)
+ assert ls.undo_layer_split(pipe) is False
+
+
+def test_undo_keeps_a_component_split_record(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ mg._PLACEMENTS[pipe] = {
+ "placement": {"vae": "cuda:1", "transformer": "cuda:0"},
+ "moved": ["vae"],
+ }
+ dit._unsloth_split_home = torch.device("cuda", 0)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ assert mg.placement_status(pipe)["placement"]["vae"] == "cuda:1"
+ ls.undo_layer_split(pipe)
+ record = mg._PLACEMENTS[pipe]
+ assert "layer_split" not in record and record["moved"] == ["vae"]
+ assert dit._unsloth_split_home == torch.device("cuda", 0) # the component split's marker stays
+
+
+def test_a_split_denoiser_is_freed_without_the_cycle_collector(monkeypatch):
+ import weakref
+
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ refs = [weakref.ref(dit), weakref.ref(dit.single_transformer_blocks[2])]
+ gc.disable()
+ try:
+ del dit, pipe
+ assert all(
+ r() is None for r in refs
+ ) # the blocks' weights go with the pipeline, as on one card
+ finally:
+ gc.enable()
+
+
+def test_memo_copies_once_and_drops_stale_entries():
+ memo = ls._Memo()
+ t = torch.ones(4)
+ a = memo.get(t, torch.float64)
+ assert a.dtype == torch.float64
+ assert memo.get(t, torch.float64) is a
+ t.add_(1) # an in-place write invalidates the copy
+ b = memo.get(t, torch.float64)
+ assert b is not a and torch.equal(b, torch.full((4,), 2.0, dtype = torch.float64))
+ assert len(memo.entries) == 1
+ del t, a, b
+ gc.collect()
+ assert not memo.entries # the source died: its copies go with it
+ with torch.inference_mode():
+ u = torch.ones(2)
+ assert memo.get(u, torch.float64) is memo.get(u, torch.float64)
+
+
+def test_move_to_device_tensor_fn_reaches_nested_tensors():
+ seen = []
+
+ def fn(t):
+ seen.append(t)
+ return t * 2
+
+ t = torch.ones(1)
+ out = mg.move_to_device({"a": (t, [t])}, torch.device("meta"), tensor_fn = fn)
+ assert len(seen) == 2 and torch.equal(out["a"][0], torch.full((1,), 2.0))
+
+
+def test_move_to_device_keeps_attribute_holders_when_asked():
+ class LayerCache:
+ def __init__(self):
+ self.k = torch.ones(1)
+
+ cache = LayerCache()
+ moved = mg.move_to_device(
+ {"c": cache}, torch.device("meta"), tensor_fn = lambda t: t, objects = False
+ )
+ assert moved["c"] is cache
+ assert mg.move_to_device(cache, torch.device("meta"), tensor_fn = lambda t: t) is not cache
+
+
+def test_cfg_parallel_declines_on_a_split(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "force")
+ assert "layer split" in mg.cfg_parallel_decline(pipe, offload_policy = "none", selection = sel)
+ from core.inference.diffusion_block_graph import install_block_graphs
+
+ handle, reason = install_block_graphs(dit)
+ assert handle is None and "weights off the GPU" in reason
+ ls.undo_layer_split(pipe)
+
+
+def test_a_whole_step_graph_armed_before_the_split_stays_eager(monkeypatch):
+ from core.inference import diffusion_cuda_graph as cg
+
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [pipe.transformer])
+ early = cg.GraphedForward(dit).enable()
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ seen = []
+ monkeypatch.setattr(early, "_eager", lambda args, kwargs: seen.append(1) or "eager")
+ assert early(torch.zeros(1), return_dict = False) == "eager" and seen
+ assert not early.poisoned and not early.stats.get("refused_host_tensor")
+ early.disable()
+ handles, reason = cg.arm_after_placement(pipe)
+ assert handles == () and "split across cards" in reason
+ ls.undo_layer_split(pipe)
+
+
+def test_a_positional_residual_runs_eager(monkeypatch):
+ torch.manual_seed(0)
+ dit = LazyDiT(dim = 1024, double = 4, single = 3)
+ x = torch.randn(2, 1024)
+ residual = [torch.randn(2, 1024) for _ in range(4)]
+ with torch.no_grad():
+ reference = dit(x, residual)
+ _, split = _install(dit, _two_card_budget(dit))
+ rec = _Recorder(monkeypatch)
+ with torch.no_grad():
+ out = dit(x, residual)
+ assert torch.equal(out, reference)
+ assert rec.out_moves == ["cuda:0"] * 5
+ assert split.eager_calls == 1 and split.lazy_calls == 0
+ with torch.no_grad():
+ dit(x, None)
+ assert split.lazy_calls == 1
+
+
+def test_routes_follow_a_relocated_denoiser(monkeypatch):
+ torch.manual_seed(0)
+ dit = LazyDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ x = torch.randn(2, 1024)
+ with torch.no_grad():
+ reference = dit(x)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ real = mg.move_to_device
+ moves: list = []
+
+ def spy(
+ obj,
+ device,
+ _depth = 0,
+ *,
+ tensor_fn = None,
+ objects = True,
+ ):
+ moves.append(str(device))
+ return real(obj, device, _depth, tensor_fn = tensor_fn, objects = objects)
+
+ monkeypatch.setattr(mg, "move_to_device", spy)
+ with torch.no_grad():
+ out = dit(x) # on the CPU: a route still aimed at cuda:1 would fail here
+ assert torch.equal(out, reference)
+ assert set(moves) <= {"cpu"}
+ assert dit._unsloth_split_home == torch.device("cuda", 0)
+ assert mg._split_home(pipe) == torch.device("cpu")
+ ls.undo_layer_split(pipe)
+
+
+def test_routes_follow_one_relocated_block(monkeypatch):
+ torch.manual_seed(0)
+ dit = LazyDiT(dim = 1024, double = 4, single = 3)
+ _install(dit, _two_card_budget(dit))
+ moved = dit.transformer_blocks[3]
+ where = {id(moved): torch.device("cuda", 0)}
+
+ def probe_device(probe, default):
+ owner = probe[0]() if probe is not None else None
+ for module in (owner,):
+ for block in dit.modules():
+ if (
+ block is moved
+ and owner is not None
+ and any(m is owner for m in block.modules())
+ ):
+ return where[id(moved)]
+ return default
+
+ rec = _Recorder(monkeypatch)
+ monkeypatch.setattr(ls, "_probe_device", probe_device)
+ with torch.no_grad():
+ dit(torch.randn(2, 1024))
+ assert rec.out_moves == ["cuda:1", "cuda:0", "cuda:0"]
+
+
+def test_generic_undo_split_removes_a_layer_split(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ assert mg.is_split(pipe)
+ mg.undo_split(pipe)
+ assert not ls.is_layer_split(dit) and not mg.is_split(pipe)
+ assert all(type(b) in (FakeBlock, FakeSingle) for _, _, b in ls.find_blocks(dit))
+ assert not dit._forward_pre_hooks and not dit._forward_hooks
+ assert pipe not in mg._PLACEMENTS
+
+
+def test_eager_calls_keep_no_copy_across_blocks(monkeypatch):
+ # inference tensors carry no version: an eager call must not reuse an earlier block's copy; a lazy call shares one per card
+ class InPlaceDiT(FakeDiT):
+ def forward(
+ self,
+ hidden_states,
+ controlnet_block_samples = None,
+ ):
+ temb = torch.ones_like(hidden_states[:1])
+ h = hidden_states
+ for block in self.transformer_blocks:
+ h = block(h, temb)
+ temb.add_(1) # written in place on the primary between blocks
+ for block in self.single_transformer_blocks:
+ h = block(h, temb)
+ return h
+
+ hits: list = []
+
+ def fake_move(
+ obj,
+ device,
+ _depth = 0,
+ *,
+ tensor_fn = None,
+ objects = True,
+ ):
+ if tensor_fn is None:
+ return obj
+
+ def walk(o):
+ if isinstance(o, torch.Tensor):
+ return tensor_fn(o)
+ if isinstance(o, (list, tuple)):
+ return type(o)(walk(v) for v in o)
+ if isinstance(o, dict):
+ return {k: walk(v) for k, v in o.items()}
+ return o
+
+ return walk(obj)
+
+ def fake_get(self, t, device):
+ hits.append(id(t) in self.entries) # a copy made for an earlier block would be served again
+ self.entries[id(t)] = (None, 0, None, {device: t})
+ return t
+
+ for cls, lazy in (
+ (InPlaceDiT, False),
+ (type("Flux2Transformer2DModel", (InPlaceDiT,), {}), True),
+ ):
+ torch.manual_seed(0)
+ dit = cls(dim = 64, double = 4, single = 3)
+ x = torch.randn(2, 64)
+ with torch.inference_mode():
+ reference = dit(x)
+ _, split = _install(dit, {0: 10**6, 1: 10**6})
+ monkeypatch.setattr(mg, "move_to_device", fake_move)
+ monkeypatch.setattr(mg, "_device_context", lambda device: contextlib.nullcontext())
+ monkeypatch.setattr(ls, "_probe_device", lambda probe, default: default)
+ monkeypatch.setattr(split, "home", torch.device("cpu"))
+ for qual, _, block in ls.find_blocks(dit):
+ block.__dict__[ls._SPLIT_ATTR].device = torch.device("cuda", 1)
+ monkeypatch.setattr(ls._Memo, "get", fake_get)
+ hits.clear()
+ with torch.inference_mode():
+ out = dit(x)
+ assert torch.equal(out, reference)
+ if lazy:
+ assert any(hits) # temb is copied once per card and shared by the blocks
+ else:
+ assert hits and not any(hits) # every block re-copies what it is given
+ monkeypatch.undo()
+
+
+def test_flex_processor_refuses_and_block_mask_moves():
+ class FlexAttnProcessor:
+ pass
+
+ dit = FakeDiT()
+ dit.attn_processors = {"transformer_blocks.0.attn.processor": FlexAttnProcessor()}
+ assert "flex attention" in ls.layer_split_refusal(dit)
+
+ class BlockMask:
+ def __init__(self, device):
+ self.device = device
+
+ def to(self, device):
+ return BlockMask(device)
+
+ mask = BlockMask("cpu")
+ moved = mg.move_to_device({"attention_mask": mask}, torch.device("meta"), objects = False)
+ assert moved["attention_mask"].device == torch.device("meta")
+
+ class KVCache: # an attribute holder the callee writes into keeps its identity
+ pass
+
+ cache = KVCache()
+ assert (
+ mg.move_to_device({"cache": cache}, torch.device("meta"), objects = False)["cache"] is cache
+ )
+
+
+def test_graph_eligibility_refuses_a_split_unet(monkeypatch):
+ from core.inference import diffusion_cuda_graph as cg
+
+ unet = FakeDiT(dim = 1024, double = 4, single = 3)
+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [unet])
+ from core.inference import diffusion_speed
+
+ monkeypatch.setattr(diffusion_speed, "_denoiser_dits", lambda pipe: [])
+ monkeypatch.setattr(diffusion_speed, "_denoiser_unet", lambda pipe: unet)
+ pipe = Pipe(unet)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(unet))
+ target = type("T", (), {"device": "cuda", "backend": "cuda"})()
+ monkeypatch.setattr(cg, "cuda_graph_disabled", lambda: False)
+ ok, reason = cg.graph_eligible(
+ target,
+ family = None,
+ pipe = pipe,
+ offload_active = False,
+ cache_active = False,
+ speed_mode = "default",
+ )
+ assert not ok and "split" in reason
+ ls.undo_layer_split(pipe)
diff --git a/studio/backend/tests/test_diffusion_layer_split_cuda.py b/studio/backend/tests/test_diffusion_layer_split_cuda.py
new file mode 100644
index 00000000000..03dfd2d9eae
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split_cuda.py
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A layer split on two real CUDA cards against one card, bitwise. Skipped below two cards."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from core.inference import diffusion_layer_split as ls
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs two CUDA cards"
+)
+
+_NL, _NS = 3, 5
+
+
+def _hidream():
+ hidream = pytest.importorskip("diffusers.models.transformers.transformer_hidream_image")
+ torch.manual_seed(0)
+ model = hidream.HiDreamImageTransformer2DModel(
+ patch_size = 2,
+ in_channels = 16,
+ num_layers = _NL,
+ num_single_layers = _NS,
+ attention_head_dim = 32,
+ num_attention_heads = 4,
+ caption_channels = [64, 48],
+ text_emb_dim = 64,
+ num_routed_experts = 4,
+ num_activated_experts = 2,
+ axes_dims_rope = (8, 12, 12),
+ max_resolution = (64, 64),
+ llama_layers = list(range(_NL + _NS)),
+ ).to(torch.bfloat16)
+ with torch.no_grad(): # weights large enough that every block moves the result
+ for p in model.parameters():
+ p.normal_(0, 0.05)
+ return model.eval()
+
+
+def _run(model):
+ g = torch.Generator().manual_seed(1)
+ inputs = dict(
+ hidden_states = torch.randn(1, 16, 32, 32, generator = g),
+ encoder_hidden_states_t5 = torch.randn(1, 16, 64, generator = g),
+ encoder_hidden_states_llama3 = torch.randn(_NL + _NS, 1, 12, 48, generator = g),
+ pooled_embeds = torch.randn(1, 64, generator = g),
+ )
+ kw = {k: v.to("cuda:0", torch.bfloat16) for k, v in inputs.items()}
+ with torch.inference_mode():
+ return (
+ model(**kw, timesteps = torch.tensor([500], device = "cuda:0"), return_dict = False)[0]
+ .float()
+ .cpu()
+ )
+
+
+class _Pipe:
+ pass
+
+
+def test_a_lazy_hidream_split_matches_one_card_bitwise():
+ assert "HiDreamImageTransformer2DModel" in ls.LAZY_CLASSES
+ reference = _run(_hidream().to("cuda:0"))
+ pipe = _Pipe()
+ pipe.transformer = _hidream().to("cuda:0")
+ blocks = [qual for qual, _, _ in ls.find_blocks(pipe.transformer)]
+ ls.place_layer_split(
+ pipe,
+ primary = 0,
+ assignment = {q: "cuda:0" if i == 0 else "cuda:1" for i, q in enumerate(blocks)},
+ )
+ try:
+ out = _run(pipe.transformer)
+ stats = ls.layer_split_stats(pipe.transformer)
+ assert {str(p.device) for p in pipe.transformer.parameters()} == {"cuda:0", "cuda:1"}
+ assert stats["lazy_calls"] == 1 and stats["eager_calls"] == 0
+ assert torch.equal(out, reference)
+ finally:
+ ls.undo_layer_split(pipe)
+
+
+def test_a_hidream_split_hops_once_per_card_boundary_and_records_per_card_segments(monkeypatch):
+ from core.inference import diffusion_block_graph as bg
+
+ monkeypatch.setitem(globals(), "_NL", 4)
+ monkeypatch.setitem(globals(), "_NS", 6)
+ reference = _run(_hidream().to("cuda:0"))
+ pipe = _Pipe()
+ pipe.transformer = _hidream().to("cuda:0")
+ on_second = {"double_stream_blocks.2.block", "double_stream_blocks.3.block"} | {
+ f"single_stream_blocks.{i}.block" for i in (3, 4, 5)
+ }
+ blocks = [qual for qual, _, _ in ls.find_blocks(pipe.transformer)]
+ assert on_second <= set(blocks) and len(blocks) == 10
+ ls.place_layer_split(
+ pipe, primary = 0, assignment = {q: "cuda:1" if q in on_second else "cuda:0" for q in blocks}
+ )
+ hops = []
+ real = ls.cross_outputs
+ monkeypatch.setattr(
+ ls, "cross_outputs", lambda out, device: hops.append(str(device)) or real(out, device)
+ )
+ try:
+ out = _run(pipe.transformer)
+ stats = ls.layer_split_stats(pipe.transformer)
+ assert hops == ["cuda:1", "cuda:0", "cuda:1", "cuda:0"]
+ assert stats["input_copies"] == 2 and stats["lazy_calls"] == 1
+ assert torch.equal(out, reference)
+ handle, why = bg.install_block_graphs(pipe.transformer, slots = False)
+ assert handle is not None, why
+ try:
+ assert [len(seg.graphs) for seg in handle.segments] == [2, 2, 3, 3]
+ finally:
+ handle.free()
+ finally:
+ ls.undo_layer_split(pipe)
diff --git a/studio/backend/tests/test_diffusion_layer_split_lifecycle.py b/studio/backend/tests/test_diffusion_layer_split_lifecycle.py
new file mode 100644
index 00000000000..9b4dbbb06c5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split_lifecycle.py
@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A layer split across the image backend's lifecycle, CPU only: the generate-time guard on the cards that run blocks
+(priced from what the plan reserved there, read off the loaded pipe even for a workflow pipeline built with
+from_pipe), and the on-demand resident hooks a stale one-card measured reserve would otherwise put on a split
+denoiser."""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+from core.inference import diffusion as D
+from core.inference import diffusion_layer_split_plan as lsp
+from core.inference import diffusion_memory as dm
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_backend import _loaded_backend, _tiny_png_b64, fake_runtime # noqa: E402,F401
+
+
+class _Pipe:
+ components: dict = {}
+
+
+def _split(
+ required = 12000,
+ weights = 9000,
+ reserve = None,
+) -> lsp.LayerSplit:
+ return lsp.LayerSplit(
+ **({} if reserve is None else {"card_reserve_mib": ((0, 1024), (1, reserve))}),
+ primary = 0,
+ cards = (0, 1),
+ ranges = (
+ lsp.BlockRange("transformer", 0, 10, 0, 9000),
+ lsp.BlockRange("transformer", 10, 20, 1, weights),
+ ),
+ text_encoders_card = None,
+ card_required_mib = ((0, 30000), (1, required)),
+ card_limit_mib = ((0, 40000), (1, 20000)),
+ card_weights_mib = ((0, 20000), (1, weights)),
+ )
+
+
+def _placed(split: lsp.LayerSplit) -> _Pipe:
+ pipe = _Pipe()
+ lsp._APPLIED[pipe] = split
+ return pipe
+
+
+def _free(monkeypatch, free_mib):
+ monkeypatch.setattr(
+ D,
+ "reclaimable_snapshot_device_memory",
+ lambda target: dm.DeviceMemory(
+ backend = "cuda",
+ device = "cuda",
+ memory_kind = "discrete_vram",
+ free_mib = free_mib,
+ total_mib = 24576,
+ ),
+ )
+
+
+def _kw(side):
+ return {
+ "width": side,
+ "height": side,
+ "batch_size": 1,
+ "family": "flux2",
+ "condition_pixels": 0,
+ }
+
+
+def test_block_card_guard_prices_the_planned_headroom_not_the_whole_request(monkeypatch):
+ pipe = _placed(_split())
+ _free(monkeypatch, 8000)
+ assert D._guard_layer_cards(pipe, object(), _kw(1024)) == 1
+ assert D._guard_layer_cards(pipe, object(), _kw(1536)) == 1
+ _free(monkeypatch, 4000)
+ assert D._guard_layer_cards(pipe, object(), _kw(1024)) == 1
+ with pytest.raises(ValueError, match = "cuda:1"):
+ D._guard_layer_cards(pipe, object(), _kw(1536))
+ assert D._guard_layer_cards(pipe, object(), {**_kw(1536), "allow_oversized": True}) == 0
+
+
+def test_block_card_guard_scales_only_the_working_set_not_the_fixed_reserve(monkeypatch):
+ pipe = _placed(_split(reserve = 2500))
+ assert pipe in lsp._APPLIED and lsp._APPLIED[pipe].working_mib(1) == (500, 2500)
+ _free(monkeypatch, 4000)
+ assert D._guard_layer_cards(pipe, object(), _kw(1536)) == 1
+ _free(monkeypatch, 2400)
+ with pytest.raises(ValueError, match = "cuda:1"):
+ D._guard_layer_cards(pipe, object(), _kw(1024))
+
+
+def test_block_card_guard_is_a_no_op_without_a_layer_split(monkeypatch):
+ _free(monkeypatch, 10)
+ assert D._guard_layer_cards(_Pipe(), object(), _kw(2048)) == 0
+ assert D._guard_layer_cards(None, object(), _kw(2048)) == 0
+
+
+def test_image_generate_guards_the_loaded_pipe_for_a_workflow_pipeline(
+ fake_runtime, tmp_path, monkeypatch
+):
+ seen = []
+ monkeypatch.setattr(
+ D,
+ "raise_on_image_activation_shortfall",
+ lambda **kw: types.SimpleNamespace(action = dm.ACTIVATION_RUN, overridden = False),
+ )
+ monkeypatch.setattr(
+ D, "_guard_layer_cards", lambda pipe, target, kwargs: seen.append(pipe) or 0
+ )
+ (tmp_path / "model.gguf").write_bytes(b"x")
+ backend = _loaded_backend(tmp_path)
+ backend.generate(
+ prompt = "a car", steps = 4, guidance = 0.0, seed = 3, init_image = _tiny_png_b64(), strength = 0.5
+ )
+ assert seen and seen[-1] is backend._state.pipe
+
+
+def test_a_layer_split_drops_the_one_card_measured_reserve():
+ class Pipe:
+ _unsloth_measured_reserve = (4096, "flux2", "default")
+
+ pipe = Pipe()
+ plain = types.SimpleNamespace(layer_split = None)
+ assert not D._drop_measured_reserve_for_layer_split(pipe, plain)
+ assert pipe._unsloth_measured_reserve
+ assert D._drop_measured_reserve_for_layer_split(
+ pipe, types.SimpleNamespace(layer_split = _split())
+ )
+ assert not pipe._unsloth_measured_reserve
+ assert (
+ dm.measured_request_extra_mib(pipe, width = 2048, height = 2048, condition_pixels = 1 << 20) == 0
+ )
diff --git a/studio/backend/tests/test_diffusion_layer_split_order.py b/studio/backend/tests/test_diffusion_layer_split_order.py
new file mode 100644
index 00000000000..be875001f78
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split_order.py
@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Layer split vs the load-time speed setup, CPU only: Studio's First-Block-Cache is installed before placement, so a
+split must remove it instead of refusing the denoiser, the load must re-read the cache state, and a decline must name
+the engine's real reason."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+
+from core.inference import diffusion as D
+from core.inference import diffusion_cache as dc
+from core.inference import diffusion_layer_split as ls
+from core.inference import diffusion_layer_split_plan as lsp
+from core.inference import diffusion_multigpu as mg
+
+from .test_diffusion_layer_split import FakeDiT, Pipe, _two_card_budget
+from .test_diffusion_layer_split_plan import _cards, _plan, _refine, engine # noqa: F401 - fixture
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(ls.LAYER_SPLIT_ENV, raising = False)
+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
+ # pin the host so the fake [0, 1] cards plan on any machine (ROCm included)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
+ monkeypatch.setattr(mg, "_capability", lambda o: None)
+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ yield
+
+
+class _Registry:
+ def __init__(self, *names):
+ self.hooks = {n: object() for n in names}
+
+
+def _hook(module, *names):
+ module._diffusers_hook = _Registry(*names)
+
+
+def _fbcache(dit):
+ """What apply_step_cache leaves: FBC hooks on the leader and the other blocks, and the marker."""
+ blocks = [b for _, _, b in ls.find_blocks(dit)]
+ _hook(blocks[0], "fbc_leader_block_hook")
+ for block in blocks[1:]:
+ _hook(block, "fbc_block_hook")
+ dit._unsloth_step_cache = dc.TC_FBCACHE
+
+
+def _fake_toggle(monkeypatch, removes = True):
+ calls = []
+
+ def toggle(pipe, *, steps, **kw):
+ calls.append(steps)
+ t = pipe.transformer
+ if removes:
+ for m in t.modules():
+ m.__dict__.pop("_diffusers_hook", None)
+ t._unsloth_step_cache = None
+ return None
+
+ monkeypatch.setattr(dc, "maybe_toggle_step_cache", toggle)
+ return calls
+
+
+def test_studio_step_cache_does_not_refuse_a_split_but_other_hooks_do():
+ dit = FakeDiT()
+ _fbcache(dit)
+ assert ls.step_cache_hooked(dit)
+ assert "carries diffusers hooks" in ls.layer_split_refusal(dit)
+ assert ls.layer_split_refusal(dit, step_cache_ok = True) is None
+ _hook(dit.proj_out, "group_offloading")
+ assert not ls.step_cache_hooked(dit)
+ assert "carries diffusers hooks" in ls.layer_split_refusal(dit, step_cache_ok = True)
+ assert not ls.step_cache_hooked(FakeDiT())
+
+
+def test_placement_removes_the_step_cache_first(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ _fbcache(dit)
+ pipe = Pipe(dit)
+ calls = _fake_toggle(monkeypatch)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ assert calls == [0] # disengaged through the cache's own teardown
+ assert ls.is_layer_split(dit) and not ls.step_cache_hooked(dit)
+ assert D._step_cache_dropped_by_layer_split(pipe)
+ ls.undo_layer_split(pipe)
+
+
+def test_a_cache_that_will_not_come_off_refuses_the_split(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ _fbcache(dit)
+ _fake_toggle(monkeypatch, removes = False)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ with pytest.raises(ls.LayerSplitError, match = "step cache"):
+ ls.place_layer_split(Pipe(dit), primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ assert not ls.is_layer_split(dit)
+
+
+def test_a_split_that_does_not_fit_leaves_the_cache_alone(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ _fbcache(dit)
+ calls = _fake_toggle(monkeypatch)
+ with pytest.raises(ls.LayerSplitError):
+ ls.place_layer_split(Pipe(dit), primary = 0, cards = [1], budgets_mib = {0: 1, 1: 1})
+ assert calls == [] and ls.step_cache_hooked(dit)
+
+
+def test_uncached_or_still_cached_pipes_are_not_reported_dropped():
+ dit = FakeDiT()
+ pipe = Pipe(dit)
+ dit._unsloth_step_cache = dc.TC_FBCACHE
+ assert not D._step_cache_dropped_by_layer_split(pipe)
+ dit._unsloth_step_cache = None
+ assert D._step_cache_dropped_by_layer_split(pipe)
+
+
+def test_the_planner_plans_through_the_step_cache():
+ dit = FakeDiT()
+ _fbcache(dit)
+ blocks = lsp.split_blocks(dit)
+ assert blocks is not None and len(blocks) == 6
+
+
+def test_a_decline_names_the_engines_reason(monkeypatch, engine): # noqa: F811
+ _cards(monkeypatch)
+ dit = FakeDiT()
+ _hook(dit.transformer_blocks[0], "group_offloading")
+
+ class HookedPipe:
+ transformer = dit
+ components: dict = {}
+
+ pipe = HookedPipe()
+ monkeypatch.setattr(lsp, "_denoiser_layouts", lambda p: None)
+ plan = _plan()
+ refined, _ = _refine(plan, pipe = pipe)
+ assert refined is plan
+ declined = lsp.layer_split_status(pipe)["declined"]
+ assert declined.startswith("denoiser cannot be split (transformer: ")
+ assert "group_offloading" in declined
+
+
+def test_the_speed_layer_is_set_up_for_a_split_the_placement_will_make():
+ plan = types.SimpleNamespace(offload_policy = "group")
+ split = types.SimpleNamespace(offload_policy = "none", layer_split = object())
+ assert D._layer_split_expected(None, plan, lambda p, pl, lg: split)
+ assert not D._layer_split_expected(
+ None, plan, lambda p, pl, lg: plan
+ ) # the one-card plan stands
+ resident = types.SimpleNamespace(offload_policy = "none")
+ assert not D._layer_split_expected(
+ None, resident, lambda p, pl, lg: split
+ ) # already resident: nothing to change
+ assert not D._layer_split_expected(
+ None, plan, lambda p, pl, lg: 1 / 0
+ ) # a failing preview keeps the plan
+
+
+def test_the_step_cache_goes_back_on_after_the_split_placed_the_blocks(monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ D,
+ "apply_step_cache",
+ lambda view, mode = None, **kw: seen.append((view, mode, kw)) or f"m{len(seen)}",
+ )
+ pipe, second = object(), object()
+ assert D._restore_step_cache_after_layer_split(pipe, (second, pipe), threshold = 0.1) == "m2"
+ assert [v for v, _m, _k in seen] == [second, pipe] and {m for _v, m, _k in seen} == {
+ dc.TC_FBCACHE
+ }
+ assert seen[0][2] == {"threshold": 0.1}
+
+
+def test_split_loads_bind_speed_records_after_placement():
+ import inspect
+
+ from core.inference import video as V
+
+ image = inspect.getsource(D.DiffusionBackend)
+ video = inspect.getsource(V.VideoBackend)
+ for src in (image, video):
+ assert "split_expected = _layer_split_expected(" in src
+ assert "offload_active = speed_offload" in src
+ assert "_restore_step_cache_after_layer_split(" in src
+ assert "engage_pinned_denoisers(" in src
+ assert "diffusion_cuda_graph.arm_block_graphs(" in video
+
+
+def test_a_resident_torchao_free_int8_dit_is_compiled_like_a_torchao_one(monkeypatch):
+ monkeypatch.setattr(D, "native_offload_host", lambda target: True)
+ monkeypatch.setattr(D, "native_quant_host", lambda target: False)
+ assert D._native_quant_needs_compile(object(), "int8", resident = True)
+ assert not D._native_quant_needs_compile(
+ object(), "int8", resident = False
+ ) # streamed: the offload path decides
+ assert not D._native_quant_needs_compile(
+ object(), None, resident = True
+ ) # torchao: the existing rule
+ monkeypatch.setattr(
+ D, "native_quant_host", lambda target: True
+ ) # ROCm / Apple keep their native policy
+ assert not D._native_quant_needs_compile(object(), "int8", resident = True)
+ monkeypatch.setattr(D, "native_quant_host", lambda target: False)
+ monkeypatch.setattr(D, "native_offload_host", lambda target: False)
+ assert not D._native_quant_needs_compile(object(), "int8", resident = True)
+
+
+def test_a_companion_split_preview_counts_as_a_resident_denoiser():
+ plan = types.SimpleNamespace(offload_policy = "group")
+ companion = types.SimpleNamespace(offload_policy = "none", secondary_ordinal = 1, layer_split = None)
+ assert D._companion_split_expected(None, plan, lambda p, pl, lg: companion)
+ assert not D._companion_split_expected(
+ None, plan, lambda p, pl, lg: plan
+ ) # the one-card plan stands
+ layered = types.SimpleNamespace(
+ offload_policy = "none", secondary_ordinal = None, layer_split = object()
+ )
+ assert not D._companion_split_expected(
+ None, plan, lambda p, pl, lg: layered
+ ) # _layer_split_expected's case
+ resident = types.SimpleNamespace(offload_policy = "none")
+ assert not D._companion_split_expected(
+ None, resident, lambda p, pl, lg: companion
+ ) # already resident
+ assert not D._companion_split_expected(
+ None, plan, lambda p, pl, lg: 1 / 0
+ ) # a failing preview keeps the plan
+
+
+def test_both_backends_price_a_companion_split_as_resident_for_the_quant_rule():
+ import inspect
+
+ from core.inference import video as V
+ for src in (inspect.getsource(D.DiffusionBackend), inspect.getsource(V.VideoBackend)):
+ rule = src[src.index("quant_forces_compile = ") :]
+ rule = rule[: rule.index("if quant_forces_compile and effective_speed == SPEED_OFF:")]
+ assert (
+ "_native_quant_needs_compile(" in rule
+ and "_companion_split_expected(pipe, plan" in rule
+ )
+
+
+def test_both_backends_force_compile_through_the_one_quant_rule():
+ import inspect
+
+ from core.inference import video as V
+ for src in (inspect.getsource(D.DiffusionBackend), inspect.getsource(V.VideoBackend)):
+ assert "quant_forces_compile = " in src and "_native_quant_needs_compile(" in src
+ assert "if quant_forces_compile and effective_speed == SPEED_OFF:" in src
+ assert src.index("split_expected = _layer_split_expected(") < src.index(
+ "quant_forces_compile = "
+ )
diff --git a/studio/backend/tests/test_diffusion_layer_split_plan.py b/studio/backend/tests/test_diffusion_layer_split_plan.py
new file mode 100644
index 00000000000..a0d7c6bbe20
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split_plan.py
@@ -0,0 +1,947 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Layer split planner (core/inference/diffusion_layer_split_plan.py), CPU only, fake card memory.
+
+The pure packer is driven with card limits directly; the post-load refinement through ``refine_plan_for_multigpu``
+with patched loaded sizes, block layouts and secondary limits; the engine is a recording fake."""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+import torch
+
+from core.inference import diffusion_layer_split_plan as lsp
+from core.inference import diffusion_memory as dm
+from core.inference import diffusion_multigpu as mg
+
+L = lsp.DenoiserLayout
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)
+ monkeypatch.delenv(lsp.LAYER_SPLIT_ENV, raising = False)
+ # pin the host so the fake [0, 1] cards plan on any machine (ROCm included)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
+ monkeypatch.setattr(mg, "_capability", lambda o: None)
+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ yield
+
+
+def _check(split, limits):
+ """Every card within its limit, every block placed exactly once in contiguous ranges."""
+ for o, req in split.card_required_mib:
+ assert req <= limits[o], (o, req, limits[o])
+ for name in dict.fromkeys(r.denoiser for r in split.ranges):
+ rs = [r for r in split.ranges if r.denoiser == name]
+ assert rs[0].start == 0
+ for a, b in zip(rs, rs[1:]):
+ assert a.stop == b.start
+ return split
+
+
+def test_packs_a_dit_too_big_for_one_card_over_two():
+ limits = {0: 18000, 1: 18000}
+ split, reason = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 23)},
+ companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 2000,
+ overhead_mib = 500,
+ )
+ assert split is not None, reason
+ _check(split, limits)
+ assert split.cards == (0, 1)
+ assert split.ranges[-1].stop == 23
+ assert split.text_encoders_card == 1
+ weights = dict(split.card_weights_mib)
+ assert sum(weights.values()) == 1000 + 23000 + 4000 + 300
+ assert weights[0] == 1000 + 300 + sum(r.mib for r in split.ranges if r.card == 0)
+ assert split.boundaries == 2
+ assert "layer split" in reason
+
+
+def test_text_encoders_too_big_for_any_card_stream_and_the_blocks_still_split():
+ kw = dict(
+ denoisers = {"transformer": L(1000, (1100,) * 54)},
+ companions = {"text_encoder": (45000, "text_encoder"), "vae": (300, "other")},
+ primary = 0,
+ card_limits = {0: 55000, 1: 55000},
+ headroom_mib = 6000,
+ overhead_mib = 1000,
+ )
+ split, reason = lsp.plan_layer_split(**kw)
+ assert split is not None, reason
+ assert split.text_encoders_streamed and split.text_encoders_card is None
+ assert split.as_public_dict()["text_encoders_streamed"] is True
+ assert {r.card for r in split.ranges} == {0, 1}
+ _check(split, kw["card_limits"])
+ kw["card_limits"] = {0: 55000, 1: 110000}
+ split, reason = lsp.plan_layer_split(**kw)
+ assert split is not None, reason
+ assert not split.text_encoders_streamed and split.text_encoders_card == 1
+
+
+def test_streamed_encoders_are_paged_not_moved_and_undo_removes_their_hooks(monkeypatch):
+ calls = {"stream": [], "engine": [], "removed": []}
+
+ class Comp(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1))
+ self.moved = []
+
+ def to(self, *a, **k):
+ self.moved.append(a[0] if a else k.get("device"))
+ return self
+
+ class Pipe:
+ def __init__(self):
+ self.transformer = Comp()
+ self.text_encoder = Comp()
+ self.vae = Comp()
+ self.components = {
+ "transformer": self.transformer,
+ "text_encoder": self.text_encoder,
+ "vae": self.vae,
+ }
+
+ pipe = Pipe()
+ split = lsp.LayerSplit(
+ primary = 0,
+ cards = (0, 1),
+ ranges = (
+ lsp.BlockRange("transformer", 0, 1, 0, 1),
+ lsp.BlockRange("transformer", 1, 2, 1, 1),
+ ),
+ text_encoders_card = None,
+ card_required_mib = ((0, 1), (1, 1)),
+ card_limit_mib = ((0, 9), (1, 9)),
+ text_encoders_streamed = True,
+ )
+ from core.inference import diffusion_layer_split as engine
+
+ monkeypatch.setattr(
+ dm,
+ "stream_text_encoders_onto",
+ lambda p, d, logger = None: calls["stream"].append(str(d)) or ["text_encoder"],
+ )
+ monkeypatch.setattr(dm, "_remove_group_offload_hooks", lambda m: calls["removed"].append(m))
+ monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])
+ monkeypatch.setattr(
+ engine, "place_layer_split", lambda p, **kw: calls["engine"].append(kw["assignment"]) or {}
+ )
+ monkeypatch.setattr(engine, "undo_layer_split", lambda p, attr = "transformer": False)
+ import contextlib
+
+ from core.inference import diffusion_fast_load
+
+ monkeypatch.setattr(
+ diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()
+ )
+ lsp.place_layer_split(pipe, primary = "cuda:0", split = split)
+ assert pipe.text_encoder.moved == [] # never moved onto a card: paged by its hooks
+ assert pipe.vae.moved == [torch.device("cuda", 0)]
+ assert calls["stream"] == ["cuda:0"]
+ assert calls["engine"] == [{"b.0": 0, "b.1": 1}]
+ record = mg._PLACEMENTS[pipe]
+ assert record["streamed"] == ["text_encoder"] and record["moved"] == []
+ assert record["placement"]["text_encoder"].startswith("cpu (streamed")
+ lsp.undo_layer_split(pipe)
+ assert calls["removed"] == [pipe.text_encoder]
+ assert pipe not in mg._PLACEMENTS
+
+
+def test_every_denoiser_keeps_a_block_off_the_primary():
+ limits = {0: 100000, 1: 6000}
+ kw = dict(
+ companions = {"text_encoder": (2000, "text_encoder"), "vae": (300, "other")},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 1000,
+ overhead_mib = 0,
+ working_set_mib = 1000,
+ secondary_overhead_mib = 0,
+ )
+ for denoisers in (
+ {"transformer": L(0, (1000,) * 10)},
+ {"transformer": L(0, (1000,) * 10), "transformer_2": L(0, (1000,) * 10)},
+ ):
+ split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)
+ assert split is not None, reason
+ _check(split, limits)
+ for name in denoisers:
+ assert any(r.card != 0 for r in split.ranges if r.denoiser == name), (
+ name,
+ split.ranges,
+ )
+
+
+def test_too_small_for_the_combined_cards_declines():
+ split, reason = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 40)},
+ companions = {"vae": (300, "other")},
+ primary = 0,
+ card_limits = {0: 18000, 1: 18000},
+ headroom_mib = 2000,
+ overhead_mib = 500,
+ )
+ assert split is None
+ assert "does not fit" in reason
+
+
+def test_no_secondary_or_unknown_blocks_declines():
+ assert (
+ lsp.plan_layer_split(
+ denoisers = {"transformer": L(0, (1000,))},
+ companions = {},
+ primary = 0,
+ card_limits = {0: 18000},
+ headroom_mib = 0,
+ overhead_mib = 0,
+ )[0]
+ is None
+ )
+ assert (
+ lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, ())},
+ companions = {},
+ primary = 0,
+ card_limits = {0: 18000, 1: 18000},
+ headroom_mib = 0,
+ overhead_mib = 0,
+ )[0]
+ is None
+ )
+
+
+def test_mixed_card_sizes_cut_in_proportion_to_free_capacity():
+ limits = {0: 20000, 1: 68000}
+ split, _ = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 60)},
+ companions = {},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 3000,
+ overhead_mib = 500,
+ )
+ _check(split, limits)
+ on = {o: sum(r.mib for r in split.ranges if r.card == o) for o in (0, 1)}
+ reserve = lsp.secondary_reserve_mib()
+ caps = {0: 20000 - 1000 - 3000 - max(500, reserve), 1: 68000 - 2000 - reserve}
+ for o in (0, 1):
+ assert abs(on[o] / caps[o] - 60000 / sum(caps.values())) < 1000 / caps[o] + 1e-9
+
+
+def test_uses_a_third_card_only_when_two_are_not_enough():
+ limits = {0: 18000, 1: 18000, 2: 18000}
+ two, _ = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 25)},
+ companions = {},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 2000,
+ overhead_mib = 500,
+ )
+ assert two.cards == (0, 1)
+ three, _ = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 40)},
+ companions = {},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 2000,
+ overhead_mib = 500,
+ )
+ _check(three, limits)
+ assert three.cards == (0, 1, 2)
+ assert three.boundaries == 3
+
+
+def test_largest_secondary_is_used_first():
+ split, _ = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 25)},
+ companions = {},
+ primary = 0,
+ card_limits = {0: 18000, 1: 9000, 2: 30000},
+ headroom_mib = 2000,
+ overhead_mib = 500,
+ )
+ assert split.cards == (0, 2)
+
+
+def test_two_experts_are_each_cut_with_the_headroom_charged_once():
+ limits = {0: 40000, 1: 40000}
+ split, reason = lsp.plan_layer_split(
+ denoisers = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)},
+ companions = {"text_encoder": (6000, "text_encoder"), "vae": (300, "other")},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 4000,
+ overhead_mib = 500,
+ )
+ assert split is not None, reason
+ _check(split, limits)
+ assert {r.denoiser for r in split.ranges if r.card == 1} == {"transformer", "transformer_2"}
+ weights = dict(split.card_weights_mib)
+ required = dict(split.card_required_mib)
+ reserve = lsp.secondary_reserve_mib()
+ assert required[0] == weights[0] + 4000 + max(500, reserve)
+ assert required[1] == weights[1] + max(1400, 1024) + reserve
+
+
+def test_phase_headroom_prices_decode_on_the_primary_and_encode_on_the_te_card():
+ split, _ = lsp.plan_layer_split(
+ denoisers = {"transformer": L(1000, (1000,) * 23)},
+ companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},
+ primary = 0,
+ card_limits = {0: 20000, 1: 20000},
+ headroom_mib = 0,
+ overhead_mib = 500,
+ phase_headroom_mib = (3000, 1500, 2500),
+ )
+ weights = dict(split.card_weights_mib)
+ required = dict(split.card_required_mib)
+ reserve = lsp.secondary_reserve_mib()
+ assert split.text_encoders_card == 1
+ assert required[0] == weights[0] + max(1500, 2500) + max(500, reserve)
+ assert required[1] == weights[1] + 3000 + reserve
+
+
+def test_secondary_is_charged_a_block_working_set_not_the_primary_activation_peak():
+ denoisers = {"transformer": L(1500, (1070,) * 56)}
+ limits = {0: 80000, 1: 80000}
+ split, reason = lsp.plan_layer_split(
+ denoisers = denoisers,
+ companions = {"vae": (200, "other")},
+ primary = 0,
+ card_limits = limits,
+ headroom_mib = 30000,
+ overhead_mib = 2048,
+ )
+ assert split is not None, reason
+ weights = dict(split.card_weights_mib)
+ required = dict(split.card_required_mib)
+ assert lsp.block_working_set_mib(denoisers, 30000) == 2140
+ assert required[1] == weights[1] + 2140 + lsp.secondary_reserve_mib()
+ assert required[0] == weights[0] + 30000 + 2048
+ assert lsp.block_working_set_mib(denoisers, 1500) == 1500
+ assert lsp.block_working_set_mib({"transformer": L(0, (100,))}, 0) == 1024
+
+
+def test_reserve_follows_comfyui_minimum_inference_memory(monkeypatch):
+ monkeypatch.setattr(lsp.sys, "platform", "linux")
+ assert lsp.secondary_reserve_mib() == 1200
+ monkeypatch.setattr(lsp.sys, "platform", "win32")
+ assert lsp.secondary_reserve_mib() == 1500
+
+
+def test_layouts_count_storage_bytes_so_quantized_blocks_are_smaller():
+ class Block(torch.nn.Module):
+ def __init__(self, dtype):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.empty(4096, 4096, dtype = dtype), requires_grad = False)
+
+ class Den(torch.nn.Module):
+ _no_split_modules = ["Block"]
+
+ def __init__(self, dtype):
+ super().__init__()
+ self.proj_out = torch.nn.Linear(4096, 1024, dtype = torch.bfloat16)
+ self.transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(3))
+ self.single_transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(2))
+
+ class Pipe:
+ transformer = Den(torch.bfloat16)
+
+ class Quant:
+ transformer = Den(torch.int8)
+
+ bf16 = lsp._denoiser_layouts(Pipe())["transformer"]
+ int8 = lsp._denoiser_layouts(Quant())["transformer"]
+ assert bf16.block_mib == (32,) * 5
+ assert int8.block_mib == (16,) * 5
+ assert bf16.stem_mib == 9
+
+
+def test_a_denoiser_without_a_block_list_has_no_layout():
+ class Pipe:
+ transformer = torch.nn.Linear(4, 4)
+
+ assert lsp._denoiser_layouts(Pipe()) is None
+
+
+class _Target:
+ supports_model_cpu_offload: bool = True
+ ordinal: int = 0
+ device: str = "cuda"
+ backend: str = "cuda"
+ is_cuda_torch_device: bool = True
+
+
+def _memory(free = 24000, total = 24576):
+ return dm.DeviceMemory(
+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
+ )
+
+
+def _plan(**kw):
+ args = dict(
+ target = _Target(),
+ device_memory = _memory(),
+ model_dense_mib = 24000,
+ companion_dense_mib = 4300,
+ text_encoder_dense_mib = 4000,
+ runtime_headroom_mib = 2000,
+ )
+ args.update(kw)
+ return dm.plan_diffusion_memory(**args)
+
+
+_BIG = {
+ "transformer": (24000, "dit"),
+ "text_encoder": (4000, "text_encoder"),
+ "vae": (300, "other"),
+}
+
+
+class _Engine:
+ def __init__(self, fail = False):
+ self.fail = fail
+ self.placed = []
+ self.undone = 0
+
+ def place(
+ self,
+ pipe,
+ *,
+ primary,
+ split,
+ logger = None,
+ ):
+ self.placed.append((primary, split))
+ if self.fail:
+ raise RuntimeError("boom")
+ return {}
+
+ def undo(self, pipe):
+ self.undone += 1
+
+
+def engine(monkeypatch):
+ eng = _Engine()
+ monkeypatch.setattr(lsp, "engine_available", lambda: True)
+ monkeypatch.setattr(lsp, "place_layer_split", eng.place)
+ monkeypatch.setattr(lsp, "undo_layer_split", eng.undo)
+ return eng
+
+
+def _cards(
+ monkeypatch,
+ sizes = _BIG,
+ sec_limit = 20000,
+ layouts = None,
+):
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: sec_limit)
+ monkeypatch.setattr(
+ mg, "best_secondary", lambda selection, probes = None: selection.secondaries[0]
+ )
+ monkeypatch.setattr(
+ lsp,
+ "_denoiser_layouts",
+ lambda pipe: layouts if layouts is not None else {"transformer": L(1000, (1000,) * 23)},
+ )
+
+
+class _Pipe:
+ """Weak-referenceable stand-in; the refinement only reads it through the patched size / layout readers."""
+
+ components: dict = {}
+
+ def to(self, *a, **k):
+ return self
+
+
+def _refine(
+ plan,
+ pipe = None,
+ ordinals = (0, 1),
+):
+ pipe = pipe if pipe is not None else _Pipe()
+ with mg.selection_scope(mg.make_selection(0, list(ordinals))):
+ return mg.refine_plan_for_multigpu(pipe, plan), pipe
+
+
+def test_one_card_offload_becomes_a_layer_split(monkeypatch, engine):
+ plan = _plan()
+ assert lsp.plan_offloads_denoiser(plan)
+ _cards(monkeypatch)
+ refined, pipe = _refine(plan)
+ assert refined is not plan
+ assert refined.offload_policy == dm.OFFLOAD_NONE
+ assert refined.layer_split is not None and refined.secondary_ordinal is None
+ assert refined.fallback_plan is plan
+ assert refined.layer_split.cards == (0, 1)
+ public = refined.as_public_dict()
+ assert public["layer_split"]["cards"] == [0, 1]
+ assert "layer_split" not in plan.as_public_dict()
+ assert any("layer split" in r for r in refined.reasons)
+
+
+def test_no_engine_keeps_the_one_card_plan(monkeypatch):
+ monkeypatch.setattr(lsp, "engine_available", lambda: False)
+ _cards(monkeypatch)
+ plan = _plan()
+ assert _refine(plan)[0] is plan
+
+
+def test_no_selection_or_kill_switches_keep_the_plan(monkeypatch, engine):
+ _cards(monkeypatch)
+ plan = _plan()
+ assert mg.refine_plan_for_multigpu(_Pipe(), plan) is plan
+ assert _refine(plan, ordinals = (0,))[0] is plan
+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")
+ assert _refine(plan)[0] is plan
+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "auto")
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
+ assert _refine(plan)[0] is plan
+ assert engine.placed == []
+
+
+def test_keep_multiple_models_loaded_gets_no_selection(monkeypatch, engine):
+ _cards(monkeypatch)
+ plan = _plan()
+ monkeypatch.setattr(mg, "models_kept_loaded", lambda: True)
+ seen = []
+
+ class Backend:
+ @mg.with_load_selection
+ def _run_load(self, **kwargs):
+ seen.append(mg.current_selection())
+ return mg.refine_plan_for_multigpu(_Pipe(), plan)
+
+ assert Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1]) is plan
+ assert seen == [None]
+
+
+def test_a_dit_that_fits_one_card_keeps_todays_plan(monkeypatch, engine):
+ small = {
+ "transformer": (8000, "dit"),
+ "text_encoder": (4000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ _cards(monkeypatch, sizes = small)
+ plan = _plan(model_dense_mib = 8000)
+ assert plan.offload_policy == dm.OFFLOAD_NONE
+ assert _refine(plan)[0] is plan
+ streamed_te = dataclasses.replace(
+ _plan(),
+ offload_policy = dm.OFFLOAD_GROUP,
+ stream_transformer = False,
+ stream_text_encoders = True,
+ )
+ _cards(monkeypatch, sec_limit = 1000) # no companion assignment either
+ assert _refine(streamed_te)[0] is streamed_te
+ assert engine.placed == []
+
+
+def test_quantized_storage_that_fits_one_card_is_a_companion_split_not_a_layer_split(
+ monkeypatch, engine
+):
+ quant = {
+ "transformer": (12000, "dit"),
+ "text_encoder": (4000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ _cards(monkeypatch, sizes = quant)
+ refined, _ = _refine(_plan())
+ assert refined.layer_split is None
+ assert refined.secondary_ordinal == 1
+
+
+def test_companion_split_is_preferred_over_a_layer_split(monkeypatch, engine):
+ fits = {
+ "transformer": (14000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ _cards(monkeypatch, sizes = fits)
+ refined, _ = _refine(_plan())
+ assert refined.secondary_ordinal == 1 and refined.layer_split is None
+
+
+def test_cards_too_small_keep_the_offload_plan_and_record_why(monkeypatch, engine):
+ _cards(monkeypatch, sec_limit = 6000)
+ plan = _plan()
+ refined, pipe = _refine(plan)
+ assert refined is plan
+ status = lsp.layer_split_status(pipe)
+ assert "does not fit" in status["declined"]
+ assert mg.placement_status(pipe)["layer_split"] == status
+
+
+def test_rocm_explicit_modes_and_legacy_cpu_offload_keep_the_plan(monkeypatch, engine):
+ _cards(monkeypatch)
+ plan = _plan()
+ monkeypatch.setattr(lsp, "_is_rocm", lambda: True)
+ assert _refine(plan)[0] is plan
+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ balanced = _plan(requested_mode = "balanced")
+ assert _refine(balanced)[0] is balanced
+ legacy = _plan(explicit_offload = True)
+ assert legacy.estimates.get("explicit_cpu_offload")
+ assert _refine(legacy)[0] is legacy
+ assert engine.placed == []
+
+
+def test_force_cuts_a_denoiser_one_card_would_hold(monkeypatch, engine):
+ small = {"transformer": (8000, "dit"), "vae": (300, "other")}
+ _cards(monkeypatch, sizes = small, layouts = {"transformer": L(500, (500,) * 15)})
+ plan = _plan(model_dense_mib = 8000, companion_dense_mib = 300, text_encoder_dense_mib = 0)
+ assert plan.offload_policy == dm.OFFLOAD_NONE
+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")
+ refined, _ = _refine(plan)
+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
+
+
+def test_video_two_expert_cut_when_neither_expert_fits_a_card(monkeypatch, engine):
+ sizes = {
+ "transformer": (28500, "dit"),
+ "transformer_2": (28500, "dit"),
+ "text_encoder": (6000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ layouts = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)}
+ _cards(monkeypatch, sizes = sizes, sec_limit = 34000, layouts = layouts)
+ plan = _plan(
+ device_memory = _memory(free = 40000, total = 40960),
+ model_dense_mib = 57000,
+ companion_dense_mib = 6300,
+ text_encoder_dense_mib = 6000,
+ )
+ refined, _ = _refine(plan, ordinals = (0, 1, 2))
+ split = refined.layer_split
+ assert split is not None
+ assert {r.denoiser for r in split.ranges} == {"transformer", "transformer_2"}
+
+
+def _layer_plan(monkeypatch, engine):
+ _cards(monkeypatch)
+ refined, pipe = _refine(_plan())
+ assert refined.layer_split is not None
+ return refined, pipe
+
+
+def test_apply_records_the_split_and_cfg_parallel_declines(monkeypatch, engine):
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ assert engine.placed[0][0] == "cuda:0" and engine.placed[0][1] is plan.layer_split
+ assert lsp.is_layer_split(pipe) and mg.is_split(pipe)
+ status = mg.placement_status(pipe)
+ assert status["layer_split"]["cards"] == [0, 1]
+ assert status["layer_split"]["boundaries"] == 2
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ assert (
+ mg.cfg_parallel_decline(pipe, offload_policy = "none", selection = mg.current_selection())
+ == "denoiser is split across cards (layer split)"
+ )
+ lsp.forget_layer_split(pipe)
+ assert not lsp.is_layer_split(pipe)
+
+
+def test_failed_placement_applies_the_one_card_fallback(monkeypatch, engine):
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ engine.fail = True
+ applied = []
+ real = dm.apply_memory_plan
+
+ def spy(p, pl, **kw):
+ applied.append(pl)
+ if pl is plan:
+ return real(p, pl, **kw)
+ return pl.offload_policy, False
+
+ monkeypatch.setattr(dm, "apply_memory_plan", spy)
+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
+ assert engine.undone == 1
+ assert applied == [plan, plan.fallback_plan]
+ assert policy == plan.fallback_plan.offload_policy
+ assert dm.applied_plan(pipe, plan) is plan.fallback_plan
+ assert not lsp.is_layer_split(pipe)
+ assert "placement failed" in lsp.layer_split_status(pipe)["declined"]
+
+
+def test_video_guard_reads_per_card_floors_from_the_layer_split(monkeypatch, engine):
+ from core.inference import video as V
+
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ assert V._video_split_card_floors(pipe) is None
+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ floors = V._video_split_card_floors(pipe)
+ split = plan.layer_split
+ weights = dict(split.card_weights_mib)
+ required = dict(split.card_required_mib)
+ reserve = dict(split.card_reserve_mib)
+ assert floors == (
+ (0, weights[0], True, True, True, None, None),
+ (1, weights[1], False, False, True, required[1] - weights[1] - reserve[1], reserve[1]),
+ )
+ assert reserve[1] > 0
+
+
+def test_video_guard_prices_a_block_card_by_its_planned_headroom():
+ from core.inference import video as V
+
+ cards = ((0, 13300, True, True, True, None), (1, 20000, False, False, True, 3200))
+ free = {0: 60000, 1: 23500}
+ kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))
+ assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw) is None
+ message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw)
+ assert message is not None and "cuda:1" in message
+ assert V.video_split_shortfall_message(cards, width = 416, height = 240, frames = 17, **kw) is None
+
+
+def test_video_guard_scales_only_the_working_set_not_the_fixed_reserve():
+ from core.inference import video as V
+
+ cards = ((0, 13300, True, True, True, None, None), (1, 20000, False, False, True, 700, 2500))
+ free = {0: 60000, 1: 24000}
+ kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))
+ assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw) is None
+ free[1] = 20000 + 700 + 2400
+ message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw)
+ assert message is not None and "cuda:1" in message
+
+
+def test_a_from_pipe_sibling_sees_the_split_through_its_shared_denoiser(monkeypatch, engine):
+ from core.inference import diffusion_layer_split as engine_mod
+
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ shared = torch.nn.Linear(2, 2)
+ pipe.transformer = shared
+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ sibling = type("Sibling", (), {"transformer": shared})()
+ monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is shared)
+ assert lsp.applied_layer_split(sibling) is plan.layer_split
+ assert lsp.secondary_block_cards(sibling) == (1,)
+ assert mg.is_split(sibling)
+ mg.undo_split(pipe)
+ assert lsp.applied_layer_split(pipe) is None and lsp.applied_layer_split(sibling) is None
+
+
+def test_a_failed_replacement_does_not_report_the_previous_split(monkeypatch, engine):
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ assert lsp.is_layer_split(pipe)
+ engine.fail = True
+ assert not lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ assert not lsp.is_layer_split(pipe) and not mg.is_split(pipe)
+ assert "placement failed" in lsp.layer_split_status(pipe)["declined"]
+
+
+def test_an_expert_wholly_on_the_primary_is_placed_plainly(monkeypatch):
+ import contextlib
+
+ from core.inference import diffusion_fast_load
+ from core.inference import diffusion_layer_split as engine_mod
+
+ class Comp(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1))
+ self.moved = []
+
+ def to(self, *a, **k):
+ self.moved.append(a[0] if a else k.get("device"))
+ return self
+
+ class Pipe:
+ def __init__(self):
+ self.transformer = Comp()
+ self.transformer_2 = Comp()
+ self.vae = Comp()
+ self.components = {
+ "transformer": self.transformer,
+ "transformer_2": self.transformer_2,
+ "vae": self.vae,
+ }
+
+ pipe = Pipe()
+ split = lsp.LayerSplit(
+ primary = 0,
+ cards = (0, 1),
+ ranges = (
+ lsp.BlockRange("transformer", 0, 1, 0, 1),
+ lsp.BlockRange("transformer", 1, 2, 1, 1),
+ lsp.BlockRange("transformer_2", 0, 2, 0, 2),
+ ),
+ text_encoders_card = None,
+ card_required_mib = ((0, 1), (1, 1)),
+ card_limit_mib = ((0, 9), (1, 9)),
+ )
+ placed = []
+ monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])
+ monkeypatch.setattr(
+ engine_mod,
+ "place_layer_split",
+ lambda p, **kw: placed.append((kw["attr"], kw["assignment"])) or {},
+ )
+ monkeypatch.setattr(
+ diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()
+ )
+ out = lsp.place_layer_split(pipe, primary = "cuda:0", split = split)
+ assert placed == [
+ ("transformer", {"b.0": 0, "b.1": 1})
+ ] # only the expert that is cut reaches the engine
+ assert out["transformer_2"]["whole_on_primary"] is True
+ assert pipe.transformer_2.moved == [torch.device("cuda", 0)]
+
+
+def test_offload_installers_refuse_or_replace_a_split(monkeypatch):
+ from core.inference import diffusion_layer_split as engine_mod
+
+ dit = torch.nn.Linear(2, 2)
+ pipe = type("P", (), {"transformer": dit, "components": {"transformer": dit}})()
+ monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is dit)
+ monkeypatch.setattr(dm, "_offload_groups", lambda m: [])
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda p: True)
+ assert dm.hook_resident_denoiser(pipe, "cuda:0") is False
+ undone = []
+ monkeypatch.setattr(lsp, "undo_layer_split", lambda p: undone.append(p))
+ dm._apply_group_offload(pipe, "cpu", None)
+ assert undone == [pipe]
+
+
+def test_image_guard_checks_every_other_card_running_blocks(monkeypatch, engine):
+ from core.inference import diffusion as D
+
+ plan, pipe = _layer_plan(monkeypatch, engine)
+ free = {"mib": 100000}
+ monkeypatch.setattr(
+ D,
+ "reclaimable_snapshot_device_memory",
+ lambda target: dm.DeviceMemory(
+ backend = "cuda",
+ device = "cuda",
+ memory_kind = "discrete_vram",
+ free_mib = free["mib"],
+ total_mib = 102400,
+ ),
+ )
+ kw = {"width": 1024, "height": 1024, "family": "flux2"}
+ assert D._guard_layer_cards(pipe, object(), kw) == 0 # not placed: nothing to check
+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
+ assert D._guard_layer_cards(pipe, object(), kw) == 1
+ free["mib"] = 10
+ with pytest.raises(ValueError, match = "cuda:1"):
+ D._guard_layer_cards(pipe, object(), kw)
+
+
+def test_two_denoisers_that_fit_only_jointly_are_cut_jointly():
+ kw = dict(
+ companions = {},
+ primary = 0,
+ card_limits = {0: 3, 1: 3},
+ headroom_mib = 0,
+ overhead_mib = 0,
+ working_set_mib = 0,
+ secondary_overhead_mib = 0,
+ )
+ denoisers = {"transformer": L(0, (1, 1)), "transformer_2": L(0, (1, 2, 1))}
+ split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)
+ assert split is not None, reason
+ per_card = {}
+ for r in split.ranges:
+ per_card[r.card] = per_card.get(r.card, 0) + r.mib
+ assert per_card[0] <= 3 and per_card[1] <= 3
+ for name, layout in denoisers.items():
+ mine = sorted((r.start, r.stop) for r in split.ranges if r.denoiser == name)
+ assert mine[0][0] == 0 and mine[-1][1] == len(layout.block_mib) # every block placed once
+ assert any(r.card != 0 for r in split.ranges if r.denoiser == name)
+ assert lsp._search_cuts([(1, 1), (1, 2, 1)], [2, 2]) is None # truly infeasible stays declined
+
+
+def test_dual_denoisers_may_keep_one_wholly_on_the_primary():
+ # A fits only on the primary, B only on the secondary: forcing both to cut would decline a valid placement
+ assert lsp._search_cuts([(2, 2), (3, 3)], [4, 6]) == [[0, 2, 2], [0, 0, 2]]
+ assert lsp._search_cuts([(2, 2), (3, 3)], [10, 0]) is None # nothing off the primary: no split
+ split, reason = lsp.plan_layer_split(
+ denoisers = {"transformer": L(0, (2, 2)), "transformer_2": L(0, (3, 3))},
+ companions = {},
+ primary = 0,
+ card_limits = {0: 4, 1: 6},
+ headroom_mib = 0,
+ overhead_mib = 0,
+ phase_headroom_mib = (0, 0, 0),
+ working_set_mib = 0,
+ secondary_overhead_mib = 0,
+ )
+ assert split is not None, reason
+ _check(split, {0: 4, 1: 6})
+ assert {(r.denoiser, r.card) for r in split.ranges} == {
+ ("transformer", 0),
+ ("transformer_2", 1),
+ }
+
+
+def test_failed_placement_drops_fused_records_left_on_another_card(monkeypatch):
+ from types import SimpleNamespace
+
+ from torch import nn
+
+ def attn(fused_device):
+ a = nn.Module()
+ a.to_q, a.to_k, a.to_v = nn.Linear(4, 4), nn.Linear(4, 4), nn.Linear(4, 4)
+ a.__dict__["_unsloth_fused_qkv"] = (
+ nn.Linear(4, 12, device = fused_device),
+ a.to_q,
+ a.to_k,
+ a.to_v,
+ )
+ return a
+
+ dit = nn.Module()
+ dit.stale, dit.intact = attn("meta"), attn("cpu")
+ pipe = SimpleNamespace(transformer = dit, to = lambda *a, **k: None)
+
+ def boom(*a, **k):
+ raise RuntimeError("out of memory")
+
+ monkeypatch.setattr(lsp, "place_layer_split", boom)
+ monkeypatch.setattr(lsp, "undo_layer_split", lambda p: None)
+ assert (
+ lsp.apply_layer_split_plan(pipe, SimpleNamespace(layer_split = object()), placement = "cuda:0")
+ is False
+ )
+ assert "_unsloth_fused_qkv" not in dit.stale.__dict__
+ assert "_unsloth_fused_qkv" in dit.intact.__dict__ # still on its parts' device: kept
+
+
+def test_an_arch_pinned_attention_backend_keeps_blocks_off_other_architectures(monkeypatch, engine):
+ # flash3 is Hopper only: a B200 secondary would reject it
+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
+
+ caps = {0: (9, 0), 1: (10, 0), 2: (9, 0)}
+ monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))
+ _cards(monkeypatch)
+ pipe = _Pipe()
+ pipe.transformer = torch.nn.Linear(2, 2)
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "_flash_3_hub")
+ plan = _plan()
+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))
+ assert refined is plan # the only secondary is another architecture: no split
+ assert "attention backend _flash_3_hub" in lsp._DECLINES[pipe]
+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1, 2))
+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards
+ setattr(
+ pipe.transformer, ATTENTION_BACKEND_ATTR, "_native_cudnn"
+ ) # runs on every admitted card
+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))
+ assert refined.layer_split is not None and 1 in refined.layer_split.cards
diff --git a/studio/backend/tests/test_diffusion_layer_split_speed.py b/studio/backend/tests/test_diffusion_layer_split_speed.py
new file mode 100644
index 00000000000..4a742a2a4f7
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_layer_split_speed.py
@@ -0,0 +1,773 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Speed paths on a layer split denoiser, CPU only: per-card block graphs, the step cache across cards, and the int8
+GEMM swap per card. The split is placed with the weights left on the CPU (``_move_weights`` stubbed); where a card
+must be read off a weight, the test names it."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+from torch import nn
+
+from core.inference import diffusion_block_graph as bg
+from core.inference import diffusion_cache as dc
+from core.inference import diffusion_cuda_graph as cg
+from core.inference import diffusion_int8_gemm as g8
+from core.inference import diffusion_layer_split as ls
+from core.inference import diffusion_multigpu as mg
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_layer_split import FakeDiT, Pipe, _two_card_budget # noqa: E402
+
+_needs_diffusers = pytest.mark.skipif(
+ importlib.util.find_spec("diffusers") is None, reason = "needs diffusers"
+)
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ for name in (ls.LAYER_SPLIT_ENV, mg.MULTIGPU_ENV, mg.CFG_PARALLEL_ENV, bg.BLOCK_GRAPHS_ENV):
+ monkeypatch.delenv(name, raising = False)
+ yield
+
+
+def _split(monkeypatch, dit):
+ pipe = Pipe(dit)
+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)
+ info = ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))
+ assert {r["device"] for r in info["ranges"]} == {"cuda:0", "cuda:1"}
+ split = ls.layer_split_of(dit)
+ card = {qual: int(dev.split(":")[1]) for qual, dev in _assignment(dit).items()}
+ return pipe, split, card
+
+
+def _assignment(dit) -> dict:
+ out = {}
+ for qual, _, block in ls.find_blocks(dit):
+ out[qual] = str(block.__dict__[ls._SPLIT_ATTR].device)
+ return out
+
+
+def test_block_graphs_record_each_block_on_its_own_card(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, card = _split(monkeypatch, dit)
+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}
+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))
+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")
+ assert handle is not None, reason
+ assert handle.cards() == {0, 1}
+ for graph in handle.graphs:
+ assert graph.shared.device_index == by_block[id(graph.block)]
+ assert graph.shared.strict_device # records only while its own card is current
+ assert handle.shared.device_index == 0 and set(handle.card_shared) == {1}
+ assert handle.describe()["cards"] == [0, 1]
+ handle.free()
+ assert not handle.graphs
+ ls.undo_layer_split(pipe)
+
+
+def test_one_card_block_graphs_keep_one_shared_state(monkeypatch):
+ dit = FakeDiT(dim = 64, double = 2, single = 2)
+ monkeypatch.setattr(
+ bg, "_block_card", lambda block: pytest.fail("read a block card on one card")
+ )
+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")
+ assert handle is not None, reason
+ assert not handle.card_shared and handle.cards() == {0}
+ assert all(g.shared is handle.shared and not g.shared.strict_device for g in handle.graphs)
+ assert "cards" not in handle.describe()
+ handle.free()
+
+
+def test_a_split_block_off_its_card_runs_its_compute(monkeypatch):
+ dit = FakeDiT(dim = 64, double = 2, single = 2)
+ shared = bg._Shared(1, None)
+ shared.strict_device = True
+ calls = []
+ graph = bg.BlockGraph(
+ dit.transformer_blocks[0], lambda *a, **k: calls.append(1) or "eager", shared
+ )
+ monkeypatch.setattr(bg._WeightView, "placement", lambda self, index: (1,))
+ monkeypatch.setattr(
+ torch.cuda, "current_device", lambda: 0
+ ) # the block's card (1) is not current
+ monkeypatch.setattr(bg, "_walk", lambda obj, live: live.append(_FakeCuda(1)) or ("k",))
+ monkeypatch.setattr(bg, "_refusal", lambda key, kwargs: None)
+ with torch.no_grad():
+ assert graph(torch.zeros(2, 64)) == "eager" and calls
+ assert graph.stats["refused_host_input"] == 1 and not graph.cache and not graph.seen
+
+
+class _FakeCuda:
+ def __init__(self, index):
+ self.device = torch.device("cuda", index)
+
+
+def test_arm_block_graphs_records_a_split_denoiser_per_block(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ armed = []
+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)
+ monkeypatch.setattr(
+ bg,
+ "install_block_graphs",
+ lambda t, **k: armed.append(t) or (types.SimpleNamespace(slots_mib = 0), "armed"),
+ )
+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])
+ applied = {"cuda_graph": False}
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")
+ handles = cg.arm_block_graphs(pipe, applied, target = target, family = None, hooked = False)
+ assert armed == [dit] and len(handles) == 1 and applied["cuda_graph"] is True
+ assert pipe._unsloth_cuda_graph_mode == "blocks"
+ assert "split denoiser" in pipe._unsloth_cuda_graph_reason
+ ls.undo_layer_split(pipe)
+
+
+def test_split_block_graphs_honour_the_kill_switch(monkeypatch):
+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "0")
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ monkeypatch.setattr(g8, "after_layer_split", lambda t, logger = None: {})
+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)
+ monkeypatch.setattr(
+ bg, "install_block_graphs", lambda t, **k: pytest.fail("armed with the kill switch set")
+ )
+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")
+ assert (
+ cg.arm_block_graphs(pipe, {"cuda_graph": True}, target = target, family = None, hooked = False)
+ == ()
+ )
+ assert "UNSLOTH_DIFFUSION_BLOCK_GRAPHS=0" in pipe._unsloth_cuda_graph_reason
+ ls.undo_layer_split(pipe)
+
+
+def test_arm_block_graphs_leaves_a_resident_one_card_denoiser_alone(monkeypatch):
+ dit = FakeDiT(dim = 64, double = 2, single = 2)
+ pipe = Pipe(dit)
+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)
+ monkeypatch.setattr(
+ bg, "install_block_graphs", lambda t, **k: pytest.fail("armed per block on one card")
+ )
+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])
+ applied = {"cuda_graph": False}
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")
+ assert cg.arm_block_graphs(pipe, applied, target = target, family = None, hooked = False) == ()
+
+
+@_needs_diffusers
+def test_split_block_classes_register_with_a_copy_of_their_base_metadata(monkeypatch):
+ from diffusers.hooks._helpers import TransformerBlockMetadata, TransformerBlockRegistry
+
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ base_meta = {}
+ for base in (
+ type(dit.transformer_blocks[0])._unsloth_layer_split_base,
+ type(dit.single_transformer_blocks[0])._unsloth_layer_split_base,
+ ):
+ meta = TransformerBlockMetadata(return_hidden_states_index = 0)
+ TransformerBlockRegistry.register(model_class = base, metadata = meta)
+ base_meta[base] = meta
+ try:
+ assert dc._register_layer_split_blocks(dit)
+ for block in (dit.transformer_blocks[0], dit.single_transformer_blocks[0]):
+ sub = type(block)
+ base = sub._unsloth_layer_split_base
+ got = TransformerBlockRegistry.get(sub)
+ assert (
+ got is not base_meta[base]
+ ) # never the base's own object: register writes _cls into it
+ assert got.return_hidden_states_index == 0
+ assert base_meta[base]._cls is base
+ finally:
+ for cls in list(TransformerBlockRegistry._registry):
+ if getattr(cls, "__name__", "") in ("FakeBlock", "FakeSingle"):
+ TransformerBlockRegistry._registry.pop(cls, None)
+ ls.undo_layer_split(pipe)
+
+
+def _tail_hook(head_output):
+ from diffusers.hooks.first_block_cache import FBCBlockHook, FBCSharedBlockState
+ from diffusers.hooks._helpers import TransformerBlockMetadata
+ from diffusers.hooks.hooks import StateManager
+
+ manager = StateManager(FBCSharedBlockState, (), {})
+ try:
+ from diffusers.hooks.hooks import CacheContext
+ manager.set_context(CacheContext(name = "cond"))
+ except ImportError: # an older diffusers keys the state by a plain name
+ manager.set_context("cond")
+ hook = FBCBlockHook(manager, is_tail = True)
+ hook._metadata = TransformerBlockMetadata(return_hidden_states_index = 0)
+ hook._metadata._cached_parameter_indices = {"hidden_states": 0}
+ state = manager.get_state()
+ state.head_block_output = head_output
+ return hook, state
+
+
+@_needs_diffusers
+def test_split_guard_leaves_a_one_card_tail_call_unchanged():
+ assert dc.install_fbcache_split_guard()
+ assert dc.install_fbcache_split_guard() # idempotent
+ from diffusers.hooks.first_block_cache import FBCBlockHook
+
+ assert getattr(FBCBlockHook.new_forward, "_unsloth_split_guard", False)
+ head = torch.randn(2, 8)
+ hook, state = _tail_hook(head)
+ x = torch.randn(2, 8)
+ fn = types.SimpleNamespace(original_forward = lambda h: h * 2)
+ hook.fn_ref = fn
+ out = hook.new_forward(None, x)
+ assert torch.equal(out, x * 2)
+ assert state.head_block_output is head
+ assert torch.equal(state.tail_block_residuals[0], x * 2 - head)
+
+
+@_needs_diffusers
+def test_split_guard_computes_the_tail_residual_on_the_tail_card_and_returns_it_home(monkeypatch):
+ dc.install_fbcache_split_guard()
+ moves = []
+
+ head = torch.randn(2, 8)
+ hook, state = _tail_hook(head)
+ x = torch.randn(2, 8)
+ seen = {}
+
+ def original_forward(h):
+ seen["head"] = state.head_block_output
+ return h * 2
+
+ hook.fn_ref = types.SimpleNamespace(original_forward = original_forward)
+ home, here = torch.device("cuda", 0), torch.device("cuda", 1)
+ monkeypatch.setattr(dc, "_first_device", lambda obj: home if obj is head else None)
+ monkeypatch.setattr(
+ hook._metadata,
+ "_get_parameter_from_args_kwargs",
+ lambda name, args, kwargs: types.SimpleNamespace(device = here),
+ )
+ copied = torch.randn(2, 8)
+ monkeypatch.setattr(
+ dc, "_tensors_to", lambda obj, device: moves.append(("head", device)) or copied
+ )
+ real_to = torch.Tensor.to
+
+ def fake_to(self, *args, **kwargs):
+ moves.append(("residual", args[0] if args else kwargs.get("device")))
+ return self
+
+ monkeypatch.setattr(torch.Tensor, "to", fake_to)
+ out = hook.new_forward(None, x)
+ monkeypatch.setattr(torch.Tensor, "to", real_to)
+ assert seen["head"] is copied # the residual subtracts the copy that sits on the tail's card
+ assert state.head_block_output is head # the shared state keeps the head card's tensor
+ assert ("head", here) in moves and ("residual", home) in moves
+ assert torch.equal(out, x * 2)
+
+
+@_needs_diffusers
+def test_apply_step_cache_engages_on_a_split_through_the_guard(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ guarded = []
+ monkeypatch.setattr(dc, "install_fbcache_split_guard", lambda: guarded.append(1) or True)
+ monkeypatch.setattr(dc, "_register_layer_split_blocks", lambda t: True)
+ monkeypatch.setattr(dc, "_pipeline_opens_cache_context", lambda p: True)
+ monkeypatch.setattr(dc, "_reuses_prefix_kv", lambda p, t: False)
+ enabled = []
+ dit.enable_cache = lambda config: enabled.append(config)
+ assert dc.apply_step_cache(pipe, mode = "fbcache") == "fbcache"
+ assert guarded and enabled
+ ls.undo_layer_split(pipe)
+
+
+def test_apply_step_cache_stays_off_on_a_split_without_the_guard(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ monkeypatch.setattr(dc, "install_fbcache_split_guard", lambda: False)
+ dit.enable_cache = lambda config: pytest.fail("cache engaged without the split guard")
+ assert dc.apply_step_cache(pipe, mode = "fbcache") is None
+ ls.undo_layer_split(pipe)
+
+
+def _int8_dit(monkeypatch, cards):
+ """Every Linear eligible; its weight's card read from ``cards`` (module -> ordinal)."""
+ dit = FakeDiT(dim = 64, double = 2, single = 2)
+ recs = {}
+ for m in dit.modules():
+ if type(m) is nn.Linear:
+ recs[id(m)] = ("v2", None, torch.ones(1), m.weight)
+ monkeypatch.setattr(g8, "_eligible", lambda m: recs.get(id(m)))
+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))
+ monkeypatch.setattr(g8, "_op", lambda: "op")
+ monkeypatch.setattr(
+ "core.inference.diffusion_convrot_quant.rotquant_device_config", lambda index: None
+ )
+ monkeypatch.setattr(g8, "_v1_act_quant_matches", lambda index: True)
+ return dit, recs
+
+
+def _card1_singles(dit) -> dict:
+ return {id(m): 1 for block in dit.single_transformer_blocks for m in block.modules()}
+
+
+def test_split_int8_swap_probes_and_swaps_each_card(monkeypatch):
+ dit, recs = _int8_dit(monkeypatch, {})
+ cards = _card1_singles(dit)
+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))
+ monkeypatch.setattr(g8, "_split_cards", lambda t: (0, 1))
+ monkeypatch.setattr("core.inference.diffusion_int8_fused.resident_cuda_device", lambda t: None)
+ probed = []
+ monkeypatch.setattr(
+ g8, "device_config", lambda index: probed.append(index) or ((1,) if index == 0 else None)
+ )
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))
+ try:
+ count = g8._swap(dit)
+ assert set(probed) == {0, 1}
+ linears = [m for m in dit.modules() if id(m) in recs]
+ on0 = [m for m in linears if cards.get(id(m), 0) == 0]
+ assert on0 and count == len(on0)
+ for m in linears:
+ if cards.get(id(m), 0) == 0:
+ assert m.__dict__[g8._REC][4] is True and g8.is_installed(m)
+ else:
+ assert not g8.is_installed(m)
+ finally:
+ g8.uninstall(dit)
+
+
+def test_split_int8_swap_refreshes_a_pre_split_onload_install(monkeypatch):
+ dit, recs = _int8_dit(monkeypatch, {})
+ cards = _card1_singles(dit)
+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))
+ monkeypatch.setattr(g8, "device_config", lambda index: (1,) if index == 0 else None)
+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())
+ try:
+ assert g8._swap_on(dit, None, 0) == len(recs)
+ moved = [m for m in dit.modules() if id(m) in recs and cards.get(id(m), 0) == 1]
+ assert moved and all(m.__dict__[g8._REC][4] is True for m in moved)
+ monkeypatch.setattr(g8, "_split_cards", lambda t: (0, 1))
+ monkeypatch.setattr(g8, "device_config", lambda index: (2,) if index == 1 else (1,))
+ assert g8._swap_on(dit, None, 1) == len(moved)
+ assert all(m.__dict__[g8._REC][4] is True for m in moved)
+ monkeypatch.setattr(g8, "device_config", lambda index: None)
+ assert (
+ g8._swap_on(dit, None, 1) == 0
+ ) # nothing to swap there and no rotation: count 0, records untouched
+ finally:
+ g8.uninstall(dit)
+
+
+def test_one_card_int8_swap_never_reads_a_weight_card(monkeypatch):
+ dit, recs = _int8_dit(monkeypatch, {})
+ monkeypatch.setattr(g8, "_weight_card", lambda m: pytest.fail("read a weight card on one card"))
+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())
+ monkeypatch.setattr(g8, "device_config", lambda index: (1,))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))
+ try:
+ assert g8._swap_on(dit, None, 0) == len(recs)
+ finally:
+ g8.uninstall(dit)
+
+
+def _swiglu_module(stale_device = "meta"):
+ from core.inference import diffusion_int8_fused as f8
+
+ m = nn.Module()
+ m.w1, m.w3, m.w2 = (
+ nn.Linear(16, 32, bias = False),
+ nn.Linear(16, 32, bias = False),
+ nn.Linear(32, 16, bias = False),
+ )
+ stale = nn.Linear(16, 64, bias = False, device = stale_device)
+ parts = (("w1", m.w1), ("w3", m.w3), ("w2", m.w2))
+ m.__dict__[f8._SWIGLU_ATTR] = (stale, parts, "w2", 32, 0, 32, 0, 0)
+ return m, stale
+
+
+def test_a_stale_fused_swiglu_is_rebuilt_on_its_parts_card_and_shares_their_storage():
+ from core.inference import diffusion_int8_fused as f8
+
+ m, stale = _swiglu_module()
+ gate = m.w1.weight.detach().clone()
+ out = f8.relocate_after_layer_split(m)
+ rec = m.__dict__[f8._SWIGLU_ATTR]
+ assert out["rebuilt"] == 1 and rec[0] is not stale
+ assert rec[0].weight.device == m.w1.weight.device
+ assert rec[0].weight.data_ptr() == m.w1.weight.data_ptr()
+ assert torch.equal(m.w1.weight, gate) and rec[1:] == m.__dict__[f8._SWIGLU_ATTR][1:]
+ assert f8.relocate_after_layer_split(m) == {
+ "rebuilt": 0,
+ "dropped": 0,
+ "restored": 0,
+ } # idempotent
+
+
+def test_a_fused_qkv_that_cannot_be_rebuilt_is_dropped():
+ from core.inference import diffusion_int8_fused as f8
+
+ attn = nn.Module()
+ attn.to_q, attn.to_k = nn.Linear(16, 16), nn.Linear(16, 16)
+ attn.to_v = nn.Linear(16, 16).to(torch.float16) # mixed dtypes: no fused Linear over the three
+ attn.__dict__["_unsloth_fused_qkv"] = (
+ nn.Linear(16, 48, device = "meta"),
+ attn.to_q,
+ attn.to_k,
+ attn.to_v,
+ )
+ out = f8.relocate_after_layer_split(attn)
+ assert out["dropped"] == 1 and "_unsloth_fused_qkv" not in attn.__dict__
+
+
+def test_fused_qkv_readers_ignore_a_fused_linear_on_another_card():
+ from core.inference import diffusion_qwenimage21_fused as q21
+ from core.inference import diffusion_zimage_fused as zf
+ for mod in (zf, q21):
+ attn = nn.Module()
+ attn.to_q, attn.to_k, attn.to_v = nn.Linear(8, 8), nn.Linear(8, 8), nn.Linear(8, 8)
+ attn.__dict__[mod._QKV_ATTR] = (
+ nn.Linear(8, 24, device = "meta"),
+ attn.to_q,
+ attn.to_k,
+ attn.to_v,
+ )
+ assert mod._qkv_intact(attn) is None
+ attn.__dict__[mod._QKV_ATTR] = (nn.Linear(8, 24), attn.to_q, attn.to_k, attn.to_v)
+ assert mod._qkv_intact(attn) is not None
+
+
+def test_after_layer_split_is_a_no_op_on_one_card(monkeypatch):
+ called = []
+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())
+ monkeypatch.setattr(
+ "core.inference.diffusion_int8_fused.relocate_after_layer_split",
+ lambda t, l = None: called.append(t),
+ )
+ assert g8.after_layer_split(nn.Linear(4, 4)) == {"cards": (), "gemm": 0}
+ assert not called
+
+
+def test_arm_block_graphs_re_homes_the_split_records_before_arming(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, _ = _split(monkeypatch, dit)
+ order = []
+ monkeypatch.setattr(
+ g8, "after_layer_split", lambda t, logger = None: order.append(("rehome", t)) or {}
+ )
+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)
+ monkeypatch.setattr(
+ bg,
+ "install_block_graphs",
+ lambda t, **k: order.append(("arm", t)) or (types.SimpleNamespace(slots_mib = 0), "armed"),
+ )
+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")
+ cg.arm_block_graphs(pipe, {"cuda_graph": False}, target = target, family = None, hooked = False)
+ assert order == [("rehome", dit), ("arm", dit)]
+ ls.undo_layer_split(pipe)
+
+
+def test_segments_are_per_card_runs_within_one_block_list(monkeypatch):
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, card = _split(monkeypatch, dit)
+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}
+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))
+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")
+ assert handle is not None, reason
+ runs = [[ls.find_blocks(dit)[i][0] for i in range(len(ls.find_blocks(dit)))]]
+ quals = {id(b): q for q, _, b in ls.find_blocks(dit)}
+ got = [[quals[id(g.block)] for g in seg.graphs] for seg in handle.segments]
+ for run in got:
+ assert len(run) >= 2
+ assert len({card[q] for q in run}) == 1
+ assert len({q.split(".")[0] for q in run}) == 1
+ assert handle.mode == "segments" and handle.describe()["segments"] == len(handle.segments)
+ assert runs # forward order list exists
+ handle.free()
+ assert not handle.segments and all(g.segment is None for g in handle.graphs)
+ ls.undo_layer_split(pipe)
+
+
+def test_segment_kill_switch_keeps_per_block_graphs(monkeypatch):
+ monkeypatch.setenv(bg.SEGMENT_GRAPHS_ENV, "0")
+ dit = FakeDiT(dim = 1024, double = 4, single = 3)
+ pipe, _, card = _split(monkeypatch, dit)
+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}
+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))
+ handle, _ = bg.install_block_graphs(dit, device = "cuda:0")
+ assert handle is not None and not handle.segments and handle.mode == "blocks"
+ handle.free()
+ ls.undo_layer_split(pipe)
+
+
+def _replaying_segment(n = 3):
+ shared = bg._Shared(1, None)
+ blocks = [nn.Linear(4, 4) for _ in range(n)]
+ graphs = [bg.BlockGraph(b, b.forward, shared) for b in blocks]
+ seg = bg.SegmentGraph(graphs, shared)
+ rec = bg._SegRecord()
+ x, temb = torch.zeros(2, 4), torch.ones(2, 4)
+ outs = [[torch.full((2, 4), float(i))] for i in range(n)]
+ for i in range(n):
+ live: list = []
+ rec.keys.append(bg._walk(((outs[i - 1][0] if i else x, temb), {}), live))
+ rec.prov.append((("e", 0), ("e", 1)) if i == 0 else (("o", i - 1, 0), ("e", 1)))
+ rec.out_specs.append(("t", 0))
+ entry = bg._SegEntry()
+ entry.record, entry.outputs, entry.replays = rec, outs, 1
+ seg.mode, seg.cur, seg.ext = "replay", entry, [x, temb]
+ seg.returned = [outs[0]]
+ seg.versions = [[bg._version(t) for t in outs[0]]]
+ seg.pos = 1
+ return seg, outs, temb
+
+
+def test_a_replayed_segment_hands_on_each_blocks_recorded_outputs():
+ seg, outs, temb = _replaying_segment()
+ mid = seg.call(1, (outs[0][0], temb), {})
+ assert mid is outs[1][0] # the next block must receive exactly this tensor
+ last = seg.call(2, (mid, temb), {})
+ assert torch.equal(last, outs[2][0]) and last is not outs[2][0] # what leaves the run is a copy
+ assert seg.pos is None and seg.stats["segment_breaks"] == 0
+
+
+def test_a_segment_breaks_when_a_block_gets_anything_else():
+ seg, outs, temb = _replaying_segment()
+ rewritten = outs[0][0] + 0 # an op between blocks: same values, another tensor
+ assert seg.call(1, (rewritten, temb), {}) is bg._NOT_HANDLED
+ assert seg.stats["segment_breaks"] == 1 and seg.pos is None
+ assert seg.call(2, (rewritten, temb), {}) is bg._NOT_HANDLED
+
+
+def test_a_segment_breaks_on_a_fresh_copy_of_a_shared_input():
+ seg, outs, temb = _replaying_segment()
+ assert seg.call(1, (outs[0][0], temb.clone()), {}) is bg._NOT_HANDLED
+ assert seg.stats["segment_breaks"] == 1
+
+
+def test_a_segment_breaks_when_an_op_between_blocks_rewrites_an_output_in_place():
+ seg, outs, temb = _replaying_segment()
+ outs[0][0].add_(100)
+ assert seg.call(1, (outs[0][0], temb), {}) is bg._NOT_HANDLED
+ assert seg.stats["segment_breaks"] == 1 and seg.pos is None
+
+
+def test_a_segment_starts_only_on_a_lazily_routed_chain(monkeypatch):
+ walked = []
+ real_walk = bg._walk
+ monkeypatch.setattr(bg, "_walk", lambda tree, live: walked.append(1) or real_walk(tree, live))
+ shared = bg._Shared(1, None)
+ root = nn.Linear(4, 4)
+ shared.root = root
+ blocks = [nn.Linear(4, 4) for _ in range(2)]
+ seg = bg.SegmentGraph([bg.BlockGraph(b, b.forward, shared) for b in blocks], shared)
+ state = types.SimpleNamespace(lazy_now = False)
+ root.__dict__[ls._SPLIT_ATTR + "_root"] = state
+ x = torch.zeros(2, 4)
+ with torch.inference_mode():
+ assert seg.call(0, (x,), {}) is bg._NOT_HANDLED
+ assert walked == [] # declined before it looked at the inputs
+ state.lazy_now = True
+ assert seg.call(0, (x,), {}) is bg._NOT_HANDLED # host tensors: declined by the next check
+ assert walked # it got past the gate
+
+
+def test_a_segment_never_starts_while_a_step_cache_is_on(monkeypatch):
+ walked = []
+ real_walk = bg._walk
+ monkeypatch.setattr(bg, "_walk", lambda tree, live: walked.append(1) or real_walk(tree, live))
+ shared = bg._Shared(1, None)
+ root = nn.Linear(4, 4)
+ shared.root = root
+ root.__dict__[ls._SPLIT_ATTR + "_root"] = types.SimpleNamespace(lazy_now = True)
+ root._unsloth_step_cache = "[email protected]"
+ seg = bg.SegmentGraph(
+ [bg.BlockGraph(nn.Linear(4, 4), lambda x: x, shared) for _ in range(2)], shared
+ )
+ with torch.inference_mode():
+ assert seg.call(0, (torch.zeros(2, 4),), {}) is bg._NOT_HANDLED
+ assert walked == []
+
+
+def test_video_generate_bypasses_graphs_after_the_auto_cache_toggle():
+ import inspect
+
+ from core.inference import video as V
+
+ src = inspect.getsource(V.VideoBackend)
+ toggle = src.index("toggled = maybe_toggle_step_cache(")
+ bypass = src.index("diffusion_cuda_graph.set_bypass(", toggle)
+ assert "cache_breaks_graph(state.transformer_cache)" in src[bypass : bypass + 200]
+ assert bypass < src.index("if static_skip:", toggle)
+
+
+@_needs_diffusers
+def test_split_guard_wraps_the_mixed_stream_tail_once():
+ from diffusers.hooks.first_block_cache import FBCBlockHook
+
+ assert dc.install_fbcache_split_guard()
+ outer = FBCBlockHook.new_forward
+ assert outer._unsloth_split_guard and outer._unsloth_mixed_stream_tail
+ inner = outer.__closure__ and [
+ c.cell_contents for c in outer.__closure__ if callable(c.cell_contents)
+ ]
+ assert any(
+ getattr(fn, "_unsloth_mixed_stream_tail", False)
+ and not getattr(fn, "_unsloth_split_guard", False)
+ for fn in inner
+ )
+ assert dc.install_fbcache_mixed_stream_tail() and dc.install_fbcache_split_guard()
+ assert FBCBlockHook.new_forward is outer # neither re-wraps the other
+
+
+def test_resident_cuda_cards_lists_every_card_of_a_split(monkeypatch):
+ from core.inference import diffusion_int8_fused as f8
+
+ monkeypatch.setattr(f8, "resident_cuda_device", lambda m: None)
+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1))
+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == (0, 1)
+ monkeypatch.setattr(g8, "_split_cards", lambda m: ())
+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == ()
+ monkeypatch.setattr(f8, "resident_cuda_device", lambda m: torch.device("cuda", 3))
+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == (3,)
+
+
+def test_qwen_real_rope_installs_on_a_split_after_probing_each_card(monkeypatch):
+ from core.inference import diffusion_int8_fused as f8
+ from core.inference import diffusion_qwenimage_rope as qr
+
+ QwenImageTransformer2DModel = type("QwenImageTransformer2DModel", (nn.Module,), {})
+ monkeypatch.setattr(qr, "disabled", lambda: False)
+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (0, 1))
+ probed, patched = [], []
+ monkeypatch.setattr(qr, "_probe", lambda index: probed.append(index) or True)
+ monkeypatch.setattr(
+ qr, "_patch_table", lambda index, logger = None: patched.append(index) or True
+ )
+ assert qr.install(QwenImageTransformer2DModel()) is True
+ assert probed == [0, 1] and patched == [0]
+ monkeypatch.setattr(qr, "_probe", lambda index: False)
+ assert qr.install(QwenImageTransformer2DModel()) is False
+
+
+def test_zimage_fused_probes_each_card_of_a_split(monkeypatch):
+ from core.inference import diffusion_int8_fused as f8
+ from core.inference import diffusion_qwenimage21_rope as q21r
+ from core.inference import diffusion_zimage_fused as zf
+
+ monkeypatch.setattr(zf, "zimage_fused_disabled", lambda: False)
+ monkeypatch.setattr(zf, "_patch_class", lambda logger = None: True)
+ monkeypatch.setitem(
+ sys.modules, zf._MODULE, types.SimpleNamespace(ZSingleStreamAttnProcessor = object)
+ )
+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (5, 6))
+ monkeypatch.setattr(q21r, "inductor_addcmul_is_fma", lambda: True)
+ monkeypatch.setattr(q21r, "probe_fusion", lambda dev: ("fma", dev.index))
+ monkeypatch.setattr(zf, "_FUSION", {})
+ out = zf.install_modules(nn.Linear(2, 2))
+ assert out["real_rope"] is True and set(zf._FUSION) == {5, 6}
+ monkeypatch.setattr(zf, "_FUSION", {5: ("fma", 5), 6: None})
+ assert zf.install_modules(nn.Linear(2, 2))["real_rope"] is False
+
+
+def test_q21_fused_checks_the_arch_of_every_split_card(monkeypatch):
+ from core.inference import diffusion_int8_fused as f8
+ from core.inference import diffusion_qwenimage21_fused as q21f
+
+ monkeypatch.setattr(q21f, "q21_convrot_fused_disabled", lambda: False)
+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (0, 1))
+ seen = []
+ monkeypatch.setattr(q21f, "arch_ok", lambda dev: seen.append(dev.index) or dev.index == 0)
+ assert q21f.install_modules(nn.Linear(2, 2)) == {"fused_qkv": 0, "out": 0}
+ assert seen == [0, 1]
+
+
+def test_hunyuan_image_and_hidream_route_lazily():
+ assert "HunyuanImageTransformer2DModel" in ls.LAZY_CLASSES
+ assert "HiDreamImageTransformer2DModel" in ls.LAZY_CLASSES
+
+
+def test_real_rope_probes_every_card_of_a_split(monkeypatch):
+ from core.inference import diffusion_qwenimage21_rope as q21r
+
+ monkeypatch.setattr(q21r, "real_rope_disabled", lambda: False)
+ monkeypatch.setattr(q21r, "_FUSION", {})
+ assert q21r.probe_split_cards((0, 1)) == () # no install probed a card: the patch is not in use
+ probed = []
+ monkeypatch.setattr(
+ q21r,
+ "probe_fusion",
+ lambda dev: probed.append(dev.index) or (None if dev.index == 2 else "fma"),
+ )
+ monkeypatch.setattr(q21r, "_FUSION", {0: "fma"})
+ assert q21r.probe_split_cards((0, 1, 2)) == (0, 1)
+ assert probed == [1, 2]
+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1))
+ seen = []
+ monkeypatch.setattr(q21r, "probe_split_cards", lambda cards: seen.append(cards) or cards)
+ assert g8.after_layer_split(nn.Linear(2, 2))["rope_cards"] == (0, 1) and seen == [(0, 1)]
+
+
+def test_split_graphs_seal_new_prompt_lengths_per_image_scope():
+ den = nn.Linear(4, 4)
+ handle = bg.BlockGraphSet(den, bg._Shared(0, None))
+ handle.track_scopes()
+ assert handle.length_varies and not handle.scope_sealed()
+ den(torch.randn(2, 4)) # the pre-hook reads this call's image scope
+ first = handle.scope
+ handle.note_capture()
+ cg.seal_lengths((handle,))
+ assert handle.scope_sealed()
+ den(torch.randn(3, 4)) # another batch: a new image scope still records
+ assert handle.scope != first and not handle.scope_sealed()
+ cg.seal_lengths((handle,)) # nothing recorded in that scope: its window stays open
+ assert not handle.scope_sealed()
+ den(torch.randn(2, 4))
+ assert handle.scope_sealed()
+ handle.reset()
+ assert not handle.scope_sealed()
+ for restore in handle.restores:
+ restore()
+ assert not den._forward_pre_hooks
+
+
+def test_a_split_records_block_graphs_for_a_prompt_length_family(monkeypatch):
+ import inspect
+
+ src = inspect.getsource(cg.arm_block_graphs)
+ assert "None if split else prompt_length_reason(pipe, family)" in src
+ assert "handle.track_scopes()" in inspect.getsource(bg.install_block_graphs)
+
+
+def test_zimage_real_rope_probes_every_card_of_a_split(monkeypatch):
+ from core.inference import diffusion_qwenimage21_rope as q21r
+ from core.inference import diffusion_zimage_fused as zf
+
+ monkeypatch.setattr(zf, "zimage_fused_disabled", lambda: False)
+ probed = []
+ monkeypatch.setattr(q21r, "inductor_addcmul_is_fma", lambda: True)
+ monkeypatch.setattr(
+ q21r,
+ "probe_fusion",
+ lambda dev: probed.append(dev.index) or (None if dev.index == 2 else "x"),
+ )
+ monkeypatch.setattr(zf, "_FUSION", {0: "x"})
+ monkeypatch.setattr(q21r, "_FUSION", {})
+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1, 2))
+ out = g8.after_layer_split(nn.Linear(2, 2))
+ assert zf._FUSION == {0: "x", 1: "x", 2: None} and probed == [1, 2]
+ assert out["zimage_rope_cards"] == (0, 1)
+ monkeypatch.setattr(zf, "_FUSION", {})
+ assert zf.probe_split_cards((0, 1)) == () # no install probed a card: the patch is not in use
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index ddc38d41a31..e388153912d 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -875,13 +875,22 @@ def test_encode_release_only_for_the_whole_tier(monkeypatch):
def test_generate_load_installs_the_encode_release():
+ import ast
+
src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
encoding = "utf-8"
)
- at = src.index("install_encode_release(pipe, plan, logger)")
- # after the placement is applied and before the int8 GEMM reads the final residency
- assert src.index("effective_policy, effective_tiling = apply_memory_plan(") < at
- assert at < src.index("if denoisers_pinned_resident(pipe):", at)
+ calls: dict = {}
+ for node in ast.walk(ast.parse(src)):
+ if isinstance(node, ast.Call):
+ fn = node.func
+ name = fn.id if isinstance(fn, ast.Name) else getattr(fn, "attr", None)
+ calls.setdefault(name, []).append(node.lineno)
+ (at,) = calls["install_encode_release"]
+ assert any(line < at for line in calls["apply_memory_plan"])
+ pinned = min(line for line in calls["denoisers_pinned_resident"] if line > at)
+ engage = min(line for line in calls["engage_pinned_denoisers"] if line > at)
+ assert at < pinned < engage
def _inner_devices(tensor):
diff --git a/studio/backend/tests/test_diffusion_prompt_length_policy.py b/studio/backend/tests/test_diffusion_prompt_length_policy.py
index cda7a40dd33..efec7bf2b34 100644
--- a/studio/backend/tests/test_diffusion_prompt_length_policy.py
+++ b/studio/backend/tests/test_diffusion_prompt_length_policy.py
@@ -71,3 +71,19 @@ def test_the_compile_arms_only_its_own_sources():
}
for name, sources in dt._FAMILY_SOURCES.items():
assert dt.PROMPT_LENGTH_POLICY[name] == sources, name
+
+
+def test_qwen_image_arms_its_sources_on_the_dynamic_tier_too():
+ class Qwen:
+ pass
+
+ class Flux3:
+ pass
+
+ Qwen.__name__ = "QwenImageTransformer2DModel"
+ Flux3.__name__ = "Flux3Transformer2DModel"
+ if not dt.supported():
+ return
+ assert dt._armed_when_dynamic(Qwen()) and not dt._armed_when_dynamic(Flux3())
+ assert dt.fingerprint(Qwen(), True) == dt.fingerprint(Qwen(), None)
+ assert dt.fingerprint(Flux3(), True) != dt.fingerprint(Flux3(), None)
diff --git a/studio/backend/tests/test_diffusion_reduction_filter_arch.py b/studio/backend/tests/test_diffusion_reduction_filter_arch.py
index ad833981fe3..1142b6b4962 100644
--- a/studio/backend/tests/test_diffusion_reduction_filter_arch.py
+++ b/studio/backend/tests/test_diffusion_reduction_filter_arch.py
@@ -6,8 +6,8 @@ one image in every fresh server there.
A few of their norm reductions get several configs that sum in different orders, and a cold-cache server benchmarks
them in its own process on near-equal timings: fresh servers rendered 2-4 variants per seed on an RTX PRO 6000, A100
-and L4. B200 servers were already deterministic for these families, so the opt-in is per (family, compute
-capability)."""
+and L4. B200 servers were deterministic for FLUX.1 and Z-Image but not Qwen-Image, so the opt-in is per (family,
+compute capability)."""
from __future__ import annotations
@@ -23,6 +23,7 @@ from core.inference import diffusion_speed as ds # noqa: E402
_SM120 = (12, 0)
_MEASURED_ARCHS = ((8, 0), (8, 9), (12, 0))
_IMAGE_FAMILIES = ("flux.1", "z-image", "qwen-image", "flux.1-kontext", "qwen-image-edit")
+_QWEN_FAMILIES = ("qwen-image", "qwen-image-edit", "qwen-image-layered")
def _family(name):
@@ -65,17 +66,31 @@ def test_image_family_opts_in_on_the_measured_archs(monkeypatch, name, cap):
def test_the_race_families_share_one_arch_list():
from core.inference import diffusion_families
+
for name in _IMAGE_FAMILIES:
- assert (
- _family(name).filter_reduction_configs_archs == diffusion_families._REDUCTION_RACE_ARCHS
+ expected = (
+ diffusion_families._QWEN_REDUCTION_RACE_ARCHS
+ if name in _QWEN_FAMILIES
+ else diffusion_families._REDUCTION_RACE_ARCHS
)
+ assert _family(name).filter_reduction_configs_archs == expected
assert set(diffusion_families._REDUCTION_RACE_ARCHS) == set(_MEASURED_ARCHS)
+ assert set(diffusion_families._QWEN_REDUCTION_RACE_ARCHS) == {*_MEASURED_ARCHS, (10, 0)}
+
+
[email protected]("name", _QWEN_FAMILIES)
+def test_qwen_image_also_pins_on_b200(monkeypatch, name):
+ _on(monkeypatch, (10, 0))
+ fam = _family(name)
+ assert cc.family_filters_reductions(fam) is True
+ assert _filter_reaching_compile(monkeypatch, fam) is True
@pytest.mark.parametrize("cap", [(10, 0), (10, 3), (9, 0), (7, 5), (12, 1), None])
@pytest.mark.parametrize("name", _IMAGE_FAMILIES)
def test_image_family_keeps_inductor_pick_elsewhere(monkeypatch, name, cap):
- # B200 (measured deterministic), B300, H100, T4, other Blackwell parts (unmeasured), no CUDA / ROCm.
+ if cap == (10, 0) and name in _QWEN_FAMILIES:
+ pytest.skip("Qwen-Image pins on B200 (test_qwen_image_also_pins_on_b200)")
_on(monkeypatch, cap)
fam = _family(name)
assert cc.family_filters_reductions(fam) is False
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 1c6d0a766b7..16eeaac4e14 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -675,7 +675,7 @@ def test_cudnn_benchmark_opt_out_image_families():
# Unmeasured families keep the benchmark.
from core.inference.diffusion_families import _FAMILIES
off = {fam.name for fam in _FAMILIES if not fam.cudnn_benchmark}
- assert off == {"qwen-image", "flux.1", "z-image", "sdxl"}
+ assert off == {"qwen-image", "flux.1", "flux.2-dev", "z-image", "sdxl"}
def test_speed_max_enables_tf32_and_fused_qkv(monkeypatch):
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 31be1b67a2f..b27436b8231 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -14078,3 +14078,48 @@ def test_an_automatic_encoder_pick_that_does_not_engage_loads_dense_without_a_re
VideoBackend().load_pipeline(
"Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", text_encoder_quant = "fp8"
)
+
+
+ "split_expected, hooked, moved",
+ [
+ (
+ True,
+ True,
+ True,
+ ), # the split fell back to a hooked one-card placement: the compile moves below the hooks
+ (True, False, False), # the split landed resident: nothing to move
+ (False, True, False), # planned offloaded: the speed layer already compiled for the hooks
+ ],
+)
+def test_a_layer_split_fallback_moves_the_compile_below_the_offload_hooks(
+ fake_runtime, tmp_path, monkeypatch, split_expected, hooked, moved
+):
+ import core.inference.video as video_mod
+
+ _shared_setup_7(monkeypatch, video_mod)
+ _stub_apply_memory_plan(monkeypatch, video_mod, policy = "model")
+ monkeypatch.setattr(
+ video_mod, "_layer_split_expected", lambda pipe, plan, refine = None: split_expected
+ )
+ monkeypatch.setattr(video_mod, "_video_denoiser_hooked", lambda pipe: hooked)
+ seen = []
+ monkeypatch.setattr(
+ video_mod,
+ "compile_pipe_below_offload_hooks",
+ lambda pipe, logger = None: seen.append(pipe) or 0,
+ )
+ speed_offload = []
+ from core.inference import diffusion_autoopt as autoopt
+
+ real_speed = autoopt.run_speed_layer
+
+ def _speed(view, target, **kw):
+ speed_offload.append(kw.get("offload_active"))
+ return real_speed(view, target, **kw)
+
+ monkeypatch.setattr(autoopt, "run_speed_layer", _speed)
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ assert speed_offload and set(speed_offload) == {not split_expected}
+ assert (seen == [backend._state.pipe]) if moved else (seen == [])