Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: split one diffusion denoiser across GPUs by block ranges (#14412)
Daniel Hanbd63cc39f62a
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 7d1c35c10a7..8bc9496a973 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -163,6 +163,7 @@ from .diffusion_memory import (MEMORY_MODE_LOW_VRAM,ACTIVATION_TILE,DeviceMemory,+ ImageActivationShortfallError,OFFLOAD_MODEL,OFFLOAD_NONE,OFFLOAD_STREAMING,@@ -282,6 +283,7 @@ from .diffusion_batched import (from .diffusion_cache import (FBCACHE_MIN_STEPS,TC_AUTO,+ TC_FBCACHE,TC_STATIC,apply_step_cache,auto_static_skip_plan,@@ -2204,6 +2206,143 @@ def _guard_vae_card(return restore, tiled+def _guard_layer_cards(pipe: Any, guard_target: Any, guard_kwargs: dict) -> int:+ """The generate-time check on every other card a layer split runs blocks on. Such a card holds only blocks: no+ text encode, no VAE decode, and one block's activations at a time, so the whole-request estimate the primary is+ checked against does not apply. It needs what the plan reserved there at the default resolution, scaled by how+ much this request's activations grow past that default. ``pipe`` is the LOADED pipeline (a from_pipe workflow+ pipeline carries no split record). Raises the guard's refusal (a ValueError); returns the number of cards+ checked. Fails open on anything unknown."""+ from .diffusion_layer_split_plan import applied_layer_split, secondary_block_cards+ from .diffusion_memory import _oversized_generate_override++ split = applied_layer_split(pipe)+ if split is None or guard_kwargs.get("allow_oversized") or _oversized_generate_override():+ return 0+ try:+ sized = dict(+ batch_size = guard_kwargs.get("batch_size") or 1, family = guard_kwargs.get("family")+ )+ needed = estimate_image_runtime_mib(+ width = guard_kwargs.get("width"),+ height = guard_kwargs.get("height"),+ condition_pixels = int(guard_kwargs.get("condition_pixels") or 0),+ **sized,+ )+ planned = estimate_image_runtime_mib(width = None, height = None, **{**sized, "batch_size": 1})+ scale = max(1.0, float(needed) / float(planned)) if planned else 1.0+ except Exception: # noqa: BLE001 - a broken estimate never blocks a generation+ return 0+ checked = 0+ for ordinal in secondary_block_cards(pipe):+ working, reserve = split.working_mib(ordinal)+ if working + reserve <= 0:+ continue+ with diffusion_device_scope(int(ordinal)):+ free = reclaimable_snapshot_device_memory(guard_target).free_mib+ if free is None:+ continue+ checked += 1+ need = int(working * scale) + reserve+ if need > int(free):+ raise ImageActivationShortfallError(+ f"Generating at {guard_kwargs.get('width')}x{guard_kwargs.get('height')} needs about "+ f"{need / 1024:.2f} GB of working memory on cuda:{int(ordinal)}, which runs part of the model split "+ f"across GPUs, but only about {int(free) / 1024:.2f} GB is free there. Generate at a smaller "+ "resolution, or free GPU memory on that GPU. To try anyway, turn on 'Allow oversized generations' "+ "under Advanced on the Images page."+ )+ return checked+++def _layer_split_expected(+ pipe: Any,+ plan: Any,+ refine: Optional[Callable[..., Any]] = None,+) -> bool:+ """Whether placement will cut the denoiser across cards: the multi-GPU refinement asked early, with the sizes+ loaded so far. The speed layer runs before placement, and on a split every block is resident on some card, so it+ is set up for that (regional compile with fullgraph, int8 kernels, graphs) and not for the one-card offload plan+ the split replaces. A split declined later lands its one-card fallback, the same path a transient OOM re-plan+ takes. ``refine``: the backend's own refinement (video declines some families)."""+ if plan is None or getattr(plan, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:+ return False+ try:+ preview = (refine or refine_plan_for_multigpu)(pipe, plan, None)+ except Exception: # noqa: BLE001 - the one-card plan decides+ return False+ return getattr(preview, "layer_split", None) is not None+++def _companion_split_expected(+ pipe: Any,+ plan: Any,+ refine: Optional[Callable[..., Any]] = None,+) -> bool:+ """Whether placement will keep the denoiser resident by moving other components to a second card (the multi-GPU+ component split), previewed like :func:`_layer_split_expected`. The one-card plan offloads there, so only this+ preview tells the quant compile rule that the denoiser ends up resident."""+ if plan is None or getattr(plan, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:+ return False+ try:+ preview = (refine or refine_plan_for_multigpu)(pipe, plan, None)+ except Exception: # noqa: BLE001 - the one-card plan decides+ return False+ return (+ getattr(preview, "secondary_ordinal", None) is not None+ and getattr(preview, "offload_policy", None) == OFFLOAD_NONE+ )+++def _native_quant_needs_compile(target: Any, native_scheme: Optional[str], resident: bool) -> bool:+ """Whether a torchao-free quantised DiT must be compiled like a torchao one: the NVIDIA int8 scheme (W8A8+ ``torch._int_mm``, chosen for an offload plan) once the DiT stays resident, on one card or split across cards,+ where eager runs it ~3.5x slower (LTX-2 split: 0.90-0.97 vs 0.26 s/step). ROCm and Apple keep their own native+ path and its compile policy; a streamed DiT keeps the offload path's."""+ return (+ native_scheme is not None+ and bool(resident)+ and native_offload_host(target)+ and not native_quant_host(target)+ )+++def _restore_step_cache_after_layer_split(+ pipe: Any, views: Any, **cache_kwargs: Any+) -> Optional[str]:+ """The split takes the load-time First-Block-Cache off before it moves the blocks; once they sit on their cards the+ cache goes back on (``diffusion_cache`` copies its residuals between the head and tail cards). Returns the mode+ that engaged on ``pipe`` (None: it declined, the denoiser runs uncached)."""+ engaged = None+ for view in views:+ mode = apply_step_cache(view, mode = TC_FBCACHE, **cache_kwargs)+ if view is pipe:+ engaged = mode+ return engaged+++def _step_cache_dropped_by_layer_split(pipe: Any) -> bool:+ """Whether the layer split placement took the load-time First-Block-Cache off ``pipe.transformer``+ (``diffusion_layer_split._drop_step_cache``): the marker is gone, so the load's cache locals are stale."""+ transformer = getattr(pipe, "transformer", None)+ return transformer is not None and not getattr(transformer, "_unsloth_step_cache", None)+++def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:+ """A layer split replaced the one-card plan whose partial residency the measured reserve priced. Left in place, a+ conditioned request past it would give the split denoiser on-demand offload hooks onto the primary+ (``hook_resident_denoiser``), pulling every block off its card. True when dropped."""+ if getattr(plan, "layer_split", None) is None or not getattr(+ pipe, "_unsloth_measured_reserve", None+ ):+ return False+ try:+ pipe._unsloth_measured_reserve = None+ except Exception: # noqa: BLE001+ return False+ return True++def _tile_for_verdict(pipe: Any,verdict: Any,@@ -8963,12 +9102,21 @@ class DiffusionBackend:# Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays# bit-identical `off`.effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")- # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF).- if (- (transformer_quant_engaged is not None or comfy_compile)- and native_scheme is None- and effective_speed == SPEED_OFF- ):+ split_expected = _layer_split_expected(pipe, plan)+ # A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower, losing to GGUF.+ quant_forces_compile = (+ transformer_quant_engaged is not None or comfy_compile+ ) and (+ native_scheme is None+ or _native_quant_needs_compile(+ target,+ native_scheme,+ split_expected+ or plan_keeps_transformer_resident(plan)+ or _companion_split_expected(pipe, plan),+ )+ )+ if quant_forces_compile and effective_speed == SPEED_OFF:logger.info("diffusion.transformer_quant: forcing speed_mode=default ""(quantized transformer must be compiled; eager is ~30x slower)"@@ -9155,6 +9303,7 @@ class DiffusionBackend:install_mps_fp16(getattr(pipe, "transformer", None), target, logger = logger)self._raise_if_load_cancelled(_load_token)+ speed_offload = plan.offload_policy != OFFLOAD_NONE and not split_expected# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the# first compiled forward pays the 25-58s compile once.if (@@ -9177,7 +9326,7 @@ class DiffusionBackend:fam,is_gguf = gguf_transformer,speed_mode = effective_speed,- offload_active = plan.offload_policy != OFFLOAD_NONE,+ offload_active = speed_offload,)or (gguf_transformer@@ -9190,9 +9339,7 @@ class DiffusionBackend:)))- and not fp16_unet_offloaded(- target, pipe, offload_active = plan.offload_policy != OFFLOAD_NONE- )+ and not fp16_unet_offloaded(target, pipe, offload_active = speed_offload)):compile_ctx = compile_cache.begin(family = fam.name,@@ -9211,9 +9358,8 @@ class DiffusionBackend:speed_mode = effective_speed,fullgraph = not cache_graph_breakand not cache_may_toggle- and plan.offload_policy == OFFLOAD_NONE,- offload_below_hooks = plan.offload_policy != OFFLOAD_NONE- and compile_below_hooks_enabled(),+ and not speed_offload,+ offload_below_hooks = speed_offload and compile_below_hooks_enabled(),),logger = logger,reduction_filter = family_filters_reductions(fam),@@ -9257,7 +9403,7 @@ class DiffusionBackend:backend = "image",speed_mode = effective_speed,is_gguf = gguf_transformer,- offload_active = plan.offload_policy != OFFLOAD_NONE,+ offload_active = speed_offload,audit = load_audit,logger = logger,apply = lambda family, log: apply_speed_optims(@@ -9268,8 +9414,9 @@ class DiffusionBackend:speed_mode = effective_speed,cache_active = cache_graph_break or cache_may_toggle,cache_engaged = cache_graph_break,- offload_active = plan.offload_policy != OFFLOAD_NONE,- denoiser_offloaded = not plan_keeps_transformer_resident(plan),+ offload_active = speed_offload,+ denoiser_offloaded = not split_expected+ and not plan_keeps_transformer_resident(plan),stream_int8_gemm = True,logger = log,),@@ -9474,6 +9621,7 @@ class DiffusionBackend:)# a transient OOM during setup may have re-planned to lower residencyplan = applied_plan(pipe, plan)+ _drop_measured_reserve_for_layer_split(pipe, plan)identity = getattr(pipe, "_unsloth_peak_identity", None)if isinstance(identity, tuple) and len(identity) == 3:# a saver the plan asked for may not have engaged: key the renders by what did@@ -9482,10 +9630,27 @@ class DiffusionBackend:pipe._unsloth_peak_identity = (identity[0], live_vae, identity[2])# placed from the planned saver's peak: requests size from the flat estimate insteadpipe._unsloth_peak_static = True+ if cache_engaged == TC_FBCACHE and _step_cache_dropped_by_layer_split(pipe):+ cache_engaged = _restore_step_cache_after_layer_split(+ pipe,+ (pipe,),+ threshold = transformer_cache_threshold,+ quant_active = cache_quant_active,+ length_changes_ok = not cache_auto,+ logger = logger,+ )+ if cache_engaged is None and getattr(plan, "layer_split", None) is not None:+ cache_may_toggle = False+ cache_reason = (+ "off: the step cache declined the denoiser split across cards"+ )# streams the whole-resident denoiser back to the flat room while the encoders runinstall_encode_release(pipe, plan, logger)# the speed layer saw only the plan; placement may have pinned every denoiser group since- if denoisers_pinned_resident(pipe):+ if (+ denoisers_pinned_resident(pipe)+ or getattr(plan, "layer_split", None) is not None+ ):engage_pinned_denoisers(pipe, speed_applied, logger)# Placement decides the graph layer: a whole-forward recording holds only while no hook moves the# denoiser; otherwise (and for a forward that is not capture-safe) its blocks are recorded instead.@@ -9529,8 +9694,7 @@ class DiffusionBackend:speed_mode,"deferred" if speed_deferred else effective_speed,"quantized transformer requires compile"- if (transformer_quant_engaged is not None or comfy_compile)- and native_scheme is None+ if quant_forces_compileand normalize_speed_mode(speed_mode) in (None, SPEED_OFF)else "auto: exact eager for the first two images; ""the compile profile engages on the 3rd"@@ -12392,6 +12556,7 @@ class DiffusionBackend:logger = logger,)verdict = raise_on_image_activation_shortfall(**guard_kwargs)+ _guard_layer_cards(state.pipe, guard_target, guard_kwargs)restore_vae = _tile_for_verdict(pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger)diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex bea93715035..10af8d16478 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -288,6 +288,7 @@ class _Shared:self.warmed: set = set()self.pool_bytes = 0self.static_bytes = 0+ self.strict_device = False_LIVE.add(self)def capture_stream(self) -> Any:@@ -411,6 +412,9 @@ class BlockGraph:self.unreplayed_placements = 0self.churned = Falseself.protect: Any = None # None until resolved, then False or the NVFP4 step controller+ self.segment: Any = None+ self.seg_index = 0+ self.owner: Any = Noneself.placements: set = set()self.refusals: dict = {}self.stats = {@@ -462,8 +466,20 @@ class BlockGraph:if _bg_capture_suppressed() or _step_recording():# a whole-step graph above records this block's kernels (diffusion_cuda_graph), not a nested replayreturn self.compute(*args, **kwargs)- if not self.enabled or self.bypassed or self.poisoned or self.churned or _bg_eager_forced():+ if (+ not self.enabled+ or self.bypassed+ or self.poisoned+ or self.churned+ or _bg_eager_forced()+ or _step_cache_live(self.shared.root)+ ):return self._eager(args, kwargs)+ segment = self.segment+ if segment is not None:+ out = segment.call(self.seg_index, args, kwargs)+ if out is not _NOT_HANDLED:+ return outtorch = _torch()if torch.is_grad_enabled():self.stats["refused_grad"] += 1@@ -490,15 +506,31 @@ class BlockGraph:# A capture never records a host read, so a replay would keep the recorded value.self.stats["refused_host_input"] += 1return self._eager(args, kwargs)+ if self.shared.strict_device and (+ torch.cuda.current_device() != self.shared.device_index+ or any(t.device.index != self.shared.device_index for t in live)+ ):+ self.stats["refused_host_input"] += 1+ return self._eager(args, kwargs)if self.protect is None:self.protect = self._protect_controller()if self.protect:# One recording per precision branch: a W4A4 replay at a W4A16 step would skip the protection.from .diffusion_cuda_graph import protect_graph_keyplacement = (placement, protect_graph_key(self.protect))+ try:+ extra = _root_key_extra(self.shared.root)+ except Exception: # noqa: BLE001 - a branch flag that cannot be read: nothing safe to replay+ self.stats["refused_object"] += 1+ return self._eager(args, kwargs)+ if extra is not None:+ placement = (placement, extra)full = (key, placement)entry = self.cache.get(full)if entry is None:+ if self.owner is not None and self.owner.scope_sealed():+ self.stats["length_eager"] = self.stats.get("length_eager", 0) + 1+ return self._eager(args, kwargs)if full not in self.seen:# First call at this layout and placement: run it for real (the warm-up), record on the next one.return self._first_sighting(full, args, kwargs)@@ -606,6 +638,8 @@ class BlockGraph:self.stats["recaptures"] += 1self.cache[full] = entryself.stats["captures"] += 1+ if self.owner is not None:+ self.owner.note_capture()from .diffusion_cuda_graph import capture_device, note_captureself._unsloth_capture_device = capture_device()@@ -657,6 +691,424 @@ class BlockGraph:exc.__traceback__ = None+SEGMENT_GRAPHS_ENV = "UNSLOTH_DIFFUSION_SEGMENT_GRAPHS"+_NOT_HANDLED = object()+++def segment_graphs_disabled() -> bool:+ return (os.environ.get(SEGMENT_GRAPHS_ENV) or "").strip().lower() in _OFF+++class _SegRecord:+ """One eager pass over a segment: each block's call layout, where each of its input tensors came from (an earlier+ block's output, or one of the first block's inputs), and its output layout."""++ __slots__ = ("keys", "prov", "out_specs")++ def __init__(self) -> None:+ self.keys: list = []+ self.prov: list = []+ self.out_specs: list = []+++class _SegEntry:+ __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays")+++def _root_key_extra(root: Any) -> Any:+ """The root's per-call branch state (``diffusion_cuda_graph.GRAPH_KEY_EXTRA_ATTR``, e.g. HunyuanVideo-1.5's+ null-mask flag a pre-hook sets): part of every block and segment key, as of the whole-step graph's, so a recording+ made on one branch never replays for the other. None without one."""+ from .diffusion_cuda_graph import GRAPH_KEY_EXTRA_ATTR++ extra = getattr(root, GRAPH_KEY_EXTRA_ATTR, None) if root is not None else None+ return ("x", extra()) if callable(extra) else None+++def _step_cache_live(root: Any) -> bool:+ """Whether a step cache (First-Block-Cache) is engaged on the denoiser: its hooks decide inside the forward, per+ step and per cond / uncond context, which a recording cannot see. Belt to the caller's per-chunk ``set_bypass``,+ as ``GraphedForward`` does: an auto cache that turns on at generate time keeps every block eager."""+ try:+ return bool(getattr(root, "_unsloth_step_cache", None)) if root is not None else False+ except Exception: # noqa: BLE001+ return False+++def _lazy_chain(root: Any) -> bool:+ """Whether this forward of a split ``root`` routes lazily: its class is one the split verified to pass each block's+ outputs straight to the next block, and the call carries no residual input. Only such a chain may replay as one+ segment, since a segment cannot see an op between blocks that rewrites a tensor in place."""+ try:+ from .diffusion_layer_split import layer_split_of+ split = layer_split_of(root)+ except Exception: # noqa: BLE001+ return False+ return bool(split is not None and split.lazy_now)+++def _version(t: Any) -> Any:+ # an inference tensor carries no version counter+ try:+ return None if t.is_inference() else t._version+ except Exception: # noqa: BLE001+ return None+++class SegmentGraph:+ """One CUDA graph over a run of blocks a layer split keeps on one card, back to back in one block list: the first+ block's call replays the whole run, and each later block's call checks that it got exactly the tensors the run+ handed on (the previous block's outputs, the first block's other inputs) and returns its recorded outputs. Any+ other call (an op between blocks rewrote a tensor, an eager boundary copied it) leaves the segment to the per-block+ path for the rest of that forward, which computes the block itself.++ Lifecycle per (layout of the first block's inputs, weight placement): the first forward records the run eagerly,+ the second warms it up once and captures it, every later one replays. All on the card's own stream and pool."""++ def __init__(+ self,+ graphs: list,+ shared: _Shared,+ max_graphs: int = MAX_GRAPHS_PER_BLOCK,+ ) -> None:+ self.graphs = list(graphs)+ self.shared = shared+ self.max_graphs = int(max_graphs)+ self.cache: "OrderedDict[tuple, _SegEntry]" = OrderedDict()+ self.seen: "OrderedDict[tuple, _SegRecord]" = OrderedDict()+ self.refused: set = set()+ self.poisoned = False+ self.enabled = True # off: every block takes its own graph's path; recordings are kept+ self.capture_error: Optional[dict] = None+ self.owner: Any = None+ self._reset_forward()+ self.stats = {+ "segment_records": 0,+ "segment_captures": 0,+ "segment_replays": 0,+ "segment_warmups": 0,+ "segment_breaks": 0,+ "segment_refusals": 0,+ "segment_fallbacks": 0,+ }+ for i, g in enumerate(self.graphs):+ g.segment = self+ g.seg_index = i+ _LIVE_BLOCKS.add(self)++ def _reset_forward(self) -> None:+ self.pos: Optional[int] = None+ self.mode: Optional[str] = None+ self.cur: Any = None+ self.ext: list = []+ self.returned: list = []+ self.versions: list = []++ def reset(self) -> None:+ for entry in self.cache.values():+ self._forget(entry)+ self.cache.clear()+ self.seen.clear()+ self.refused.clear()+ self._reset_forward()++ def detach(self) -> None:+ self.reset()+ for g in self.graphs:+ if g.segment is self:+ g.segment = None++ def _forget(self, entry: _SegEntry) -> None:+ entry.graph = None+ entry.outputs = None+ self.shared.drop(getattr(entry, "slots", ()))++ def _placement(self) -> Optional[tuple]:+ out = []+ for g in self.graphs:+ p = g.weights.placement(self.shared.device_index)+ if p is None:+ return None+ out.append(p)+ return tuple(out)++ def call(self, i: int, args: tuple, kwargs: dict) -> Any:+ if self.poisoned or not self.enabled:+ return _NOT_HANDLED+ if i == 0:+ return self._start(args, kwargs)+ if self.pos != i:+ return _NOT_HANDLED+ return self._next(i, args, kwargs)++ def _start(self, args: tuple, kwargs: dict) -> Any:+ self._reset_forward()+ torch = _torch()+ if (+ torch.is_grad_enabled()+ or not _lazy_chain(self.shared.root)+ or _step_cache_live(self.shared.root)+ ):+ return _NOT_HANDLED+ live: list = []+ try:+ key = _walk((args, kwargs), live)+ extra = _root_key_extra(self.shared.root)+ except Exception: # noqa: BLE001+ return _NOT_HANDLED+ if _refusal(key, kwargs) or any(t.device.type != "cuda" for t in live):+ return _NOT_HANDLED+ if torch.cuda.current_device() != self.shared.device_index or any(+ t.device.index != self.shared.device_index for t in live+ ):+ return _NOT_HANDLED+ for g in self.graphs:+ if g.protect is None:+ g.protect = g._protect_controller()+ if g.poisoned or g.churned or g.bypassed or g.protect:+ return _NOT_HANDLED+ placement = self._placement()+ if placement is None:+ return _NOT_HANDLED+ full = (key, placement) if extra is None else (key, placement, extra)+ if full in self.refused:+ return _NOT_HANDLED+ entry = self.cache.get(full)+ if entry is None and self.owner is not None and self.owner.scope_sealed():+ return _NOT_HANDLED # its blocks run ungraphed too (BlockGraph.owner)+ if entry is None and full in self.seen:+ entry = self._capture(full, live)+ if entry is None:+ return _NOT_HANDLED+ if entry is not None:+ self.cache.move_to_end(full)+ for dst, src in zip(entry.static_in, live):+ if dst.data_ptr() != src.data_ptr():+ dst.copy_(src)+ entry.graph.replay()+ entry.replays += 1+ self.stats["segment_replays"] += 1+ self.mode, self.cur, self.ext = "replay", entry, live+ return self._hand_out(0, entry)+ rec = _SegRecord()+ out = self.graphs[0].compute(*args, **kwargs)+ flat: list = []+ try:+ spec = _flatten(out, flat)+ except Exception: # noqa: BLE001+ self.refused.add(full)+ self.stats["segment_refusals"] += 1+ return out+ rec.keys.append(key)+ rec.prov.append(tuple(("e", e) for e in range(len(live))))+ rec.out_specs.append(spec)+ self.mode, self.cur, self.ext = "record", (full, rec), live+ self.returned = [flat]+ self.versions = [[_version(t) for t in flat]]+ self.pos = 1+ self.graphs[0].stats["eager_calls"] += 1+ return out++ def _provenance(self, live: list) -> Optional[tuple]:+ out = []+ for t in live:+ found = None+ for j in range(len(self.returned) - 1, -1, -1):+ for m, r in enumerate(self.returned[j]):+ if t is r:+ if _version(t) != self.versions[j][m]:+ return None # rewritten in place since its block handed it on+ found = ("o", j, m)+ break+ if found is not None:+ break+ if found is None:+ for e, x in enumerate(self.ext):+ if t is x:+ found = ("e", e)+ break+ if found is None:+ return None+ out.append(found)+ return tuple(out)++ def _next(self, i: int, args: tuple, kwargs: dict) -> Any:+ live: list = []+ try:+ key = _walk((args, kwargs), live)+ except Exception: # noqa: BLE001+ self._abandon()+ return _NOT_HANDLED+ prov = self._provenance(live)+ last = i == len(self.graphs) - 1+ if self.mode == "record":+ full, rec = self.cur+ if prov is None or _refusal(key, kwargs):+ self.refused.add(full)+ self.stats["segment_refusals"] += 1+ self._abandon()+ return _NOT_HANDLED+ out = self.graphs[i].compute(*args, **kwargs)+ flat: list = []+ try:+ spec = _flatten(out, flat)+ except Exception: # noqa: BLE001+ self.refused.add(full)+ self.stats["segment_refusals"] += 1+ self._abandon()+ return out+ rec.keys.append(key)+ rec.prov.append(prov)+ rec.out_specs.append(spec)+ self.returned.append(flat)+ self.versions.append([_version(t) for t in flat])+ self.graphs[i].stats["eager_calls"] += 1+ if last:+ self.seen[full] = rec+ while len(self.seen) > 4:+ self.seen.popitem(last = False)+ self.stats["segment_records"] += 1+ self._reset_forward()+ else:+ self.pos = i + 1+ return out+ entry = self.cur+ if prov is None or key != entry.record.keys[i] or prov != entry.record.prov[i]:+ self.stats["segment_breaks"] += 1+ self._abandon()+ return _NOT_HANDLED+ return self._hand_out(i, entry)++ def _hand_out(self, i: int, entry: _SegEntry) -> Any:+ tensors = entry.outputs[i]+ last = i == len(self.graphs) - 1+ if last:+ tensors = [t.clone() for t in tensors]+ self._reset_forward()+ else:+ if i == 0:+ self.returned = [tensors]+ self.versions = [[_version(t) for t in tensors]]+ else:+ self.returned.append(tensors)+ self.versions.append([_version(t) for t in tensors])+ self.pos = i + 1+ self.graphs[i].stats["segment_hits"] = self.graphs[i].stats.get("segment_hits", 0) + 1+ return _rebuild(entry.record.out_specs[i], tensors)++ def _abandon(self) -> None:+ self._reset_forward()++ def _chain(self, rec: _SegRecord, ext: list) -> list:+ results: list = []+ for i, g in enumerate(self.graphs):+ tensors = [ext[p[1]] if p[0] == "e" else results[p[1]][p[2]] for p in rec.prov[i]]+ call_args, call_kwargs = _unwalk(rec.keys[i], iter(tensors))+ out = g.compute(*call_args, **call_kwargs)+ flat: list = []+ if _flatten(out, flat) != rec.out_specs[i]:+ raise RuntimeError("segment block output layout changed between calls")+ results.append(flat)+ return results++ def _capture(self, full: tuple, live: list) -> Optional[_SegEntry]:+ from .diffusion_cuda_graph import (+ _capturing,+ capture_device,+ note_capture,+ refuse_if_exhausted,+ retire_failed_capture,+ )++ torch = _torch()+ rec = self.seen.pop(full)+ entry = _SegEntry()+ slot = ("segment", id(self), full[0])+ try:+ entry.static_in = self.shared.statics_for(slot, live)+ for dst, src in zip(entry.static_in, live):+ dst.copy_(src)+ # once, on the compute stream: a guard the static buffers trip recompiles here, never inside the capture+ self._chain(rec, entry.static_in)+ self.stats["segment_warmups"] += 1+ current = torch.cuda.current_stream()+ stream = self.shared.capture_stream()+ stream.wait_stream(current)+ refuse_if_exhausted(self.shared.logger)+ graph = torch.cuda.CUDAGraph()+ pool = self.shared.graph_pool()+ with torch.cuda.stream(stream):+ before = torch.cuda.memory_reserved()+ with _capturing():+ try:+ graph.capture_begin(pool = pool, capture_error_mode = "thread_local")+ try:+ outputs = self._chain(rec, entry.static_in)+ finally:+ graph.capture_end()+ except BaseException as exc:+ retire_failed_capture(graph, pool, exc)+ if self.shared.pool == pool:+ self.shared.pool = None+ raise+ current.wait_stream(stream)+ self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)+ except Exception as exc: # noqa: BLE001 - this layout stays per block for the load's life+ self.capture_error = {"type": type(exc).__name__, "msg": str(exc)[:4000]}+ self.refused.add(full)+ self.stats["segment_fallbacks"] += 1+ if self.shared.logger is not None:+ self.shared.logger.warning(+ "diffusion.block_graph: recording a %d-block segment failed (%s: %s); its blocks record one by one",+ len(self.graphs),+ type(exc).__name__,+ exc,+ )+ return None+ entry.graph = graph+ entry.outputs = outputs+ entry.record = rec+ entry.slots = (slot,)+ entry.replays = 0+ self.shared.hold(entry.slots)+ self.cache[full] = entry+ self.stats["segment_captures"] += 1+ if self.owner is not None:+ self.owner.note_capture()+ self._unsloth_capture_device = capture_device()+ note_capture(self._unsloth_capture_device)+ while len(self.cache) > self.max_graphs:+ _, evicted = self.cache.popitem(last = False)+ self._forget(evicted)+ return entry+++def _segments(transformer: Any, graphs: list) -> list:+ """Runs of at least two armed blocks a layer split keeps on one card, back to back in one block list (forward+ order), as (graphs, shared) pairs."""+ try:+ from .diffusion_layer_split import find_blocks+ except Exception: # noqa: BLE001+ return []+ by_block = {id(g.block): g for g in graphs}+ runs: list = []+ current: list = []+ current_key = None+ for _qual, group, block in find_blocks(transformer):+ g = by_block.get(id(block))+ key = (group, g.shared.device_index, id(g.shared)) if g is not None else None+ if g is not None and key == current_key:+ current.append(g)+ continue+ if len(current) > 1:+ runs.append(current)+ current = [g] if g is not None else []+ current_key = key+ if len(current) > 1:+ runs.append(current)+ return [(run, run[0].shared) for run in runs]++def _group_offload_hook(module: Any) -> Any:try:from diffusers.hooks import group_offloading as go@@ -708,6 +1160,8 @@ class BlockGraphSet:) -> None:self.module = transformerself.shared = shared+ self.card_shared: dict = {}+ self.segments: list = []self.logger = loggerself.graphs: list = []self.restores: list = []@@ -717,6 +1171,54 @@ class BlockGraphSet:self._prefetcher = Noneself.mode = "blocks"self.max_graphs = MAX_GRAPHS_PER_BLOCK+ self.length_varies = False+ self.scope: Any = None+ self._sealed_scopes: set = set()+ self._scope_captures: dict = {}+ self._render_scopes: set = set()++ def scope_sealed(self) -> bool:+ return self.length_varies and self.scope in self._sealed_scopes++ def note_capture(self) -> None:+ if self.length_varies:+ self._scope_captures[self.scope] = self._scope_captures.get(self.scope, 0) + 1++ def track_scopes(self) -> None:+ """Read each denoiser call's image scope (resolution, batch, references) before its blocks run."""+ from .diffusion_cuda_graph import _image_scope++ def _pre(module: Any, args: Any, kwargs: Any) -> None:+ try:+ self.scope = _image_scope(args, kwargs)+ except Exception: # noqa: BLE001 - an unkeyable call: no scope, nothing seals+ self.scope = None+ self._render_scopes.add(self.scope)++ self.length_varies = True+ handle = self.module.register_forward_pre_hook(_pre, with_kwargs = True)+ self.restores.append(handle.remove)++ def shared_for(self, index: int) -> _Shared:+ """The pool, stream and static buffers of card ``index``: the primary's own, else one per other card."""+ if self.shared.device_index is None:+ self.shared.device_index = index+ if self.shared.device_index == index:+ self.shared.strict_device = True+ return self.shared+ state = self.card_shared.get(index)+ if state is None:+ state = _Shared(index, self.logger)+ state.root = self.shared.root+ state.strict_device = True+ self.card_shared[index] = state+ return state++ def cards(self) -> set:+ return {s.device_index for s in self._all_shared()}++ def _all_shared(self) -> list:+ return [self.shared, *self.card_shared.values()]@propertydef cache(self) -> dict:@@ -732,6 +1234,12 @@ class BlockGraphSet:for g in self.graphs:for k, v in g.stats.items():total[k] = total.get(k, 0) + int(v)+ for seg in self.segments:+ for k, v in seg.stats.items():+ total[k] = total.get(k, 0) + int(v)+ total["captures"] = total.get("captures", 0) + total.get("segment_captures", 0)+ total["warmups"] = total.get("warmups", 0) + total.get("segment_warmups", 0)+ total["replays"] = total.get("replays", 0) + total.get("segment_replays", 0)# The whole-forward wrapper's names, so diffusion_cuda_graph.stats aggregates both kinds alike.total["refused_host_tensor"] = total.get("refused_host_weight", 0)total["cap_skips"] = total.get("evictions", 0)@@ -755,13 +1263,22 @@ class BlockGraphSet:return selfdef reset(self) -> "BlockGraphSet":+ self._sealed_scopes.clear()+ self._scope_captures.clear()+ self._render_scopes.clear()+ for seg in self.segments:+ seg.reset()for g in self.graphs:g.reset()- self.shared.release()+ for state in self._all_shared():+ state.release()_release_cached()return selfdef free(self) -> "BlockGraphSet":+ for seg in self.segments:+ seg.detach()+ self.segments.clear()for g in self.graphs:g.reset()g.enabled = False@@ -772,7 +1289,8 @@ class BlockGraphSet:passself.restores.clear()self.graphs.clear()- self.shared.release()+ for state in self._all_shared():+ state.release()self.enabled = Falsetry:from .diffusion_offload_prefetch import module_prefetcher@@ -801,8 +1319,18 @@ class BlockGraphSet:getattr(getattr(self, "_prefetcher", None), "slot_bytes", 0) or 0)>> 20,- "pool_mib": int(self.shared.pool_bytes >> 20),- "static_mib": int(self.shared.static_bytes >> 20),+ "pool_mib": int(sum(st.pool_bytes for st in self._all_shared()) >> 20),+ "static_mib": int(sum(st.static_bytes for st in self._all_shared()) >> 20),+ **({"cards": sorted(self.cards())} if self.card_shared else {}),+ **(+ {+ "segments": len(self.segments),+ "segment_blocks": sum(len(seg.graphs) for seg in self.segments),+ "segment_graphs": sum(len(seg.cache) for seg in self.segments),+ }+ if self.segments+ else {}+ ),"capture_error": Noneif not self.capture_errorelse {@@ -840,7 +1368,7 @@ class BlockGraphSet:"""Device memory this layer holds that the caching allocator cannot hand to anything else."""pf = getattr(self, "_prefetcher", None)slots = int(getattr(pf, "slot_bytes", 0) or 0) if pf is not None else 0- return int(self.shared.pool_bytes + self.shared.static_bytes + slots)+ return int(sum(st.pool_bytes + st.static_bytes for st in self._all_shared()) + slots)def _release_cached() -> None:@@ -869,6 +1397,21 @@ def _device_index(transformer: Any, fallback: Any = None) -> Optional[int]:return None+def _block_card(block: Any) -> Optional[int]:+ """The CUDA ordinal every weight of ``block`` sits on (a layer split places each block whole), else None."""+ index: Any = _UNSEEN+ for t in _WeightView(block).tensors:+ for leaf in _leaves(t):+ dev = leaf.device+ if dev.type != "cuda" or dev.index is None:+ return None+ if index is _UNSEEN:+ index = dev.index+ elif dev.index != index:+ return None+ return None if index is _UNSEEN else int(index)++def install_block_graphs(transformer: Any,*,@@ -882,7 +1425,10 @@ def install_block_graphs(blocks = repeated_blocks(transformer)if not blocks:return None, "no repeated blocks to record"+ from .diffusion_layer_split import is_layer_split+torch = _torch()+ split = is_layer_split(transformer)shared = _Shared(_device_index(transformer, device), logger)shared.root = transformerhandle = BlockGraphSet(transformer, shared, logger)@@ -895,6 +1441,13 @@ def install_block_graphs(continuecompiled = getattr(block, "_compiled_call_impl", None)hook = _group_offload_hook(block)+ block_shared = shared+ if split:+ index = _block_card(block)+ if index is None:+ refused["weights off the GPU"] = refused.get("weights off the GPU", 0) + 1+ continue+ block_shared = handle.shared_for(index)if hook is not None and compiled is not None:# compiled through its hooks (UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS=0): the compute is inside a dynamo framewhy = "compiled through its offload hooks"@@ -919,14 +1472,14 @@ def install_block_graphs(refused["a stacked hook chain"] = refused.get("a stacked hook chain", 0) + 1continuecompute = target.forward- graph = BlockGraph(block, compute, shared)+ graph = BlockGraph(block, compute, block_shared)target.forward = graphdef restore(ref: Any = target, fn: Any = compute) -> None:if ref.forward is not fn:ref.forward = fnelif compiled is not None:- graph = BlockGraph(block, compiled, shared)+ graph = BlockGraph(block, compiled, block_shared)block._compiled_call_impl = graphdef restore(@@ -939,7 +1492,7 @@ def install_block_graphs(else:slot = block.__dict__.get("forward")fwd = slot if slot is not None else type(block).forward.__get__(block)- graph = BlockGraph(block, fwd, shared)+ graph = BlockGraph(block, fwd, block_shared)block.__dict__["forward"] = graphdef restore(@@ -964,6 +1517,16 @@ def install_block_graphs(", ".join(f"{n} block(s): {w}" for w, n in refused.items()) or "no block could be armed")return None, reason+ if split and not segment_graphs_disabled():+ handle.segments = [+ SegmentGraph(run, run_shared)+ for run, run_shared in _segments(transformer, handle.graphs)+ ]+ handle.mode = "segments" if handle.segments else handle.mode+ if split and _prompt_length_varies(transformer):+ handle.track_scopes()+ for unit in (*handle.graphs, *handle.segments):+ unit.owner = handleif slots and torch.cuda.is_available():try:from .diffusion_offload_prefetch import module_prefetcher@@ -977,10 +1540,20 @@ def install_block_graphs(logger.warning("diffusion.block_graph: slot ring unavailable (%s)", exc)if logger is not None:logger.info(- "diffusion.block_graph: armed %d of %d %s block(s)%s%s",+ "diffusion.block_graph: armed %d of %d %s block(s)%s%s%s",len(handle.graphs),len(blocks),type(transformer).__name__,+ (+ f" over cards {sorted(handle.cards())}"+ + (+ f" as {len(handle.segments)} per-card segment graph(s)"+ if handle.segments+ else ""+ )+ )+ if split+ else "",f"; streamed blocks read a {handle.slots_mib} MiB slot ring"if handle.slots_mibelse "",@@ -989,6 +1562,14 @@ def install_block_graphs(return handle, "armed"+def _prompt_length_varies(transformer: Any) -> bool:+ try:+ from .diffusion_cuda_graph import _length_varies+ return _length_varies(transformer)+ except Exception: # noqa: BLE001+ return False++COMPILE_BELOW_HOOKS_ENV = "UNSLOTH_DIFFUSION_COMPILE_BELOW_HOOKS"diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.pyindex 94c1fc8daee..24c90b12229 100644--- a/studio/backend/core/inference/diffusion_cache.py+++ b/studio/backend/core/inference/diffusion_cache.py@@ -432,6 +432,7 @@ def _transformer_blocks_registered(transformer: Any, logger: Any = None) -> boolif not callable(named_children):return Trueregister_unregistered_transformer_blocks(logger)+ _register_layer_split_blocks(transformer)blocks = [getattr(block, "_orig_mod", block)for name, child in named_children()@@ -501,6 +502,107 @@ def install_fbcache_length_guard() -> bool:return True+def _register_layer_split_blocks(transformer: Any) -> bool:+ """Register a layer split's block classes (same-named subclasses of the real blocks) with their base's metadata,+ so the cache finds them. A copy, never the base's own object: ``register`` writes ``_cls`` into it. True when+ every split block class is registered."""+ try:+ import dataclasses++ from diffusers.hooks._helpers import TransformerBlockRegistry+ except Exception: # noqa: BLE001 - no registry: enable_cache decides+ return True+ ok = True+ for cls in {type(m) for m in transformer.modules()}:+ base = cls.__dict__.get("_unsloth_layer_split_base")+ if base is None:+ continue+ try:+ TransformerBlockRegistry.get(cls)+ continue+ except Exception: # noqa: BLE001 - not registered yet+ pass+ try:+ meta = TransformerBlockRegistry.get(base)+ except Exception: # noqa: BLE001 - the base is not a cacheable block either+ continue+ try:+ TransformerBlockRegistry.register(+ model_class = cls,+ metadata = dataclasses.replace(meta, _cls = None, _cached_parameter_indices = None),+ )+ except Exception: # noqa: BLE001+ ok = False+ return ok+++def _tensors_to(obj: Any, device: Any) -> Any:+ if isinstance(obj, (list, tuple)):+ return type(obj)(_tensors_to(x, device) for x in obj)+ to = getattr(obj, "to", None)+ return to(device) if callable(to) else obj+++def _first_device(obj: Any) -> Any:+ if isinstance(obj, (list, tuple)):+ for x in obj:+ dev = _first_device(x)+ if dev is not None:+ return dev+ return None+ return getattr(obj, "device", None)+++def install_fbcache_split_guard() -> bool:+ """Make First-Block-Cache's tail hook work when the tail block runs on another card than the head block (a layer+ split): the head block's output is copied to the tail's card for the residual, and the residual goes back to the+ head's card, where the head hook adds it on a cached step. Copies are exact, so the cache decides and renders as+ on one card. Calls whose tensors share a card take the original path unchanged. Process-wide and idempotent;+ must run before ``enable_cache`` (diffusers binds ``new_forward`` when it registers the hook). False when this+ diffusers has no FBCache block hook."""+ try:+ from diffusers.hooks.first_block_cache import FBCBlockHook+ except Exception: # noqa: BLE001 - no FBCache in this diffusers+ return False+ install_fbcache_mixed_stream_tail()+ original = getattr(FBCBlockHook, "new_forward", None)+ if original is None:+ return False+ if getattr(original, "_unsloth_split_guard", False):+ return True++ def new_forward(self, module, *args, **kwargs):+ if not getattr(self, "is_tail", False):+ return original(self, module, *args, **kwargs)+ state = self.state_manager.get_state()+ head = state.head_block_output+ home = _first_device(head)+ try:+ here = self._metadata._get_parameter_from_args_kwargs(+ "hidden_states", args, kwargs+ ).device+ except Exception: # noqa: BLE001 - the original raises the same way+ here = None+ if not state.should_compute or home is None or here is None or here == home:+ return original(self, module, *args, **kwargs)+ state.head_block_output = _tensors_to(head, here)+ try:+ out = original(self, module, *args, **kwargs)+ finally:+ state.head_block_output = head+ residuals = state.tail_block_residuals+ if residuals is not None:+ state.tail_block_residuals = tuple(+ r.to(home) if r is not None else None for r in residuals+ )+ return out++ new_forward._unsloth_split_guard = True+ new_forward._unsloth_mixed_stream_tail = getattr(original, "_unsloth_mixed_stream_tail", False)+ FBCBlockHook.new_forward = new_forward+ return True++def _split_mixed_stream_tail_residuals(tail_hidden: Any, head_output: Any) -> Optional[tuple]:"""Tail residuals for a dual-stream head and a single-stream tail, or None when unmappable.@@ -675,6 +777,14 @@ def apply_step_cache(transformer = getattr(pipe, "transformer", None)if transformer is None:return None+ from .diffusion_layer_split import is_layer_split++ if is_layer_split(transformer) and not (+ install_fbcache_split_guard() and _register_layer_split_blocks(transformer)+ ):+ # without the guard the tail hook subtracts the head block's output (primary) from its own (another card)+ _warn(logger, mode, RuntimeError("denoiser is split across cards; running uncached"))+ return None# Before enable_cache, which is what raises on a block class the registry has never seen.register_unregistered_transformer_blocks(logger)# FLUX.2's dual-stream head + single-stream tail crashes the stock tail hook on the first step.diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 00ac17cbd71..5f96f9b226f 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -647,6 +647,31 @@ class _Entry:)+def _layer_split(module: Any) -> bool:+ from .diffusion_layer_split import is_layer_split+ return is_layer_split(module)+++def _rehome_after_layer_split(pipe: Any, logger: Any) -> None:+ """Before any recording or first forward: the int8 / fused records the speed layer built on one card follow the+ split's blocks to their cards (``diffusion_int8_gemm.after_layer_split``)."""+ try:+ from .diffusion_int8_gemm import after_layer_split+ for module in _denoiser_modules(pipe):+ if _layer_split(module):+ after_layer_split(module, logger)+ except Exception as exc: # noqa: BLE001 - the stock paths stay correct+ _warn(logger, "layer split re-home", exc)+++def _layer_split_pipe(pipe: Any) -> bool:+ """Any denoiser of ``pipe`` is split across cards by block ranges."""+ try:+ return any(_layer_split(m) for m in _denoiser_modules(pipe))+ except Exception: # noqa: BLE001+ return False++class GraphedForward:"""A callable that replaces one denoiser module's ``forward`` with a CUDA-graph replay.@@ -1054,6 +1079,7 @@ class GraphedForward:or self.poisoned# Belt to the caller's per-chunk bypass: one step's cond and uncond share a key.or getattr(self.module, "_unsloth_step_cache", None)+ or _layer_split(self.module)):return self._eager(args, kwargs)@@ -1899,7 +1925,12 @@ def arm_after_placement(handles: list = []refusals: list = []modes: list = []+ from .diffusion_layer_split import is_layer_split+for module in _denoiser_modules(pipe):+ if is_layer_split(module):+ refusals.append(f"{type(module).__name__}: denoiser is split across cards")+ continueplacement, why = offload_placement(module)if why is not None:refusals.append(f"{type(module).__name__}: {why}")@@ -2027,6 +2058,11 @@ def graph_eligible(# Lazy: ``diffusion_speed`` imports this module, so a module-level import here is a cycle.from .diffusion_speed import _denoiser_dits, _denoiser_unet+ from .diffusion_layer_split import is_layer_split++ if any(is_layer_split(m) for m in _denoiser_modules(pipe)):+ return False, "denoiser is split across cards"+# A whole-compiled U-Net serves from ``_compiled_call_impl``: GraphedCompiledCall captures there.if not _denoiser_dits(pipe) and _denoiser_unet(pipe) is None:return False, "no denoiser transformer"@@ -2335,6 +2371,11 @@ def arm_block_graphs(whole = [h for h in handles if isinstance(h, GraphedForward)]prior = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")step = [h for h in whole if h.placement is not None or h.plan is not None]+ split = _layer_split_pipe(pipe)+ if split:+ # a whole-step capture records one card; a split's blocks record per block on their own card+ step = []+ _rehome_after_layer_split(pipe, logger)if whole and len(step) == len(whole):# The whole step stays primary; where per-block graphs are the default they arm for the calls it leaves eager.pipe._unsloth_cuda_graph_mode = "step"@@ -2344,13 +2385,13 @@ def arm_block_graphs(if why is None:_attach_block_fallback(pipe, step, target, hooked, pinned, logger)return handles- if not hooked and (whole or "not capture-safe" not in prior):+ if not split and not hooked and (whole or "not capture-safe" not in prior):# Nothing moves the denoiser: the speed layer's decision stands, except a forward that is not capture-safe,# whose blocks still record.return handles- why = _block_basics(- target, family, cache_engaged, speed_mode, family_default- ) or prompt_length_reason(pipe, family)+ why = _block_basics(target, family, cache_engaged, speed_mode, family_default) or (+ None if split else prompt_length_reason(pipe, family)+ )if whole:uninstall_all(whole, logger = logger)pipe._unsloth_cuda_graph_mode = None@@ -2414,6 +2455,8 @@ def arm_block_graphs(if hooked and pinnedelse "offloaded denoiser"if hooked+ else "split denoiser (each card's blocks on that card)"+ if splitelse "denoiser")slots = sum(int(getattr(h, "slots_mib", 0) or 0) for h in armed)diff --git a/studio/backend/core/inference/diffusion_dynamic_text.py b/studio/backend/core/inference/diffusion_dynamic_text.pyindex 8f913cd52d4..bbf8541c470 100644--- a/studio/backend/core/inference/diffusion_dynamic_text.py+++ b/studio/backend/core/inference/diffusion_dynamic_text.py@@ -129,6 +129,13 @@ _FAMILY_SOURCES: dict[str, tuple[str, ...]] = {"MiniMaxH3Transformer3DModel": _MINIMAX_H3_SOURCES,}+_ARMED_WHEN_DYNAMIC: frozenset[str] = frozenset({"QwenImageTransformer2DModel"})+++def _armed_when_dynamic(transformer: Any) -> bool:+ return type(transformer).__name__ in _ARMED_WHEN_DYNAMIC++# Unbacked: a backed symbol specialises size 1, and H3's temb has 1 row on step 1, 2 after (a second compile)._FAMILY_UNBACKED: dict[str, tuple[str, ...]] = {"MiniMaxH3Transformer3DModel": ("L['temb']",),@@ -182,11 +189,11 @@ def sources_for(transformer: Any) -> tuple[str, ...]:def fingerprint(transformer: Any, dynamic: Any) -> Optional[str]:- if dynamic is True:+ if dynamic is True and not _armed_when_dynamic(transformer):# dynamic=True already makes every dim dynamic; only the unbacked sources are armed (see install).unbacked = unbacked_sources_for(transformer)return "unbacked:" + ",".join(unbacked) if unbacked else None- if dynamic is not None:+ if dynamic is not None and dynamic is not True:return Nonesources = sources_for(transformer)if not sources or not supported():@@ -273,7 +280,8 @@ def install(*,dynamic: Any = None,) -> bool:- """Arm sources for a ``dynamic`` compile: None arms dynamic + unbacked, True only unbacked, False nothing."""+ """Arm sources for a ``dynamic`` compile: None arms dynamic + unbacked, True only unbacked (dynamic + unbacked+ for :data:`_ARMED_WHEN_DYNAMIC`), False nothing."""if dynamic is False:return Falseif getattr(transformer, "_unsloth_dynamic_text", None) is not None:@@ -286,7 +294,9 @@ def install(if logger is not None:logger.warning("diffusion.dynamic_text: slice identity fix failed: %s", exc)unbacked = unbacked_sources_for(transformer)- sources = sources_for(transformer) if dynamic is None else ()+ sources = (+ sources_for(transformer) if dynamic is None or _armed_when_dynamic(transformer) else ()+ )cfg = _compiler_config()if not (sources or unbacked) or cfg is None:return Falsediff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex cd6672dfc79..f04e4eee464 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -260,6 +260,8 @@ class DiffusionFamily:# Fresh servers rendered 2-4 variants of one seed here (FLUX.1, Z-Image, Qwen-Image; the edit siblings share their DiT):# near-tied norm-reduction configs, benchmarked per process, sum in different orders. B200 measured deterministic._REDUCTION_RACE_ARCHS: tuple[tuple[int, int], ...] = ((8, 0), (8, 9), (12, 0))+# Qwen-Image races on B200 too (torch 2.14): fresh processes rendered one seed 26 dB apart; with the filter 6/6 matched bitwise.+_QWEN_REDUCTION_RACE_ARCHS: tuple[tuple[int, int], ...] = _REDUCTION_RACE_ARCHS + ((10, 0),)# --- Shared FamilyProfile data (see family_profile.FamilyProfile) ---@@ -611,6 +613,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (label = "FLUX.2-dev",profile = FamilyProfile(te_int8_skip = (3, 0), preview_factors = "flux2"),pipeline_class = "Flux2Pipeline",+ # cudnn.benchmark's per-process VAE conv pick rendered 5 variants of one seed over 7 fresh B200 processes.+ cudnn_benchmark = False,uses_negative_prompt = False,transformer_class = "Flux2Transformer2DModel",base_repo = "black-forest-labs/FLUX.2-dev",@@ -684,7 +688,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (te_int8_skip = _QWENIMAGE_TE_INT8_SKIP,preview_factors = "qwen_image",),- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,comfy_flow_shift = 3.1, # ComfyUI ModelSamplingAuraFlow 3.1 (Qwen-Image-Edit 2511 template)comfy_flow_shift_variants = (("qwen-image-edit-2511", 3.1),@@ -757,7 +761,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (scheme_deny = _QWENIMAGE_SCHEME_DENY,te_int8_skip = _QWENIMAGE_TE_INT8_SKIP,),- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,# ComfyUI's Image to Layers template: ModelSamplingAuraFlow 1, 2 layers, the input scaled to 640.comfy_flow_shift = 1.0,layer_count = 2,@@ -805,7 +809,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (preview_factors = "qwen_image",auto_static_skip = (("qwen/qwen-image", (("default", 2), ("max", 3), ("steps", 20))),),),- filter_reduction_configs_archs = _REDUCTION_RACE_ARCHS,+ filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,comfy_flow_shift = 3.1, # ComfyUI ModelSamplingAuraFlow 3.1 (Qwen-Image templates)pipeline_class = "QwenImagePipeline",transformer_class = "QwenImageTransformer2DModel",diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex aa18f95d337..908c14b2609 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -1334,7 +1334,10 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->ok = ok and hidden_states.dtype == torch.bfloat16if ok:fused_in, parts, down_name, n, gate_col, value_col, in_group, down_group = rec- ok = all(_get(self, nm) is mod for nm, mod in parts)+ ok = (+ all(_get(self, nm) is mod for nm, mod in parts)+ and fused_in.weight.device == hidden_states.device+ )if not ok:return type(self).forward(self, hidden_states, *args, **kwargs)down = _get(self, down_name)@@ -1500,6 +1503,18 @@ def resident_cuda_device(module: Any) -> Any:return dev if dev.type == "cuda" else None+def resident_cuda_cards(module: Any) -> tuple:+ """The CUDA ordinals ``module``'s weights sit on: its one resident card, or every card of a layer split (each+ block on one card). Empty when any weight is off the GPU, or on ROCm."""+ dev = resident_cuda_device(module)+ if dev is not None:+ import torch+ return (dev.index if dev.index is not None else torch.cuda.current_device(),)+ from .diffusion_int8_gemm import _split_cards++ return _split_cards(module)++def run_on_first_call(module: Any, key: str, fn: Any) -> None:"""Run ``fn(module)`` once, eagerly, right before ``module``'s first forward (after load-time placement movedthe weights, before the regional compile traces the blocks and before any CUDA-graph capture). Idempotent per key."""@@ -1599,8 +1614,12 @@ def _has_eligible(transformer: Any) -> bool:def _finalize(transformer: Any, logger: Any = None) -> int:global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES, _INTMM_MODULEdev = resident_cuda_device(transformer)+ cards: tuple = ()if dev is None:- return 0+ from .diffusion_int8_gemm import _split_cards+ cards = _split_cards(transformer)+ if not cards:+ return 0# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.if not _has_eligible(transformer):return 0@@ -1612,23 +1631,33 @@ def _finalize(transformer: Any, logger: Any = None) -> int:_INTMM_MODULE = intmmimport torch- if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):- return 0+ if cards:+ usable = {index for index in cards if _device_ok(index)}+ if not usable:+ return 0+ indices = sorted(usable)+ else:+ if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):+ return 0+ indices = [dev.index if dev.index is not None else torch.cuda.current_device()]count = 0rearm = False- index = dev.index if dev.index is not None else torch.cuda.current_device()- actq = _act_quant_op() if _act_quant_device_ok(index) else None+ actq_ok = {index: _act_quant_device_ok(index) for index in indices}+ actq = _act_quant_op() if any(actq_ok.values()) else Nonewith _LOCK:_OP_HANDLE = _op()- if actq is not None:- _ACTQ_HANDLE = actq- _ACTQ_DEVICES = _ACTQ_DEVICES | {index}- else:- _ACTQ_DEVICES = _ACTQ_DEVICES - {index}+ for index, ok in actq_ok.items():+ if ok and actq is not None:+ _ACTQ_HANDLE = actq+ _ACTQ_DEVICES = _ACTQ_DEVICES | {index}+ else:+ _ACTQ_DEVICES = _ACTQ_DEVICES - {index}for _name, module in transformer.named_modules():if _MARK in module.__dict__:count += 1continue+ if cards and _module_card(module) not in usable:+ continue # its card failed the kernel probe (or holds no weights of it): stock pathif _ff_eligible(module):fn = _ff_forwardelif _prepare_swiglu(module):@@ -1668,6 +1697,102 @@ def _finalize(transformer: Any, logger: Any = None) -> int:return count+def _module_card(module: Any) -> Optional[int]:+ for t in module.parameters():+ dev = t.device+ return dev.index if dev.type == "cuda" else None+ return None+++_OFF_TREE_RECORDS = (+ (_SWIGLU_ATTR, "swiglu"),+ ("_unsloth_fused_qkv", "qkv"),+ ("_unsloth_q21_fused_qkv", "qkv"),+)+++def relocate_after_layer_split(transformer: Any, logger: Any = None) -> dict:+ """A layer split moved blocks to other cards after their fused records were built on one card. Each off-tree+ fused Linear (which shared its parts' storage there) is rebuilt on its parts' new card, freeing the stale copy, or+ its record is dropped (the module then runs its stock path). Regions on a card whose fused kernels fail the probe+ get their stock forward back. Returns counts; idempotent. Must run before the first forward (the regional compile+ traces what it finds)."""+ from .diffusion_zimage_fused import _fuse_linears, _share_storage++ out = {"rebuilt": 0, "dropped": 0, "restored": 0}+ global _ACTQ_DEVICES+ cards = {}+ with _LOCK:+ for module in transformer.modules():+ for attr, kind in _OFF_TREE_RECORDS:+ rec = module.__dict__.get(attr)+ if rec is None:+ continue+ if kind == "swiglu":+ parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None+ else:+ parts = list(rec[1:4])+ if parts is None:+ continue # the in-projection is the module's own Linear: it moved with the block+ fused = rec[0]+ here = parts[0].weight.device+ if fused.weight.device == here:+ continue+ rebuilt = _fuse_linears(parts)+ if rebuilt is not None and _share_storage(rebuilt, parts):+ module.__dict__[attr] = (rebuilt, *rec[1:])+ out["rebuilt"] += 1+ else:+ module.__dict__.pop(attr, None)+ out["dropped"] += 1+ if _MARK in module.__dict__:+ index = _module_card(module)+ if index is not None:+ cards.setdefault(index, []).append(module)+ for index, modules in cards.items():+ if _device_ok(index):+ if _act_quant_device_ok(index) and _ACTQ_HANDLE is not None:+ _ACTQ_DEVICES = _ACTQ_DEVICES | {index}+ continue+ for module in modules:+ _restore_module(module)+ out["restored"] += 1+ if logger is not None and any(out.values()):+ logger.info(+ "diffusion.int8_fused: layer split re-homed fused records (%d rebuilt, %d dropped, %d regions stock)",+ out["rebuilt"],+ out["dropped"],+ out["restored"],+ )+ return out+++def drop_stranded_fused_records(transformer: Any) -> int:+ """Drop fused records whose parts left their card after a failed split (``pipe.to("cpu")`` cannot free them).+ Returns how many were dropped."""+ dropped = 0+ with _LOCK:+ for module in transformer.modules():+ for attr, kind in _OFF_TREE_RECORDS:+ rec = module.__dict__.get(attr)+ if rec is None:+ continue+ if kind == "swiglu":+ parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None+ else:+ parts = list(rec[1:4])+ if parts is None:+ continue # the in-projection is the module's own Linear: it moved with the block+ try:+ stranded = any(p.weight.device != rec[0].weight.device for p in parts)+ except Exception: # noqa: BLE001 - an unreadable record cannot be trusted either+ stranded = True+ if stranded:+ module.__dict__.pop(attr, None)+ dropped += 1+ return dropped++def _hooked_forward_slot(module: Any) -> Optional[tuple]:"""(fn_ref, attribute) holding the module's own forward in the innermost ``HookFunctionReference``, else None."""registry = getattr(module, "_diffusers_hook", None)@@ -1735,21 +1860,25 @@ def uninstall(transformer: Any = None) -> None:cancel_first_call(transformer, "int8_fused")with _LOCK:for module in transformer.modules():- prev = module.__dict__.pop(_MARK, None)- if prev is None:- continue- slot = _hooked_forward_slot(module)- if slot is not None and _is_fused_forward(getattr(*slot), module):- # Swapped under a hook, or a cache engaged after the swap: restore the hook's inner, keep its wrapper.- setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))- elif _is_fused_forward(module.__dict__.get("forward"), module):- if prev is _NO_PREV:- module.__dict__.pop("forward", None)- else:- module.forward = prev- _disarm_fused_inner(module, prev)- module.__dict__.pop("_unsloth_i8_addcmul", None)- module.__dict__.pop(_SWIGLU_ATTR, None)+ _restore_module(module)+++def _restore_module(module: Any) -> None:+ """One region's stock forward back (caller holds ``_LOCK`` or owns the module)."""+ prev = module.__dict__.pop(_MARK, None)+ if prev is None:+ return+ slot = _hooked_forward_slot(module)+ if slot is not None and _is_fused_forward(getattr(*slot), module):+ setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))+ elif _is_fused_forward(module.__dict__.get("forward"), module):+ if prev is _NO_PREV:+ module.__dict__.pop("forward", None)+ else:+ module.forward = prev+ _disarm_fused_inner(module, prev)+ module.__dict__.pop("_unsloth_i8_addcmul", None)+ module.__dict__.pop(_SWIGLU_ATTR, None)def is_installed(module: Any) -> bool:diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex d60673ff69c..484ce6d2703 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -767,7 +767,6 @@ def _swap(logger: Any = None,device: Any = None,) -> int:- global _OP_HANDLEfrom .diffusion_int8_fused import resident_cuda_deviceimport torch@@ -785,12 +784,50 @@ def _swap(return 0else:dev = resident_cuda_device(transformer)+ if dev is None:+ cards = _split_cards(transformer)+ if cards:+ count = sum(_swap_on(transformer, logger, index) for index in cards)+ return countif dev is None:return 0index = dev.index if dev.index is not None else torch.cuda.current_device()+ return _swap_on(transformer, logger, index)+++def _split_cards(transformer: Any) -> tuple:+ """The CUDA ordinals of a layer-split denoiser whose every weight sits on a card (NVIDIA), else ()."""+ try:+ import torch++ from .diffusion_layer_split import is_layer_split++ if getattr(torch.version, "hip", None) or not is_layer_split(transformer):+ return ()+ devices = {p.device for p in transformer.parameters()}+ except Exception: # noqa: BLE001+ return ()+ if not devices or any(d.type != "cuda" or d.index is None for d in devices):+ return ()+ return tuple(sorted(d.index for d in devices))+++def _weight_card(module: Any) -> Optional[int]:+ weight = getattr(module, "weight", None)+ dev = getattr(weight, "device", None)+ return dev.index if dev is not None and dev.type == "cuda" else None+++def _swap_on(transformer: Any, logger: Any, index: int) -> int:+ """Swap the eligible int8 Linears of ``transformer`` against card ``index``'s probed kernels. On a layer split+ (several cards) only the Linears whose weight sits on ``index``."""+ global _OP_HANDLE+ import torch++ multi = len(_split_cards(transformer)) > 1recs = [(m, _eligible(m)) for m in transformer.modules()]- recs = [(m, r) for m, r in recs if r is not None]+ recs = [(m, r) for m, r in recs if r is not None and (not multi or _weight_card(m) == index)]if not recs:return 0from .diffusion_convrot_quant import rotquant_device_config@@ -812,6 +849,14 @@ def _swap(return 0for module, rec in recs:if _MARK in module.__dict__:+ if multi:+ module.__dict__[_REC] = (+ rec[0],+ rec[1],+ bool(rotq and rec[1] is not None),+ rec[3],+ gemm,+ )count += 1continue# kind, rotation group, fused rotation + act quant on, the Parameter (no payload alias), fused GEMM on@@ -836,6 +881,45 @@ def _swap(return count+def after_layer_split(transformer: Any, logger: Any = None) -> dict:+ """Run once a layer split placed ``transformer``'s blocks, before its first forward: re-home the fused int8 / QKV+ records built while it sat on one card (``diffusion_int8_fused.relocate_after_layer_split``), and probe each card's+ GEMM tiles so the int8 Linears swapped before the split run their own card's kernel; each RoPE patch's probe map+ covers every card. Idempotent; a no-op on a denoiser that is not split."""+ out: dict = {"cards": (), "gemm": 0}+ cards = _split_cards(transformer)+ if len(cards) < 2:+ return out+ out["cards"] = cards+ try:+ from .diffusion_int8_fused import relocate_after_layer_split+ out.update(relocate_after_layer_split(transformer, logger))+ except Exception as exc: # noqa: BLE001 - the stock paths stay correct+ if logger is not None:+ logger.warning(+ "diffusion.int8_gemm: fused record re-home after the layer split failed (%s)", exc+ )+ try:+ from .diffusion_qwenimage21_rope import probe_split_cards+ out["rope_cards"] = probe_split_cards(cards)+ except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE+ pass+ try:+ from .diffusion_zimage_fused import probe_split_cards as zimage_probe+ out["zimage_rope_cards"] = zimage_probe(cards)+ except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE+ pass+ if int8_gemm_mode() != "off" and any(_MARK in m.__dict__ for m in transformer.modules()):+ try:+ out["gemm"] = sum(_swap_on(transformer, logger, index) for index in cards)+ except Exception as exc: # noqa: BLE001+ if logger is not None:+ logger.warning(+ "diffusion.int8_gemm: per-card swap after the layer split failed (%s)", exc+ )+ return out++def uninstall(transformer: Any = None) -> None:if transformer is None:returndiff --git a/studio/backend/core/inference/diffusion_layer_split.py b/studio/backend/core/inference/diffusion_layer_split.pynew file mode 100644index 00000000000..767b99565e0--- /dev/null+++ b/studio/backend/core/inference/diffusion_layer_split.py@@ -0,0 +1,898 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""One denoiser split across several CUDA cards by contiguous ranges of its repeated blocks (layer split).++For a DiT that does not fit one selected card: each repeated block lives whole on one card, everything else of the+denoiser (embedders, modulation, final norm and projection) stays on the primary. Blocks run one after another, so+this buys memory, not speed; the weights never move after placement and only activations cross a card boundary.++The boundary is a same-named subclass of each block's class whose ``__call__`` moves tensor arguments to the block's+card before ``nn.Module.__call__`` runs. It runs outside ``_compiled_call_impl``, so a regional compile only ever sees+tensors on the block's own card (no graph break; the shared block code compiles once per card). Tensors computed once+per forward on the primary (rotary tables, modulation, masks) are copied once per card per forward, not per block.++Outputs: in lazy mode a block's outputs move only where the next block of its list sits on another card, and home at+the end of the list. That is only correct when the denoiser's loop does no tensor op of its own between blocks, so it+is limited to the classes below and to calls without a residual input (ControlNet, VACE hints); every other call is+eager: outputs return to the primary after every off-primary block. Both are exact copies, so the image matches a+one-card run bit for bit.++Every route reads where the block's weights and the denoiser's own weights are NOW, per call, so a later ``.to()`` of+the pipeline or of the denoiser (eviction to the CPU, a move onto one card) keeps the forward on one device, as the+component boundary does.++Kill switch ``UNSLOTH_DIFFUSION_LAYER_SPLIT=0`` (never split a denoiser; the planner offloads as before); ``force``+splits across the selected cards even when one card would hold the denoiser (for proving bit-identity).+"""++from __future__ import annotations++import dataclasses+import inspect+import os+import threading+import weakref+from typing import Any, Mapping, Optional, Sequence++LAYER_SPLIT_ENV = "UNSLOTH_DIFFUSION_LAYER_SPLIT"++# Denoisers whose forward passes block outputs straight to the next block (checked against diffusers' loops).+LAZY_CLASSES: frozenset[str] = frozenset(+ {+ "Flux2Transformer2DModel",+ "Flux3Transformer2DModel",+ "FluxTransformer2DModel",+ "HunyuanImageTransformer2DModel",+ "QwenImageTransformer2DModel",+ "QwenImage21Transformer2DModel",+ "ZImageTransformer2DModel",+ "WanTransformer3DModel",+ "LTX2VideoTransformer3DModel",+ # HiDream's llama states / cats follow hidden_states.device (bitwise vs one card, test_diffusion_layer_split_cuda.py).+ "HiDreamImageTransformer2DModel",+ "HunyuanVideo15Transformer3DModel",+ "Ideogram4Transformer2DModel",+ "Krea2Transformer2DModel",+ "Lumina2Transformer2DModel",+ "MiniMaxH3Transformer3DModel",+ }+)+# Block lists run in another order than registered (forward order, so each card is entered once).+FORWARD_ORDER: dict[str, tuple[str, ...]] = {+ "Flux3Transformer2DModel": (+ "txt_mode_blocks",+ "mode_blocks.image",+ "mode_blocks.image_cond",+ "single_transformer_blocks",+ ),+}+# A non-None keyword whose name contains one of these feeds a root-level add between blocks: eager for that call.+_RESIDUAL_KWARG_PARTS: tuple[str, ...] = (+ "controlnet",+ "control_hidden",+ "residual",+ "hint",+ "vace",+)++_STEP_CACHE_HOOKS: frozenset[str] = frozenset({"fbc_leader_block_hook", "fbc_block_hook"})++_SPLIT_ATTR = "_unsloth_layer_split"+_SUBCLASSES: dict[type, type] = {}+_SUBCLASS_LOCK = threading.Lock()+++class LayerSplitError(RuntimeError):+ """The denoiser cannot be split (no repeated blocks, hooks that own its placement, or the budgets are too small).+ The caller falls back to its one-card plan."""+++def layer_split_mode() -> str:+ """``auto`` (default: split only a denoiser that fits no single selected card), ``off``, or ``force``.+ ``UNSLOTH_DIFFUSION_MULTIGPU=0`` turns it off too."""+ from .diffusion_multigpu import multigpu_mode++ if multigpu_mode() == "off":+ return "off"+ raw = (os.environ.get(LAYER_SPLIT_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return "off"+ if raw == "force":+ return "force"+ return "auto"+++def _torch():+ import torch+ return torch+++def _leaves(t: Any) -> list:+ from .diffusion_block_graph import _leaves as leaves+ return leaves(t)+++def module_bytes(module: Any) -> int:+ """Bytes the module's weights take on a card: tensor subclasses (torchao) count their inner tensors."""+ seen: set = set()+ total = 0+ for t in (*module.parameters(), *module.buffers()):+ if id(t) in seen:+ continue+ seen.add(id(t))+ for leaf in _leaves(t):+ try:+ total += leaf.numel() * leaf.element_size()+ except Exception: # noqa: BLE001 - opaque leaf: count the wrapper+ total += t.numel() * t.element_size()+ break+ return total+++def _block_list_name(qual: str) -> str:+ """The block list a repeated block belongs to: the module path before its last list index. A list entry that+ wraps the block (HiDream's ``double_stream_blocks.3.block``) belongs to ``double_stream_blocks``, not to a group+ of its own, so consecutive entries route lazily and record as one segment."""+ parts = qual.split(".")+ for i in range(len(parts) - 1, -1, -1):+ if parts[i].isdigit():+ return ".".join(parts[:i])+ return qual.rpartition(".")[0]+++def find_blocks(transformer: Any) -> list[tuple[str, str, Any]]:+ """(qualified name, list name, block) for every repeated block, in forward order (registration order unless+ :data:`FORWARD_ORDER` names the lists). Nested repeated blocks are part of their outer block."""+ names: tuple[str, ...] = ()+ try:+ from .diffusion_regional_compile import ensure_repeated_blocks+ names = tuple(ensure_repeated_blocks(transformer) or ())+ except Exception: # noqa: BLE001 - fall back to what the class declares+ pass+ if not names:+ names = tuple(getattr(transformer, "_repeated_blocks", None) or ())+ if not names:+ names = tuple(getattr(transformer, "_no_split_modules", None) or ())+ wanted = set(names)+ out: list[tuple[str, str, Any]] = []+ inside: Optional[str] = None+ for qual, module in transformer.named_modules():+ if module is transformer:+ continue+ if inside is not None and qual.startswith(inside + "."):+ continue+ inside = None+ base = getattr(type(module), "_unsloth_layer_split_base", type(module))+ if base.__name__ in wanted:+ out.append((qual, _block_list_name(qual), module))+ inside = qual+ order = FORWARD_ORDER.get(type(transformer).__name__)+ if order:+ rank = {g: i for i, g in enumerate(order)}+ out.sort(+ key = lambda item: rank.get(item[1], len(rank))+ ) # stable: keeps each list's own order+ return out+++def _module_device(module: Any) -> Any:+ for t in module.parameters():+ return t.device+ for t in module.buffers():+ return t.device+ return None+++def _flex_processor(transformer: Any) -> Optional[str]:+ try:+ processors = getattr(transformer, "attn_processors", None) or {}+ except Exception: # noqa: BLE001 - no processor registry: nothing flex to find+ return None+ for processor in processors.values():+ name = type(processor).__name__+ if "Flex" in name:+ return name+ return None+++def _diffusers_hook_names(module: Any) -> set:+ registry = getattr(module, "_diffusers_hook", None)+ return set(getattr(registry, "hooks", None) or {}) if registry is not None else set()+++def step_cache_hooked(transformer: Any) -> bool:+ """Whether Studio's First-Block-Cache (``diffusion_cache``) is the ONLY diffusers hook on ``transformer``. It keeps+ its residuals on one card, so a split removes it first (:func:`_drop_step_cache`) instead of refusing."""+ try:+ names = set().union(*(_diffusers_hook_names(m) for m in transformer.modules()))+ except Exception: # noqa: BLE001 - not a torch module+ return False+ return bool(names) and names <= _STEP_CACHE_HOOKS+++def layer_split_refusal(transformer: Any, *, step_cache_ok: bool = False) -> Optional[str]:+ """Why this denoiser cannot be split, else None. ``step_cache_ok``: Studio's own First-Block-Cache hooks do not+ refuse (placement removes them)."""+ torch = _torch()+ if not isinstance(transformer, torch.nn.Module):+ return "no denoiser module"+ if getattr(transformer, "_compiled_call_impl", None) is not None:+ return "denoiser is compiled whole"+ cache_only = step_cache_ok and step_cache_hooked(transformer)+ for name, sub in transformer.named_modules():+ if getattr(sub, "_hf_hook", None) is not None:+ return f"{name or '<root>'} carries an accelerate hook"+ hooks = _diffusers_hook_names(sub)+ if hooks and not cache_only:+ return f"{name or '<root>'} carries diffusers hooks ({', '.join(sorted(hooks))})"+ for t in (*transformer.parameters(), *transformer.buffers()):+ if t.device.type == "meta":+ return "denoiser has meta weights"+ break+ flex = _flex_processor(transformer)+ if flex is not None:+ # its BlockMask's mask_mod closes over tensors built on the primary, which no boundary copy reaches+ return f"{flex} drives flex attention through a BlockMask built on the primary"+ if len(find_blocks(transformer)) < 2:+ return "denoiser has fewer than two repeated blocks"+ return None+++class LayerRange:+ device: str+ blocks: list[str]+ mib: int++ def describe(self) -> dict:+ return {+ "device": self.device,+ "count": len(self.blocks),+ "first": self.blocks[0] if self.blocks else None,+ "last": self.blocks[-1] if self.blocks else None,+ "mib": self.mib,+ }+++class LayerSplitPlan:+ primary: str+ rest_mib: int+ ranges: list[LayerRange]+ assignment: dict[str, str] # block qualified name -> device++ def per_card_mib(self) -> dict[str, int]:+ out: dict[str, int] = {self.primary: self.rest_mib}+ for r in self.ranges:+ out[r.device] = out.get(r.device, 0) + r.mib+ return out+++def _cuda(ordinal_or_device: Any) -> Any:+ torch = _torch()+ if isinstance(ordinal_or_device, int):+ return torch.device("cuda", ordinal_or_device)+ dev = torch.device(ordinal_or_device)+ if dev.type == "cuda" and dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ return dev+++def plan_layer_split(+ transformer: Any,+ *,+ primary: Any,+ cards: Sequence[Any],+ budgets_mib: Mapping[Any, int],+ strategy: str = "fill",+) -> LayerSplitPlan:+ """Contiguous block ranges over ``cards`` (primary first; added if missing), in forward order.++ ``budgets_mib``: denoiser weights each card may take (keys: ordinal or device string); the primary's budget also+ pays for the non-block modules. ``fill`` packs each card up to its budget before the next (fewest cards, one+ crossing per used card); ``balance`` spreads the blocks in proportion to the budgets. Raises+ :class:`LayerSplitError` when the blocks do not fit."""+ primary_dev = _cuda(primary)+ order = [primary_dev]+ for c in cards:+ dev = _cuda(c)+ if dev not in order:+ order.append(dev)+ if len(order) < 2:+ raise LayerSplitError("a layer split needs at least two cards")++ def _budget(dev: Any) -> int:+ for key in (dev.index, str(dev), dev):+ if key in budgets_mib:+ return int(budgets_mib[key])+ raise LayerSplitError(f"no budget for {dev}")++ mib = 1 << 20+ blocks = find_blocks(transformer)+ if len(blocks) < 2:+ raise LayerSplitError("denoiser has fewer than two repeated blocks")+ sizes = [module_bytes(b) for _, _, b in blocks]+ rest = max(0, module_bytes(transformer) - sum(sizes))+ budgets = [_budget(d) * mib for d in order]+ budgets[0] -= rest+ if budgets[0] < 0:+ raise LayerSplitError(+ f"the primary's budget ({_budget(order[0])} MiB) does not cover the non-block modules "+ f"({rest // mib} MiB)"+ )+ total = sum(sizes)+ if total > sum(max(0, b) for b in budgets):+ raise LayerSplitError(+ f"blocks need {total // mib} MiB, the cards' budgets give {sum(max(0, b) for b in budgets) // mib} MiB"+ )+ if strategy == "balance":+ weight = sum(max(0, b) for b in budgets)+ targets = [min(b, total * max(0, b) / weight) for b in budgets]+ elif strategy == "fill":+ targets = list(budgets)+ else:+ raise LayerSplitError(f"unknown strategy {strategy!r}")++ assignment: dict[str, str] = {}+ ranges: list[LayerRange] = []+ card = 0+ used = 0+ current: list[str] = []+ current_bytes = 0++ def _close() -> None:+ if current:+ ranges.append(LayerRange(str(order[card]), list(current), current_bytes // mib))++ for (qual, _, _), size in zip(blocks, sizes):+ while card < len(order) - 1 and (+ used + size > budgets[card] or (strategy == "balance" and used >= targets[card])+ ):+ _close()+ current, current_bytes, used = [], 0, 0+ card += 1+ if used + size > budgets[card]:+ raise LayerSplitError(+ f"block {qual} ({size // mib} MiB) does not fit any remaining card"+ )+ assignment[qual] = str(order[card])+ current.append(qual)+ current_bytes += size+ used += size+ _close()+ return LayerSplitPlan(str(primary_dev), rest // mib, ranges, assignment)+++def assigned_plan(+ transformer: Any, *, primary: Any, assignment: Mapping[str, Any]+) -> LayerSplitPlan:+ """A plan from an explicit block -> card map (qualified names from :func:`find_blocks`; card = ordinal or device).+ Every block must be assigned. Raises :class:`LayerSplitError` on an unknown or missing block."""+ primary_dev = _cuda(primary)+ mib = 1 << 20+ blocks = find_blocks(transformer)+ if len(blocks) < 2:+ raise LayerSplitError("denoiser has fewer than two repeated blocks")+ known = {qual for qual, _, _ in blocks}+ unknown = sorted(set(assignment) - known)+ if unknown:+ raise LayerSplitError(f"assignment names unknown blocks ({', '.join(unknown[:3])})")+ sizes = [module_bytes(b) for _, _, b in blocks]+ rest = max(0, module_bytes(transformer) - sum(sizes))+ out: dict[str, str] = {}+ ranges: list[LayerRange] = []+ for (qual, _, _), size in zip(blocks, sizes):+ if qual not in assignment:+ raise LayerSplitError(f"block {qual} has no card")+ dev = str(_cuda(assignment[qual]))+ out[qual] = dev+ if ranges and ranges[-1].device == dev:+ ranges[-1].blocks.append(qual)+ ranges[-1].mib += size // mib+ else:+ ranges.append(LayerRange(dev, [qual], size // mib))+ if len({r.device for r in ranges} | {str(primary_dev)}) < 2:+ raise LayerSplitError("assignment keeps every block on the primary")+ return LayerSplitPlan(str(primary_dev), rest // mib, ranges, out)+++def cross_copy(t: Any, device: Any) -> Any:+ """Every input copy onto a block's card goes through here (the memo and the per-call hand-off): the one place to+ redirect, e.g. into a per-card graph's static buffer."""+ return t.to(device)+++def cross_outputs(out: Any, device: Any) -> Any:+ """Every block output sent to another card (the next block's, or home) goes through here."""+ from .diffusion_multigpu import move_to_device+ return move_to_device(out, device, objects = False)+++class _Memo:+ """Copies of one forward's tensors per card, keyed by the source's id and checked against a weak reference, its+ data pointer and its version, so a freed or rewritten source never serves a stale copy. Inference tensors (every+ render) keep no version, so the memo is only trusted across blocks in lazy mode, where the denoiser does no tensor+ op of its own between blocks; an eager call clears it after every block (``_split_call``)."""++ def __init__(self) -> None:+ self.entries: dict[int, tuple[Any, int, Any, dict]] = {}+ self.copies = 0++ def clear(self) -> None:+ self.entries.clear()++ @staticmethod+ def _version(t: Any) -> Any:+ try:+ return t._version+ except Exception: # noqa: BLE001 - inference tensors keep no version counter+ return None++ def get(self, t: Any, device: Any) -> Any:+ torch = _torch()+ if type(t) is not torch.Tensor and not isinstance(t, torch.nn.Parameter):+ self.copies += 1+ return cross_copy(t, device) # tensor subclasses: no stable data pointer to key on+ key = id(t)+ hit = self.entries.get(key)+ if hit is not None:+ ref, ptr, version, copies = hit+ if ref() is t and ptr == t.data_ptr() and version == self._version(t):+ moved = copies.get(device)+ if moved is None:+ self.copies += 1+ moved = copies[device] = cross_copy(t, device)+ return moved+ self.copies += 1+ moved = cross_copy(t, device)+ entries = self.entries++ def _drop(_ref: Any, key: int = key) -> None:+ current = entries.get(key)+ if current is not None and current[0] is _ref:+ entries.pop(key, None)++ try:+ ref = weakref.ref(t, _drop)+ except TypeError:+ return moved+ entries[key] = (ref, t.data_ptr(), self._version(t), {device: moved})+ return moved+++def _probe(module: Any) -> Optional[tuple]:+ """(weak module, kind, name) of one tensor ``module`` itself or a descendant holds, read per call for its current+ device; None when it holds none. ``skip``: ids of submodules (the blocks) whose tensors are not the module's."""+ return _probe_outside(module, frozenset())+++def _probe_outside(module: Any, skip: frozenset) -> Optional[tuple]:+ stack = [module]+ while stack:+ current = stack.pop()+ if id(current) in skip:+ continue+ for kind in ("_parameters", "_buffers"):+ for name, tensor in getattr(current, kind, {}).items():+ if tensor is not None:+ return (weakref.ref(current), kind, name)+ stack.extend(reversed(list(current.children())))+ return None+++def _probe_device(probe: Optional[tuple], default: Any) -> Any:+ if probe is None:+ return default+ owner = probe[0]()+ tensor = getattr(owner, probe[1], {}).get(probe[2]) if owner is not None else None+ return default if tensor is None else tensor.device+++class _BlockState:+ device: Any # the card placement put the block on (used only when the block holds no tensor)+ next_block: (+ Any # weak reference to the next block of the same list, or None at the end of the list+ )+ split: "_Split"+ probe: Optional[tuple] = None++ def current(self) -> Any:+ return _probe_device(self.probe, self.device)+++class _Split:+ def __init__(self, transformer: Any, home: Any, lazy_class: bool) -> None:+ self.transformer = weakref.ref(transformer)+ self.home = home+ self.lazy_class = lazy_class+ self.lazy_now = False+ self.memo = _Memo()+ self.handles: list = []+ # weak: blocks hold their state, which holds this split; a strong list would make every split a cycle+ self.blocks: list[tuple[Any, type]] = []+ self.owns_home = False+ self.home_probe: Optional[tuple] = None+ self.calls = 0+ self.crossings = 0+ self.lazy_calls = 0+ self.eager_calls = 0++ def current_home(self) -> Any:+ """Where the denoiser's own (non-block) weights are now: the placement's primary until something moves them."""+ return _probe_device(self.home_probe, self.home)++ def move_in(+ self,+ obj: Any,+ device: Any,+ home: Any = None,+ ) -> Any:+ from .diffusion_multigpu import move_to_device++ home = self.home if home is None else home+ memo = self.memo++ def _tensor(t: Any) -> Any:+ self.crossings += 1+ if t.device == home:+ return memo.get(t, device)+ memo.copies += 1+ return cross_copy(t, device)++ return move_to_device(obj, device, tensor_fn = _tensor, objects = False)+++def _split_call(self: Any, *args: Any, **kwargs: Any) -> Any:+ state: Optional[_BlockState] = self.__dict__.get(_SPLIT_ATTR)+ base_call = type(self)._unsloth_layer_split_base.__call__+ if state is None:+ return base_call(self, *args, **kwargs)+ from .diffusion_multigpu import _device_context++ split = state.split+ device = state.current()+ home = split.current_home()+ args, kwargs = split.move_in((args, kwargs), device, home)+ with _device_context(device):+ out = base_call(self, *args, **kwargs)+ target = home if device != home else None+ if split.lazy_now:+ nxt = state.next_block() if state.next_block is not None else None+ nxt_state = nxt.__dict__.get(_SPLIT_ATTR) if nxt is not None else None+ if nxt_state is not None:+ nxt_device = nxt_state.current()+ target = nxt_device if nxt_device != device else None+ else:+ # eager: an in-place write between blocks is invisible (inference tensors carry no version), so no copy outlives its block+ split.memo.clear()+ if target is not None:+ out = cross_outputs(out, target)+ return out+++def _split_class(cls: type) -> type:+ base = getattr(cls, "_unsloth_layer_split_base", None)+ if base is not None:+ return cls+ with _SUBCLASS_LOCK:+ sub = _SUBCLASSES.get(cls)+ if sub is None:+ # Same name and module: class-name checks (``_repeated_blocks``, block graph / cache lookups) still match.+ sub = type(+ cls.__name__,+ (cls,),+ {+ "__call__": _split_call,+ "__module__": cls.__module__,+ "__qualname__": cls.__qualname__,+ "_unsloth_layer_split_base": cls,+ },+ )+ _SUBCLASSES[cls] = sub+ return sub+++_POSITIONAL_NAMES: "weakref.WeakKeyDictionary[type, tuple]" = weakref.WeakKeyDictionary()+++def _positional_names(module: Any) -> tuple:+ """Names of ``module.forward``'s positional parameters in order, cached per class."""+ cls = type(module)+ names = _POSITIONAL_NAMES.get(cls)+ if names is None:+ try:+ params = inspect.signature(cls.forward).parameters.values()+ names = tuple(+ p.name+ for p in params+ if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.name != "self"+ )+ except (TypeError, ValueError):+ names = ()+ _POSITIONAL_NAMES[cls] = names+ return names+++def _is_residual_name(name: Any) -> bool:+ lowered = str(name).lower()+ return any(part in lowered for part in _RESIDUAL_KWARG_PARTS)+++def _residual_call(+ kwargs: Mapping[str, Any],+ args: Sequence[Any] = (),+ module: Any = None,+) -> bool:+ """The call feeds a residual the denoiser adds between blocks (ControlNet / VACE hints), by keyword or by position."""+ for key, value in kwargs.items():+ if value is not None and _is_residual_name(key):+ return True+ if args and module is not None:+ names = _positional_names(module)+ for name, value in zip(names, args):+ if value is not None and _is_residual_name(name):+ return True+ if len(args) > len(names) and any(_is_residual_name(n) for n in names):+ return True # *args past the named slots: cannot tell, so stay exact+ return False+++def _move_rest(transformer: Any, block_ids: set, device: Any, logger: Any) -> None:+ """Every parameter and buffer of ``transformer`` outside the blocks to ``device``."""+ from .diffusion_fast_load import fast_upload++ owners = []+ stack = [transformer]+ while stack:+ module = stack.pop()+ if id(module) in block_ids:+ continue+ owners.append(module)+ stack.extend(module.children())+ with fast_upload(owners, device, logger = logger):+ for module in owners:+ module._apply(lambda t: t.to(device), recurse = False)+++def _move_weights(+ transformer: Any, plan: LayerSplitPlan, blocks: list, home: Any, logger: Any+) -> None:+ """Each block once to its card (blocks first, so the primary only ever holds its own share), then the rest."""+ torch = _torch()+ from .diffusion_fast_load import fast_upload++ by_device: dict[str, list] = {}+ for qual, _, block in blocks:+ by_device.setdefault(plan.assignment[qual], []).append(block)+ for dev_name, mods in by_device.items():+ dev = torch.device(dev_name)+ with fast_upload(mods, dev, logger = logger):+ for block in mods:+ block.to(dev)+ _move_rest(transformer, {id(b) for _, _, b in blocks}, home, logger)+++def _install_boundary(transformer: Any, plan: LayerSplitPlan, blocks: list, split: _Split) -> None:+ torch = _torch()+ for i, (qual, group, block) in enumerate(blocks):+ dev = torch.device(plan.assignment[qual])+ nxt = blocks[i + 1] if i + 1 < len(blocks) else None+ nxt_ref = weakref.ref(nxt[2]) if nxt is not None and nxt[1] == group else None+ original = type(block)+ block.__dict__[_SPLIT_ATTR] = _BlockState(dev, nxt_ref, split, _probe(block))+ block.__class__ = _split_class(original)+ split.blocks.append(+ (weakref.ref(block), getattr(original, "_unsloth_layer_split_base", original))+ )+ split.home_probe = _probe_outside(transformer, frozenset(id(b) for _, _, b in blocks))++ def _pre(module: Any, args: Any, kwargs: Any) -> None:+ split.memo.clear()+ split.calls += 1+ split.lazy_now = split.lazy_class and not _residual_call(kwargs, args, module)+ if split.lazy_now:+ split.lazy_calls += 1+ else:+ split.eager_calls += 1++ def _post(module: Any, args: Any, kwargs: Any, output: Any) -> None:+ split.memo.clear()+ split.lazy_now = False++ split.handles.append(transformer.register_forward_pre_hook(_pre, with_kwargs = True))+ split.handles.append(+ transformer.register_forward_hook(_post, with_kwargs = True, always_call = True)+ )+ transformer.__dict__[_SPLIT_ATTR + "_root"] = split+ split.owns_home = "_unsloth_split_home" not in transformer.__dict__+ transformer._unsloth_split_home = split.home+++def layer_split_of(module: Any) -> Optional[_Split]:+ if module is None:+ return None+ return getattr(module, "__dict__", {}).get(_SPLIT_ATTR + "_root")+++def is_layer_split(module: Any) -> bool:+ return layer_split_of(module) is not None+++def _drop_step_cache(pipe: Any, attr: str, transformer: Any, logger: Any) -> bool:+ """Take Studio's First-Block-Cache off ``transformer`` before a split (it keeps residuals on one card and its+ hooks own the blocks' forward). The load re-reads ``_unsloth_step_cache`` afterwards, so status reports it off.+ Raises :class:`LayerSplitError` when the hooks cannot be verified gone. True when a cache was removed."""+ if not step_cache_hooked(transformer):+ return False+ from . import diffusion_cache++ view = pipe if attr == "transformer" else type("_View", (), {"transformer": transformer})()+ diffusion_cache.maybe_toggle_step_cache(view, steps = 0, logger = logger)+ if step_cache_hooked(transformer) or getattr(transformer, "_unsloth_step_cache", None):+ raise LayerSplitError("the step cache on the denoiser could not be removed")+ if logger is not None:+ logger.info(+ "diffusion.layer_split: step cache removed from %s (it keeps residuals on one card)",+ attr,+ )+ return True+++def place_layer_split(+ pipe: Any,+ *,+ primary: Any,+ cards: Sequence[Any] = (),+ budgets_mib: Optional[Mapping[Any, int]] = None,+ strategy: str = "fill",+ attr: str = "transformer",+ assignment: Optional[Mapping[str, Any]] = None,+ logger: Any = None,+) -> dict[str, Any]:+ """Split ``pipe.<attr>`` across ``cards`` (see :func:`plan_layer_split`), or by an explicit block -> card+ ``assignment`` (see :func:`assigned_plan`), and place it. Other components are left where they are. Returns the placement summary also shown by ``placement_status``. Raises :class:`LayerSplitError`+ (nothing moved) when the split is refused or does not fit; on a failure while moving, the boundary is removed and+ the error re-raised, and the caller re-places the pipeline."""+ torch = _torch()+ from . import diffusion_multigpu as mg++ if layer_split_mode() == "off":+ raise LayerSplitError(f"{LAYER_SPLIT_ENV}=0 or {mg.MULTIGPU_ENV}=0")+ transformer = getattr(pipe, attr, None)+ why = layer_split_refusal(transformer, step_cache_ok = True)+ if why is not None:+ raise LayerSplitError(why)+ if is_layer_split(transformer):+ undo_layer_split(pipe, attr = attr)+ if assignment is not None:+ assigned_plan(+ transformer, primary = primary, assignment = assignment+ ) # validate before touching the cache+ else:+ plan_layer_split(+ transformer, primary = primary, cards = cards, budgets_mib = budgets_mib, strategy = strategy+ )+ _drop_step_cache(pipe, attr, transformer, logger)+ if assignment is not None:+ plan = assigned_plan(transformer, primary = primary, assignment = assignment)+ strategy = "assigned"+ else:+ plan = plan_layer_split(+ transformer, primary = primary, cards = cards, budgets_mib = budgets_mib, strategy = strategy+ )+ home = torch.device(plan.primary)+ blocks = find_blocks(transformer)+ split = _Split(transformer, home, type(transformer).__name__ in LAZY_CLASSES)+ try:+ _move_weights(transformer, plan, blocks, home, logger)+ _install_boundary(transformer, plan, blocks, split)+ mg._SPLIT_EVER_PLACED = True+ mg.install_execution_device_patch()+ except Exception:+ _remove_boundary(transformer, split)+ raise++ info = {+ "primary": plan.primary,+ "strategy": strategy,+ "denoiser": attr,+ "rest_mib": plan.rest_mib,+ "ranges": [r.describe() for r in plan.ranges],+ "per_card_mib": plan.per_card_mib(),+ "lazy": split.lazy_class,+ }+ split.info = info # type: ignore[attr-defined]+ try:+ record = mg._PLACEMENTS.get(pipe)+ if record is None:+ record = {+ "placement": {+ str(n): str(_module_device(m))+ for n, m in (getattr(pipe, "components", {}) or {}).items()+ if isinstance(m, torch.nn.Module) and m is not transformer+ },+ "primary": plan.primary,+ }+ mg._PLACEMENTS[pipe] = record+ record.setdefault("placement", {})[attr] = "+".join(r.device for r in plan.ranges)+ record["layer_split"] = info+ except TypeError: # not weak-referenceable+ pass+ if logger is not None:+ logger.info(+ "diffusion.layer_split: %s split %s",+ type(transformer).__name__,+ ", ".join(f"{r.device}: {len(r.blocks)} blocks {r.mib} MiB" for r in plan.ranges),+ )+ return info+++def _remove_boundary(transformer: Any, split: _Split) -> None:+ for handle in split.handles:+ try:+ handle.remove()+ except Exception: # noqa: BLE001+ pass+ split.handles.clear()+ for ref, base in split.blocks:+ block = ref()+ if block is None:+ continue+ block.__dict__.pop(_SPLIT_ATTR, None)+ if getattr(type(block), "_unsloth_layer_split_base", None) is base:+ block.__class__ = base+ split.blocks.clear()+ split.memo.clear()+ if transformer is not None:+ transformer.__dict__.pop(_SPLIT_ATTR + "_root", None)+ if split.owns_home:+ transformer.__dict__.pop("_unsloth_split_home", None)+++def undo_layer_split(+ pipe: Any,+ *,+ attr: str = "transformer",+ device: Any = None,+) -> bool:+ """Remove the boundary (weights stay where they are unless ``device`` is given, then the denoiser moves there).+ True when a split was removed."""+ from . import diffusion_multigpu as mg++ transformer = getattr(pipe, attr, None)+ split = layer_split_of(transformer)+ if split is None:+ return False+ _remove_boundary(transformer, split)+ try:+ record = mg._PLACEMENTS.get(pipe)+ except TypeError:+ record = None+ if record is not None:+ record.pop("layer_split", None)+ if not record.get("moved"): # no component split under it: the record was ours+ mg._PLACEMENTS.pop(pipe, None)+ else:+ record.get("placement", {}).pop(attr, None)+ if device is not None:+ transformer.to(device)+ return True+++def layer_split_stats(module: Any) -> Optional[dict]:+ split = layer_split_of(module)+ if split is None:+ return None+ out = dict(getattr(split, "info", {}) or {})+ out.update(+ {+ "calls": split.calls,+ "lazy_calls": split.lazy_calls,+ "eager_calls": split.eager_calls,+ "input_handoffs": split.crossings,+ "input_copies": split.memo.copies,+ }+ )+ return outdiff --git a/studio/backend/core/inference/diffusion_layer_split_plan.py b/studio/backend/core/inference/diffusion_layer_split_plan.pynew file mode 100644index 00000000000..3a8656a119a--- /dev/null+++ b/studio/backend/core/inference/diffusion_layer_split_plan.py@@ -0,0 +1,854 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Layer split planner: a denoiser too big for one card, cut into contiguous block ranges across the selected cards.++Order of preference on a multi-card selection: companion split (diffusion_multigpu), then a layer split, then the+one-card offload plan. A layer split is only considered when the one-card plan offloads the DENOISER itself and the+companion assignment found nothing; it is priced with the same per-card reserves as the companion split. The VAE and+every non-text-encoder companion stay on the primary (the decode and its tile verdict stay where today's guard reads+them); the text encoders may move to any used card. One card, Keep multiple models loaded, ROCm and the kill switches+keep today's plan unchanged.++The engine (block placement + device-crossing hooks) is a separate piece; this module only defines the interface it+must provide (``split_blocks``, ``engine_available``, ``place_layer_split``, ``undo_layer_split``) with stubs that keep+production behaviour unchanged until it lands.++``UNSLOTH_DIFFUSION_LAYER_SPLIT=0`` turns only this off; ``force`` also cuts a denoiser one card would hold resident+(for proving the engine on cards that are too big to need it).+"""++from __future__ import annotations++import contextlib+import dataclasses+import sys+import weakref+from typing import Any, Callable, Mapping, Optional, Sequence++LAYER_SPLIT_ENV = "UNSLOTH_DIFFUSION_LAYER_SPLIT"+_TE_ROLE = "text_encoder"+_DIT_ROLE = "dit"+_DENOISER_OFFLOAD_POLICIES = ("model", "sequential", "streaming")+# A block-only secondary holds one block's working set (B200 1024px: 0.6 GiB klein-4B, 1.2 GiB FLUX.2-dev):+# priced as twice the largest block, at least 1 GiB, never above the plan's denoise headroom.+_WORKING_SET_BLOCKS = 2+_WORKING_SET_MIN_MIB = 1024+_TE_ONLY_HEADROOM_MIB = (+ 1024 # diffusion_multigpu.TEXT_ENCODER_HEADROOM_MIB: an encoder-only phase on a card+)+++def secondary_reserve_mib() -> int:+ """ComfyUI's per-card floor (``minimum_inference_memory``: 0.8 GiB + EXTRA_RESERVED_VRAM, 400 MiB on Linux, 600 MiB+ on Windows plus 100 MiB on a 16 GB+ card, taken here unconditionally) for a card that only runs blocks. The primary+ keeps Studio's base overhead when that is larger."""+ return 800 + (700 if sys.platform == "win32" else 400)+++def block_working_set_mib(+ denoisers: Mapping[str, "DenoiserLayout"], denoise_headroom_mib: int+) -> int:+ largest = max((max(d.block_mib) for d in denoisers.values() if d.block_mib), default = 0)+ estimate = max(_WORKING_SET_MIN_MIB, _WORKING_SET_BLOCKS * int(largest))+ return min(estimate, int(denoise_headroom_mib)) if denoise_headroom_mib > 0 else estimate+++def layer_split_mode() -> str:+ """``off`` / ``auto`` (default) / ``force``; off whenever the whole multi-GPU lever is off."""+ from .diffusion_layer_split import layer_split_mode as engine_mode+ return engine_mode()++[email protected](frozen = True)+class BlockRange:+ denoiser: str+ start: int+ stop: int+ card: int+ mib: int++[email protected](frozen = True)+class LayerSplit:+ primary: int+ cards: tuple[int, ...]+ ranges: tuple[BlockRange, ...]+ text_encoders_card: Optional[int]+ card_required_mib: tuple[tuple[int, int], ...]+ card_limit_mib: tuple[tuple[int, int], ...]+ card_weights_mib: tuple[tuple[int, int], ...] = ()+ reason: str = ""+ text_encoders_streamed: bool = False+ card_reserve_mib: tuple[tuple[int, int], ...] = ()++ def working_mib(self, ordinal: int) -> tuple[int, int]:+ """(activation working set, fixed reserve) the plan held on card ``ordinal`` beside its weights, at the default+ request. A request larger than that scales the first only."""+ required = dict(self.card_required_mib).get(ordinal, 0)+ weights = dict(self.card_weights_mib).get(ordinal, 0)+ reserve = dict(self.card_reserve_mib).get(ordinal, 0)+ return max(0, int(required) - int(weights) - int(reserve)), int(reserve)++ @property+ def boundaries(self) -> int:+ """Device crossings per pass of every denoiser, the return of the output to the primary included."""+ out = 0+ for name in dict.fromkeys(r.denoiser for r in self.ranges):+ path = [+ self.primary,+ *(r.card for r in self.ranges if r.denoiser == name),+ self.primary,+ ]+ out += sum(1 for a, b in zip(path, path[1:]) if a != b)+ return out++ def card_of(self, denoiser: str, index: int) -> Optional[int]:+ for r in self.ranges:+ if r.denoiser == denoiser and r.start <= index < r.stop:+ return r.card+ return None++ def as_public_dict(self) -> dict[str, Any]:+ return {+ "primary": self.primary,+ "cards": list(self.cards),+ "ranges": [dataclasses.asdict(r) for r in self.ranges],+ "text_encoders_card": self.text_encoders_card,+ "text_encoders_streamed": self.text_encoders_streamed,+ "card_required_mib": {str(o): m for o, m in self.card_required_mib},+ "card_limit_mib": {str(o): m for o, m in self.card_limit_mib},+ "card_weights_mib": {str(o): m for o, m in self.card_weights_mib},+ "card_reserve_mib": {str(o): m for o, m in self.card_reserve_mib},+ "boundaries": self.boundaries,+ "reason": self.reason,+ }++[email protected](frozen = True)+class DenoiserLayout:+ """Storage MiB of one denoiser: ``stem`` (everything outside the sequential blocks) and each block in order."""++ stem_mib: int+ block_mib: tuple[int, ...]+++def _partition(blocks: Sequence[int], capacities: Sequence[int]) -> Optional[list[int]]:+ """Cut points (len(capacities) + 1, first 0, last len(blocks)) of a contiguous partition with part i <= capacity i,+ proportional to the capacities when that fits, else greedy first-fit. None when nothing fits."""+ n = len(blocks)+ total = sum(blocks)+ cap_sum = sum(max(0, c) for c in capacities)+ if n == 0:+ return [0] * (len(capacities) + 1)+ if cap_sum < total:+ return None++ def check(cuts: list[int]) -> bool:+ return all(+ sum(blocks[cuts[i] : cuts[i + 1]]) <= capacities[i] for i in range(len(capacities))+ )++ cuts = [0]+ acc = 0+ prefix = [0]+ for b in blocks:+ prefix.append(prefix[-1] + b)+ for i, cap in enumerate(capacities[:-1]):+ acc += max(0, cap)+ target = total * acc / cap_sum+ best = min(range(cuts[-1], n + 1), key = lambda j: (abs(prefix[j] - target), j))+ cuts.append(best)+ cuts.append(n)+ if check(cuts):+ return cuts+ cuts = [0]+ for cap in capacities:+ j = cuts[-1]+ used = 0+ while j < n and used + blocks[j] <= cap:+ used += blocks[j]+ j += 1+ cuts.append(j)+ if cuts[-1] != n:+ return None+ return cuts+++_SEARCH_BUDGET = 200_000+++def _search_cuts(+ layouts: Sequence[Sequence[int]], capacities: Sequence[int]+) -> Optional[list[list[int]]]:+ """Cut points for several denoisers sharing the cards, by backtracking: each denoiser contiguous over the cards in+ order, at least one keeping a block off the primary (a denoiser wholly on the primary is placed there unsplit),+ each card's parts within its capacity. The fallback when cutting each denoiser in turn by proportion leaves a later one no fit.+ None when nothing fits within the search budget."""+ k = len(capacities)+ prefixes = []+ for blocks in layouts:+ prefix = [0]+ for b in blocks:+ prefix.append(prefix[-1] + int(b))+ prefixes.append(prefix)+ visited = [0]++ def options(d: int, remaining: list[int]):+ prefix = prefixes[d]+ n = len(prefix) - 1++ def rec(i: int, start: int, cuts: list[int]):+ if i == k - 1:+ if prefix[n] - prefix[start] <= remaining[i]:+ yield [*cuts, n]+ return+ for j in range(start, n + 1):+ if prefix[j] - prefix[start] > remaining[i]:+ break+ yield from rec(i + 1, j, [*cuts, j])++ yield from rec(0, 0, [0])++ def dfs(d: int, remaining: list[int], split: bool) -> Optional[list[list[int]]]:+ if d == len(layouts):+ return [] if split else None+ prefix = prefixes[d]+ n = len(prefix) - 1+ for cuts in options(d, remaining):+ visited[0] += 1+ if visited[0] > _SEARCH_BUDGET:+ return None+ left = [remaining[i] - (prefix[cuts[i + 1]] - prefix[cuts[i]]) for i in range(k)]+ rest = dfs(d + 1, left, split or cuts[1] < n)+ if rest is not None:+ return [cuts, *rest]+ return None++ return dfs(0, [int(c) for c in capacities], False)+++def plan_layer_split(+ *,+ denoisers: Mapping[str, DenoiserLayout],+ companions: Mapping[str, tuple[int, str]],+ primary: int,+ card_limits: Mapping[int, int],+ headroom_mib: int,+ overhead_mib: int,+ phase_headroom_mib: Optional[tuple[int, int, int]] = None,+ secondary_overhead_mib: Optional[int] = None,+ working_set_mib: Optional[int] = None,+) -> tuple[Optional[LayerSplit], str]:+ """Pure decision: ``(LayerSplit, reason)`` or ``(None, decline reason)``.++ ``card_limits`` maps every usable card (the primary included) to its resident limit. Fewest cards first: the+ primary, then the secondaries by limit (largest first). ``phase_headroom_mib`` = ``(encode, denoise, decode)``.++ The primary holds the stems, the VAE, the latents and the activation peak: it is charged the plan's full headroom+ and ``max(overhead_mib, secondary reserve)``. A secondary runs blocks only: one block's working set+ (``working_set_mib``, default :func:`block_working_set_mib`) plus ``secondary_overhead_mib`` (default+ :func:`secondary_reserve_mib`), and the encode headroom when the text encoders live there."""+ if primary not in card_limits:+ return None, "primary card limit unknown"+ if not denoisers or any(not d.block_mib for d in denoisers.values()):+ return None, "denoiser blocks unknown"+ secondaries = sorted(+ (o for o in card_limits if o != primary), key = lambda o: (-card_limits[o], o)+ )+ if not secondaries:+ return None, "no secondary card"+ stems = sum(d.stem_mib for d in denoisers.values())+ te_mib = sum(m for m, role in companions.values() if role == _TE_ROLE)+ other_mib = sum(m for m, role in companions.values() if role not in (_TE_ROLE, _DIT_ROLE))+ if phase_headroom_mib is not None:+ encode, denoise, decode = (int(x or 0) for x in phase_headroom_mib)+ else:+ denoise = decode = int(headroom_mib)+ encode = _TE_ONLY_HEADROOM_MIB+ sec_reserve = (+ secondary_reserve_mib() if secondary_overhead_mib is None else int(secondary_overhead_mib)+ )+ pri_reserve = max(int(overhead_mib), sec_reserve)+ working = (+ block_working_set_mib(denoisers, denoise)+ if working_set_mib is None+ else int(working_set_mib)+ )+ reserve = {primary: pri_reserve, **{o: sec_reserve for o in secondaries}}+ blocks_total = sum(sum(d.block_mib) for d in denoisers.values())+ need_total = blocks_total + stems + te_mib + other_mib+ best_shortfall: Optional[int] = None+ for k in range(2, len(secondaries) + 2):+ cards = [primary, *secondaries[: k - 1]]+ # TE home: last card first, primary last; streaming the TE from host (one paged encode per prompt) is the last resort+ te_homes = (+ [(o, False) for o in reversed(cards)] + [(None, True)] if te_mib else [(None, False)]+ )+ for te_home, te_streamed in te_homes:+ fixed = {o: 0 for o in cards}+ head = {o: working for o in cards}+ fixed[primary] += stems + other_mib+ head[primary] = max(denoise, decode)+ if te_home is not None:+ fixed[te_home] += te_mib+ head[te_home] = max(head[te_home], encode)+ elif te_streamed:+ head[primary] = max(head[primary], encode)+ caps = [card_limits[o] - fixed[o] - head[o] - reserve[o] for o in cards]+ if any(c < 0 for c in caps):+ continue+ ranges: list[BlockRange] = []+ remaining = list(caps)+ ok = True+ names = list(denoisers)+ for idx, name in enumerate(names):+ layout = denoisers[name]+ left = sum(sum(denoisers[n].block_mib) for n in names[idx:])+ share = sum(layout.block_mib) / left if left else 1.0+ caps_d = [int(r * share) if idx < len(names) - 1 else r for r in remaining]+ cuts = _partition(layout.block_mib, caps_d) or _partition(+ layout.block_mib, remaining+ )+ n = len(layout.block_mib)+ if cuts is not None and n > 1 and cuts[1] == n:+ # the cut rounded every block onto the primary, which the engine refuses: move its last block to the furthest card with room+ room = [+ c+ for c in range(len(cards) - 1, 0, -1)+ if layout.block_mib[-1] <= remaining[c]+ ]+ cuts = (+ [0, *([n - 1] * room[0]), *([n] * (len(cards) - room[0]))] if room else None+ )+ if cuts is None:+ ok = False+ break+ for i, card in enumerate(cards):+ lo, hi = cuts[i], cuts[i + 1]+ if hi > lo:+ mib = sum(layout.block_mib[lo:hi])+ ranges.append(BlockRange(name, lo, hi, card, mib))+ remaining[i] -= mib+ if not ok and len(names) > 1:+ # cut in turn, an earlier denoiser can take the room a later one needed: search the cuts jointly+ joint = _search_cuts([denoisers[n].block_mib for n in names], caps)+ if joint is not None:+ ok, ranges, remaining = True, [], list(caps)+ for name, cuts in zip(names, joint):+ blocks = denoisers[name].block_mib+ for i, card in enumerate(cards):+ lo, hi = cuts[i], cuts[i + 1]+ if hi > lo:+ mib = sum(blocks[lo:hi])+ ranges.append(BlockRange(name, lo, hi, card, mib))+ remaining[i] -= mib+ if not ok:+ continue+ used = {r.card for r in ranges} | {primary}+ if te_home is not None:+ used.add(te_home)+ if len(used) < 2:+ continue # everything landed on the primary: no split needed (and the caller only asks when it is)+ required = {+ o: card_limits[o] - caps[i] + (caps[i] - remaining[i])+ for i, o in enumerate(cards)+ if o in used+ }+ split = LayerSplit(+ primary = primary,+ cards = tuple(o for o in cards if o in used),+ ranges = tuple(ranges),+ text_encoders_card = te_home,+ text_encoders_streamed = te_streamed,+ card_required_mib = tuple(sorted(required.items())),+ card_limit_mib = tuple(sorted((o, int(card_limits[o])) for o in used)),+ card_weights_mib = tuple(+ sorted(+ (o, fixed[o] + caps[i] - remaining[i])+ for i, o in enumerate(cards)+ if o in used+ )+ ),+ card_reserve_mib = tuple(sorted((o, int(reserve[o])) for o in used)),+ )+ per_card = ", ".join(+ f"cuda:{o} {required[o]}/{card_limits[o]} MiB" for o in split.cards+ )+ reason = f"multi-GPU layer split: {', '.join(names)} cut over {len(split.cards)} cards ({per_card})"+ return dataclasses.replace(split, reason = reason), reason+ available = (+ sum(card_limits[o] - reserve[o] for o in cards)+ - max(denoise, decode)+ - (k - 1) * working+ )+ shortfall = need_total - available+ best_shortfall = shortfall if best_shortfall is None else min(best_shortfall, shortfall)+ return None, (+ f"layer split does not fit the selected cards ({need_total} MiB of weights, "+ f"short by about {max(0, best_shortfall or 0)} MiB after the per-card reserves)"+ )+++_BLOCK_LIST_ATTRS: tuple[str, ...] = (+ "transformer_blocks",+ "single_transformer_blocks",+ "blocks",+ "layers",+)+++def split_blocks(module: Any) -> Optional[list[tuple[str, Any]]]:+ """The denoiser's repeated blocks in forward order as ``(qualified name, module)`` (the engine's own list, so a+ plan's indices name the blocks it places); None when the engine cannot cut this denoiser."""+ from .diffusion_layer_split import find_blocks, layer_split_refusal+ try:+ if layer_split_refusal(module, step_cache_ok = True) is not None:+ return None+ return [(qual, block) for qual, _, block in find_blocks(module)] or None+ except Exception: # noqa: BLE001 - no torch, or a module the engine cannot walk+ return None+++def engine_available() -> bool:+ """Whether the layer split engine can run here (CUDA torch)."""+ try:+ import torch+ return bool(torch.cuda.is_available())+ except Exception: # noqa: BLE001+ return False+++def _denoisers(split: LayerSplit) -> list[str]:+ return list(dict.fromkeys(r.denoiser for r in split.ranges))+++def place_layer_split(+ pipe: Any,+ *,+ primary: str,+ split: LayerSplit,+ logger: Any = None,+) -> dict:+ """Place every block range on its card, the stems and non-TE companions on ``primary``, the text encoders on+ ``split.text_encoders_card``, and install the device-crossing hooks. Raises on failure."""+ import torch++ from . import diffusion_multigpu as mg+ from .diffusion_fast_load import fast_upload+ from .diffusion_layer_split import LayerSplitError+ from .diffusion_layer_split import place_layer_split as engine_place++ primary_dev = torch.device(primary)+ if primary_dev.type == "cuda" and primary_dev.index is None:+ primary_dev = torch.device("cuda", torch.cuda.current_device())+ names = _denoisers(split)+ te_card = split.text_encoders_card+ te_dev = torch.device("cuda", int(te_card)) if te_card is not None else primary_dev+ components = {+ str(n): m+ for n, m in (getattr(pipe, "components", {}) or {}).items()+ if isinstance(m, torch.nn.Module) and str(n) not in names+ }+ streamed = (+ [n for n in components if n.startswith("text_encoder")]+ if split.text_encoders_streamed+ else []+ )+ moved = [+ n+ for n in components+ if n.startswith("text_encoder") and te_dev != primary_dev and n not in streamed+ ]+ placement: dict[str, str] = {}+ for name, module in components.items():+ if name in streamed:+ continue+ device = te_dev if name in moved else primary_dev+ with fast_upload([module], device, logger = logger):+ module.to(device)+ placement[name] = str(device)+ if streamed:+ from .diffusion_memory import stream_text_encoders_onto+ for name in stream_text_encoders_onto(pipe, primary_dev, logger):+ placement[name] = f"cpu (streamed to {primary_dev})"+ for name in moved:+ mg.install_boundary(components[name], primary_dev)+ if moved or streamed:+ for name, module in components.items():+ if name not in moved and name not in streamed:+ module._unsloth_split_home = primary_dev+ with contextlib.suppress(TypeError):+ mg._PLACEMENTS[pipe] = {+ "placement": placement,+ "primary": str(primary_dev),+ "secondary": str(te_dev) if moved else None,+ "moved": moved,+ "streamed": streamed,+ }+ out: dict[str, Any] = {}+ for name in names:+ module = getattr(pipe, name, None)+ blocks = split_blocks(module)+ if not blocks:+ raise LayerSplitError(f"{name}: no blocks to split")+ assignment: dict[str, int] = {}+ for r in split.ranges:+ if r.denoiser == name:+ for qual, _ in blocks[r.start : r.stop]:+ assignment[qual] = int(r.card)+ if assignment and all(card == primary_dev.index for card in assignment.values()):+ with fast_upload([module], primary_dev, logger = logger):+ module.to(primary_dev)+ out[name] = {"primary": str(primary_dev), "denoiser": name, "whole_on_primary": True}+ continue+ out[name] = engine_place(+ pipe, primary = primary_dev, attr = name, assignment = assignment, logger = logger+ )+ return out+++def undo_layer_split(pipe: Any) -> None:+ """Remove the crossing hooks a (partial) ``place_layer_split`` installed. Never raises."""+ from . import diffusion_multigpu as mg+ from .diffusion_layer_split import undo_layer_split as engine_undo++ try:+ record = dict(+ mg._PLACEMENTS.get(pipe) or {}+ ) # read first: the engine's undo drops a record it owns+ except TypeError:+ record = {}+ for name in mg.DENOISER_ATTRS:+ with contextlib.suppress(Exception):+ engine_undo(pipe, attr = name)+ with contextlib.suppress(Exception):+ if record.get("streamed"):+ from .diffusion_memory import _remove_group_offload_hooks+ for name in record["streamed"]:+ module = getattr(pipe, name, None)+ if module is not None:+ _remove_group_offload_hooks(module)+ with contextlib.suppress(Exception):+ mg.undo_split(pipe)+++_APPLIED: "weakref.WeakKeyDictionary[Any, LayerSplit]" = weakref.WeakKeyDictionary()+_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()+++def _record_decline(pipe: Any, reason: str, logger: Any) -> None:+ with contextlib.suppress(TypeError):+ _DECLINES[pipe] = reason+ if logger is not None:+ logger.info("diffusion.multigpu: layer split not used (%s)", reason)+++def _is_rocm() -> bool:+ try:+ import torch+ return bool(getattr(torch.version, "hip", None))+ except Exception: # noqa: BLE001+ return False+++def plan_offloads_denoiser(plan: Any) -> bool:+ """The one-card plan pages the denoiser's weights per call (what a layer split exists to remove)."""+ policy = getattr(plan, "offload_policy", "none")+ if policy in _DENOISER_OFFLOAD_POLICIES:+ return True+ return policy == "group" and bool(getattr(plan, "stream_transformer", True))+++def _denoiser_layouts(pipe: Any) -> Optional[dict[str, DenoiserLayout]]:+ from .diffusion_memory import _storage_nbytes+ from .diffusion_multigpu import DENOISER_ATTRS++ mib = 1 << 20+ out: dict[str, DenoiserLayout] = {}+ for name in DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ if module is None:+ continue+ blocks = split_blocks(module)+ if not blocks:+ return None+ seen: set[int] = set()++ def size(m: Any) -> int:+ nbytes = 0+ for t in (*m.parameters(), *m.buffers()):+ if id(t) in seen:+ continue+ seen.add(id(t))+ nbytes += sum(_storage_nbytes(t))+ return nbytes++ block_bytes = [size(b) for _, b in blocks]+ stem_bytes = size(module) # what is left after the blocks claimed theirs+ out[name] = DenoiserLayout(+ stem_mib = -(-stem_bytes // mib), block_mib = tuple(-(-b // mib) for b in block_bytes)+ )+ return out or None+++def _layout_refusal(pipe: Any) -> str:+ """The engine's reason the first unsplittable denoiser of ``pipe`` cannot be cut (for the decline status)."""+ from .diffusion_layer_split import layer_split_refusal+ from .diffusion_multigpu import DENOISER_ATTRS++ for name in DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ if module is None:+ continue+ try:+ why = layer_split_refusal(module, step_cache_ok = True)+ except Exception as exc: # noqa: BLE001+ why = f"{type(exc).__name__}: {exc}"+ if why is not None:+ return f"{name}: {why}"+ return "no sequential block list the engine can cut"+++# Backends gated on the primary's arch only (flash3, flash4, Sage, FA2): split only onto identical cards.+_PRIMARY_ARCH_BACKENDS = frozenset({"_flash_3_hub", "flash_4_hub", "sage", "sage_hub", "flash"})+++def _backend_cards(+ pipe: Any, primary: int, secondaries: Sequence[int]+) -> tuple[tuple[int, ...], Optional[str]]:+ """``(secondaries the engaged attention backend runs on, that backend)``; a card of unknown capability is kept."""+ from . import diffusion_multigpu as mg++ backend = None+ with contextlib.suppress(Exception):+ from .diffusion_attention import ATTENTION_BACKEND_ATTR+ for name in mg.DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ tag = getattr(module, ATTENTION_BACKEND_ATTR, None) if module is not None else None+ if tag in _PRIMARY_ARCH_BACKENDS:+ backend = tag+ break+ cards = tuple(int(o) for o in secondaries)+ if backend is None:+ return cards, None+ need = mg._capability(primary)+ if need is None:+ return cards, backend+ return tuple(o for o in cards if (mg._capability(o) or need) == need), backend+++def refine_plan_for_layer_split(+ pipe: Any,+ plan: Any,+ *,+ selection: Any,+ sizes: Mapping[str, tuple[int, str]],+ primary_limit_mib: int,+ secondary_limit_mib: Callable[[int], Optional[int]],+ headroom_mib: int,+ overhead_mib: int,+ logger: Any = None,+) -> Any:+ """Called by ``refine_plan_for_multigpu`` once the companion assignment found nothing (or under ``force``). The+ plan unchanged unless a layer split fits; then a resident plan carrying ``layer_split`` and the one-card plan as+ ``fallback_plan``. The caller has already checked the selection, mode, explicit offload and CUDA gates."""+ mode = layer_split_mode()+ if mode == "off" or not engine_available():+ return plan+ if not plan_offloads_denoiser(plan) and mode != "force":+ return plan+ if _is_rocm():+ _record_decline(pipe, "ROCm (layer split is CUDA only)", logger)+ return plan+ layouts = _denoiser_layouts(pipe)+ if layouts is None:+ _record_decline(pipe, f"denoiser cannot be split ({_layout_refusal(pipe)})", logger)+ return plan+ limits: dict[int, int] = {int(selection.primary): int(primary_limit_mib)}+ secondaries, backend = _backend_cards(pipe, int(selection.primary), selection.secondaries)+ if not secondaries:+ _record_decline(+ pipe,+ f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",+ logger,+ )+ return plan+ for o in secondaries:+ limit = secondary_limit_mib(int(o))+ if limit is not None:+ limits[int(o)] = int(limit)+ companions = {n: v for n, v in sizes.items() if v[1] != _DIT_ROLE}+ split, reason = plan_layer_split(+ denoisers = layouts,+ companions = companions,+ primary = int(selection.primary),+ card_limits = limits,+ headroom_mib = int(headroom_mib),+ overhead_mib = int(overhead_mib),+ )+ if split is None:+ _record_decline(pipe, reason, logger)+ return plan+ reason = f"{reason}; single card would use '{plan.offload_policy}'"+ if logger is not None:+ logger.info("diffusion.multigpu: %s", reason)+ with contextlib.suppress(TypeError):+ _DECLINES.pop(pipe, None)+ return dataclasses.replace(+ plan,+ offload_policy = "none",+ vae_tiling = plan.vae_tiling,+ vae_slicing = plan.vae_slicing,+ stream_text_encoders = False,+ stream_transformer = True,+ resident_transformer_mib = None,+ resident_text_encoder_mib = None,+ reasons = tuple(plan.reasons) + (reason,),+ layer_split = dataclasses.replace(split, reason = reason),+ fallback_plan = plan,+ )+++def apply_layer_split_plan(+ pipe: Any,+ plan: Any,+ *,+ placement: str,+ logger: Any = None,+) -> bool:+ """Place a layer split plan. True when placed; on any failure the split is undone, the pipe returns to the host and+ False is returned (the caller then applies ``fallback_plan``)."""+ split = getattr(plan, "layer_split", None)+ if split is None:+ return False+ try:+ place_layer_split(pipe, primary = placement, split = split, logger = logger)+ except Exception as exc: # noqa: BLE001 - fall back to one card+ if logger is not None:+ logger.warning(+ "diffusion.multigpu: layer split placement failed (%s); falling back to one card",+ exc,+ )+ with contextlib.suppress(Exception):+ undo_layer_split(pipe)+ forget_applied_layer_split(pipe)+ with contextlib.suppress(Exception):+ pipe.to("cpu")+ _drop_stranded_fused_records(pipe)+ with contextlib.suppress(Exception):+ import torch+ torch.cuda.empty_cache()+ _record_decline(pipe, f"placement failed ({exc})", None)+ return False+ with contextlib.suppress(TypeError):+ _APPLIED[pipe] = split+ for name in _denoisers(split):+ module = getattr(pipe, name, None)+ if module is not None:+ module.__dict__[_RECORD_ATTR] = split+ return True+++_RECORD_ATTR = "_unsloth_layer_split_record"+++def _drop_stranded_fused_records(pipe: Any) -> None:+ """Drop fused records a failed placement left on a card. Never raises."""+ from .diffusion_multigpu import DENOISER_ATTRS+ for name in DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ if module is None:+ continue+ with contextlib.suppress(Exception):+ from .diffusion_int8_fused import drop_stranded_fused_records+ drop_stranded_fused_records(module)+++def applied_layer_split(pipe: Any) -> Optional[LayerSplit]:+ """The split placed on ``pipe``, or on a denoiser it shares with the pipeline that placed it."""+ if pipe is None:+ return None+ try:+ split = _APPLIED.get(pipe)+ except TypeError:+ split = None+ if split is not None:+ return split+ from .diffusion_layer_split import is_layer_split as engine_split+ from .diffusion_multigpu import DENOISER_ATTRS++ for name in DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ record = getattr(module, "__dict__", {}).get(_RECORD_ATTR) if module is not None else None+ if record is not None and engine_split(module):+ return record+ return None+++def forget_applied_layer_split(pipe: Any) -> None:+ """Teardown: the placed split's record goes (the pipeline's and its denoisers'), a decline reason stays."""+ from .diffusion_multigpu import DENOISER_ATTRS++ with contextlib.suppress(TypeError):+ _APPLIED.pop(pipe, None)+ for name in DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ if module is not None:+ getattr(module, "__dict__", {}).pop(_RECORD_ATTR, None)+++def is_layer_split(pipe: Any) -> bool:+ return applied_layer_split(pipe) is not None+++def forget_layer_split(pipe: Any) -> None:+ """Unload / re-place: drop the record (the engine's ``undo_layer_split`` removes the hooks)."""+ forget_applied_layer_split(pipe)+ with contextlib.suppress(TypeError):+ _DECLINES.pop(pipe, None)+++def layer_split_status(pipe: Any) -> Optional[dict[str, Any]]:+ """``status()["multi_gpu"]["layer_split"]``: the applied split, ``{"declined": reason}`` after an attempt that did+ not engage, or None when none was considered."""+ split = applied_layer_split(pipe)+ if split is not None:+ return split.as_public_dict()+ try:+ declined = _DECLINES.get(pipe) if pipe is not None else None+ except TypeError:+ declined = None+ return {"declined": declined} if declined is not None else None+++def layer_split_card_floors(pipe: Any) -> Optional[tuple]:+ """``vram_split_cards`` for the video guard: ``(ordinal, resident MiB, decodes, home, denoises, block headroom+ MiB, reserve MiB)`` per card. The VAE and the stems stay on the primary, and every card runs blocks. The last two+ fields are what the plan held on a card that holds only blocks: one block's working set at the family's default+ request (it scales with the request) and the fixed per-card reserve (it does not); None on the primary, which the+ whole-request estimate prices."""+ split = applied_layer_split(pipe)+ if split is None:+ return None+ weights = dict(split.card_weights_mib)+ return tuple(+ (+ o,+ int(weights.get(o, 0)),+ o == split.primary,+ o == split.primary,+ True,+ None if o == split.primary else split.working_mib(o)[0],+ None if o == split.primary else split.working_mib(o)[1],+ )+ for o in split.cards+ )+++def secondary_block_cards(pipe: Any) -> tuple[int, ...]:+ """Cards other than the primary that run blocks (each needs its own generate-time activation check)."""+ split = applied_layer_split(pipe)+ if split is None:+ return ()+ return tuple(sorted({r.card for r in split.ranges if r.card != split.primary}))diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 54d092e751a..82b777cfad1 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -621,6 +621,7 @@ class MemoryPlan:# The single-card plan a split replaced, applied on the primary if the split placement fails (a torchao denoiser# cannot take the whole-module offload a blind fallback would choose).fallback_plan: Optional["MemoryPlan"] = field(default = None, compare = False, repr = False)+ layer_split: Optional[Any] = None@propertydef engages_offload(self) -> bool:@@ -643,6 +644,11 @@ class MemoryPlan:"secondary_components": (list(self.secondary_components) if self.secondary_components is not None else None),+ **(+ {"layer_split": self.layer_split.as_public_dict()}+ if self.layer_split is not None+ else {}+ ),}@@ -3513,6 +3519,12 @@ def resident_group_mib(pipe: Any) -> int:return total >> 20+def _any_layer_split(pipe: Any) -> bool:+ from .diffusion_layer_split import is_layer_split+ from .diffusion_multigpu import DENOISER_ATTRS+ return any(is_layer_split(getattr(pipe, name, None)) for name in DENOISER_ATTRS)++def hook_resident_denoiser(pipe: Any,device: Any,@@ -3530,6 +3542,7 @@ def hook_resident_denoiser(transformer is Noneor _offload_groups(transformer)or not _pipe_denoisers_hold_torchao(pipe)+ or _any_layer_split(pipe)):return Falseimport torch@@ -3976,11 +3989,21 @@ def apply_memory_plan()_fallback_to_model_offload()policy = OFFLOAD_MODEL- elif getattr(plan, "secondary_ordinal", None) is not None:- from .diffusion_multigpu import apply_split_plan- if not apply_split_plan(pipe, plan, placement = placement, logger = logger):+ elif (+ getattr(plan, "secondary_ordinal", None) is not None+ or getattr(plan, "layer_split", None) is not None+ ):+ if getattr(plan, "layer_split", None) is not None:+ from .diffusion_layer_split_plan import apply_layer_split_plan as _apply_split+ else:+ from .diffusion_multigpu import apply_split_plan as _apply_split+ if not _apply_split(pipe, plan, placement = placement, logger = logger):fallback = getattr(plan, "fallback_plan", None)- if fallback is not None and getattr(fallback, "secondary_ordinal", None) is None:+ if (+ fallback is not None+ and getattr(fallback, "secondary_ordinal", None) is None+ and getattr(fallback, "layer_split", None) is None+ ):# the policy one card would have used (group offload keeps a torchao denoiser loadable)result = apply_memory_plan(pipe, fallback, device = device, placement_device = placement_device, logger = logger@@ -4001,6 +4024,7 @@ def apply_memory_plan(secondary_ordinal = None,secondary_components = None,fallback_plan = None,+ layer_split = None,)except Exception: # noqa: BLE001pass@@ -5033,6 +5057,11 @@ def _apply_group_offload(return Falseif not stream_transformer and not stream_text_encoders:return False+ if _any_layer_split(pipe):+ from .diffusion_layer_split_plan import undo_layer_split+ undo_layer_split(pipe)+ if logger is not None:+ logger.info("diffusion.memory: removed the layer split before applying group offload")installed = 0# every component's torch forward hooks before this call: an OOM after hooks exist unwinds to this statehook_snapshots: list = []@@ -5257,6 +5286,62 @@ def _apply_group_offload(return False+def stream_text_encoders_onto(+ pipe: Any,+ device: Any,+ logger: Any = None,+) -> list[str]:+ """Leaf-level group offload of every ``text_encoder*`` onto ``device`` and nothing else: the denoisers and the other+ components are not touched (a layer split places them; ``_apply_group_offload`` would move every unstreamed+ component onto one card). A text encoder runs once, before step 0, so it pages in for that encode. Returns the+ streamed names; raises (with this call's hooks removed) when an encoder refuses, and the caller falls back."""+ import inspect++ import torch+ from diffusers.hooks import apply_group_offloading++ install_group_offload_buffer_restore()+ install_group_offload_hooks_eager()+ onload = torch.device(device)+ use_stream = onload.type == "cuda"+ kwargs: dict[str, Any] = {+ "onload_device": onload,+ "offload_device": torch.device("cpu"),+ "offload_type": "leaf_level",+ "use_stream": use_stream,+ }+ params = inspect.signature(apply_group_offloading).parameters+ if use_stream:+ if "non_blocking" in params:+ kwargs["non_blocking"] = True+ if "record_stream" in params:+ kwargs["record_stream"] = True+ encoders = {+ str(n): m+ for n, m in (getattr(pipe, "components", {}) or {}).items()+ if str(n).startswith("text_encoder") and isinstance(m, torch.nn.Module)+ }+ if "low_cpu_mem_usage" in params:+ pin = _streamed_pin_plan(0, sum(_module_host_mib(m) for m in encoders.values()), logger)+ kwargs["low_cpu_mem_usage"] = not pin[1]+ done: list[str] = []+ try:+ for name, module in encoders.items():+ module.to("cpu")+ done.append(name)+ apply_group_offloading(module, **kwargs)+ _pin_vision_embedding_device(module)+ except Exception:+ for name in done:+ _remove_group_offload_hooks(encoders[name])+ raise+ if logger is not None and done:+ logger.info(+ "diffusion.memory: streaming %s onto %s for the encode", ", ".join(done), onload+ )+ return done++# Generate-time activation guard. The load-time plan cannot know the output resolution: a model is loaded once and# then generates at whatever size the sliders say, so ``_plan_memory`` budgets the 1024x1024 default. That is the# right call for PLACEMENT, but it means a request for a much larger frame is never checked against anything, so thisdiff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex 1f5d12dcfa9..fdda8d74802 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -9,6 +9,9 @@ Three levers, all at a component or denoiser-call boundary, never inside a modelother companions sit on a secondary card, when that removes offload on the primary. Inputs move to a module's owncard at its entry points and outputs come back to the primary, so the pipeline still sees one device. Copies areexact, so the output is bit-identical to a single-card resident run.+* Layer split (``diffusion_layer_split``): a DiT that fits no single selected card is split by contiguous ranges of+ its repeated blocks, the rest of it on the primary. Activations cross at the range boundaries only; outputs are+ bit-identical to a one-card run.* Second denoiser (after load): a two-DiT pipeline (an MoE ``transformer_2``) keeps one expert per card when bothexperts do not fit one card.* CFG-parallel (generate): when guidance runs separate cond / uncond denoiser calls on the same latents, every odd@@ -17,6 +20,8 @@ Three levers, all at a component or denoiser-call boundary, never inside a modelstep stay on the primary. Branches are recognised from the call boundary alone (same latent tensor object inconsecutive calls), so true CFG, rescale and multi-branch guidance work unchanged and a guidance-distilled model(one call per step) never builds a replica.+* Layer split (after load, diffusion_layer_split_plan): when one card would offload the denoiser itself and no+ component assignment fits, the denoiser's blocks are cut across the cards. Excludes CFG-parallel.Kill switches: ``UNSLOTH_DIFFUSION_MULTIGPU=0`` (everything here; one card as before), ``UNSLOTH_DIFFUSION_CFG_PARALLEL=0``(only the replica). ``UNSLOTH_DIFFUSION_MULTIGPU=force`` splits the companions even when one card would hold@@ -425,13 +430,20 @@ def refine_plan_for_multigpu("""Post-load: an offloading plan becomes a resident split when the LOADED components can be shared by the cards.No-op without a selection, for explicit offload modes, a plan already resident (unless forced) or split, unified memory- or an unknown budget. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""+ or an unknown budget. When no component assignment fits, a layer split (diffusion_layer_split_plan) is tried before+ keeping the offload plan. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""selection = current_selection()if selection is None or plan is None:return plan- if getattr(plan, "secondary_ordinal", None) is not None:+ if (+ getattr(plan, "secondary_ordinal", None) is not None+ or getattr(plan, "layer_split", None) is not None+ ):return plan- if plan.offload_policy == "none" and multigpu_mode() != "force":+ from .diffusion_layer_split_plan import layer_split_mode, refine_plan_for_layer_split++ layer_force = layer_split_mode() == "force"+ if plan.offload_policy == "none" and multigpu_mode() != "force" and not layer_force:return planif getattr(plan, "requested_mode", None) not in ("auto", "fast"):return plan@@ -469,6 +481,22 @@ def refine_plan_for_multigpu(or 0)overhead = int(plan.estimates.get("base_overhead_mib") or 0)++ def layer_split_or_plan() -> Any:+ return refine_plan_for_layer_split(+ pipe,+ plan,+ selection = selection,+ sizes = sizes,+ primary_limit_mib = int(primary_limit),+ secondary_limit_mib = lambda o: secondary_resident_limit_mib(o, mode),+ headroom_mib = headroom,+ overhead_mib = overhead,+ logger = logger,+ )++ if layer_force:+ return layer_split_or_plan()moved = choose_component_assignment(sizes,headroom_mib = headroom,@@ -477,7 +505,7 @@ def refine_plan_for_multigpu(secondary_limit_mib = int(sec_limit),)if moved is None:- return plan+ return layer_split_or_plan()reason = (f"multi-GPU (loaded sizes): {', '.join(moved)} on cuda:{secondary}, the rest resident on "f"cuda:{selection.primary}; single card would use '{plan.offload_policy}'"@@ -538,10 +566,18 @@ def move_to_device(obj: Any,device: Any,_depth: int = 0,+ *,+ tensor_fn: Optional[Callable[[Any], Any]] = None,+ objects: bool = True,) -> Any:+ """Tensors in ``obj`` (containers, dataclasses, ModelOutputs, plain attribute holders) on ``device``.+ ``tensor_fn(t)`` replaces ``t.to(device)`` for a tensor that is elsewhere. ``objects=False`` leaves dataclasses+ and attribute holders as they are (a per-layer KV cache the callee writes into must keep its identity)."""torch = _torch()if isinstance(obj, torch.Tensor):- return obj if obj.device == device else obj.to(device)+ if obj.device == device:+ return obj+ return obj.to(device) if tensor_fn is None else tensor_fn(obj)if (_depth > 6or obj is None@@ -553,28 +589,39 @@ def move_to_device(try:out = copy.copy(obj)for key, value in obj.items():- moved = move_to_device(value, device, _depth + 1)+ moved = move_to_device(+ value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects+ )if moved is not value:out[key] = movedreturn outexcept Exception: # noqa: BLE001- return type(obj)((k, move_to_device(v, device, _depth + 1)) for k, v in obj.items())+ return type(obj)(+ (k, move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects))+ for k, v in obj.items()+ )if isinstance(obj, tuple):- moved = [move_to_device(v, device, _depth + 1) for v in obj]+ moved = [+ move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj+ ]if all(a is b for a, b in zip(moved, obj)):return objif hasattr(obj, "_fields"):return type(obj)(*moved)return type(obj)(moved) if type(obj) is not tuple else tuple(moved)if isinstance(obj, list):- return [move_to_device(v, device, _depth + 1) for v in obj]- if isinstance(obj, torch.nn.Module):+ return [+ move_to_device(v, device, _depth + 1, tensor_fn = tensor_fn, objects = objects) for v in obj+ ]+ if type(obj).__name__ == "BlockMask" and callable(getattr(obj, "to", None)):+ return obj.to(device)+ if isinstance(obj, torch.nn.Module) or not objects:return objif dataclasses.is_dataclass(obj) and not isinstance(obj, type):changes = {}for f in dataclasses.fields(obj):value = getattr(obj, f.name, None)- moved = move_to_device(value, device, _depth + 1)+ moved = move_to_device(value, device, _depth + 1, tensor_fn = tensor_fn, objects = objects)if moved is not value:changes[f.name] = movedif not changes:@@ -601,7 +648,7 @@ def move_to_device(out = copy.copy(obj)for k, v in state.items():if isinstance(v, torch.Tensor) and v.device != device:- setattr(out, k, v.to(device))+ setattr(out, k, v.to(device) if tensor_fn is None else tensor_fn(v))return outreturn obj@@ -687,6 +734,12 @@ def _split_home(pipe: Any) -> Any:return Noneif not isinstance(components, dict):return None+ from .diffusion_layer_split import layer_split_of++ for module in components.values():+ layered = layer_split_of(module)+ if layered is not None:+ return layered.current_home()for module in components.values():home = getattr(module, "_unsloth_split_home", None)if home is not None:@@ -803,7 +856,18 @@ def place_split(def undo_split(pipe: Any) -> None:+ """Remove every split placed on ``pipe``: a layer split of any denoiser (its block boundary and the planner's+ record) and the component boundaries. Weights stay where they are."""torch = _torch()+ from .diffusion_layer_split import undo_layer_split as undo_layers+ from .diffusion_layer_split_plan import forget_applied_layer_split++ for name in DENOISER_ATTRS:+ try:+ undo_layers(pipe, attr = name)+ except Exception: # noqa: BLE001 - best effort; the component boundaries below still come off+ pass+ forget_applied_layer_split(pipe)for module in (getattr(pipe, "components", {}) or {}).values():if isinstance(module, torch.nn.Module):remove_boundary(module)@@ -850,12 +914,17 @@ def apply_split_plan(def is_split(pipe: Any) -> bool:+ """A companion split or a layer split is placed on ``pipe``."""if pipe is None:return Falsetry:- return pipe in _PLACEMENTS+ if pipe in _PLACEMENTS:+ return Trueexcept TypeError:return False+ from .diffusion_layer_split_plan import is_layer_split++ return is_layer_split(pipe)def placement_status(pipe: Any) -> Optional[dict[str, Any]]:@@ -867,9 +936,28 @@ def placement_status(pipe: Any) -> Optional[dict[str, Any]]:declined = _CFG_DECLINES.get(pipe)except TypeError:return None- if info is None and cfg is None and declined is None:+ from .diffusion_layer_split_plan import layer_split_status++ layer = layer_split_status(pipe)+ if info is None and cfg is None and declined is None and layer is None:return Noneout: dict[str, Any] = dict(info or {})+ from .diffusion_layer_split import layer_split_stats++ live = {+ name: stats+ for name in DENOISER_ATTRS+ if (stats := layer_split_stats(getattr(pipe, name, None))) is not None+ }+ if layer is not None:+ out["layer_split"] = dict(layer)+ if live:+ out["layer_split"]["live"] = live+ elif (+ "layer_split" in out+ ): # placed by the engine directly (no planner record): its own summary, live+ engine = out["layer_split"]+ out["layer_split"] = dict(live.get(engine.get("denoiser", "transformer")) or engine)if cfg is not None:out["cfg_parallel"] = cfg.describe()elif declined is not None:@@ -1919,6 +2007,11 @@ def cfg_parallel_decline(mode = cfg_parallel_mode()if mode == "off":return f"{CFG_PARALLEL_ENV}=0"+ from .diffusion_layer_split_plan import is_layer_split++ # A replica is a whole denoiser on one card, which a denoiser cut across cards cannot be.+ if is_layer_split(pipe):+ return "denoiser is split across cards (layer split)"# A replica takes a whole denoiser's worth of another card; automatic placement only borrows one to avoid offload.if selection.automatic and mode != "force":return f"automatic GPU placement (select several cards or set {CFG_PARALLEL_ENV}=force)"@@ -1931,6 +2024,10 @@ def cfg_parallel_decline(torch = _torch()if not isinstance(denoiser, torch.nn.Module):return "no DiT denoiser (a U-Net batches its guidance branches)"+ from .diffusion_layer_split import is_layer_split++ if is_layer_split(denoiser):+ return "denoiser is split across cards (layer split)"# Whole-module / sequential offload page the denoiser per call; a group / streaming plan is fine only when it left# the denoiser itself resident (encoders streamed), which shows as a denoiser without offload hooks.if offload_policy in ("model", "sequential"):diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.pyindex 8b9d6917d04..b75e64047c6 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py@@ -91,6 +91,8 @@ def _qkv_intact(attn: Any) -> Any:lin, q, k, v = recif attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:return None+ if lin.weight.device != q.weight.device:+ return None # a layer split moved the parts after the fusion; the stale fused copy is on another cardreturn lin@@ -191,9 +193,9 @@ def install(return resultif offload_active or type(transformer).__name__ != "QwenImage21Transformer2DModel":return result- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call- if resident_cuda_device(transformer) is not None:+ if resident_cuda_cards(transformer):return install_modules(transformer, logger)run_on_first_call(transformer, "q21_convrot_fused", lambda t: install_modules(t, logger))return {"fused_qkv": None, "out": None}@@ -203,10 +205,12 @@ def install_modules(root: Any, logger: Any = None) -> dict:result = {"fused_qkv": 0, "out": 0}if q21_convrot_fused_disabled():return result- from .diffusion_int8_fused import resident_cuda_device+ import torch++ from .diffusion_int8_fused import resident_cuda_cards- dev = resident_cuda_device(root)- if dev is None or not arch_ok(dev):+ cards = resident_cuda_cards(root) # one card, or each card of a layer split+ if not cards or not all(arch_ok(torch.device("cuda", index)) for index in cards):return resultfrom .diffusion_zimage_fused import _fuse_linears, _share_storage@@ -217,6 +221,8 @@ def install_modules(root: Any, logger: Any = None) -> dict:parts = [getattr(module, n, None) for n in ("to_q", "to_k", "to_v")]if any(p is None for p in parts) or not _rotated_group(parts):continue+ if len({p.weight.device for p in parts}) != 1:+ continuetodo.append((module, parts))if not todo or not _patch_prepare(logger):return resultdiff --git a/studio/backend/core/inference/diffusion_qwenimage21_rope.py b/studio/backend/core/inference/diffusion_qwenimage21_rope.pyindex 982e7cdcdcf..4089b786062 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_rope.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_rope.py@@ -229,6 +229,23 @@ def install(logger: Any = None, device: Any = None) -> bool:return True+def probe_split_cards(cards: Any) -> tuple:+ """A layer split runs its blocks on several cards, and the real-arithmetic wrapper reads each card's own probe+ (an unprobed card keeps the complex form). Probes every card not yet probed; nothing when no install probed a+ card (the patch is not in use). Returns the cards whose probe passed."""+ if real_rope_disabled() or not _FUSION:+ return ()+ import torch++ for index in cards:+ if index not in _FUSION:+ try:+ _FUSION[index] = probe_fusion(torch.device("cuda", int(index)))+ except Exception: # noqa: BLE001 - that card keeps the complex form+ _FUSION[index] = None+ return tuple(index for index in cards if _FUSION.get(index) is not None)++def uninstall() -> None:with _LOCK:for mod, stock in list(_INSTALLED.items()):diff --git a/studio/backend/core/inference/diffusion_qwenimage_rope.py b/studio/backend/core/inference/diffusion_qwenimage_rope.pyindex 93473b4c7c0..4a97058cb33 100644--- a/studio/backend/core/inference/diffusion_qwenimage_rope.py+++ b/studio/backend/core/inference/diffusion_qwenimage_rope.py@@ -145,19 +145,14 @@ def install(transformer: Any, logger: Any = None) -> bool:return Falseif type(transformer).__name__ != "QwenImageTransformer2DModel":return False- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_calldef _finish(t: Any) -> bool:- dev = resident_cuda_device(t)- if dev is None:- return False- import torch-- return _patch_table(- dev.index if dev.index is not None else torch.cuda.current_device(), logger- )+ cards = resident_cuda_cards(t)+ passed = [index for index in cards if _probe(index)]+ return bool(passed) and _patch_table(passed[0], logger)- if resident_cuda_device(transformer) is not None:+ if resident_cuda_cards(transformer):return _finish(transformer)run_on_first_call(transformer, "qwenimage_rope", _finish)return Truediff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.pyindex 2b84b0b04e5..ea283ee7194 100644--- a/studio/backend/core/inference/diffusion_zimage_fused.py+++ b/studio/backend/core/inference/diffusion_zimage_fused.py@@ -73,6 +73,8 @@ def _qkv_intact(attn: Any) -> Any:lin, q, k, v = recif attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:return None+ if lin.weight.device != q.weight.device:+ return None # a layer split moved the parts after the fusion; the stale fused copy is on another cardreturn lin@@ -274,9 +276,9 @@ def install(return {"real_rope": False, "fused_qkv": 0}if not _patch_class(logger):return {"real_rope": False, "fused_qkv": 0}- from .diffusion_int8_fused import resident_cuda_device, run_on_first_call+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call- if resident_cuda_device(transformer) is not None:+ if resident_cuda_cards(transformer):return install_modules(transformer, logger, fuse_qkv = not offload_active)run_on_first_call(transformer,@@ -320,15 +322,18 @@ def install_modules(import torch- from .diffusion_int8_fused import resident_cuda_device+ from .diffusion_int8_fused import resident_cuda_cardscls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor- dev = resident_cuda_device(root)- if dev is None:+ cards = resident_cuda_cards(+ root+ ) # one card, or each card of a layer split (the processor reads its query's card)+ if not cards:return resultwith _LOCK:- index = dev.index if dev.index is not None else torch.cuda.current_device()- if index not in _FUSION:+ for index in cards:+ if index in _FUSION:+ continuetry:from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion_FUSION[index] = (@@ -336,7 +341,7 @@ def install_modules()except Exception: # noqa: BLE001_FUSION[index] = None- result["real_rope"] = _FUSION[index] is not None+ result["real_rope"] = all(_FUSION[index] is not None for index in cards)for _name, module in root.named_modules():if not fuse_qkv or type(getattr(module, "processor", None)) is not cls:continue@@ -360,6 +365,28 @@ def install_modules(return result+def probe_split_cards(cards: Any) -> tuple:+ """Probe the split's other cards (an unprobed card takes the complex RoPE the regional compile cannot lower).+ Returns the cards whose probe passed."""+ if zimage_fused_disabled() or not _FUSION:+ return ()+ import torch++ with _LOCK:+ for index in cards:+ index = int(index)+ if index in _FUSION:+ continue+ try:+ from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion+ _FUSION[index] = (+ probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None+ )+ except Exception: # noqa: BLE001 - that card keeps the complex form+ _FUSION[index] = None+ return tuple(index for index in cards if _FUSION.get(int(index)) is not None)++def uninstall(transformer: Any = None) -> None:"""Restore the stock processor; drop the fused Linears (the per-row views stay valid) and pending install."""if transformer is not None:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex adac540845e..3bf8ce46be8 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -73,6 +73,7 @@ from .video_moe_pair import moe_expert_of, moe_partner_filename, moe_pick_pairsfrom .diffusion_cache import (FBCACHE_MIN_STEPS,TC_AUTO,+ TC_FBCACHE,TC_STATIC,apply_step_cache,auto_static_skip_entry,@@ -166,6 +167,7 @@ from .diffusion_speed import (SPEED_OFF,apply_speed_optims,arm_graphs_after_placement,+ engage_pinned_denoisers,rocm_probe_load_scoped,resolve_speed_mode,restore_backend_flags,@@ -205,9 +207,18 @@ from .diffusion_transformer_quant import (stored_denoiser_precision,transformer_is_quantised,)-from .diffusion import _family_override_resolved, _memory_request_forces_offload+from .diffusion import (+ _family_override_resolved,+ _companion_split_expected,+ _layer_split_expected,+ _memory_request_forces_offload,+ _native_quant_needs_compile,+ _restore_step_cache_after_layer_split,+ _step_cache_dropped_by_layer_split,+)from .diffusion_native_quant import native_quant_reasonfrom .diffusion_batched import is_oom_error+from .diffusion_block_graph import compile_pipe_below_offload_hooksfrom .diffusion_precision import (effective_te_quant,normalize_te_quant,@@ -3436,6 +3447,11 @@ def _video_split_card_floors(pipe: Any) -> Optional[tuple]:"""``((ordinal, resident MiB, runs the decode), ...)`` for a pipeline split across cards, else None. A split planreports no offload policy, so the offload floor is None and the request guard would be skipped; this records whateach card holds so the guard can run per card."""+ from .diffusion_layer_split_plan import layer_split_card_floors++ layered = layer_split_card_floors(pipe)+ if layered is not None:+ return layeredstatus = multigpu_placement_status(pipe) if multigpu_is_split(pipe) else Noneplacement = (status or {}).get("placement") or {}if not placement:@@ -3491,16 +3507,39 @@ def video_split_shortfall_message(width: Optional[int] = None,height: Optional[int] = None,frames: Optional[int] = None,+ planned_shape: Optional[tuple] = None,) -> Optional[str]:"""The offload guard run on every card of a split: each card's resident weights plus the base headroom, thedecoded-clip share on the card that runs the VAE decode, and on the denoiser's card, when the decode ran- elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. The first refusal,- else None."""+ elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. A layer split's+ block-only card (a sixth field: the headroom its plan reserved at ``planned_shape`` = (width, height, frames))+ is charged that headroom, scaled by how many more tokens this request carries, plus its fixed reserve (a seventh+ field, unscaled). The first refusal, else None."""for card in cards:ordinal, floor_mib, decodes = card[:3]home = len(card) > 3 and bool(card[3])denoises = bool(card[4]) if len(card) > 4 else homeshaped = bool(width and height and frames)+ block_headroom = card[5] if len(card) > 5 else None+ block_reserve = int(card[6] or 0) if len(card) > 6 else 0+ if block_headroom is not None and not home and not decodes:+ scale = 1.0+ if shaped and planned_shape and all(planned_shape):+ planned = int(planned_shape[0]) * int(planned_shape[1]) * int(planned_shape[2])+ scale = max(1.0, int(width) * int(height) * int(frames) / float(planned))+ headroom = int(int(block_headroom) * scale) + block_reserve+ required = int(floor_mib) + headroom+ available = int(available_mib(int(ordinal)))+ if available < required:+ shape = f" for {width}x{height} at {frames} frames" if shaped else ""+ return (+ f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "+ f"placement (split across GPUs, cuda:{int(ordinal)}: {int(floor_mib) / 1024:.1f} GiB of model "+ f"blocks plus {headroom / 1024:.1f} GiB for their activations); {available / 1024:.1f} GiB is "+ "available. Free VRAM held by other processes, generate a shorter or smaller clip, or set the "+ "DiT precision lower."+ )+ continueframes_mib = (-(-int(width) * int(height) * int(frames) * 3 * 4 // (1 << 20)) if shaped else 0)@@ -9125,13 +9164,20 @@ class VideoBackend:effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf", dense_default = SPEED_DEFAULT)- # A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant- # engaged but speed was off.- if (- transformer_quant_engaged is not None- and native_scheme is None- and effective_speed == SPEED_OFF- ):+ video_refine = lambda p, pl, lg: _video_refine_for_multigpu(fam, p, pl, lg) # noqa: E731+ split_expected = _layer_split_expected(pipe, plan, video_refine)+ # A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower.+ quant_forces_compile = transformer_quant_engaged is not None and (+ native_scheme is None+ or _native_quant_needs_compile(+ target,+ native_scheme,+ split_expected+ or plan.offload_policy == "none"+ or _companion_split_expected(pipe, plan, video_refine),+ )+ )+ if quant_forces_compile and effective_speed == SPEED_OFF:logger.info("video.transformer_quant: forcing speed_mode=default ""(quantized transformer must be compiled; eager is ~30x slower)"@@ -9278,6 +9324,7 @@ class VideoBackend:from .video_hv15_vae import install_vectorised_causal_maskhv15_mask_engaged = install_vectorised_causal_mask(pipe, logger = logger) > 0speed_optims: tuple = ()+ speed_offload = plan.offload_policy != "none" and not split_expectedfor view in views:# Both helpers act on ``view.transformer``; call once per view (engaged values match, so record the first).# is_gguf needs kind==gguf AND no quant engaged.@@ -9305,7 +9352,7 @@ class VideoBackend:backend = "video",speed_mode = effective_speed,is_gguf = gguf_transformer,- offload_active = plan.offload_policy != "none",+ offload_active = speed_offload,audit = load_audit,apply = lambda family, log, view = view: apply_speed_optims(view,@@ -9316,7 +9363,7 @@ class VideoBackend:# An auto cache that could still engage also drops fullgraph (FBCache under a fullgraph-compiled# DiT crashes)cache_active = cache_graph_break or cache_may_toggle,- offload_active = plan.offload_policy != "none",+ offload_active = speed_offload,cuda_graph_default = False,logger = log,),@@ -9373,7 +9420,21 @@ class VideoBackend:# a transient OOM during setup may have re-planned to lower residencyplanned = planplan = applied_plan(pipe, plan)- if getattr(planned, "secondary_ordinal", None) is not None and plan is not planned:+ if cache_engaged == TC_FBCACHE and _step_cache_dropped_by_layer_split(pipe):+ cache_engaged = _restore_step_cache_after_layer_split(+ pipe,+ views,+ threshold = transformer_cache_threshold,+ quant_active = cache_quant_active,+ logger = logger,+ )+ if cache_engaged is None and getattr(plan, "layer_split", None) is not None:+ cache_may_toggle = False+ cache_reason = "off: the step cache declined the denoiser split across cards"+ if (+ getattr(planned, "secondary_ordinal", None) is not None+ or getattr(planned, "layer_split", None) is not None+ ) and plan is not planned:vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:@@ -9428,6 +9489,9 @@ class VideoBackend:speed_optims += ("vae_untiled_when_fits",)# Wan's decode also grows within a single tile, which tiling alone cannot bound.install_decoder_sync(pipe, target, logger = logger)+ if not speed_offload and _video_denoiser_hooked(pipe):+ # compiled for a resident denoiser but it landed hooked: move the compile below the offload hooks (as the image backend)+ compile_pipe_below_offload_hooks(pipe, logger)# A family that opts in to graphs arms them against the placement that landed (offload hooks included).graph_applied = arm_graphs_after_placement(pipe, {"cuda_graph": "cuda_graph" in speed_optims}, logger@@ -9436,6 +9500,27 @@ class VideoBackend:speed_optims += ("cuda_graph",)elif not graph_applied.get("cuda_graph"):speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")+ if getattr(plan, "layer_split", None) is not None:+ split_applied = {k: True for k in speed_optims}+ engage_pinned_denoisers(pipe, split_applied, logger)+ from . import diffusion_cuda_graph++ diffusion_cuda_graph.arm_block_graphs(+ pipe,+ split_applied,+ target = target,+ family = fam,+ hooked = _video_denoiser_hooked(pipe),+ cache_engaged = bool(cache_graph_break),+ speed_mode = effective_speed,+ family_default = False,+ logger = logger,+ )+ for key in ("int8_gemm", "cuda_graph"):+ if split_applied.get(key) and key not in speed_optims:+ speed_optims += (key,)+ elif not split_applied.get(key):+ speed_optims = tuple(o for o in speed_optims if o != key)# Last, so the bf16 entry wraps whatever decode path the steps above installed. Not on SPEED_OFF.if getattr(fam, "vae_force_fp32", False) and effective_speed != SPEED_OFF:vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)@@ -9458,7 +9543,7 @@ class VideoBackend:speed_mode,effective_speed,"quantized transformer requires compile"- if transformer_quant_engaged is not None and native_scheme is None+ if quant_forces_compileelse "clip denoises amortise the one-time compile within a single run"if speed_mode is Noneelse "requested",@@ -11685,6 +11770,13 @@ class VideoBackend:width = width,height = height,frames = frames,+ planned_shape = (+ fam.resolution_presets[0][0],+ fam.resolution_presets[0][1],+ fam.default_num_frames,+ )+ if getattr(fam, "resolution_presets", None)+ else None,)if shortfall is not None:raise RuntimeError(shortfall)@@ -12169,6 +12261,13 @@ class VideoBackend:+ ("reaches" if toggled else "is below")+ f" {FBCACHE_MIN_STEPS}")+ # A step cache keys residuals on cond / uncond context, invisible to a graph key: graphs armed with the cache off run eager.+ graph_handles = tuple(getattr(state.pipe, "_unsloth_cuda_graphs", ()) or ())+ if graph_handles:+ from . import diffusion_cuda_graph+ diffusion_cuda_graph.set_bypass(+ graph_handles, cache_breaks_graph(state.transformer_cache)+ )if static_skip:# Auto skip was measured on text-to-video only.conditioned = bool(first_pil is not None or last_pil is not None or references)diff --git a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json b/studio/backend/tests/fixtures/diffusion_autoopt_parity.jsonindex 0dc592cb74b..b42b00ac69d 100644--- a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json+++ b/studio/backend/tests/fixtures/diffusion_autoopt_parity.json@@ -65,30 +65,30 @@"image|flux.1|off|float16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},"image|flux.1|off|float16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},"image|flux.1|off|float16|offload=1|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|bfloat16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|bfloat16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|bfloat16|offload=1|gguf=1": {"applied":[["compiled_dequant","cudnn_benchmark","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|float16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|float16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|default|float16|offload=1|gguf=1": {"applied":[["compiled_dequant","cudnn_benchmark"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|bfloat16|offload=0|gguf=0": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|bfloat16|offload=0|gguf=1": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|bfloat16|offload=1|gguf=0": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|bfloat16|offload=1|gguf=1": {"applied":[["cudnn_benchmark","fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|float16|offload=0|gguf=0": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|float16|offload=0|gguf=1": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|float16|offload=1|gguf=0": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|eager|float16|offload=1|gguf=1": {"applied":[["cudnn_benchmark"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|bfloat16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|bfloat16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|bfloat16|offload=1|gguf=1": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|float16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|float16|offload=1|gguf=0": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},- "image|flux.2-dev|max|float16|offload=1|gguf=1": {"applied":[["compiled","cudnn_benchmark","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|bfloat16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph","fp16_accum"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|bfloat16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|bfloat16|offload=1|gguf=1": {"applied":[["compiled_dequant","fp16_accum"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":true}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|float16|offload=0|gguf=1": {"applied":[["compiled_dequant","cuda_graph"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|float16|offload=1|gguf=0": {"applied":[["compiled"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":false,"compile_calls":[{"dynamic":true,"fullgraph":false}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|default|float16|offload=1|gguf=1": {"applied":[["compiled_dequant"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|bfloat16|offload=0|gguf=0": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|bfloat16|offload=0|gguf=1": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|bfloat16|offload=1|gguf=0": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|bfloat16|offload=1|gguf=1": {"applied":[["fp16_accum"]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|float16|offload=0|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|float16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|float16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|eager|float16|offload=1|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|bfloat16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|bfloat16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|bfloat16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|bfloat16|offload=1|gguf=1": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|float16|offload=0|gguf=0": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|float16|offload=0|gguf=1": {"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"graph_after_placement":false,"graph_reason":"eligible","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":true,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|float16|offload=1|gguf=0": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},+ "image|flux.2-dev|max|float16|offload=1|gguf=1": {"applied":[["compiled","fp16_accum","tf32"]],"graph_after_placement":true,"graph_reason":"armed after placement","keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":true,"compile_calls":[{"dynamic":null,"fullgraph":false,"mode":"max-autotune-no-cudagraphs"}],"int8_gemm":false,"static":false}},"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":true,"qwenimage21_fast_step":false,"rocm_fused":false}},"image|flux.2-dev|off|bfloat16|offload=0|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},"image|flux.2-dev|off|bfloat16|offload=0|gguf=1": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},"image|flux.2-dev|off|bfloat16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{"flux2_rope":false,"qwenimage21_fast_step":false,"rocm_fused":false}},diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.jsonindex 3daa6adeae7..1280e2062d8 100644--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json@@ -306,7 +306,7 @@]},"flux.2-dev": {- "cudnn_benchmark": true,+ "cudnn_benchmark": false,"filter_reduction_configs": false,"filter_reduction_configs_archs": []},@@ -390,6 +390,10 @@[12,0+ ],+ [+ 10,+ 0]]},@@ -413,6 +417,10 @@[12,0+ ],+ [+ 10,+ 0]]},@@ -431,6 +439,10 @@[12,0+ ],+ [+ 10,+ 0]]},diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.pyindex db0426f7806..31d366f8225 100644--- a/studio/backend/tests/test_diffusion_block_graph.py+++ b/studio/backend/tests/test_diffusion_block_graph.py@@ -172,6 +172,56 @@ def test_a_host_tensor_input_is_never_recorded():handle.free()+def test_a_block_graph_keys_on_the_roots_branch_flag():+ _cuda()+ block = Block().cuda()+ root = types.SimpleNamespace(masked = False)+ root._unsloth_graph_key_extra = lambda: root.masked++ def compute(x):+ y = block.lin(x)+ return y * 0.5 if root.masked else y++ shared = bg._Shared(torch.cuda.current_device(), None)+ shared.root = root+ graph = bg.BlockGraph(block, compute, shared)+ x = torch.randn(4, 64, device = "cuda")+ with torch.inference_mode():+ for _ in range(3):+ assert torch.equal(graph(x), compute(x))+ assert graph.stats["captures"] == 1+ root.masked = True+ for _ in range(3):+ assert torch.equal(graph(x), compute(x))+ assert graph.stats["captures"] == 2+++def test_a_block_graph_runs_eager_while_a_step_cache_is_on():+ _cuda()+ block = Block().cuda()+ root = types.SimpleNamespace(_unsloth_step_cache = None)++ def compute(x):+ y = block.lin(x)+ return (+ y * 0.5 if root._unsloth_step_cache else y+ ) # what the cache hooks change inside the forward++ shared = bg._Shared(torch.cuda.current_device(), None)+ shared.root = root+ graph = bg.BlockGraph(block, compute, shared)+ x = torch.randn(4, 64, device = "cuda")+ with torch.inference_mode():+ for _ in range(3):+ assert torch.equal(graph(x), compute(x))+ assert graph.stats["captures"] == 1+ replays = graph.stats["replays"]+ root._unsloth_step_cache = "[email protected]"+ for _ in range(3):+ assert torch.equal(graph(x), compute(x))+ assert graph.stats["captures"] == 1 and graph.stats["replays"] == replays++def test_weight_placement_is_none_while_a_weight_is_on_the_host():view = bg._WeightView(Block())assert view.placement(None) is Nonediff --git a/studio/backend/tests/test_diffusion_layer_split.py b/studio/backend/tests/test_diffusion_layer_split.pynew file mode 100644index 00000000000..9158c364036--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split.py@@ -0,0 +1,711 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Layer split of one denoiser across cards (core/inference/diffusion_layer_split.py), CPU only.++The planner runs on real module sizes. The boundary is exercised with the blocks assigned to ``cuda:0`` / ``cuda:1``+while their weights stay on the CPU: the cross-card copy is replaced by a recorder, which checks where every block's+inputs and outputs would go without a second card."""++from __future__ import annotations++import contextlib+import gc++import pytest+import torch+from torch import nn++from core.inference import diffusion_layer_split as ls+from core.inference import diffusion_multigpu as mg+++class FakeBlock(nn.Module):+ def __init__(self, dim: int = 8):+ super().__init__()+ self.lin = nn.Linear(dim, dim)++ def forward(+ self,+ hidden_states,+ temb,+ rotary = None,+ ):+ out = self.lin(hidden_states) * temb+ if rotary is not None:+ out = out + rotary[0]+ return out+++class FakeSingle(nn.Module):+ def __init__(self, dim: int = 8):+ super().__init__()+ self.lin = nn.Linear(dim, dim)+ self.inner = FakeBlock(dim) # a repeated class nested in a block stays with its block++ def forward(self, hidden_states, temb):+ return self.inner(self.lin(hidden_states), temb)+++class FakeDiT(nn.Module):+ _repeated_blocks = ["FakeBlock", "FakeSingle"]++ def __init__(+ self,+ dim: int = 8,+ double: int = 3,+ single: int = 3,+ ):+ super().__init__()+ self.x_embedder = nn.Linear(dim, dim)+ self.transformer_blocks = nn.ModuleList(FakeBlock(dim) for _ in range(double))+ self.single_transformer_blocks = nn.ModuleList(FakeSingle(dim) for _ in range(single))+ self.proj_out = nn.Linear(dim, dim)++ def forward(+ self,+ hidden_states,+ controlnet_block_samples = None,+ ):+ h = self.x_embedder(hidden_states)+ temb = torch.ones_like(h[:1])+ rotary = (torch.zeros_like(h), torch.zeros_like(h))+ for i, block in enumerate(self.transformer_blocks):+ h = block(h, temb, rotary)+ if controlnet_block_samples is not None:+ h = h + controlnet_block_samples[i]+ for block in self.single_transformer_blocks:+ h = block(h, temb)+ return self.proj_out(h)+++LazyDiT = type("Flux2Transformer2DModel", (FakeDiT,), {})+++class Pipe:+ def __init__(self, transformer):+ self.transformer = transformer+ self.vae = nn.Linear(2, 2)++ @property+ def components(self):+ return {"transformer": self.transformer, "vae": self.vae}++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(ls.LAYER_SPLIT_ENV, raising = False)+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)+ yield+++def _mib(module) -> int:+ return ls.module_bytes(module) / (1 << 20)+++def test_mode_parsing(monkeypatch):+ assert ls.layer_split_mode() == "auto"+ for raw in ("0", "off", "False"):+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, raw)+ assert ls.layer_split_mode() == "off"+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, "force")+ assert ls.layer_split_mode() == "force"+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")+ assert ls.layer_split_mode() == "off"+++def test_find_blocks_in_registration_order_and_skips_nested():+ dit = FakeDiT()+ found = ls.find_blocks(dit)+ assert [q for q, _, _ in found] == [+ "transformer_blocks.0",+ "transformer_blocks.1",+ "transformer_blocks.2",+ "single_transformer_blocks.0",+ "single_transformer_blocks.1",+ "single_transformer_blocks.2",+ ]+ assert {g for _, g, _ in found} == {"transformer_blocks", "single_transformer_blocks"}+++def test_find_blocks_follows_the_forward_order_of_a_listed_class(monkeypatch):+ class ModeDiT(nn.Module):+ _repeated_blocks = ["FakeBlock"]++ def __init__(self):+ super().__init__()+ self.mode_blocks = nn.ModuleDict(+ {"image": nn.ModuleList(FakeBlock() for _ in range(2))}+ )+ self.txt_mode_blocks = nn.ModuleList(FakeBlock() for _ in range(2))++ monkeypatch.setitem(ls.FORWARD_ORDER, "ModeDiT", ("txt_mode_blocks", "mode_blocks.image"))+ assert [q for q, _, _ in ls.find_blocks(ModeDiT())] == [+ "txt_mode_blocks.0",+ "txt_mode_blocks.1",+ "mode_blocks.image.0",+ "mode_blocks.image.1",+ ]+++def test_flux3_forward_order_names_its_real_lists():+ try:+ from diffusers.models.transformers import transformer_flux3+ except Exception: # noqa: BLE001 - this diffusers has no FLUX.3+ pytest.skip("diffusers without FLUX.3")+ import inspect++ src = inspect.getsource(transformer_flux3)+ for group in ls.FORWARD_ORDER["Flux3Transformer2DModel"]:+ head = group.split(".")[0]+ assert f"self.{head}" in src+++def test_a_wrapped_list_entry_groups_with_its_list():+ assert ls._block_list_name("double_stream_blocks.3.block") == "double_stream_blocks"+ assert ls._block_list_name("transformer_blocks.7") == "transformer_blocks"+ assert ls._block_list_name("mode_blocks.image.2") == "mode_blocks.image"+ assert ls._block_list_name("stages.1.blocks.0") == "stages.1.blocks"++ class Wrap(nn.Module):+ def __init__(self):+ super().__init__()+ self.block = FakeBlock(8)++ class Wrapped(nn.Module):+ _repeated_blocks = ["FakeBlock"]++ def __init__(self):+ super().__init__()+ self.double = nn.ModuleList([Wrap() for _ in range(3)])+ self.single = nn.ModuleList([Wrap() for _ in range(2)])++ groups = [g for _q, g, _b in ls.find_blocks(Wrapped())]+ assert groups == ["double"] * 3 + ["single"] * 2+++def test_module_bytes_counts_parameters_and_buffers_once():+ lin = nn.Linear(4, 4)+ lin.register_buffer("scale", torch.ones(3, dtype = torch.float16))+ lin.alias = lin.weight # the same tensor twice+ assert ls.module_bytes(lin) == (16 + 4) * 4 + 3 * 2+++def test_plan_fill_packs_the_primary_first():+ dit = FakeDiT(dim = 1024, double = 4, single = 4)+ block = _mib(dit.transformer_blocks[0])+ single = _mib(dit.single_transformer_blocks[0])+ rest = _mib(dit) - 4 * block - 4 * single+ budget0 = int(rest + 3 * block + 0.5 * block + 1)+ plan = ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: budget0, 1: 100})+ assert plan.primary == "cuda:0"+ assert [r.device for r in plan.ranges] == ["cuda:0", "cuda:1"]+ assert plan.ranges[0].blocks == [f"transformer_blocks.{i}" for i in range(3)]+ assert plan.ranges[1].blocks[0] == "transformer_blocks.3"+ assert len(plan.ranges[1].blocks) == 5+ assert set(plan.assignment.values()) == {"cuda:0", "cuda:1"}+ plan = ls.plan_layer_split(+ dit, primary = "cuda:0", cards = ["cuda:1"], budgets_mib = {"cuda:0": 100, "cuda:1": 100}+ )+ assert [r.device for r in plan.ranges] == ["cuda:0"]+++def test_plan_balance_spreads_by_budget():+ dit = FakeDiT(dim = 1024, double = 4, single = 4)+ plan = ls.plan_layer_split(+ dit, primary = 0, cards = [1], budgets_mib = {0: 100, 1: 100}, strategy = "balance"+ )+ counts = [len(r.blocks) for r in plan.ranges]+ assert counts and abs(counts[0] - counts[1]) <= 2 and sum(counts) == 8+++def test_plan_refusals():+ dit = FakeDiT(dim = 1024, double = 4, single = 4)+ with pytest.raises(ls.LayerSplitError, match = "two cards"):+ ls.plan_layer_split(dit, primary = 0, cards = [0], budgets_mib = {0: 100})+ with pytest.raises(ls.LayerSplitError, match = "no budget"):+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 100})+ with pytest.raises(ls.LayerSplitError, match = "non-block"):+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 0, 1: 100})+ with pytest.raises(ls.LayerSplitError, match = "blocks need"):+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 10, 1: 1})+ with pytest.raises(ls.LayerSplitError, match = "strategy"):+ ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = {0: 100, 1: 100}, strategy = "x")+++def test_refusal_reasons():+ assert ls.layer_split_refusal(None) == "no denoiser module"+ dit = FakeDiT()+ assert ls.layer_split_refusal(dit) is None+ dit.transformer_blocks[0]._hf_hook = object()+ assert "accelerate" in ls.layer_split_refusal(dit)+ dit = FakeDiT()+ dit.compile()+ assert ls.layer_split_refusal(dit) == "denoiser is compiled whole"+ assert "fewer than two" in ls.layer_split_refusal(FakeDiT(double = 1, single = 0))+++def test_off_switch_refuses(monkeypatch):+ monkeypatch.setenv(ls.LAYER_SPLIT_ENV, "0")+ with pytest.raises(ls.LayerSplitError, match = ls.LAYER_SPLIT_ENV):+ ls.place_layer_split(Pipe(FakeDiT()), primary = 0, cards = [1], budgets_mib = {0: 9, 1: 9})+++class _Recorder:+ """Stands in for the cross-card copies: records (what, device) and leaves the CPU tensors where they are."""++ def __init__(+ self,+ monkeypatch,+ simulate_cards = True,+ ):+ self.out_moves: list = []+ self.in_moves: list = []+ real = mg.move_to_device++ def fake_move(+ obj,+ device,+ _depth = 0,+ *,+ tensor_fn = None,+ objects = True,+ ):+ assert (+ objects is False+ ) # a split never copies attribute holders (KV caches keep their identity)+ if tensor_fn is None:+ self.out_moves.append(str(device))+ else:++ def counted(t):+ self.in_moves.append(str(device))+ return t++ return real(obj, torch.device("meta"), _depth, tensor_fn = counted, objects = False)+ return obj++ monkeypatch.setattr(mg, "move_to_device", fake_move)+ monkeypatch.setattr(mg, "_device_context", lambda device: contextlib.nullcontext())+ if simulate_cards:+ monkeypatch.setattr(ls, "_probe_device", lambda probe, default: default)+++def _install(dit, budgets):+ plan = ls.plan_layer_split(dit, primary = 0, cards = [1], budgets_mib = budgets)+ split = ls._Split(dit, torch.device("cuda", 0), type(dit).__name__ in ls.LAZY_CLASSES)+ ls._install_boundary(dit, plan, ls.find_blocks(dit), split)+ return plan, split+++def _two_card_budget(dit):+ block = _mib(dit.transformer_blocks[0])+ rest = _mib(dit) - sum(_mib(b) for _, _, b in ls.find_blocks(dit))+ return {0: int(rest + 2 * block + 0.5 * block + 1), 1: 100}+++def test_lazy_routing_moves_only_at_card_and_list_boundaries(monkeypatch):+ torch.manual_seed(0)+ dit = LazyDiT(dim = 1024, double = 4, single = 3)+ x = torch.randn(2, 1024)+ with torch.no_grad():+ reference = dit(x)+ plan, split = _install(dit, _two_card_budget(dit))+ assert plan.ranges[0].blocks == ["transformer_blocks.0", "transformer_blocks.1"]+ rec = _Recorder(monkeypatch)+ with torch.no_grad():+ out = dit(x)+ assert torch.equal(out, reference)+ assert rec.out_moves == ["cuda:1", "cuda:0", "cuda:0"]+ assert split.lazy_calls == 1 and split.eager_calls == 0+ assert not split.lazy_now # reset after the forward+++def test_eager_routing_with_a_residual_kwarg(monkeypatch):+ torch.manual_seed(0)+ dit = LazyDiT(dim = 1024, double = 4, single = 3)+ x = torch.randn(2, 1024)+ residual = [torch.randn(2, 1024) for _ in range(4)]+ with torch.no_grad():+ reference = dit(x, controlnet_block_samples = residual)+ _, split = _install(dit, _two_card_budget(dit))+ rec = _Recorder(monkeypatch)+ with torch.no_grad():+ out = dit(x, controlnet_block_samples = residual)+ assert torch.equal(out, reference)+ assert rec.out_moves == ["cuda:0"] * 5+ assert split.eager_calls == 1+++def test_unlisted_class_is_always_eager(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ _, split = _install(dit, _two_card_budget(dit))+ rec = _Recorder(monkeypatch)+ with torch.no_grad():+ dit(torch.randn(2, 1024))+ assert rec.out_moves == ["cuda:0"] * 5+ assert split.lazy_calls == 0+++def test_class_swap_keeps_names_and_undo_restores(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ budgets = _two_card_budget(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ info = ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = budgets)+ assert [r["count"] for r in info["ranges"]] == [2, 5]+ block = dit.transformer_blocks[3]+ assert type(block).__name__ == "FakeBlock" and type(block) is not FakeBlock+ assert isinstance(block, FakeBlock)+ from core.inference.diffusion_block_graph import repeated_blocks++ assert len(repeated_blocks(dit)) == len(+ list(m for m in dit.modules() if isinstance(m, (FakeBlock, FakeSingle)))+ )+ assert ls.is_layer_split(dit) and mg.is_split(pipe)+ assert dit._unsloth_split_home == torch.device("cuda", 0)+ status = mg.placement_status(pipe)+ assert status["placement"]["transformer"] == "cuda:0+cuda:1"+ assert status["layer_split"]["per_card_mib"].keys() == {"cuda:0", "cuda:1"}+ assert status["layer_split"]["calls"] == 0+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = budgets)+ assert type(block).__mro__[1] is FakeBlock++ assert ls.undo_layer_split(pipe)+ assert all(type(b) in (FakeBlock, FakeSingle) for _, _, b in ls.find_blocks(dit))+ assert not dit._forward_pre_hooks and not dit._forward_hooks+ assert "_unsloth_split_home" not in dit.__dict__+ assert not ls.is_layer_split(dit) and not mg.is_split(pipe)+ assert ls.undo_layer_split(pipe) is False+++def test_undo_keeps_a_component_split_record(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ mg._PLACEMENTS[pipe] = {+ "placement": {"vae": "cuda:1", "transformer": "cuda:0"},+ "moved": ["vae"],+ }+ dit._unsloth_split_home = torch.device("cuda", 0)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ assert mg.placement_status(pipe)["placement"]["vae"] == "cuda:1"+ ls.undo_layer_split(pipe)+ record = mg._PLACEMENTS[pipe]+ assert "layer_split" not in record and record["moved"] == ["vae"]+ assert dit._unsloth_split_home == torch.device("cuda", 0) # the component split's marker stays+++def test_a_split_denoiser_is_freed_without_the_cycle_collector(monkeypatch):+ import weakref++ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ refs = [weakref.ref(dit), weakref.ref(dit.single_transformer_blocks[2])]+ gc.disable()+ try:+ del dit, pipe+ assert all(+ r() is None for r in refs+ ) # the blocks' weights go with the pipeline, as on one card+ finally:+ gc.enable()+++def test_memo_copies_once_and_drops_stale_entries():+ memo = ls._Memo()+ t = torch.ones(4)+ a = memo.get(t, torch.float64)+ assert a.dtype == torch.float64+ assert memo.get(t, torch.float64) is a+ t.add_(1) # an in-place write invalidates the copy+ b = memo.get(t, torch.float64)+ assert b is not a and torch.equal(b, torch.full((4,), 2.0, dtype = torch.float64))+ assert len(memo.entries) == 1+ del t, a, b+ gc.collect()+ assert not memo.entries # the source died: its copies go with it+ with torch.inference_mode():+ u = torch.ones(2)+ assert memo.get(u, torch.float64) is memo.get(u, torch.float64)+++def test_move_to_device_tensor_fn_reaches_nested_tensors():+ seen = []++ def fn(t):+ seen.append(t)+ return t * 2++ t = torch.ones(1)+ out = mg.move_to_device({"a": (t, [t])}, torch.device("meta"), tensor_fn = fn)+ assert len(seen) == 2 and torch.equal(out["a"][0], torch.full((1,), 2.0))+++def test_move_to_device_keeps_attribute_holders_when_asked():+ class LayerCache:+ def __init__(self):+ self.k = torch.ones(1)++ cache = LayerCache()+ moved = mg.move_to_device(+ {"c": cache}, torch.device("meta"), tensor_fn = lambda t: t, objects = False+ )+ assert moved["c"] is cache+ assert mg.move_to_device(cache, torch.device("meta"), tensor_fn = lambda t: t) is not cache+++def test_cfg_parallel_declines_on_a_split(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "force")+ assert "layer split" in mg.cfg_parallel_decline(pipe, offload_policy = "none", selection = sel)+ from core.inference.diffusion_block_graph import install_block_graphs++ handle, reason = install_block_graphs(dit)+ assert handle is None and "weights off the GPU" in reason+ ls.undo_layer_split(pipe)+++def test_a_whole_step_graph_armed_before_the_split_stays_eager(monkeypatch):+ from core.inference import diffusion_cuda_graph as cg++ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [pipe.transformer])+ early = cg.GraphedForward(dit).enable()+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ seen = []+ monkeypatch.setattr(early, "_eager", lambda args, kwargs: seen.append(1) or "eager")+ assert early(torch.zeros(1), return_dict = False) == "eager" and seen+ assert not early.poisoned and not early.stats.get("refused_host_tensor")+ early.disable()+ handles, reason = cg.arm_after_placement(pipe)+ assert handles == () and "split across cards" in reason+ ls.undo_layer_split(pipe)+++def test_a_positional_residual_runs_eager(monkeypatch):+ torch.manual_seed(0)+ dit = LazyDiT(dim = 1024, double = 4, single = 3)+ x = torch.randn(2, 1024)+ residual = [torch.randn(2, 1024) for _ in range(4)]+ with torch.no_grad():+ reference = dit(x, residual)+ _, split = _install(dit, _two_card_budget(dit))+ rec = _Recorder(monkeypatch)+ with torch.no_grad():+ out = dit(x, residual)+ assert torch.equal(out, reference)+ assert rec.out_moves == ["cuda:0"] * 5+ assert split.eager_calls == 1 and split.lazy_calls == 0+ with torch.no_grad():+ dit(x, None)+ assert split.lazy_calls == 1+++def test_routes_follow_a_relocated_denoiser(monkeypatch):+ torch.manual_seed(0)+ dit = LazyDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ x = torch.randn(2, 1024)+ with torch.no_grad():+ reference = dit(x)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ real = mg.move_to_device+ moves: list = []++ def spy(+ obj,+ device,+ _depth = 0,+ *,+ tensor_fn = None,+ objects = True,+ ):+ moves.append(str(device))+ return real(obj, device, _depth, tensor_fn = tensor_fn, objects = objects)++ monkeypatch.setattr(mg, "move_to_device", spy)+ with torch.no_grad():+ out = dit(x) # on the CPU: a route still aimed at cuda:1 would fail here+ assert torch.equal(out, reference)+ assert set(moves) <= {"cpu"}+ assert dit._unsloth_split_home == torch.device("cuda", 0)+ assert mg._split_home(pipe) == torch.device("cpu")+ ls.undo_layer_split(pipe)+++def test_routes_follow_one_relocated_block(monkeypatch):+ torch.manual_seed(0)+ dit = LazyDiT(dim = 1024, double = 4, single = 3)+ _install(dit, _two_card_budget(dit))+ moved = dit.transformer_blocks[3]+ where = {id(moved): torch.device("cuda", 0)}++ def probe_device(probe, default):+ owner = probe[0]() if probe is not None else None+ for module in (owner,):+ for block in dit.modules():+ if (+ block is moved+ and owner is not None+ and any(m is owner for m in block.modules())+ ):+ return where[id(moved)]+ return default++ rec = _Recorder(monkeypatch)+ monkeypatch.setattr(ls, "_probe_device", probe_device)+ with torch.no_grad():+ dit(torch.randn(2, 1024))+ assert rec.out_moves == ["cuda:1", "cuda:0", "cuda:0"]+++def test_generic_undo_split_removes_a_layer_split(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ assert mg.is_split(pipe)+ mg.undo_split(pipe)+ assert not ls.is_layer_split(dit) and not mg.is_split(pipe)+ assert all(type(b) in (FakeBlock, FakeSingle) for _, _, b in ls.find_blocks(dit))+ assert not dit._forward_pre_hooks and not dit._forward_hooks+ assert pipe not in mg._PLACEMENTS+++def test_eager_calls_keep_no_copy_across_blocks(monkeypatch):+ # inference tensors carry no version: an eager call must not reuse an earlier block's copy; a lazy call shares one per card+ class InPlaceDiT(FakeDiT):+ def forward(+ self,+ hidden_states,+ controlnet_block_samples = None,+ ):+ temb = torch.ones_like(hidden_states[:1])+ h = hidden_states+ for block in self.transformer_blocks:+ h = block(h, temb)+ temb.add_(1) # written in place on the primary between blocks+ for block in self.single_transformer_blocks:+ h = block(h, temb)+ return h++ hits: list = []++ def fake_move(+ obj,+ device,+ _depth = 0,+ *,+ tensor_fn = None,+ objects = True,+ ):+ if tensor_fn is None:+ return obj++ def walk(o):+ if isinstance(o, torch.Tensor):+ return tensor_fn(o)+ if isinstance(o, (list, tuple)):+ return type(o)(walk(v) for v in o)+ if isinstance(o, dict):+ return {k: walk(v) for k, v in o.items()}+ return o++ return walk(obj)++ def fake_get(self, t, device):+ hits.append(id(t) in self.entries) # a copy made for an earlier block would be served again+ self.entries[id(t)] = (None, 0, None, {device: t})+ return t++ for cls, lazy in (+ (InPlaceDiT, False),+ (type("Flux2Transformer2DModel", (InPlaceDiT,), {}), True),+ ):+ torch.manual_seed(0)+ dit = cls(dim = 64, double = 4, single = 3)+ x = torch.randn(2, 64)+ with torch.inference_mode():+ reference = dit(x)+ _, split = _install(dit, {0: 10**6, 1: 10**6})+ monkeypatch.setattr(mg, "move_to_device", fake_move)+ monkeypatch.setattr(mg, "_device_context", lambda device: contextlib.nullcontext())+ monkeypatch.setattr(ls, "_probe_device", lambda probe, default: default)+ monkeypatch.setattr(split, "home", torch.device("cpu"))+ for qual, _, block in ls.find_blocks(dit):+ block.__dict__[ls._SPLIT_ATTR].device = torch.device("cuda", 1)+ monkeypatch.setattr(ls._Memo, "get", fake_get)+ hits.clear()+ with torch.inference_mode():+ out = dit(x)+ assert torch.equal(out, reference)+ if lazy:+ assert any(hits) # temb is copied once per card and shared by the blocks+ else:+ assert hits and not any(hits) # every block re-copies what it is given+ monkeypatch.undo()+++def test_flex_processor_refuses_and_block_mask_moves():+ class FlexAttnProcessor:+ pass++ dit = FakeDiT()+ dit.attn_processors = {"transformer_blocks.0.attn.processor": FlexAttnProcessor()}+ assert "flex attention" in ls.layer_split_refusal(dit)++ class BlockMask:+ def __init__(self, device):+ self.device = device++ def to(self, device):+ return BlockMask(device)++ mask = BlockMask("cpu")+ moved = mg.move_to_device({"attention_mask": mask}, torch.device("meta"), objects = False)+ assert moved["attention_mask"].device == torch.device("meta")++ class KVCache: # an attribute holder the callee writes into keeps its identity+ pass++ cache = KVCache()+ assert (+ mg.move_to_device({"cache": cache}, torch.device("meta"), objects = False)["cache"] is cache+ )+++def test_graph_eligibility_refuses_a_split_unet(monkeypatch):+ from core.inference import diffusion_cuda_graph as cg++ unet = FakeDiT(dim = 1024, double = 4, single = 3)+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [unet])+ from core.inference import diffusion_speed++ monkeypatch.setattr(diffusion_speed, "_denoiser_dits", lambda pipe: [])+ monkeypatch.setattr(diffusion_speed, "_denoiser_unet", lambda pipe: unet)+ pipe = Pipe(unet)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(unet))+ target = type("T", (), {"device": "cuda", "backend": "cuda"})()+ monkeypatch.setattr(cg, "cuda_graph_disabled", lambda: False)+ ok, reason = cg.graph_eligible(+ target,+ family = None,+ pipe = pipe,+ offload_active = False,+ cache_active = False,+ speed_mode = "default",+ )+ assert not ok and "split" in reason+ ls.undo_layer_split(pipe)diff --git a/studio/backend/tests/test_diffusion_layer_split_cuda.py b/studio/backend/tests/test_diffusion_layer_split_cuda.pynew file mode 100644index 00000000000..03dfd2d9eae--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split_cuda.py@@ -0,0 +1,120 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A layer split on two real CUDA cards against one card, bitwise. Skipped below two cards."""++from __future__ import annotations++import pytest+import torch++from core.inference import diffusion_layer_split as ls++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs two CUDA cards"+)++_NL, _NS = 3, 5+++def _hidream():+ hidream = pytest.importorskip("diffusers.models.transformers.transformer_hidream_image")+ torch.manual_seed(0)+ model = hidream.HiDreamImageTransformer2DModel(+ patch_size = 2,+ in_channels = 16,+ num_layers = _NL,+ num_single_layers = _NS,+ attention_head_dim = 32,+ num_attention_heads = 4,+ caption_channels = [64, 48],+ text_emb_dim = 64,+ num_routed_experts = 4,+ num_activated_experts = 2,+ axes_dims_rope = (8, 12, 12),+ max_resolution = (64, 64),+ llama_layers = list(range(_NL + _NS)),+ ).to(torch.bfloat16)+ with torch.no_grad(): # weights large enough that every block moves the result+ for p in model.parameters():+ p.normal_(0, 0.05)+ return model.eval()+++def _run(model):+ g = torch.Generator().manual_seed(1)+ inputs = dict(+ hidden_states = torch.randn(1, 16, 32, 32, generator = g),+ encoder_hidden_states_t5 = torch.randn(1, 16, 64, generator = g),+ encoder_hidden_states_llama3 = torch.randn(_NL + _NS, 1, 12, 48, generator = g),+ pooled_embeds = torch.randn(1, 64, generator = g),+ )+ kw = {k: v.to("cuda:0", torch.bfloat16) for k, v in inputs.items()}+ with torch.inference_mode():+ return (+ model(**kw, timesteps = torch.tensor([500], device = "cuda:0"), return_dict = False)[0]+ .float()+ .cpu()+ )+++class _Pipe:+ pass+++def test_a_lazy_hidream_split_matches_one_card_bitwise():+ assert "HiDreamImageTransformer2DModel" in ls.LAZY_CLASSES+ reference = _run(_hidream().to("cuda:0"))+ pipe = _Pipe()+ pipe.transformer = _hidream().to("cuda:0")+ blocks = [qual for qual, _, _ in ls.find_blocks(pipe.transformer)]+ ls.place_layer_split(+ pipe,+ primary = 0,+ assignment = {q: "cuda:0" if i == 0 else "cuda:1" for i, q in enumerate(blocks)},+ )+ try:+ out = _run(pipe.transformer)+ stats = ls.layer_split_stats(pipe.transformer)+ assert {str(p.device) for p in pipe.transformer.parameters()} == {"cuda:0", "cuda:1"}+ assert stats["lazy_calls"] == 1 and stats["eager_calls"] == 0+ assert torch.equal(out, reference)+ finally:+ ls.undo_layer_split(pipe)+++def test_a_hidream_split_hops_once_per_card_boundary_and_records_per_card_segments(monkeypatch):+ from core.inference import diffusion_block_graph as bg++ monkeypatch.setitem(globals(), "_NL", 4)+ monkeypatch.setitem(globals(), "_NS", 6)+ reference = _run(_hidream().to("cuda:0"))+ pipe = _Pipe()+ pipe.transformer = _hidream().to("cuda:0")+ on_second = {"double_stream_blocks.2.block", "double_stream_blocks.3.block"} | {+ f"single_stream_blocks.{i}.block" for i in (3, 4, 5)+ }+ blocks = [qual for qual, _, _ in ls.find_blocks(pipe.transformer)]+ assert on_second <= set(blocks) and len(blocks) == 10+ ls.place_layer_split(+ pipe, primary = 0, assignment = {q: "cuda:1" if q in on_second else "cuda:0" for q in blocks}+ )+ hops = []+ real = ls.cross_outputs+ monkeypatch.setattr(+ ls, "cross_outputs", lambda out, device: hops.append(str(device)) or real(out, device)+ )+ try:+ out = _run(pipe.transformer)+ stats = ls.layer_split_stats(pipe.transformer)+ assert hops == ["cuda:1", "cuda:0", "cuda:1", "cuda:0"]+ assert stats["input_copies"] == 2 and stats["lazy_calls"] == 1+ assert torch.equal(out, reference)+ handle, why = bg.install_block_graphs(pipe.transformer, slots = False)+ assert handle is not None, why+ try:+ assert [len(seg.graphs) for seg in handle.segments] == [2, 2, 3, 3]+ finally:+ handle.free()+ finally:+ ls.undo_layer_split(pipe)diff --git a/studio/backend/tests/test_diffusion_layer_split_lifecycle.py b/studio/backend/tests/test_diffusion_layer_split_lifecycle.pynew file mode 100644index 00000000000..9b4dbbb06c5--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split_lifecycle.py@@ -0,0 +1,144 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A layer split across the image backend's lifecycle, CPU only: the generate-time guard on the cards that run blocks+(priced from what the plan reserved there, read off the loaded pipe even for a workflow pipeline built with+from_pipe), and the on-demand resident hooks a stale one-card measured reserve would otherwise put on a split+denoiser."""++from __future__ import annotations++import sys+import types+from pathlib import Path++import pytest++from core.inference import diffusion as D+from core.inference import diffusion_layer_split_plan as lsp+from core.inference import diffusion_memory as dm++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_backend import _loaded_backend, _tiny_png_b64, fake_runtime # noqa: E402,F401+++class _Pipe:+ components: dict = {}+++def _split(+ required = 12000,+ weights = 9000,+ reserve = None,+) -> lsp.LayerSplit:+ return lsp.LayerSplit(+ **({} if reserve is None else {"card_reserve_mib": ((0, 1024), (1, reserve))}),+ primary = 0,+ cards = (0, 1),+ ranges = (+ lsp.BlockRange("transformer", 0, 10, 0, 9000),+ lsp.BlockRange("transformer", 10, 20, 1, weights),+ ),+ text_encoders_card = None,+ card_required_mib = ((0, 30000), (1, required)),+ card_limit_mib = ((0, 40000), (1, 20000)),+ card_weights_mib = ((0, 20000), (1, weights)),+ )+++def _placed(split: lsp.LayerSplit) -> _Pipe:+ pipe = _Pipe()+ lsp._APPLIED[pipe] = split+ return pipe+++def _free(monkeypatch, free_mib):+ monkeypatch.setattr(+ D,+ "reclaimable_snapshot_device_memory",+ lambda target: dm.DeviceMemory(+ backend = "cuda",+ device = "cuda",+ memory_kind = "discrete_vram",+ free_mib = free_mib,+ total_mib = 24576,+ ),+ )+++def _kw(side):+ return {+ "width": side,+ "height": side,+ "batch_size": 1,+ "family": "flux2",+ "condition_pixels": 0,+ }+++def test_block_card_guard_prices_the_planned_headroom_not_the_whole_request(monkeypatch):+ pipe = _placed(_split())+ _free(monkeypatch, 8000)+ assert D._guard_layer_cards(pipe, object(), _kw(1024)) == 1+ assert D._guard_layer_cards(pipe, object(), _kw(1536)) == 1+ _free(monkeypatch, 4000)+ assert D._guard_layer_cards(pipe, object(), _kw(1024)) == 1+ with pytest.raises(ValueError, match = "cuda:1"):+ D._guard_layer_cards(pipe, object(), _kw(1536))+ assert D._guard_layer_cards(pipe, object(), {**_kw(1536), "allow_oversized": True}) == 0+++def test_block_card_guard_scales_only_the_working_set_not_the_fixed_reserve(monkeypatch):+ pipe = _placed(_split(reserve = 2500))+ assert pipe in lsp._APPLIED and lsp._APPLIED[pipe].working_mib(1) == (500, 2500)+ _free(monkeypatch, 4000)+ assert D._guard_layer_cards(pipe, object(), _kw(1536)) == 1+ _free(monkeypatch, 2400)+ with pytest.raises(ValueError, match = "cuda:1"):+ D._guard_layer_cards(pipe, object(), _kw(1024))+++def test_block_card_guard_is_a_no_op_without_a_layer_split(monkeypatch):+ _free(monkeypatch, 10)+ assert D._guard_layer_cards(_Pipe(), object(), _kw(2048)) == 0+ assert D._guard_layer_cards(None, object(), _kw(2048)) == 0+++def test_image_generate_guards_the_loaded_pipe_for_a_workflow_pipeline(+ fake_runtime, tmp_path, monkeypatch+):+ seen = []+ monkeypatch.setattr(+ D,+ "raise_on_image_activation_shortfall",+ lambda **kw: types.SimpleNamespace(action = dm.ACTIVATION_RUN, overridden = False),+ )+ monkeypatch.setattr(+ D, "_guard_layer_cards", lambda pipe, target, kwargs: seen.append(pipe) or 0+ )+ (tmp_path / "model.gguf").write_bytes(b"x")+ backend = _loaded_backend(tmp_path)+ backend.generate(+ prompt = "a car", steps = 4, guidance = 0.0, seed = 3, init_image = _tiny_png_b64(), strength = 0.5+ )+ assert seen and seen[-1] is backend._state.pipe+++def test_a_layer_split_drops_the_one_card_measured_reserve():+ class Pipe:+ _unsloth_measured_reserve = (4096, "flux2", "default")++ pipe = Pipe()+ plain = types.SimpleNamespace(layer_split = None)+ assert not D._drop_measured_reserve_for_layer_split(pipe, plain)+ assert pipe._unsloth_measured_reserve+ assert D._drop_measured_reserve_for_layer_split(+ pipe, types.SimpleNamespace(layer_split = _split())+ )+ assert not pipe._unsloth_measured_reserve+ assert (+ dm.measured_request_extra_mib(pipe, width = 2048, height = 2048, condition_pixels = 1 << 20) == 0+ )diff --git a/studio/backend/tests/test_diffusion_layer_split_order.py b/studio/backend/tests/test_diffusion_layer_split_order.pynew file mode 100644index 00000000000..be875001f78--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split_order.py@@ -0,0 +1,258 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Layer split vs the load-time speed setup, CPU only: Studio's First-Block-Cache is installed before placement, so a+split must remove it instead of refusing the denoiser, the load must re-read the cache state, and a decline must name+the engine's real reason."""++from __future__ import annotations++import types++import pytest+import torch++from core.inference import diffusion as D+from core.inference import diffusion_cache as dc+from core.inference import diffusion_layer_split as ls+from core.inference import diffusion_layer_split_plan as lsp+from core.inference import diffusion_multigpu as mg++from .test_diffusion_layer_split import FakeDiT, Pipe, _two_card_budget+from .test_diffusion_layer_split_plan import _cards, _plan, _refine, engine # noqa: F401 - fixture++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(ls.LAYER_SPLIT_ENV, raising = False)+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)+ # pin the host so the fake [0, 1] cards plan on any machine (ROCm included)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)+ monkeypatch.setattr(mg, "_capability", lambda o: None)+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ yield+++class _Registry:+ def __init__(self, *names):+ self.hooks = {n: object() for n in names}+++def _hook(module, *names):+ module._diffusers_hook = _Registry(*names)+++def _fbcache(dit):+ """What apply_step_cache leaves: FBC hooks on the leader and the other blocks, and the marker."""+ blocks = [b for _, _, b in ls.find_blocks(dit)]+ _hook(blocks[0], "fbc_leader_block_hook")+ for block in blocks[1:]:+ _hook(block, "fbc_block_hook")+ dit._unsloth_step_cache = dc.TC_FBCACHE+++def _fake_toggle(monkeypatch, removes = True):+ calls = []++ def toggle(pipe, *, steps, **kw):+ calls.append(steps)+ t = pipe.transformer+ if removes:+ for m in t.modules():+ m.__dict__.pop("_diffusers_hook", None)+ t._unsloth_step_cache = None+ return None++ monkeypatch.setattr(dc, "maybe_toggle_step_cache", toggle)+ return calls+++def test_studio_step_cache_does_not_refuse_a_split_but_other_hooks_do():+ dit = FakeDiT()+ _fbcache(dit)+ assert ls.step_cache_hooked(dit)+ assert "carries diffusers hooks" in ls.layer_split_refusal(dit)+ assert ls.layer_split_refusal(dit, step_cache_ok = True) is None+ _hook(dit.proj_out, "group_offloading")+ assert not ls.step_cache_hooked(dit)+ assert "carries diffusers hooks" in ls.layer_split_refusal(dit, step_cache_ok = True)+ assert not ls.step_cache_hooked(FakeDiT())+++def test_placement_removes_the_step_cache_first(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ _fbcache(dit)+ pipe = Pipe(dit)+ calls = _fake_toggle(monkeypatch)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ assert calls == [0] # disengaged through the cache's own teardown+ assert ls.is_layer_split(dit) and not ls.step_cache_hooked(dit)+ assert D._step_cache_dropped_by_layer_split(pipe)+ ls.undo_layer_split(pipe)+++def test_a_cache_that_will_not_come_off_refuses_the_split(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ _fbcache(dit)+ _fake_toggle(monkeypatch, removes = False)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ with pytest.raises(ls.LayerSplitError, match = "step cache"):+ ls.place_layer_split(Pipe(dit), primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ assert not ls.is_layer_split(dit)+++def test_a_split_that_does_not_fit_leaves_the_cache_alone(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ _fbcache(dit)+ calls = _fake_toggle(monkeypatch)+ with pytest.raises(ls.LayerSplitError):+ ls.place_layer_split(Pipe(dit), primary = 0, cards = [1], budgets_mib = {0: 1, 1: 1})+ assert calls == [] and ls.step_cache_hooked(dit)+++def test_uncached_or_still_cached_pipes_are_not_reported_dropped():+ dit = FakeDiT()+ pipe = Pipe(dit)+ dit._unsloth_step_cache = dc.TC_FBCACHE+ assert not D._step_cache_dropped_by_layer_split(pipe)+ dit._unsloth_step_cache = None+ assert D._step_cache_dropped_by_layer_split(pipe)+++def test_the_planner_plans_through_the_step_cache():+ dit = FakeDiT()+ _fbcache(dit)+ blocks = lsp.split_blocks(dit)+ assert blocks is not None and len(blocks) == 6+++def test_a_decline_names_the_engines_reason(monkeypatch, engine): # noqa: F811+ _cards(monkeypatch)+ dit = FakeDiT()+ _hook(dit.transformer_blocks[0], "group_offloading")++ class HookedPipe:+ transformer = dit+ components: dict = {}++ pipe = HookedPipe()+ monkeypatch.setattr(lsp, "_denoiser_layouts", lambda p: None)+ plan = _plan()+ refined, _ = _refine(plan, pipe = pipe)+ assert refined is plan+ declined = lsp.layer_split_status(pipe)["declined"]+ assert declined.startswith("denoiser cannot be split (transformer: ")+ assert "group_offloading" in declined+++def test_the_speed_layer_is_set_up_for_a_split_the_placement_will_make():+ plan = types.SimpleNamespace(offload_policy = "group")+ split = types.SimpleNamespace(offload_policy = "none", layer_split = object())+ assert D._layer_split_expected(None, plan, lambda p, pl, lg: split)+ assert not D._layer_split_expected(+ None, plan, lambda p, pl, lg: plan+ ) # the one-card plan stands+ resident = types.SimpleNamespace(offload_policy = "none")+ assert not D._layer_split_expected(+ None, resident, lambda p, pl, lg: split+ ) # already resident: nothing to change+ assert not D._layer_split_expected(+ None, plan, lambda p, pl, lg: 1 / 0+ ) # a failing preview keeps the plan+++def test_the_step_cache_goes_back_on_after_the_split_placed_the_blocks(monkeypatch):+ seen = []+ monkeypatch.setattr(+ D,+ "apply_step_cache",+ lambda view, mode = None, **kw: seen.append((view, mode, kw)) or f"m{len(seen)}",+ )+ pipe, second = object(), object()+ assert D._restore_step_cache_after_layer_split(pipe, (second, pipe), threshold = 0.1) == "m2"+ assert [v for v, _m, _k in seen] == [second, pipe] and {m for _v, m, _k in seen} == {+ dc.TC_FBCACHE+ }+ assert seen[0][2] == {"threshold": 0.1}+++def test_split_loads_bind_speed_records_after_placement():+ import inspect++ from core.inference import video as V++ image = inspect.getsource(D.DiffusionBackend)+ video = inspect.getsource(V.VideoBackend)+ for src in (image, video):+ assert "split_expected = _layer_split_expected(" in src+ assert "offload_active = speed_offload" in src+ assert "_restore_step_cache_after_layer_split(" in src+ assert "engage_pinned_denoisers(" in src+ assert "diffusion_cuda_graph.arm_block_graphs(" in video+++def test_a_resident_torchao_free_int8_dit_is_compiled_like_a_torchao_one(monkeypatch):+ monkeypatch.setattr(D, "native_offload_host", lambda target: True)+ monkeypatch.setattr(D, "native_quant_host", lambda target: False)+ assert D._native_quant_needs_compile(object(), "int8", resident = True)+ assert not D._native_quant_needs_compile(+ object(), "int8", resident = False+ ) # streamed: the offload path decides+ assert not D._native_quant_needs_compile(+ object(), None, resident = True+ ) # torchao: the existing rule+ monkeypatch.setattr(+ D, "native_quant_host", lambda target: True+ ) # ROCm / Apple keep their native policy+ assert not D._native_quant_needs_compile(object(), "int8", resident = True)+ monkeypatch.setattr(D, "native_quant_host", lambda target: False)+ monkeypatch.setattr(D, "native_offload_host", lambda target: False)+ assert not D._native_quant_needs_compile(object(), "int8", resident = True)+++def test_a_companion_split_preview_counts_as_a_resident_denoiser():+ plan = types.SimpleNamespace(offload_policy = "group")+ companion = types.SimpleNamespace(offload_policy = "none", secondary_ordinal = 1, layer_split = None)+ assert D._companion_split_expected(None, plan, lambda p, pl, lg: companion)+ assert not D._companion_split_expected(+ None, plan, lambda p, pl, lg: plan+ ) # the one-card plan stands+ layered = types.SimpleNamespace(+ offload_policy = "none", secondary_ordinal = None, layer_split = object()+ )+ assert not D._companion_split_expected(+ None, plan, lambda p, pl, lg: layered+ ) # _layer_split_expected's case+ resident = types.SimpleNamespace(offload_policy = "none")+ assert not D._companion_split_expected(+ None, resident, lambda p, pl, lg: companion+ ) # already resident+ assert not D._companion_split_expected(+ None, plan, lambda p, pl, lg: 1 / 0+ ) # a failing preview keeps the plan+++def test_both_backends_price_a_companion_split_as_resident_for_the_quant_rule():+ import inspect++ from core.inference import video as V+ for src in (inspect.getsource(D.DiffusionBackend), inspect.getsource(V.VideoBackend)):+ rule = src[src.index("quant_forces_compile = ") :]+ rule = rule[: rule.index("if quant_forces_compile and effective_speed == SPEED_OFF:")]+ assert (+ "_native_quant_needs_compile(" in rule+ and "_companion_split_expected(pipe, plan" in rule+ )+++def test_both_backends_force_compile_through_the_one_quant_rule():+ import inspect++ from core.inference import video as V+ for src in (inspect.getsource(D.DiffusionBackend), inspect.getsource(V.VideoBackend)):+ assert "quant_forces_compile = " in src and "_native_quant_needs_compile(" in src+ assert "if quant_forces_compile and effective_speed == SPEED_OFF:" in src+ assert src.index("split_expected = _layer_split_expected(") < src.index(+ "quant_forces_compile = "+ )diff --git a/studio/backend/tests/test_diffusion_layer_split_plan.py b/studio/backend/tests/test_diffusion_layer_split_plan.pynew file mode 100644index 00000000000..a0d7c6bbe20--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split_plan.py@@ -0,0 +1,947 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Layer split planner (core/inference/diffusion_layer_split_plan.py), CPU only, fake card memory.++The pure packer is driven with card limits directly; the post-load refinement through ``refine_plan_for_multigpu``+with patched loaded sizes, block layouts and secondary limits; the engine is a recording fake."""++from __future__ import annotations++import dataclasses++import pytest+import torch++from core.inference import diffusion_layer_split_plan as lsp+from core.inference import diffusion_memory as dm+from core.inference import diffusion_multigpu as mg++L = lsp.DenoiserLayout++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)+ monkeypatch.delenv(lsp.LAYER_SPLIT_ENV, raising = False)+ # pin the host so the fake [0, 1] cards plan on any machine (ROCm included)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)+ monkeypatch.setattr(mg, "_capability", lambda o: None)+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ yield+++def _check(split, limits):+ """Every card within its limit, every block placed exactly once in contiguous ranges."""+ for o, req in split.card_required_mib:+ assert req <= limits[o], (o, req, limits[o])+ for name in dict.fromkeys(r.denoiser for r in split.ranges):+ rs = [r for r in split.ranges if r.denoiser == name]+ assert rs[0].start == 0+ for a, b in zip(rs, rs[1:]):+ assert a.stop == b.start+ return split+++def test_packs_a_dit_too_big_for_one_card_over_two():+ limits = {0: 18000, 1: 18000}+ split, reason = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 23)},+ companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},+ primary = 0,+ card_limits = limits,+ headroom_mib = 2000,+ overhead_mib = 500,+ )+ assert split is not None, reason+ _check(split, limits)+ assert split.cards == (0, 1)+ assert split.ranges[-1].stop == 23+ assert split.text_encoders_card == 1+ weights = dict(split.card_weights_mib)+ assert sum(weights.values()) == 1000 + 23000 + 4000 + 300+ assert weights[0] == 1000 + 300 + sum(r.mib for r in split.ranges if r.card == 0)+ assert split.boundaries == 2+ assert "layer split" in reason+++def test_text_encoders_too_big_for_any_card_stream_and_the_blocks_still_split():+ kw = dict(+ denoisers = {"transformer": L(1000, (1100,) * 54)},+ companions = {"text_encoder": (45000, "text_encoder"), "vae": (300, "other")},+ primary = 0,+ card_limits = {0: 55000, 1: 55000},+ headroom_mib = 6000,+ overhead_mib = 1000,+ )+ split, reason = lsp.plan_layer_split(**kw)+ assert split is not None, reason+ assert split.text_encoders_streamed and split.text_encoders_card is None+ assert split.as_public_dict()["text_encoders_streamed"] is True+ assert {r.card for r in split.ranges} == {0, 1}+ _check(split, kw["card_limits"])+ kw["card_limits"] = {0: 55000, 1: 110000}+ split, reason = lsp.plan_layer_split(**kw)+ assert split is not None, reason+ assert not split.text_encoders_streamed and split.text_encoders_card == 1+++def test_streamed_encoders_are_paged_not_moved_and_undo_removes_their_hooks(monkeypatch):+ calls = {"stream": [], "engine": [], "removed": []}++ class Comp(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1))+ self.moved = []++ def to(self, *a, **k):+ self.moved.append(a[0] if a else k.get("device"))+ return self++ class Pipe:+ def __init__(self):+ self.transformer = Comp()+ self.text_encoder = Comp()+ self.vae = Comp()+ self.components = {+ "transformer": self.transformer,+ "text_encoder": self.text_encoder,+ "vae": self.vae,+ }++ pipe = Pipe()+ split = lsp.LayerSplit(+ primary = 0,+ cards = (0, 1),+ ranges = (+ lsp.BlockRange("transformer", 0, 1, 0, 1),+ lsp.BlockRange("transformer", 1, 2, 1, 1),+ ),+ text_encoders_card = None,+ card_required_mib = ((0, 1), (1, 1)),+ card_limit_mib = ((0, 9), (1, 9)),+ text_encoders_streamed = True,+ )+ from core.inference import diffusion_layer_split as engine++ monkeypatch.setattr(+ dm,+ "stream_text_encoders_onto",+ lambda p, d, logger = None: calls["stream"].append(str(d)) or ["text_encoder"],+ )+ monkeypatch.setattr(dm, "_remove_group_offload_hooks", lambda m: calls["removed"].append(m))+ monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])+ monkeypatch.setattr(+ engine, "place_layer_split", lambda p, **kw: calls["engine"].append(kw["assignment"]) or {}+ )+ monkeypatch.setattr(engine, "undo_layer_split", lambda p, attr = "transformer": False)+ import contextlib++ from core.inference import diffusion_fast_load++ monkeypatch.setattr(+ diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()+ )+ lsp.place_layer_split(pipe, primary = "cuda:0", split = split)+ assert pipe.text_encoder.moved == [] # never moved onto a card: paged by its hooks+ assert pipe.vae.moved == [torch.device("cuda", 0)]+ assert calls["stream"] == ["cuda:0"]+ assert calls["engine"] == [{"b.0": 0, "b.1": 1}]+ record = mg._PLACEMENTS[pipe]+ assert record["streamed"] == ["text_encoder"] and record["moved"] == []+ assert record["placement"]["text_encoder"].startswith("cpu (streamed")+ lsp.undo_layer_split(pipe)+ assert calls["removed"] == [pipe.text_encoder]+ assert pipe not in mg._PLACEMENTS+++def test_every_denoiser_keeps_a_block_off_the_primary():+ limits = {0: 100000, 1: 6000}+ kw = dict(+ companions = {"text_encoder": (2000, "text_encoder"), "vae": (300, "other")},+ primary = 0,+ card_limits = limits,+ headroom_mib = 1000,+ overhead_mib = 0,+ working_set_mib = 1000,+ secondary_overhead_mib = 0,+ )+ for denoisers in (+ {"transformer": L(0, (1000,) * 10)},+ {"transformer": L(0, (1000,) * 10), "transformer_2": L(0, (1000,) * 10)},+ ):+ split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)+ assert split is not None, reason+ _check(split, limits)+ for name in denoisers:+ assert any(r.card != 0 for r in split.ranges if r.denoiser == name), (+ name,+ split.ranges,+ )+++def test_too_small_for_the_combined_cards_declines():+ split, reason = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 40)},+ companions = {"vae": (300, "other")},+ primary = 0,+ card_limits = {0: 18000, 1: 18000},+ headroom_mib = 2000,+ overhead_mib = 500,+ )+ assert split is None+ assert "does not fit" in reason+++def test_no_secondary_or_unknown_blocks_declines():+ assert (+ lsp.plan_layer_split(+ denoisers = {"transformer": L(0, (1000,))},+ companions = {},+ primary = 0,+ card_limits = {0: 18000},+ headroom_mib = 0,+ overhead_mib = 0,+ )[0]+ is None+ )+ assert (+ lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, ())},+ companions = {},+ primary = 0,+ card_limits = {0: 18000, 1: 18000},+ headroom_mib = 0,+ overhead_mib = 0,+ )[0]+ is None+ )+++def test_mixed_card_sizes_cut_in_proportion_to_free_capacity():+ limits = {0: 20000, 1: 68000}+ split, _ = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 60)},+ companions = {},+ primary = 0,+ card_limits = limits,+ headroom_mib = 3000,+ overhead_mib = 500,+ )+ _check(split, limits)+ on = {o: sum(r.mib for r in split.ranges if r.card == o) for o in (0, 1)}+ reserve = lsp.secondary_reserve_mib()+ caps = {0: 20000 - 1000 - 3000 - max(500, reserve), 1: 68000 - 2000 - reserve}+ for o in (0, 1):+ assert abs(on[o] / caps[o] - 60000 / sum(caps.values())) < 1000 / caps[o] + 1e-9+++def test_uses_a_third_card_only_when_two_are_not_enough():+ limits = {0: 18000, 1: 18000, 2: 18000}+ two, _ = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 25)},+ companions = {},+ primary = 0,+ card_limits = limits,+ headroom_mib = 2000,+ overhead_mib = 500,+ )+ assert two.cards == (0, 1)+ three, _ = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 40)},+ companions = {},+ primary = 0,+ card_limits = limits,+ headroom_mib = 2000,+ overhead_mib = 500,+ )+ _check(three, limits)+ assert three.cards == (0, 1, 2)+ assert three.boundaries == 3+++def test_largest_secondary_is_used_first():+ split, _ = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 25)},+ companions = {},+ primary = 0,+ card_limits = {0: 18000, 1: 9000, 2: 30000},+ headroom_mib = 2000,+ overhead_mib = 500,+ )+ assert split.cards == (0, 2)+++def test_two_experts_are_each_cut_with_the_headroom_charged_once():+ limits = {0: 40000, 1: 40000}+ split, reason = lsp.plan_layer_split(+ denoisers = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)},+ companions = {"text_encoder": (6000, "text_encoder"), "vae": (300, "other")},+ primary = 0,+ card_limits = limits,+ headroom_mib = 4000,+ overhead_mib = 500,+ )+ assert split is not None, reason+ _check(split, limits)+ assert {r.denoiser for r in split.ranges if r.card == 1} == {"transformer", "transformer_2"}+ weights = dict(split.card_weights_mib)+ required = dict(split.card_required_mib)+ reserve = lsp.secondary_reserve_mib()+ assert required[0] == weights[0] + 4000 + max(500, reserve)+ assert required[1] == weights[1] + max(1400, 1024) + reserve+++def test_phase_headroom_prices_decode_on_the_primary_and_encode_on_the_te_card():+ split, _ = lsp.plan_layer_split(+ denoisers = {"transformer": L(1000, (1000,) * 23)},+ companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},+ primary = 0,+ card_limits = {0: 20000, 1: 20000},+ headroom_mib = 0,+ overhead_mib = 500,+ phase_headroom_mib = (3000, 1500, 2500),+ )+ weights = dict(split.card_weights_mib)+ required = dict(split.card_required_mib)+ reserve = lsp.secondary_reserve_mib()+ assert split.text_encoders_card == 1+ assert required[0] == weights[0] + max(1500, 2500) + max(500, reserve)+ assert required[1] == weights[1] + 3000 + reserve+++def test_secondary_is_charged_a_block_working_set_not_the_primary_activation_peak():+ denoisers = {"transformer": L(1500, (1070,) * 56)}+ limits = {0: 80000, 1: 80000}+ split, reason = lsp.plan_layer_split(+ denoisers = denoisers,+ companions = {"vae": (200, "other")},+ primary = 0,+ card_limits = limits,+ headroom_mib = 30000,+ overhead_mib = 2048,+ )+ assert split is not None, reason+ weights = dict(split.card_weights_mib)+ required = dict(split.card_required_mib)+ assert lsp.block_working_set_mib(denoisers, 30000) == 2140+ assert required[1] == weights[1] + 2140 + lsp.secondary_reserve_mib()+ assert required[0] == weights[0] + 30000 + 2048+ assert lsp.block_working_set_mib(denoisers, 1500) == 1500+ assert lsp.block_working_set_mib({"transformer": L(0, (100,))}, 0) == 1024+++def test_reserve_follows_comfyui_minimum_inference_memory(monkeypatch):+ monkeypatch.setattr(lsp.sys, "platform", "linux")+ assert lsp.secondary_reserve_mib() == 1200+ monkeypatch.setattr(lsp.sys, "platform", "win32")+ assert lsp.secondary_reserve_mib() == 1500+++def test_layouts_count_storage_bytes_so_quantized_blocks_are_smaller():+ class Block(torch.nn.Module):+ def __init__(self, dtype):+ super().__init__()+ self.w = torch.nn.Parameter(torch.empty(4096, 4096, dtype = dtype), requires_grad = False)++ class Den(torch.nn.Module):+ _no_split_modules = ["Block"]++ def __init__(self, dtype):+ super().__init__()+ self.proj_out = torch.nn.Linear(4096, 1024, dtype = torch.bfloat16)+ self.transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(3))+ self.single_transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(2))++ class Pipe:+ transformer = Den(torch.bfloat16)++ class Quant:+ transformer = Den(torch.int8)++ bf16 = lsp._denoiser_layouts(Pipe())["transformer"]+ int8 = lsp._denoiser_layouts(Quant())["transformer"]+ assert bf16.block_mib == (32,) * 5+ assert int8.block_mib == (16,) * 5+ assert bf16.stem_mib == 9+++def test_a_denoiser_without_a_block_list_has_no_layout():+ class Pipe:+ transformer = torch.nn.Linear(4, 4)++ assert lsp._denoiser_layouts(Pipe()) is None+++class _Target:+ supports_model_cpu_offload: bool = True+ ordinal: int = 0+ device: str = "cuda"+ backend: str = "cuda"+ is_cuda_torch_device: bool = True+++def _memory(free = 24000, total = 24576):+ return dm.DeviceMemory(+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total+ )+++def _plan(**kw):+ args = dict(+ target = _Target(),+ device_memory = _memory(),+ model_dense_mib = 24000,+ companion_dense_mib = 4300,+ text_encoder_dense_mib = 4000,+ runtime_headroom_mib = 2000,+ )+ args.update(kw)+ return dm.plan_diffusion_memory(**args)+++_BIG = {+ "transformer": (24000, "dit"),+ "text_encoder": (4000, "text_encoder"),+ "vae": (300, "other"),+}+++class _Engine:+ def __init__(self, fail = False):+ self.fail = fail+ self.placed = []+ self.undone = 0++ def place(+ self,+ pipe,+ *,+ primary,+ split,+ logger = None,+ ):+ self.placed.append((primary, split))+ if self.fail:+ raise RuntimeError("boom")+ return {}++ def undo(self, pipe):+ self.undone += 1+++def engine(monkeypatch):+ eng = _Engine()+ monkeypatch.setattr(lsp, "engine_available", lambda: True)+ monkeypatch.setattr(lsp, "place_layer_split", eng.place)+ monkeypatch.setattr(lsp, "undo_layer_split", eng.undo)+ return eng+++def _cards(+ monkeypatch,+ sizes = _BIG,+ sec_limit = 20000,+ layouts = None,+):+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: sec_limit)+ monkeypatch.setattr(+ mg, "best_secondary", lambda selection, probes = None: selection.secondaries[0]+ )+ monkeypatch.setattr(+ lsp,+ "_denoiser_layouts",+ lambda pipe: layouts if layouts is not None else {"transformer": L(1000, (1000,) * 23)},+ )+++class _Pipe:+ """Weak-referenceable stand-in; the refinement only reads it through the patched size / layout readers."""++ components: dict = {}++ def to(self, *a, **k):+ return self+++def _refine(+ plan,+ pipe = None,+ ordinals = (0, 1),+):+ pipe = pipe if pipe is not None else _Pipe()+ with mg.selection_scope(mg.make_selection(0, list(ordinals))):+ return mg.refine_plan_for_multigpu(pipe, plan), pipe+++def test_one_card_offload_becomes_a_layer_split(monkeypatch, engine):+ plan = _plan()+ assert lsp.plan_offloads_denoiser(plan)+ _cards(monkeypatch)+ refined, pipe = _refine(plan)+ assert refined is not plan+ assert refined.offload_policy == dm.OFFLOAD_NONE+ assert refined.layer_split is not None and refined.secondary_ordinal is None+ assert refined.fallback_plan is plan+ assert refined.layer_split.cards == (0, 1)+ public = refined.as_public_dict()+ assert public["layer_split"]["cards"] == [0, 1]+ assert "layer_split" not in plan.as_public_dict()+ assert any("layer split" in r for r in refined.reasons)+++def test_no_engine_keeps_the_one_card_plan(monkeypatch):+ monkeypatch.setattr(lsp, "engine_available", lambda: False)+ _cards(monkeypatch)+ plan = _plan()+ assert _refine(plan)[0] is plan+++def test_no_selection_or_kill_switches_keep_the_plan(monkeypatch, engine):+ _cards(monkeypatch)+ plan = _plan()+ assert mg.refine_plan_for_multigpu(_Pipe(), plan) is plan+ assert _refine(plan, ordinals = (0,))[0] is plan+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")+ assert _refine(plan)[0] is plan+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "auto")+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")+ assert _refine(plan)[0] is plan+ assert engine.placed == []+++def test_keep_multiple_models_loaded_gets_no_selection(monkeypatch, engine):+ _cards(monkeypatch)+ plan = _plan()+ monkeypatch.setattr(mg, "models_kept_loaded", lambda: True)+ seen = []++ class Backend:+ @mg.with_load_selection+ def _run_load(self, **kwargs):+ seen.append(mg.current_selection())+ return mg.refine_plan_for_multigpu(_Pipe(), plan)++ assert Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1]) is plan+ assert seen == [None]+++def test_a_dit_that_fits_one_card_keeps_todays_plan(monkeypatch, engine):+ small = {+ "transformer": (8000, "dit"),+ "text_encoder": (4000, "text_encoder"),+ "vae": (300, "other"),+ }+ _cards(monkeypatch, sizes = small)+ plan = _plan(model_dense_mib = 8000)+ assert plan.offload_policy == dm.OFFLOAD_NONE+ assert _refine(plan)[0] is plan+ streamed_te = dataclasses.replace(+ _plan(),+ offload_policy = dm.OFFLOAD_GROUP,+ stream_transformer = False,+ stream_text_encoders = True,+ )+ _cards(monkeypatch, sec_limit = 1000) # no companion assignment either+ assert _refine(streamed_te)[0] is streamed_te+ assert engine.placed == []+++def test_quantized_storage_that_fits_one_card_is_a_companion_split_not_a_layer_split(+ monkeypatch, engine+):+ quant = {+ "transformer": (12000, "dit"),+ "text_encoder": (4000, "text_encoder"),+ "vae": (300, "other"),+ }+ _cards(monkeypatch, sizes = quant)+ refined, _ = _refine(_plan())+ assert refined.layer_split is None+ assert refined.secondary_ordinal == 1+++def test_companion_split_is_preferred_over_a_layer_split(monkeypatch, engine):+ fits = {+ "transformer": (14000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ _cards(monkeypatch, sizes = fits)+ refined, _ = _refine(_plan())+ assert refined.secondary_ordinal == 1 and refined.layer_split is None+++def test_cards_too_small_keep_the_offload_plan_and_record_why(monkeypatch, engine):+ _cards(monkeypatch, sec_limit = 6000)+ plan = _plan()+ refined, pipe = _refine(plan)+ assert refined is plan+ status = lsp.layer_split_status(pipe)+ assert "does not fit" in status["declined"]+ assert mg.placement_status(pipe)["layer_split"] == status+++def test_rocm_explicit_modes_and_legacy_cpu_offload_keep_the_plan(monkeypatch, engine):+ _cards(monkeypatch)+ plan = _plan()+ monkeypatch.setattr(lsp, "_is_rocm", lambda: True)+ assert _refine(plan)[0] is plan+ monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ balanced = _plan(requested_mode = "balanced")+ assert _refine(balanced)[0] is balanced+ legacy = _plan(explicit_offload = True)+ assert legacy.estimates.get("explicit_cpu_offload")+ assert _refine(legacy)[0] is legacy+ assert engine.placed == []+++def test_force_cuts_a_denoiser_one_card_would_hold(monkeypatch, engine):+ small = {"transformer": (8000, "dit"), "vae": (300, "other")}+ _cards(monkeypatch, sizes = small, layouts = {"transformer": L(500, (500,) * 15)})+ plan = _plan(model_dense_mib = 8000, companion_dense_mib = 300, text_encoder_dense_mib = 0)+ assert plan.offload_policy == dm.OFFLOAD_NONE+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")+ refined, _ = _refine(plan)+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)+++def test_video_two_expert_cut_when_neither_expert_fits_a_card(monkeypatch, engine):+ sizes = {+ "transformer": (28500, "dit"),+ "transformer_2": (28500, "dit"),+ "text_encoder": (6000, "text_encoder"),+ "vae": (300, "other"),+ }+ layouts = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)}+ _cards(monkeypatch, sizes = sizes, sec_limit = 34000, layouts = layouts)+ plan = _plan(+ device_memory = _memory(free = 40000, total = 40960),+ model_dense_mib = 57000,+ companion_dense_mib = 6300,+ text_encoder_dense_mib = 6000,+ )+ refined, _ = _refine(plan, ordinals = (0, 1, 2))+ split = refined.layer_split+ assert split is not None+ assert {r.denoiser for r in split.ranges} == {"transformer", "transformer_2"}+++def _layer_plan(monkeypatch, engine):+ _cards(monkeypatch)+ refined, pipe = _refine(_plan())+ assert refined.layer_split is not None+ return refined, pipe+++def test_apply_records_the_split_and_cfg_parallel_declines(monkeypatch, engine):+ plan, pipe = _layer_plan(monkeypatch, engine)+ assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ assert engine.placed[0][0] == "cuda:0" and engine.placed[0][1] is plan.layer_split+ assert lsp.is_layer_split(pipe) and mg.is_split(pipe)+ status = mg.placement_status(pipe)+ assert status["layer_split"]["cards"] == [0, 1]+ assert status["layer_split"]["boundaries"] == 2+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ assert (+ mg.cfg_parallel_decline(pipe, offload_policy = "none", selection = mg.current_selection())+ == "denoiser is split across cards (layer split)"+ )+ lsp.forget_layer_split(pipe)+ assert not lsp.is_layer_split(pipe)+++def test_failed_placement_applies_the_one_card_fallback(monkeypatch, engine):+ plan, pipe = _layer_plan(monkeypatch, engine)+ engine.fail = True+ applied = []+ real = dm.apply_memory_plan++ def spy(p, pl, **kw):+ applied.append(pl)+ if pl is plan:+ return real(p, pl, **kw)+ return pl.offload_policy, False++ monkeypatch.setattr(dm, "apply_memory_plan", spy)+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")+ assert engine.undone == 1+ assert applied == [plan, plan.fallback_plan]+ assert policy == plan.fallback_plan.offload_policy+ assert dm.applied_plan(pipe, plan) is plan.fallback_plan+ assert not lsp.is_layer_split(pipe)+ assert "placement failed" in lsp.layer_split_status(pipe)["declined"]+++def test_video_guard_reads_per_card_floors_from_the_layer_split(monkeypatch, engine):+ from core.inference import video as V++ plan, pipe = _layer_plan(monkeypatch, engine)+ assert V._video_split_card_floors(pipe) is None+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ floors = V._video_split_card_floors(pipe)+ split = plan.layer_split+ weights = dict(split.card_weights_mib)+ required = dict(split.card_required_mib)+ reserve = dict(split.card_reserve_mib)+ assert floors == (+ (0, weights[0], True, True, True, None, None),+ (1, weights[1], False, False, True, required[1] - weights[1] - reserve[1], reserve[1]),+ )+ assert reserve[1] > 0+++def test_video_guard_prices_a_block_card_by_its_planned_headroom():+ from core.inference import video as V++ cards = ((0, 13300, True, True, True, None), (1, 20000, False, False, True, 3200))+ free = {0: 60000, 1: 23500}+ kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))+ assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw) is None+ message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw)+ assert message is not None and "cuda:1" in message+ assert V.video_split_shortfall_message(cards, width = 416, height = 240, frames = 17, **kw) is None+++def test_video_guard_scales_only_the_working_set_not_the_fixed_reserve():+ from core.inference import video as V++ cards = ((0, 13300, True, True, True, None, None), (1, 20000, False, False, True, 700, 2500))+ free = {0: 60000, 1: 24000}+ kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))+ assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw) is None+ free[1] = 20000 + 700 + 2400+ message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw)+ assert message is not None and "cuda:1" in message+++def test_a_from_pipe_sibling_sees_the_split_through_its_shared_denoiser(monkeypatch, engine):+ from core.inference import diffusion_layer_split as engine_mod++ plan, pipe = _layer_plan(monkeypatch, engine)+ shared = torch.nn.Linear(2, 2)+ pipe.transformer = shared+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ sibling = type("Sibling", (), {"transformer": shared})()+ monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is shared)+ assert lsp.applied_layer_split(sibling) is plan.layer_split+ assert lsp.secondary_block_cards(sibling) == (1,)+ assert mg.is_split(sibling)+ mg.undo_split(pipe)+ assert lsp.applied_layer_split(pipe) is None and lsp.applied_layer_split(sibling) is None+++def test_a_failed_replacement_does_not_report_the_previous_split(monkeypatch, engine):+ plan, pipe = _layer_plan(monkeypatch, engine)+ assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ assert lsp.is_layer_split(pipe)+ engine.fail = True+ assert not lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ assert not lsp.is_layer_split(pipe) and not mg.is_split(pipe)+ assert "placement failed" in lsp.layer_split_status(pipe)["declined"]+++def test_an_expert_wholly_on_the_primary_is_placed_plainly(monkeypatch):+ import contextlib++ from core.inference import diffusion_fast_load+ from core.inference import diffusion_layer_split as engine_mod++ class Comp(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1))+ self.moved = []++ def to(self, *a, **k):+ self.moved.append(a[0] if a else k.get("device"))+ return self++ class Pipe:+ def __init__(self):+ self.transformer = Comp()+ self.transformer_2 = Comp()+ self.vae = Comp()+ self.components = {+ "transformer": self.transformer,+ "transformer_2": self.transformer_2,+ "vae": self.vae,+ }++ pipe = Pipe()+ split = lsp.LayerSplit(+ primary = 0,+ cards = (0, 1),+ ranges = (+ lsp.BlockRange("transformer", 0, 1, 0, 1),+ lsp.BlockRange("transformer", 1, 2, 1, 1),+ lsp.BlockRange("transformer_2", 0, 2, 0, 2),+ ),+ text_encoders_card = None,+ card_required_mib = ((0, 1), (1, 1)),+ card_limit_mib = ((0, 9), (1, 9)),+ )+ placed = []+ monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])+ monkeypatch.setattr(+ engine_mod,+ "place_layer_split",+ lambda p, **kw: placed.append((kw["attr"], kw["assignment"])) or {},+ )+ monkeypatch.setattr(+ diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()+ )+ out = lsp.place_layer_split(pipe, primary = "cuda:0", split = split)+ assert placed == [+ ("transformer", {"b.0": 0, "b.1": 1})+ ] # only the expert that is cut reaches the engine+ assert out["transformer_2"]["whole_on_primary"] is True+ assert pipe.transformer_2.moved == [torch.device("cuda", 0)]+++def test_offload_installers_refuse_or_replace_a_split(monkeypatch):+ from core.inference import diffusion_layer_split as engine_mod++ dit = torch.nn.Linear(2, 2)+ pipe = type("P", (), {"transformer": dit, "components": {"transformer": dit}})()+ monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is dit)+ monkeypatch.setattr(dm, "_offload_groups", lambda m: [])+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda p: True)+ assert dm.hook_resident_denoiser(pipe, "cuda:0") is False+ undone = []+ monkeypatch.setattr(lsp, "undo_layer_split", lambda p: undone.append(p))+ dm._apply_group_offload(pipe, "cpu", None)+ assert undone == [pipe]+++def test_image_guard_checks_every_other_card_running_blocks(monkeypatch, engine):+ from core.inference import diffusion as D++ plan, pipe = _layer_plan(monkeypatch, engine)+ free = {"mib": 100000}+ monkeypatch.setattr(+ D,+ "reclaimable_snapshot_device_memory",+ lambda target: dm.DeviceMemory(+ backend = "cuda",+ device = "cuda",+ memory_kind = "discrete_vram",+ free_mib = free["mib"],+ total_mib = 102400,+ ),+ )+ kw = {"width": 1024, "height": 1024, "family": "flux2"}+ assert D._guard_layer_cards(pipe, object(), kw) == 0 # not placed: nothing to check+ lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")+ assert D._guard_layer_cards(pipe, object(), kw) == 1+ free["mib"] = 10+ with pytest.raises(ValueError, match = "cuda:1"):+ D._guard_layer_cards(pipe, object(), kw)+++def test_two_denoisers_that_fit_only_jointly_are_cut_jointly():+ kw = dict(+ companions = {},+ primary = 0,+ card_limits = {0: 3, 1: 3},+ headroom_mib = 0,+ overhead_mib = 0,+ working_set_mib = 0,+ secondary_overhead_mib = 0,+ )+ denoisers = {"transformer": L(0, (1, 1)), "transformer_2": L(0, (1, 2, 1))}+ split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)+ assert split is not None, reason+ per_card = {}+ for r in split.ranges:+ per_card[r.card] = per_card.get(r.card, 0) + r.mib+ assert per_card[0] <= 3 and per_card[1] <= 3+ for name, layout in denoisers.items():+ mine = sorted((r.start, r.stop) for r in split.ranges if r.denoiser == name)+ assert mine[0][0] == 0 and mine[-1][1] == len(layout.block_mib) # every block placed once+ assert any(r.card != 0 for r in split.ranges if r.denoiser == name)+ assert lsp._search_cuts([(1, 1), (1, 2, 1)], [2, 2]) is None # truly infeasible stays declined+++def test_dual_denoisers_may_keep_one_wholly_on_the_primary():+ # A fits only on the primary, B only on the secondary: forcing both to cut would decline a valid placement+ assert lsp._search_cuts([(2, 2), (3, 3)], [4, 6]) == [[0, 2, 2], [0, 0, 2]]+ assert lsp._search_cuts([(2, 2), (3, 3)], [10, 0]) is None # nothing off the primary: no split+ split, reason = lsp.plan_layer_split(+ denoisers = {"transformer": L(0, (2, 2)), "transformer_2": L(0, (3, 3))},+ companions = {},+ primary = 0,+ card_limits = {0: 4, 1: 6},+ headroom_mib = 0,+ overhead_mib = 0,+ phase_headroom_mib = (0, 0, 0),+ working_set_mib = 0,+ secondary_overhead_mib = 0,+ )+ assert split is not None, reason+ _check(split, {0: 4, 1: 6})+ assert {(r.denoiser, r.card) for r in split.ranges} == {+ ("transformer", 0),+ ("transformer_2", 1),+ }+++def test_failed_placement_drops_fused_records_left_on_another_card(monkeypatch):+ from types import SimpleNamespace++ from torch import nn++ def attn(fused_device):+ a = nn.Module()+ a.to_q, a.to_k, a.to_v = nn.Linear(4, 4), nn.Linear(4, 4), nn.Linear(4, 4)+ a.__dict__["_unsloth_fused_qkv"] = (+ nn.Linear(4, 12, device = fused_device),+ a.to_q,+ a.to_k,+ a.to_v,+ )+ return a++ dit = nn.Module()+ dit.stale, dit.intact = attn("meta"), attn("cpu")+ pipe = SimpleNamespace(transformer = dit, to = lambda *a, **k: None)++ def boom(*a, **k):+ raise RuntimeError("out of memory")++ monkeypatch.setattr(lsp, "place_layer_split", boom)+ monkeypatch.setattr(lsp, "undo_layer_split", lambda p: None)+ assert (+ lsp.apply_layer_split_plan(pipe, SimpleNamespace(layer_split = object()), placement = "cuda:0")+ is False+ )+ assert "_unsloth_fused_qkv" not in dit.stale.__dict__+ assert "_unsloth_fused_qkv" in dit.intact.__dict__ # still on its parts' device: kept+++def test_an_arch_pinned_attention_backend_keeps_blocks_off_other_architectures(monkeypatch, engine):+ # flash3 is Hopper only: a B200 secondary would reject it+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR++ caps = {0: (9, 0), 1: (10, 0), 2: (9, 0)}+ monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))+ _cards(monkeypatch)+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2)+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "_flash_3_hub")+ plan = _plan()+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))+ assert refined is plan # the only secondary is another architecture: no split+ assert "attention backend _flash_3_hub" in lsp._DECLINES[pipe]+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1, 2))+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards+ setattr(+ pipe.transformer, ATTENTION_BACKEND_ATTR, "_native_cudnn"+ ) # runs on every admitted card+ refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))+ assert refined.layer_split is not None and 1 in refined.layer_split.cardsdiff --git a/studio/backend/tests/test_diffusion_layer_split_speed.py b/studio/backend/tests/test_diffusion_layer_split_speed.pynew file mode 100644index 00000000000..4a742a2a4f7--- /dev/null+++ b/studio/backend/tests/test_diffusion_layer_split_speed.py@@ -0,0 +1,773 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Speed paths on a layer split denoiser, CPU only: per-card block graphs, the step cache across cards, and the int8+GEMM swap per card. The split is placed with the weights left on the CPU (``_move_weights`` stubbed); where a card+must be read off a weight, the test names it."""++from __future__ import annotations++import importlib.util+import sys+import types+from pathlib import Path++import pytest+import torch+from torch import nn++from core.inference import diffusion_block_graph as bg+from core.inference import diffusion_cache as dc+from core.inference import diffusion_cuda_graph as cg+from core.inference import diffusion_int8_gemm as g8+from core.inference import diffusion_layer_split as ls+from core.inference import diffusion_multigpu as mg++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_layer_split import FakeDiT, Pipe, _two_card_budget # noqa: E402++_needs_diffusers = pytest.mark.skipif(+ importlib.util.find_spec("diffusers") is None, reason = "needs diffusers"+)++[email protected](autouse = True)+def _env(monkeypatch):+ for name in (ls.LAYER_SPLIT_ENV, mg.MULTIGPU_ENV, mg.CFG_PARALLEL_ENV, bg.BLOCK_GRAPHS_ENV):+ monkeypatch.delenv(name, raising = False)+ yield+++def _split(monkeypatch, dit):+ pipe = Pipe(dit)+ monkeypatch.setattr(ls, "_move_weights", lambda *a, **k: None)+ info = ls.place_layer_split(pipe, primary = 0, cards = [1], budgets_mib = _two_card_budget(dit))+ assert {r["device"] for r in info["ranges"]} == {"cuda:0", "cuda:1"}+ split = ls.layer_split_of(dit)+ card = {qual: int(dev.split(":")[1]) for qual, dev in _assignment(dit).items()}+ return pipe, split, card+++def _assignment(dit) -> dict:+ out = {}+ for qual, _, block in ls.find_blocks(dit):+ out[qual] = str(block.__dict__[ls._SPLIT_ATTR].device)+ return out+++def test_block_graphs_record_each_block_on_its_own_card(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, card = _split(monkeypatch, dit)+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")+ assert handle is not None, reason+ assert handle.cards() == {0, 1}+ for graph in handle.graphs:+ assert graph.shared.device_index == by_block[id(graph.block)]+ assert graph.shared.strict_device # records only while its own card is current+ assert handle.shared.device_index == 0 and set(handle.card_shared) == {1}+ assert handle.describe()["cards"] == [0, 1]+ handle.free()+ assert not handle.graphs+ ls.undo_layer_split(pipe)+++def test_one_card_block_graphs_keep_one_shared_state(monkeypatch):+ dit = FakeDiT(dim = 64, double = 2, single = 2)+ monkeypatch.setattr(+ bg, "_block_card", lambda block: pytest.fail("read a block card on one card")+ )+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")+ assert handle is not None, reason+ assert not handle.card_shared and handle.cards() == {0}+ assert all(g.shared is handle.shared and not g.shared.strict_device for g in handle.graphs)+ assert "cards" not in handle.describe()+ handle.free()+++def test_a_split_block_off_its_card_runs_its_compute(monkeypatch):+ dit = FakeDiT(dim = 64, double = 2, single = 2)+ shared = bg._Shared(1, None)+ shared.strict_device = True+ calls = []+ graph = bg.BlockGraph(+ dit.transformer_blocks[0], lambda *a, **k: calls.append(1) or "eager", shared+ )+ monkeypatch.setattr(bg._WeightView, "placement", lambda self, index: (1,))+ monkeypatch.setattr(+ torch.cuda, "current_device", lambda: 0+ ) # the block's card (1) is not current+ monkeypatch.setattr(bg, "_walk", lambda obj, live: live.append(_FakeCuda(1)) or ("k",))+ monkeypatch.setattr(bg, "_refusal", lambda key, kwargs: None)+ with torch.no_grad():+ assert graph(torch.zeros(2, 64)) == "eager" and calls+ assert graph.stats["refused_host_input"] == 1 and not graph.cache and not graph.seen+++class _FakeCuda:+ def __init__(self, index):+ self.device = torch.device("cuda", index)+++def test_arm_block_graphs_records_a_split_denoiser_per_block(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ armed = []+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)+ monkeypatch.setattr(+ bg,+ "install_block_graphs",+ lambda t, **k: armed.append(t) or (types.SimpleNamespace(slots_mib = 0), "armed"),+ )+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])+ applied = {"cuda_graph": False}+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")+ handles = cg.arm_block_graphs(pipe, applied, target = target, family = None, hooked = False)+ assert armed == [dit] and len(handles) == 1 and applied["cuda_graph"] is True+ assert pipe._unsloth_cuda_graph_mode == "blocks"+ assert "split denoiser" in pipe._unsloth_cuda_graph_reason+ ls.undo_layer_split(pipe)+++def test_split_block_graphs_honour_the_kill_switch(monkeypatch):+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "0")+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ monkeypatch.setattr(g8, "after_layer_split", lambda t, logger = None: {})+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)+ monkeypatch.setattr(+ bg, "install_block_graphs", lambda t, **k: pytest.fail("armed with the kill switch set")+ )+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")+ assert (+ cg.arm_block_graphs(pipe, {"cuda_graph": True}, target = target, family = None, hooked = False)+ == ()+ )+ assert "UNSLOTH_DIFFUSION_BLOCK_GRAPHS=0" in pipe._unsloth_cuda_graph_reason+ ls.undo_layer_split(pipe)+++def test_arm_block_graphs_leaves_a_resident_one_card_denoiser_alone(monkeypatch):+ dit = FakeDiT(dim = 64, double = 2, single = 2)+ pipe = Pipe(dit)+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)+ monkeypatch.setattr(+ bg, "install_block_graphs", lambda t, **k: pytest.fail("armed per block on one card")+ )+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])+ applied = {"cuda_graph": False}+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")+ assert cg.arm_block_graphs(pipe, applied, target = target, family = None, hooked = False) == ()+++@_needs_diffusers+def test_split_block_classes_register_with_a_copy_of_their_base_metadata(monkeypatch):+ from diffusers.hooks._helpers import TransformerBlockMetadata, TransformerBlockRegistry++ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ base_meta = {}+ for base in (+ type(dit.transformer_blocks[0])._unsloth_layer_split_base,+ type(dit.single_transformer_blocks[0])._unsloth_layer_split_base,+ ):+ meta = TransformerBlockMetadata(return_hidden_states_index = 0)+ TransformerBlockRegistry.register(model_class = base, metadata = meta)+ base_meta[base] = meta+ try:+ assert dc._register_layer_split_blocks(dit)+ for block in (dit.transformer_blocks[0], dit.single_transformer_blocks[0]):+ sub = type(block)+ base = sub._unsloth_layer_split_base+ got = TransformerBlockRegistry.get(sub)+ assert (+ got is not base_meta[base]+ ) # never the base's own object: register writes _cls into it+ assert got.return_hidden_states_index == 0+ assert base_meta[base]._cls is base+ finally:+ for cls in list(TransformerBlockRegistry._registry):+ if getattr(cls, "__name__", "") in ("FakeBlock", "FakeSingle"):+ TransformerBlockRegistry._registry.pop(cls, None)+ ls.undo_layer_split(pipe)+++def _tail_hook(head_output):+ from diffusers.hooks.first_block_cache import FBCBlockHook, FBCSharedBlockState+ from diffusers.hooks._helpers import TransformerBlockMetadata+ from diffusers.hooks.hooks import StateManager++ manager = StateManager(FBCSharedBlockState, (), {})+ try:+ from diffusers.hooks.hooks import CacheContext+ manager.set_context(CacheContext(name = "cond"))+ except ImportError: # an older diffusers keys the state by a plain name+ manager.set_context("cond")+ hook = FBCBlockHook(manager, is_tail = True)+ hook._metadata = TransformerBlockMetadata(return_hidden_states_index = 0)+ hook._metadata._cached_parameter_indices = {"hidden_states": 0}+ state = manager.get_state()+ state.head_block_output = head_output+ return hook, state+++@_needs_diffusers+def test_split_guard_leaves_a_one_card_tail_call_unchanged():+ assert dc.install_fbcache_split_guard()+ assert dc.install_fbcache_split_guard() # idempotent+ from diffusers.hooks.first_block_cache import FBCBlockHook++ assert getattr(FBCBlockHook.new_forward, "_unsloth_split_guard", False)+ head = torch.randn(2, 8)+ hook, state = _tail_hook(head)+ x = torch.randn(2, 8)+ fn = types.SimpleNamespace(original_forward = lambda h: h * 2)+ hook.fn_ref = fn+ out = hook.new_forward(None, x)+ assert torch.equal(out, x * 2)+ assert state.head_block_output is head+ assert torch.equal(state.tail_block_residuals[0], x * 2 - head)+++@_needs_diffusers+def test_split_guard_computes_the_tail_residual_on_the_tail_card_and_returns_it_home(monkeypatch):+ dc.install_fbcache_split_guard()+ moves = []++ head = torch.randn(2, 8)+ hook, state = _tail_hook(head)+ x = torch.randn(2, 8)+ seen = {}++ def original_forward(h):+ seen["head"] = state.head_block_output+ return h * 2++ hook.fn_ref = types.SimpleNamespace(original_forward = original_forward)+ home, here = torch.device("cuda", 0), torch.device("cuda", 1)+ monkeypatch.setattr(dc, "_first_device", lambda obj: home if obj is head else None)+ monkeypatch.setattr(+ hook._metadata,+ "_get_parameter_from_args_kwargs",+ lambda name, args, kwargs: types.SimpleNamespace(device = here),+ )+ copied = torch.randn(2, 8)+ monkeypatch.setattr(+ dc, "_tensors_to", lambda obj, device: moves.append(("head", device)) or copied+ )+ real_to = torch.Tensor.to++ def fake_to(self, *args, **kwargs):+ moves.append(("residual", args[0] if args else kwargs.get("device")))+ return self++ monkeypatch.setattr(torch.Tensor, "to", fake_to)+ out = hook.new_forward(None, x)+ monkeypatch.setattr(torch.Tensor, "to", real_to)+ assert seen["head"] is copied # the residual subtracts the copy that sits on the tail's card+ assert state.head_block_output is head # the shared state keeps the head card's tensor+ assert ("head", here) in moves and ("residual", home) in moves+ assert torch.equal(out, x * 2)+++@_needs_diffusers+def test_apply_step_cache_engages_on_a_split_through_the_guard(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ guarded = []+ monkeypatch.setattr(dc, "install_fbcache_split_guard", lambda: guarded.append(1) or True)+ monkeypatch.setattr(dc, "_register_layer_split_blocks", lambda t: True)+ monkeypatch.setattr(dc, "_pipeline_opens_cache_context", lambda p: True)+ monkeypatch.setattr(dc, "_reuses_prefix_kv", lambda p, t: False)+ enabled = []+ dit.enable_cache = lambda config: enabled.append(config)+ assert dc.apply_step_cache(pipe, mode = "fbcache") == "fbcache"+ assert guarded and enabled+ ls.undo_layer_split(pipe)+++def test_apply_step_cache_stays_off_on_a_split_without_the_guard(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ monkeypatch.setattr(dc, "install_fbcache_split_guard", lambda: False)+ dit.enable_cache = lambda config: pytest.fail("cache engaged without the split guard")+ assert dc.apply_step_cache(pipe, mode = "fbcache") is None+ ls.undo_layer_split(pipe)+++def _int8_dit(monkeypatch, cards):+ """Every Linear eligible; its weight's card read from ``cards`` (module -> ordinal)."""+ dit = FakeDiT(dim = 64, double = 2, single = 2)+ recs = {}+ for m in dit.modules():+ if type(m) is nn.Linear:+ recs[id(m)] = ("v2", None, torch.ones(1), m.weight)+ monkeypatch.setattr(g8, "_eligible", lambda m: recs.get(id(m)))+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))+ monkeypatch.setattr(g8, "_op", lambda: "op")+ monkeypatch.setattr(+ "core.inference.diffusion_convrot_quant.rotquant_device_config", lambda index: None+ )+ monkeypatch.setattr(g8, "_v1_act_quant_matches", lambda index: True)+ return dit, recs+++def _card1_singles(dit) -> dict:+ return {id(m): 1 for block in dit.single_transformer_blocks for m in block.modules()}+++def test_split_int8_swap_probes_and_swaps_each_card(monkeypatch):+ dit, recs = _int8_dit(monkeypatch, {})+ cards = _card1_singles(dit)+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))+ monkeypatch.setattr(g8, "_split_cards", lambda t: (0, 1))+ monkeypatch.setattr("core.inference.diffusion_int8_fused.resident_cuda_device", lambda t: None)+ probed = []+ monkeypatch.setattr(+ g8, "device_config", lambda index: probed.append(index) or ((1,) if index == 0 else None)+ )+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))+ try:+ count = g8._swap(dit)+ assert set(probed) == {0, 1}+ linears = [m for m in dit.modules() if id(m) in recs]+ on0 = [m for m in linears if cards.get(id(m), 0) == 0]+ assert on0 and count == len(on0)+ for m in linears:+ if cards.get(id(m), 0) == 0:+ assert m.__dict__[g8._REC][4] is True and g8.is_installed(m)+ else:+ assert not g8.is_installed(m)+ finally:+ g8.uninstall(dit)+++def test_split_int8_swap_refreshes_a_pre_split_onload_install(monkeypatch):+ dit, recs = _int8_dit(monkeypatch, {})+ cards = _card1_singles(dit)+ monkeypatch.setattr(g8, "_weight_card", lambda m: cards.get(id(m), 0))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))+ monkeypatch.setattr(g8, "device_config", lambda index: (1,) if index == 0 else None)+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())+ try:+ assert g8._swap_on(dit, None, 0) == len(recs)+ moved = [m for m in dit.modules() if id(m) in recs and cards.get(id(m), 0) == 1]+ assert moved and all(m.__dict__[g8._REC][4] is True for m in moved)+ monkeypatch.setattr(g8, "_split_cards", lambda t: (0, 1))+ monkeypatch.setattr(g8, "device_config", lambda index: (2,) if index == 1 else (1,))+ assert g8._swap_on(dit, None, 1) == len(moved)+ assert all(m.__dict__[g8._REC][4] is True for m in moved)+ monkeypatch.setattr(g8, "device_config", lambda index: None)+ assert (+ g8._swap_on(dit, None, 1) == 0+ ) # nothing to swap there and no rotation: count 0, records untouched+ finally:+ g8.uninstall(dit)+++def test_one_card_int8_swap_never_reads_a_weight_card(monkeypatch):+ dit, recs = _int8_dit(monkeypatch, {})+ monkeypatch.setattr(g8, "_weight_card", lambda m: pytest.fail("read a weight card on one card"))+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())+ monkeypatch.setattr(g8, "device_config", lambda index: (1,))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))+ try:+ assert g8._swap_on(dit, None, 0) == len(recs)+ finally:+ g8.uninstall(dit)+++def _swiglu_module(stale_device = "meta"):+ from core.inference import diffusion_int8_fused as f8++ m = nn.Module()+ m.w1, m.w3, m.w2 = (+ nn.Linear(16, 32, bias = False),+ nn.Linear(16, 32, bias = False),+ nn.Linear(32, 16, bias = False),+ )+ stale = nn.Linear(16, 64, bias = False, device = stale_device)+ parts = (("w1", m.w1), ("w3", m.w3), ("w2", m.w2))+ m.__dict__[f8._SWIGLU_ATTR] = (stale, parts, "w2", 32, 0, 32, 0, 0)+ return m, stale+++def test_a_stale_fused_swiglu_is_rebuilt_on_its_parts_card_and_shares_their_storage():+ from core.inference import diffusion_int8_fused as f8++ m, stale = _swiglu_module()+ gate = m.w1.weight.detach().clone()+ out = f8.relocate_after_layer_split(m)+ rec = m.__dict__[f8._SWIGLU_ATTR]+ assert out["rebuilt"] == 1 and rec[0] is not stale+ assert rec[0].weight.device == m.w1.weight.device+ assert rec[0].weight.data_ptr() == m.w1.weight.data_ptr()+ assert torch.equal(m.w1.weight, gate) and rec[1:] == m.__dict__[f8._SWIGLU_ATTR][1:]+ assert f8.relocate_after_layer_split(m) == {+ "rebuilt": 0,+ "dropped": 0,+ "restored": 0,+ } # idempotent+++def test_a_fused_qkv_that_cannot_be_rebuilt_is_dropped():+ from core.inference import diffusion_int8_fused as f8++ attn = nn.Module()+ attn.to_q, attn.to_k = nn.Linear(16, 16), nn.Linear(16, 16)+ attn.to_v = nn.Linear(16, 16).to(torch.float16) # mixed dtypes: no fused Linear over the three+ attn.__dict__["_unsloth_fused_qkv"] = (+ nn.Linear(16, 48, device = "meta"),+ attn.to_q,+ attn.to_k,+ attn.to_v,+ )+ out = f8.relocate_after_layer_split(attn)+ assert out["dropped"] == 1 and "_unsloth_fused_qkv" not in attn.__dict__+++def test_fused_qkv_readers_ignore_a_fused_linear_on_another_card():+ from core.inference import diffusion_qwenimage21_fused as q21+ from core.inference import diffusion_zimage_fused as zf+ for mod in (zf, q21):+ attn = nn.Module()+ attn.to_q, attn.to_k, attn.to_v = nn.Linear(8, 8), nn.Linear(8, 8), nn.Linear(8, 8)+ attn.__dict__[mod._QKV_ATTR] = (+ nn.Linear(8, 24, device = "meta"),+ attn.to_q,+ attn.to_k,+ attn.to_v,+ )+ assert mod._qkv_intact(attn) is None+ attn.__dict__[mod._QKV_ATTR] = (nn.Linear(8, 24), attn.to_q, attn.to_k, attn.to_v)+ assert mod._qkv_intact(attn) is not None+++def test_after_layer_split_is_a_no_op_on_one_card(monkeypatch):+ called = []+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())+ monkeypatch.setattr(+ "core.inference.diffusion_int8_fused.relocate_after_layer_split",+ lambda t, l = None: called.append(t),+ )+ assert g8.after_layer_split(nn.Linear(4, 4)) == {"cards": (), "gemm": 0}+ assert not called+++def test_arm_block_graphs_re_homes_the_split_records_before_arming(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, _ = _split(monkeypatch, dit)+ order = []+ monkeypatch.setattr(+ g8, "after_layer_split", lambda t, logger = None: order.append(("rehome", t)) or {}+ )+ monkeypatch.setattr(cg, "_block_basics", lambda *a, **k: None)+ monkeypatch.setattr(+ bg,+ "install_block_graphs",+ lambda t, **k: order.append(("arm", t)) or (types.SimpleNamespace(slots_mib = 0), "armed"),+ )+ monkeypatch.setattr("core.inference.diffusion_speed._denoiser_dits", lambda p: [p.transformer])+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:0")+ cg.arm_block_graphs(pipe, {"cuda_graph": False}, target = target, family = None, hooked = False)+ assert order == [("rehome", dit), ("arm", dit)]+ ls.undo_layer_split(pipe)+++def test_segments_are_per_card_runs_within_one_block_list(monkeypatch):+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, card = _split(monkeypatch, dit)+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))+ handle, reason = bg.install_block_graphs(dit, device = "cuda:0")+ assert handle is not None, reason+ runs = [[ls.find_blocks(dit)[i][0] for i in range(len(ls.find_blocks(dit)))]]+ quals = {id(b): q for q, _, b in ls.find_blocks(dit)}+ got = [[quals[id(g.block)] for g in seg.graphs] for seg in handle.segments]+ for run in got:+ assert len(run) >= 2+ assert len({card[q] for q in run}) == 1+ assert len({q.split(".")[0] for q in run}) == 1+ assert handle.mode == "segments" and handle.describe()["segments"] == len(handle.segments)+ assert runs # forward order list exists+ handle.free()+ assert not handle.segments and all(g.segment is None for g in handle.graphs)+ ls.undo_layer_split(pipe)+++def test_segment_kill_switch_keeps_per_block_graphs(monkeypatch):+ monkeypatch.setenv(bg.SEGMENT_GRAPHS_ENV, "0")+ dit = FakeDiT(dim = 1024, double = 4, single = 3)+ pipe, _, card = _split(monkeypatch, dit)+ by_block = {id(b): card[q] for q, _, b in ls.find_blocks(dit)}+ monkeypatch.setattr(bg, "_block_card", lambda block: by_block.get(id(block)))+ handle, _ = bg.install_block_graphs(dit, device = "cuda:0")+ assert handle is not None and not handle.segments and handle.mode == "blocks"+ handle.free()+ ls.undo_layer_split(pipe)+++def _replaying_segment(n = 3):+ shared = bg._Shared(1, None)+ blocks = [nn.Linear(4, 4) for _ in range(n)]+ graphs = [bg.BlockGraph(b, b.forward, shared) for b in blocks]+ seg = bg.SegmentGraph(graphs, shared)+ rec = bg._SegRecord()+ x, temb = torch.zeros(2, 4), torch.ones(2, 4)+ outs = [[torch.full((2, 4), float(i))] for i in range(n)]+ for i in range(n):+ live: list = []+ rec.keys.append(bg._walk(((outs[i - 1][0] if i else x, temb), {}), live))+ rec.prov.append((("e", 0), ("e", 1)) if i == 0 else (("o", i - 1, 0), ("e", 1)))+ rec.out_specs.append(("t", 0))+ entry = bg._SegEntry()+ entry.record, entry.outputs, entry.replays = rec, outs, 1+ seg.mode, seg.cur, seg.ext = "replay", entry, [x, temb]+ seg.returned = [outs[0]]+ seg.versions = [[bg._version(t) for t in outs[0]]]+ seg.pos = 1+ return seg, outs, temb+++def test_a_replayed_segment_hands_on_each_blocks_recorded_outputs():+ seg, outs, temb = _replaying_segment()+ mid = seg.call(1, (outs[0][0], temb), {})+ assert mid is outs[1][0] # the next block must receive exactly this tensor+ last = seg.call(2, (mid, temb), {})+ assert torch.equal(last, outs[2][0]) and last is not outs[2][0] # what leaves the run is a copy+ assert seg.pos is None and seg.stats["segment_breaks"] == 0+++def test_a_segment_breaks_when_a_block_gets_anything_else():+ seg, outs, temb = _replaying_segment()+ rewritten = outs[0][0] + 0 # an op between blocks: same values, another tensor+ assert seg.call(1, (rewritten, temb), {}) is bg._NOT_HANDLED+ assert seg.stats["segment_breaks"] == 1 and seg.pos is None+ assert seg.call(2, (rewritten, temb), {}) is bg._NOT_HANDLED+++def test_a_segment_breaks_on_a_fresh_copy_of_a_shared_input():+ seg, outs, temb = _replaying_segment()+ assert seg.call(1, (outs[0][0], temb.clone()), {}) is bg._NOT_HANDLED+ assert seg.stats["segment_breaks"] == 1+++def test_a_segment_breaks_when_an_op_between_blocks_rewrites_an_output_in_place():+ seg, outs, temb = _replaying_segment()+ outs[0][0].add_(100)+ assert seg.call(1, (outs[0][0], temb), {}) is bg._NOT_HANDLED+ assert seg.stats["segment_breaks"] == 1 and seg.pos is None+++def test_a_segment_starts_only_on_a_lazily_routed_chain(monkeypatch):+ walked = []+ real_walk = bg._walk+ monkeypatch.setattr(bg, "_walk", lambda tree, live: walked.append(1) or real_walk(tree, live))+ shared = bg._Shared(1, None)+ root = nn.Linear(4, 4)+ shared.root = root+ blocks = [nn.Linear(4, 4) for _ in range(2)]+ seg = bg.SegmentGraph([bg.BlockGraph(b, b.forward, shared) for b in blocks], shared)+ state = types.SimpleNamespace(lazy_now = False)+ root.__dict__[ls._SPLIT_ATTR + "_root"] = state+ x = torch.zeros(2, 4)+ with torch.inference_mode():+ assert seg.call(0, (x,), {}) is bg._NOT_HANDLED+ assert walked == [] # declined before it looked at the inputs+ state.lazy_now = True+ assert seg.call(0, (x,), {}) is bg._NOT_HANDLED # host tensors: declined by the next check+ assert walked # it got past the gate+++def test_a_segment_never_starts_while_a_step_cache_is_on(monkeypatch):+ walked = []+ real_walk = bg._walk+ monkeypatch.setattr(bg, "_walk", lambda tree, live: walked.append(1) or real_walk(tree, live))+ shared = bg._Shared(1, None)+ root = nn.Linear(4, 4)+ shared.root = root+ root.__dict__[ls._SPLIT_ATTR + "_root"] = types.SimpleNamespace(lazy_now = True)+ root._unsloth_step_cache = "[email protected]"+ seg = bg.SegmentGraph(+ [bg.BlockGraph(nn.Linear(4, 4), lambda x: x, shared) for _ in range(2)], shared+ )+ with torch.inference_mode():+ assert seg.call(0, (torch.zeros(2, 4),), {}) is bg._NOT_HANDLED+ assert walked == []+++def test_video_generate_bypasses_graphs_after_the_auto_cache_toggle():+ import inspect++ from core.inference import video as V++ src = inspect.getsource(V.VideoBackend)+ toggle = src.index("toggled = maybe_toggle_step_cache(")+ bypass = src.index("diffusion_cuda_graph.set_bypass(", toggle)+ assert "cache_breaks_graph(state.transformer_cache)" in src[bypass : bypass + 200]+ assert bypass < src.index("if static_skip:", toggle)+++@_needs_diffusers+def test_split_guard_wraps_the_mixed_stream_tail_once():+ from diffusers.hooks.first_block_cache import FBCBlockHook++ assert dc.install_fbcache_split_guard()+ outer = FBCBlockHook.new_forward+ assert outer._unsloth_split_guard and outer._unsloth_mixed_stream_tail+ inner = outer.__closure__ and [+ c.cell_contents for c in outer.__closure__ if callable(c.cell_contents)+ ]+ assert any(+ getattr(fn, "_unsloth_mixed_stream_tail", False)+ and not getattr(fn, "_unsloth_split_guard", False)+ for fn in inner+ )+ assert dc.install_fbcache_mixed_stream_tail() and dc.install_fbcache_split_guard()+ assert FBCBlockHook.new_forward is outer # neither re-wraps the other+++def test_resident_cuda_cards_lists_every_card_of_a_split(monkeypatch):+ from core.inference import diffusion_int8_fused as f8++ monkeypatch.setattr(f8, "resident_cuda_device", lambda m: None)+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1))+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == (0, 1)+ monkeypatch.setattr(g8, "_split_cards", lambda m: ())+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == ()+ monkeypatch.setattr(f8, "resident_cuda_device", lambda m: torch.device("cuda", 3))+ assert f8.resident_cuda_cards(nn.Linear(2, 2)) == (3,)+++def test_qwen_real_rope_installs_on_a_split_after_probing_each_card(monkeypatch):+ from core.inference import diffusion_int8_fused as f8+ from core.inference import diffusion_qwenimage_rope as qr++ QwenImageTransformer2DModel = type("QwenImageTransformer2DModel", (nn.Module,), {})+ monkeypatch.setattr(qr, "disabled", lambda: False)+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (0, 1))+ probed, patched = [], []+ monkeypatch.setattr(qr, "_probe", lambda index: probed.append(index) or True)+ monkeypatch.setattr(+ qr, "_patch_table", lambda index, logger = None: patched.append(index) or True+ )+ assert qr.install(QwenImageTransformer2DModel()) is True+ assert probed == [0, 1] and patched == [0]+ monkeypatch.setattr(qr, "_probe", lambda index: False)+ assert qr.install(QwenImageTransformer2DModel()) is False+++def test_zimage_fused_probes_each_card_of_a_split(monkeypatch):+ from core.inference import diffusion_int8_fused as f8+ from core.inference import diffusion_qwenimage21_rope as q21r+ from core.inference import diffusion_zimage_fused as zf++ monkeypatch.setattr(zf, "zimage_fused_disabled", lambda: False)+ monkeypatch.setattr(zf, "_patch_class", lambda logger = None: True)+ monkeypatch.setitem(+ sys.modules, zf._MODULE, types.SimpleNamespace(ZSingleStreamAttnProcessor = object)+ )+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (5, 6))+ monkeypatch.setattr(q21r, "inductor_addcmul_is_fma", lambda: True)+ monkeypatch.setattr(q21r, "probe_fusion", lambda dev: ("fma", dev.index))+ monkeypatch.setattr(zf, "_FUSION", {})+ out = zf.install_modules(nn.Linear(2, 2))+ assert out["real_rope"] is True and set(zf._FUSION) == {5, 6}+ monkeypatch.setattr(zf, "_FUSION", {5: ("fma", 5), 6: None})+ assert zf.install_modules(nn.Linear(2, 2))["real_rope"] is False+++def test_q21_fused_checks_the_arch_of_every_split_card(monkeypatch):+ from core.inference import diffusion_int8_fused as f8+ from core.inference import diffusion_qwenimage21_fused as q21f++ monkeypatch.setattr(q21f, "q21_convrot_fused_disabled", lambda: False)+ monkeypatch.setattr(f8, "resident_cuda_cards", lambda m: (0, 1))+ seen = []+ monkeypatch.setattr(q21f, "arch_ok", lambda dev: seen.append(dev.index) or dev.index == 0)+ assert q21f.install_modules(nn.Linear(2, 2)) == {"fused_qkv": 0, "out": 0}+ assert seen == [0, 1]+++def test_hunyuan_image_and_hidream_route_lazily():+ assert "HunyuanImageTransformer2DModel" in ls.LAZY_CLASSES+ assert "HiDreamImageTransformer2DModel" in ls.LAZY_CLASSES+++def test_real_rope_probes_every_card_of_a_split(monkeypatch):+ from core.inference import diffusion_qwenimage21_rope as q21r++ monkeypatch.setattr(q21r, "real_rope_disabled", lambda: False)+ monkeypatch.setattr(q21r, "_FUSION", {})+ assert q21r.probe_split_cards((0, 1)) == () # no install probed a card: the patch is not in use+ probed = []+ monkeypatch.setattr(+ q21r,+ "probe_fusion",+ lambda dev: probed.append(dev.index) or (None if dev.index == 2 else "fma"),+ )+ monkeypatch.setattr(q21r, "_FUSION", {0: "fma"})+ assert q21r.probe_split_cards((0, 1, 2)) == (0, 1)+ assert probed == [1, 2]+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1))+ seen = []+ monkeypatch.setattr(q21r, "probe_split_cards", lambda cards: seen.append(cards) or cards)+ assert g8.after_layer_split(nn.Linear(2, 2))["rope_cards"] == (0, 1) and seen == [(0, 1)]+++def test_split_graphs_seal_new_prompt_lengths_per_image_scope():+ den = nn.Linear(4, 4)+ handle = bg.BlockGraphSet(den, bg._Shared(0, None))+ handle.track_scopes()+ assert handle.length_varies and not handle.scope_sealed()+ den(torch.randn(2, 4)) # the pre-hook reads this call's image scope+ first = handle.scope+ handle.note_capture()+ cg.seal_lengths((handle,))+ assert handle.scope_sealed()+ den(torch.randn(3, 4)) # another batch: a new image scope still records+ assert handle.scope != first and not handle.scope_sealed()+ cg.seal_lengths((handle,)) # nothing recorded in that scope: its window stays open+ assert not handle.scope_sealed()+ den(torch.randn(2, 4))+ assert handle.scope_sealed()+ handle.reset()+ assert not handle.scope_sealed()+ for restore in handle.restores:+ restore()+ assert not den._forward_pre_hooks+++def test_a_split_records_block_graphs_for_a_prompt_length_family(monkeypatch):+ import inspect++ src = inspect.getsource(cg.arm_block_graphs)+ assert "None if split else prompt_length_reason(pipe, family)" in src+ assert "handle.track_scopes()" in inspect.getsource(bg.install_block_graphs)+++def test_zimage_real_rope_probes_every_card_of_a_split(monkeypatch):+ from core.inference import diffusion_qwenimage21_rope as q21r+ from core.inference import diffusion_zimage_fused as zf++ monkeypatch.setattr(zf, "zimage_fused_disabled", lambda: False)+ probed = []+ monkeypatch.setattr(q21r, "inductor_addcmul_is_fma", lambda: True)+ monkeypatch.setattr(+ q21r,+ "probe_fusion",+ lambda dev: probed.append(dev.index) or (None if dev.index == 2 else "x"),+ )+ monkeypatch.setattr(zf, "_FUSION", {0: "x"})+ monkeypatch.setattr(q21r, "_FUSION", {})+ monkeypatch.setattr(g8, "_split_cards", lambda m: (0, 1, 2))+ out = g8.after_layer_split(nn.Linear(2, 2))+ assert zf._FUSION == {0: "x", 1: "x", 2: None} and probed == [1, 2]+ assert out["zimage_rope_cards"] == (0, 1)+ monkeypatch.setattr(zf, "_FUSION", {})+ assert zf.probe_split_cards((0, 1)) == () # no install probed a card: the patch is not in usediff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex ddc38d41a31..e388153912d 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -875,13 +875,22 @@ def test_encode_release_only_for_the_whole_tier(monkeypatch):def test_generate_load_installs_the_encode_release():+ import ast+src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(encoding = "utf-8")- at = src.index("install_encode_release(pipe, plan, logger)")- # after the placement is applied and before the int8 GEMM reads the final residency- assert src.index("effective_policy, effective_tiling = apply_memory_plan(") < at- assert at < src.index("if denoisers_pinned_resident(pipe):", at)+ calls: dict = {}+ for node in ast.walk(ast.parse(src)):+ if isinstance(node, ast.Call):+ fn = node.func+ name = fn.id if isinstance(fn, ast.Name) else getattr(fn, "attr", None)+ calls.setdefault(name, []).append(node.lineno)+ (at,) = calls["install_encode_release"]+ assert any(line < at for line in calls["apply_memory_plan"])+ pinned = min(line for line in calls["denoisers_pinned_resident"] if line > at)+ engage = min(line for line in calls["engage_pinned_denoisers"] if line > at)+ assert at < pinned < engagedef _inner_devices(tensor):diff --git a/studio/backend/tests/test_diffusion_prompt_length_policy.py b/studio/backend/tests/test_diffusion_prompt_length_policy.pyindex cda7a40dd33..efec7bf2b34 100644--- a/studio/backend/tests/test_diffusion_prompt_length_policy.py+++ b/studio/backend/tests/test_diffusion_prompt_length_policy.py@@ -71,3 +71,19 @@ def test_the_compile_arms_only_its_own_sources():}for name, sources in dt._FAMILY_SOURCES.items():assert dt.PROMPT_LENGTH_POLICY[name] == sources, name+++def test_qwen_image_arms_its_sources_on_the_dynamic_tier_too():+ class Qwen:+ pass++ class Flux3:+ pass++ Qwen.__name__ = "QwenImageTransformer2DModel"+ Flux3.__name__ = "Flux3Transformer2DModel"+ if not dt.supported():+ return+ assert dt._armed_when_dynamic(Qwen()) and not dt._armed_when_dynamic(Flux3())+ assert dt.fingerprint(Qwen(), True) == dt.fingerprint(Qwen(), None)+ assert dt.fingerprint(Flux3(), True) != dt.fingerprint(Flux3(), None)diff --git a/studio/backend/tests/test_diffusion_reduction_filter_arch.py b/studio/backend/tests/test_diffusion_reduction_filter_arch.pyindex ad833981fe3..1142b6b4962 100644--- a/studio/backend/tests/test_diffusion_reduction_filter_arch.py+++ b/studio/backend/tests/test_diffusion_reduction_filter_arch.py@@ -6,8 +6,8 @@ one image in every fresh server there.A few of their norm reductions get several configs that sum in different orders, and a cold-cache server benchmarksthem in its own process on near-equal timings: fresh servers rendered 2-4 variants per seed on an RTX PRO 6000, A100-and L4. B200 servers were already deterministic for these families, so the opt-in is per (family, compute-capability)."""+and L4. B200 servers were deterministic for FLUX.1 and Z-Image but not Qwen-Image, so the opt-in is per (family,+compute capability)."""from __future__ import annotations@@ -23,6 +23,7 @@ from core.inference import diffusion_speed as ds # noqa: E402_SM120 = (12, 0)_MEASURED_ARCHS = ((8, 0), (8, 9), (12, 0))_IMAGE_FAMILIES = ("flux.1", "z-image", "qwen-image", "flux.1-kontext", "qwen-image-edit")+_QWEN_FAMILIES = ("qwen-image", "qwen-image-edit", "qwen-image-layered")def _family(name):@@ -65,17 +66,31 @@ def test_image_family_opts_in_on_the_measured_archs(monkeypatch, name, cap):def test_the_race_families_share_one_arch_list():from core.inference import diffusion_families+for name in _IMAGE_FAMILIES:- assert (- _family(name).filter_reduction_configs_archs == diffusion_families._REDUCTION_RACE_ARCHS+ expected = (+ diffusion_families._QWEN_REDUCTION_RACE_ARCHS+ if name in _QWEN_FAMILIES+ else diffusion_families._REDUCTION_RACE_ARCHS)+ assert _family(name).filter_reduction_configs_archs == expectedassert set(diffusion_families._REDUCTION_RACE_ARCHS) == set(_MEASURED_ARCHS)+ assert set(diffusion_families._QWEN_REDUCTION_RACE_ARCHS) == {*_MEASURED_ARCHS, (10, 0)}++[email protected]("name", _QWEN_FAMILIES)+def test_qwen_image_also_pins_on_b200(monkeypatch, name):+ _on(monkeypatch, (10, 0))+ fam = _family(name)+ assert cc.family_filters_reductions(fam) is True+ assert _filter_reaching_compile(monkeypatch, fam) is True@pytest.mark.parametrize("cap", [(10, 0), (10, 3), (9, 0), (7, 5), (12, 1), None])@pytest.mark.parametrize("name", _IMAGE_FAMILIES)def test_image_family_keeps_inductor_pick_elsewhere(monkeypatch, name, cap):- # B200 (measured deterministic), B300, H100, T4, other Blackwell parts (unmeasured), no CUDA / ROCm.+ if cap == (10, 0) and name in _QWEN_FAMILIES:+ pytest.skip("Qwen-Image pins on B200 (test_qwen_image_also_pins_on_b200)")_on(monkeypatch, cap)fam = _family(name)assert cc.family_filters_reductions(fam) is Falsediff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 1c6d0a766b7..16eeaac4e14 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -675,7 +675,7 @@ def test_cudnn_benchmark_opt_out_image_families():# Unmeasured families keep the benchmark.from core.inference.diffusion_families import _FAMILIESoff = {fam.name for fam in _FAMILIES if not fam.cudnn_benchmark}- assert off == {"qwen-image", "flux.1", "z-image", "sdxl"}+ assert off == {"qwen-image", "flux.1", "flux.2-dev", "z-image", "sdxl"}def test_speed_max_enables_tf32_and_fused_qkv(monkeypatch):diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 31be1b67a2f..b27436b8231 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -14078,3 +14078,48 @@ def test_an_automatic_encoder_pick_that_does_not_engage_loads_dense_without_a_reVideoBackend().load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", text_encoder_quant = "fp8")+++ "split_expected, hooked, moved",+ [+ (+ True,+ True,+ True,+ ), # the split fell back to a hooked one-card placement: the compile moves below the hooks+ (True, False, False), # the split landed resident: nothing to move+ (False, True, False), # planned offloaded: the speed layer already compiled for the hooks+ ],+)+def test_a_layer_split_fallback_moves_the_compile_below_the_offload_hooks(+ fake_runtime, tmp_path, monkeypatch, split_expected, hooked, moved+):+ import core.inference.video as video_mod++ _shared_setup_7(monkeypatch, video_mod)+ _stub_apply_memory_plan(monkeypatch, video_mod, policy = "model")+ monkeypatch.setattr(+ video_mod, "_layer_split_expected", lambda pipe, plan, refine = None: split_expected+ )+ monkeypatch.setattr(video_mod, "_video_denoiser_hooked", lambda pipe: hooked)+ seen = []+ monkeypatch.setattr(+ video_mod,+ "compile_pipe_below_offload_hooks",+ lambda pipe, logger = None: seen.append(pipe) or 0,+ )+ speed_offload = []+ from core.inference import diffusion_autoopt as autoopt++ real_speed = autoopt.run_speed_layer++ def _speed(view, target, **kw):+ speed_offload.append(kw.get("offload_active"))+ return real_speed(view, target, **kw)++ monkeypatch.setattr(autoopt, "run_speed_layer", _speed)+ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ assert speed_offload and set(speed_offload) == {not split_expected}+ assert (seen == [backend._state.pipe]) if moved else (seen == [])