Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: multi-GPU component placement and CFG-parallel (#13725)

Daniel Han811c08e782cf
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 7c1978bf850..99094681eed 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -106,6 +106,19 @@ from .diffusion_device import (
auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
+ resolve_selected_cuda_ordinals,
+)
+from .diffusion_multigpu import (
+ AUTOMATIC_ORDINALS,
+ end_cfg_render,
+ drop_displaced_cfg_router,
+ install_cfg_parallel,
+ multigpu_mode,
+ invalidate_cfg_replica,
+ placement_status as multigpu_placement_status,
+ refine_plan_for_multigpu,
+ uninstall_cfg_parallel,
+ with_load_selection,
)
from .diffusion_conditioning import (
LocalizedEdit,
@@ -2104,6 +2117,129 @@ def _plans_at_requested_speed(fn: Callable[..., Any]) -> Callable[..., Any]:
return wrapper
+def _vae_on_other_card(pipe: Any, guard_target: Any) -> Any:
+ """The CUDA device of ``pipe.vae`` when a multi-GPU split put it on another card than ``guard_target``, else None."""
+ from .diffusion_multigpu import _module_device, _split_home, is_split
+
+ if not is_split(pipe) and _split_home(pipe) is None:
+ return None
+ vae = getattr(pipe, "vae", None)
+ try:
+ device = _module_device(vae) if vae is not None else None
+ except Exception: # noqa: BLE001 - an unreadable VAE keeps the primary's verdict
+ return None
+ if device is None or getattr(device, "type", None) != "cuda":
+ return None
+ if device.index in (None, placed_cuda_ordinal(guard_target)):
+ return None
+ return device
+
+
+def _guard_vae_card(
+ pipe: Any, guard_target: Any, guard_kwargs: dict, restore_vae: Any, logger: Any
+) -> tuple[Any, bool]:
+ """Re-run the generate-time activation guard on the VAE's card when a multi-GPU split put it on another card than
+ the denoiser: the decode allocates there, and the primary's free memory says nothing about it. Engages VAE tiling
+ when only a tiled decode fits that card (unless already tiled) and raises the guard's refusal when nothing fits.
+ Returns ``(restore_vae, tiled)`` for a tiling it engaged, else ``(None, False)``. Conservative: the whole request's
+ estimate is checked against the VAE's card."""
+ device = _vae_on_other_card(pipe, guard_target)
+ if device is None:
+ return None, False
+ with diffusion_device_scope(device.index):
+ vae_kwargs = {
+ **guard_kwargs,
+ "device_memory": reclaimable_snapshot_device_memory(guard_target),
+ }
+ verdict = raise_on_image_activation_shortfall(**vae_kwargs)
+ if verdict.action != ACTIVATION_TILE or restore_vae is not None:
+ return None, False
+ restore, tiled = engage_vae_tiling(pipe, logger = logger)
+ try:
+ if not tiled and not verdict.overridden:
+ raise_on_image_activation_shortfall(**{**vae_kwargs, "vae_tile_side": None})
+ elif (
+ int(vae_kwargs.get("batch_size") or 1) > 1
+ and vae_kwargs.get("vae_sliced")
+ and not verdict.overridden
+ and not vae_is_sliced(getattr(pipe, "vae", None))
+ ):
+ raise_on_image_activation_shortfall(**{**vae_kwargs, "vae_sliced": False})
+ except ValueError:
+ # The refusal: the caller never receives the handle, so undo the tiling here.
+ if restore is not None:
+ restore()
+ raise
+ except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe and keep the tiling, as the caller does
+ (logger or get_logger(__name__)).warning(
+ "diffusion.generate: VAE card headroom re-check skipped (%s)", exc
+ )
+ except BaseException:
+ if restore is not None:
+ restore()
+ raise
+ return restore, tiled
+
+
+def _tile_for_verdict(
+ pipe: Any,
+ verdict: Any,
+ guard_target: Any,
+ guard_kwargs: dict,
+ guard_batch: int,
+ restore_vae: Any,
+ logger: Any,
+) -> Any:
+ """Engage VAE tiling when the guard says only a tiled decode fits; returns the VAE restore handle. A multi-GPU split
+ that decodes on another card is decided by that card's verdict alone: the primary never runs the decode, so its
+ tile verdict must not switch the secondary's VAE to the (not bit-identical) tiled path."""
+ vae_restore, vae_card_tiled = _guard_vae_card(
+ pipe, guard_target, guard_kwargs, restore_vae, logger
+ )
+ if vae_card_tiled:
+ return vae_restore
+ if verdict.action != ACTIVATION_TILE or _vae_on_other_card(pipe, guard_target):
+ return restore_vae
+ restore_vae, vae_tiled = engage_vae_tiling(pipe, logger = logger)
+ try:
+ if not vae_tiled and not verdict.overridden:
+ # Passed only because it would be tiled; tiling failed, so refuse on the untiled figure.
+ raise_on_image_activation_shortfall(**{**guard_kwargs, "vae_tile_side": None})
+ elif (
+ guard_batch > 1
+ and guard_kwargs["vae_sliced"]
+ and not verdict.overridden
+ and not vae_is_sliced(getattr(pipe, "vae", None))
+ ):
+ raise_on_image_activation_shortfall(**{**guard_kwargs, "vae_sliced": False})
+ except ValueError:
+ # the caller never receives the handle on a refusal: undo the tiling here
+ if restore_vae is not None:
+ restore_vae()
+ raise
+ except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe and keep the tiling, as generate() does
+ (logger or get_logger(__name__)).warning(
+ "diffusion.generate: activation headroom re-check skipped (%s)", exc
+ )
+ except BaseException:
+ if restore_vae is not None:
+ restore_vae()
+ raise
+ return restore_vae
+
+
+def _controlnet_streams(state: Any) -> bool:
+ """Whether a ControlNet added at generate time streams from host memory instead of loading whole beside the base
+ model: under any offload policy, a calibrated placement (budgets only the base model), and a multi-GPU split
+ (reports "none", but its fit priced only the base pipeline, which one card would have offloaded)."""
+ from .diffusion_multigpu import is_split
+ return (
+ getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE
+ or bool(getattr(state, "calibrated_placement", False))
+ or is_split(getattr(state, "pipe", None))
+ )
+
+
def _calibrated_activation(fam: Any, target: Any) -> Any:
"""Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel."""
if getattr(target, "backend", None) != "cuda" or getattr(target, "vendor", None) != "nvidia":
@@ -4137,6 +4273,16 @@ class DiffusionBackend:
if resolve_diffusion_device_target().device == "cuda"
else None
)
+ gpu_ordinals = (
+ resolve_selected_cuda_ordinals(gpu_ids)
+ if gpu_ids
+ and len(set(gpu_ids)) > 1
+ and multigpu_mode() != "off"
+ and resolve_diffusion_device_target().device == "cuda"
+ else AUTOMATIC_ORDINALS
+ if not gpu_ids
+ else []
+ )
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
@@ -4222,11 +4368,14 @@ class DiffusionBackend:
vae_file = vae_file,
_load_token = token,
_cancel_event = cancel_event,
+ _gpu_ordinals = gpu_ordinals,
),
daemon = True,
).start()
return self.status()
+ # The multi-GPU selection (begin_load's ``_gpu_ordinals``) is visible to this load's planner and placement only.
+ @with_load_selection
@scoped_local_files_only
def _run_load(self, **kwargs: Any) -> None:
token = kwargs.get("_load_token")
@@ -9236,6 +9385,7 @@ class DiffusionBackend:
self._raise_if_load_cancelled(_load_token)
# a refinement may have re-tiered the plan; routed encoders need group offload
plan = self._small_host_plan(pipe, plan, fam, logger)
+ plan = refine_plan_for_multigpu(pipe, plan, logger)
effective_policy, effective_tiling = apply_memory_plan(
pipe,
plan,
@@ -9453,6 +9603,8 @@ class DiffusionBackend:
load_bg_compile = (
bg_compile.arm(bg_module, logger = logger) if bg_module is not None else None
)
+ # After every forward wrapper of the speed / graph layers, so the replica sees the final module.
+ install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)
te_releaser = te_release.maybe_install(pipe, plan, logger = logger)
state = _LoadState(
pipe = pipe,
@@ -10997,11 +11149,14 @@ class DiffusionBackend:
# Placement follows the base offload policy (resident base -> resident, offloaded -> group offload).
# Calibrated tiers budget only the base model: ControlNet must stream, no resident fallback.
calibrated = bool(getattr(state, "calibrated_placement", False))
- if (getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE or calibrated) and (
- _offload_controlnet_module(cn_model, state.device, logger)
+ from .diffusion_multigpu import is_split
+
+ # A split's resident fit also priced the base pipeline alone (one card would have offloaded it).
+ if _controlnet_streams(state) and _offload_controlnet_module(
+ cn_model, state.device, logger
):
pass
- elif calibrated:
+ elif calibrated or is_split(state.pipe):
del cn_model
clear_gpu_cache()
raise ValueError(
@@ -11357,6 +11512,7 @@ class DiffusionBackend:
try:
self._engage_deferred_profile(state)
except BaseException:
+ invalidate_cfg_replica(state.pipe, "the deferred speed profile failed part way", logger)
if installed:
uninstall_static_step_skip(state.pipe)
object.__setattr__(state, "transformer_cache", None)
@@ -11364,6 +11520,13 @@ class DiffusionBackend:
tc.clear()
tc.update(tc_before)
raise
+ # the profile patched / compiled the source denoiser: a replica built during the eager images is stale
+ invalidate_cfg_replica(
+ state.pipe, "the deferred speed profile changed the denoiser", logger
+ )
+ drop_displaced_cfg_router(
+ state.pipe, "the deferred speed profile replaced the denoiser's forward", logger
+ )
# Outside load_pipeline (3rd generation): its own probe scope, so one OOM is not re-probed by plan and apply.
@rocm_probe_load_scoped
@@ -12088,22 +12251,9 @@ class DiffusionBackend:
logger = logger,
)
verdict = raise_on_image_activation_shortfall(**guard_kwargs)
- if verdict.action == ACTIVATION_TILE:
- restore_vae, vae_tiled = engage_vae_tiling(pipe, logger = logger)
- if not vae_tiled and not verdict.overridden:
- # Passed only because it would be tiled; tiling failed, so refuse on the untiled figure.
- raise_on_image_activation_shortfall(
- **{**guard_kwargs, "vae_tile_side": None}
- )
- elif (
- guard_batch > 1
- and guard_kwargs["vae_sliced"]
- and not verdict.overridden
- and not vae_is_sliced(getattr(pipe, "vae", None))
- ):
- raise_on_image_activation_shortfall(
- **{**guard_kwargs, "vae_sliced": False}
- )
+ restore_vae = _tile_for_verdict(
+ pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger
+ )
except ValueError:
raise # the refusal itself: the route turns this into a 400 with the reason
except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe
@@ -12547,6 +12697,7 @@ class DiffusionBackend:
logger.debug("diffusion.step_skip: reset failed: %s", exc)
if restore_vae is not None:
restore_vae()
+ end_cfg_render(getattr(state, "pipe", None))
if restore_resident is not None:
try:
restore_resident()
@@ -12722,6 +12873,7 @@ class DiffusionBackend:
if prebuild is not None:
prebuild.cancel_all()
compile_cache.restore(state.compile_cache_ctx, logger = logger)
+ uninstall_cfg_parallel(state.pipe)
# Before clear_gpu_cache(), or the graph pool stays reserved for the life of the process.
cuda_graph.uninstall_all(state.cuda_graphs)
uninstall_static_step_skip(state.pipe)
@@ -12830,6 +12982,7 @@ class DiffusionBackend:
"attention_backend": state.attention_backend,
"transformer_cache": state.transformer_cache,
"transformer_cache_stats": static_skip_stats(state.pipe),
+ "multi_gpu": multigpu_placement_status(state.pipe),
"resolved": resolved,
"generation_defaults": _generation_defaults_for(
state.repo_id,
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index d6ff36eb24e..664d6eb8215 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -338,8 +338,9 @@ def resolve_selected_cuda_ordinal(
order (``4,5`` -> torch 0,1; ``1,0`` reverses them), hence going through the hardware layer
that owns the mask.
- Neither engine shards a checkpoint, so several cards still resolve to one: most free VRAM
- wins, as ``auto_select_gpu_ids`` already does for training, ties to the lowest ordinal. Taking
+ Neither engine shards a checkpoint, so several cards resolve to one PRIMARY card (the
+ multi-GPU layer, diffusion_multigpu, may place companions or a denoiser replica on the
+ others, see :func:`resolve_selected_cuda_ordinals`): most free VRAM wins, as ``auto_select_gpu_ids`` already does for training, ties to the lowest ordinal. Taking
the FIRST id instead would land on ordinal 0 whenever everything is selected, i.e. the small
card on the mixed boxes this exists for. Resolved ONCE per load and carried, never re-derived:
free VRAM moves the moment the checkpoint lands.
@@ -352,9 +353,36 @@ def resolve_selected_cuda_ordinal(
still run -- they read the mask and nvidia-smi -- so the single card the UI sends resolves and
only a multi-card pick comes back None.
"""
+ ordinals = resolve_selected_cuda_ordinals(gpu_ids)
+ if not ordinals:
+ return None
+ if len(ordinals) == 1:
+ return ordinals[0]
+ if not allow_ranking:
+ return None
+ return rank_cuda_ordinals(ordinals)[0]
+
+
+def rank_cuda_ordinals(ordinals: list[int]) -> list[int]:
+ """``ordinals`` by free VRAM, most first, ties to the lowest ordinal (an unreadable card sorts last)."""
+
+ def _free_vram(ordinal: int) -> int:
+ try:
+ import torch
+ return int(torch.cuda.mem_get_info(ordinal)[0])
+ except Exception: # noqa: BLE001 -- an unreadable card sorts last rather than failing the load
+ return -1
+
+ return sorted(ordinals, key = lambda ordinal: (-_free_vram(ordinal), ordinal))
+
+
+def resolve_selected_cuda_ordinals(gpu_ids: Optional[list[int]]) -> list[int]:
+ """Every torch ordinal of the selection, in ascending order (validated and translated through the mask exactly as
+ :func:`resolve_selected_cuda_ordinal` does). Empty for no selection; raises ValueError for one this host cannot
+ honour. The multi-GPU layer (diffusion_multigpu) uses the cards beyond the primary."""
wanted = sorted({int(gpu_id) for gpu_id in gpu_ids or ()})
if not wanted:
- return None
+ return []
try:
from utils.hardware.hardware import (
get_parent_visible_gpu_ids,
@@ -372,19 +400,7 @@ def resolve_selected_cuda_ordinal(
f"Requested GPU {wanted} but none of them are visible to this process "
f"(visible: {visible}). Clear the GPU selection to use the default device."
)
- if len(ordinals) == 1:
- return ordinals[0]
- if not allow_ranking:
- return None
-
- def _free_vram(ordinal: int) -> int:
- try:
- import torch
- return int(torch.cuda.mem_get_info(ordinal)[0])
- except Exception: # noqa: BLE001 -- an unreadable card sorts last rather than failing the load
- return -1
-
- return max(ordinals, key = lambda ordinal: (_free_vram(ordinal), -ordinal))
+ return sorted(dict.fromkeys(ordinals))
_UNPLANNED = object()
@@ -486,6 +502,37 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
return 0
+def smi_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
+ """(free, total) MiB of torch ordinal ``ordinal`` from nvidia-smi, so a card can be sized without a CUDA context
+ on it. (None, None) wherever ``resolve_auto_cuda_ordinal`` would not trust the rows (ROCm, a CUDA order other
+ than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). Never raises."""
+ try:
+ from utils.hardware import hardware as hw
+
+ if hw.IS_ROCM or not hw._cuda_order_matches_smi():
+ return None, None
+ visible = hw.get_parent_visible_gpu_ids()
+ if not 0 <= int(ordinal) < len(visible):
+ return None, None
+ index = visible[int(ordinal)]
+ if hw._integrated_cuda_rows(visible)[0]:
+ return None, None
+ mig = hw._nvidia_mig_gpu_indices_or_none()
+ if mig is None or index in mig:
+ return None, None
+ result = hw._smi_visible_utilization(hw._get_parent_visible_gpu_spec())
+ for device in (result or {}).get("devices", []):
+ if device.get("index") != index:
+ continue
+ total, used = device.get("vram_total_gb"), device.get("vram_used_gb")
+ if total is None or used is None:
+ return None, None
+ return int(max(float(total) - float(used), 0.0) * 1024), int(float(total) * 1024)
+ except Exception: # noqa: BLE001 -- an unreadable card is simply not used
+ pass
+ return None, None
+
+
def auto_cuda_ranking_possible_cached() -> Optional[bool]:
"""``auto_cuda_ranking_possible`` when it can be answered from state already in memory, else None.
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index ece7a4317e8..3a294c9790c 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import functools
import os
import sys
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
from typing import Any, Callable, Optional
@@ -614,6 +614,11 @@ class MemoryPlan:
# MiB of the streamed denoiser / encoders kept resident (whole groups); None streams every group.
resident_transformer_mib: Optional[int] = None
resident_text_encoder_mib: Optional[int] = None
+ secondary_ordinal: Optional[int] = None
+ secondary_components: Optional[tuple[str, ...]] = None
+ # The single-card plan a split replaced, applied on the primary if the split placement fails (a torchao denoiser
+ # cannot take the whole-module offload a blind fallback would choose).
+ fallback_plan: Optional["MemoryPlan"] = field(default = None, compare = False, repr = False)
@property
def engages_offload(self) -> bool:
@@ -632,6 +637,10 @@ class MemoryPlan:
"stream_transformer": self.stream_transformer,
"resident_transformer_mib": self.resident_transformer_mib,
"resident_text_encoder_mib": self.resident_text_encoder_mib,
+ "secondary_ordinal": self.secondary_ordinal,
+ "secondary_components": (
+ list(self.secondary_components) if self.secondary_components is not None else None
+ ),
}
@@ -1965,6 +1974,8 @@ def plan_diffusion_memory(
reasons.append("companions exceed budget; whole-module offload of every component")
# The legacy cpu_offload flag applies only when no memory_mode was supplied, so an explicit `fast` stays resident.
+ if explicit_offload and normalize_memory_mode(requested_mode) is None:
+ estimates["explicit_cpu_offload"] = 1
if (
explicit_offload
and normalize_memory_mode(requested_mode) is None
@@ -2003,18 +2014,117 @@ def plan_diffusion_memory(
any_offload = policy != OFFLOAD_NONE
reasons.append(reason)
estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(tile)
- return MemoryPlan(
+ split = _multigpu_companion_split(
+ target = target,
+ device_memory = device_memory,
+ mode = mode,
+ requested_mode = requested_mode,
+ explicit_offload = explicit_offload,
+ can_offload = can_offload,
+ policy = policy,
+ budget = budget,
+ model_dense_mib = model_dense_mib,
+ companion_dense_mib = companion_dense_mib,
+ runtime_headroom_mib = runtime_headroom_mib,
+ base_overhead_mib = base_overhead_mib,
+ text_encoder_dense_mib = text_encoder_dense_mib,
+ calibrated_activation = calibrated_activation,
+ )
+ single_card = MemoryPlan(
requested_mode = mode,
offload_policy = policy,
vae_tiling = tile,
vae_slicing = any_offload,
device_memory = device_memory,
- estimates = estimates,
+ estimates = dict(estimates),
reasons = tuple(reasons),
# only ever meaningful under group offload; every other tier already places the encoders
stream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,
stream_transformer = stream_transformer or policy != OFFLOAD_GROUP,
)
+ if split is not None:
+ estimates["multigpu_primary_required_mib"] = split.primary_required_mib
+ estimates["multigpu_secondary_required_mib"] = split.secondary_required_mib
+ return MemoryPlan(
+ requested_mode = mode,
+ offload_policy = OFFLOAD_NONE,
+ vae_tiling = False,
+ vae_slicing = False,
+ device_memory = device_memory,
+ estimates = estimates,
+ reasons = tuple(reasons) + (split.reason,),
+ secondary_ordinal = split.secondary_ordinal,
+ secondary_components = split.components,
+ fallback_plan = single_card,
+ )
+ return single_card
+
+
+def _multigpu_companion_split(
+ *,
+ target: Any,
+ device_memory: DeviceMemory,
+ mode: str,
+ requested_mode: Optional[str],
+ explicit_offload: bool,
+ can_offload: bool,
+ policy: str,
+ budget: Optional[int],
+ model_dense_mib: Optional[int],
+ companion_dense_mib: Optional[int],
+ runtime_headroom_mib: int,
+ base_overhead_mib: int,
+ text_encoder_dense_mib: Optional[int] = None,
+ calibrated_activation: Optional[CalibratedImageActivation] = None,
+) -> Any:
+ """The multi-GPU companion split for this plan, or None (one card: every existing decision unchanged). Only for
+ ``auto`` / ``fast`` on a discrete CUDA card that is the primary of a multi-card selection; explicit offload modes
+ and the legacy ``cpu_offload`` flag keep their meaning."""
+ from .diffusion_multigpu import (
+ current_selection,
+ plan_companion_split,
+ secondary_resident_limit_mib,
+ )
+
+ selection = current_selection()
+ if selection is None or not can_offload or device_memory.is_unified:
+ return None
+ if getattr(device_memory, "device", None) != "cuda" or budget is None:
+ return None
+ ordinal = getattr(target, "ordinal", None)
+ if ordinal is None and selection.automatic:
+ from .diffusion_device import placed_cuda_ordinal
+ ordinal = placed_cuda_ordinal(target)
+ if ordinal != selection.primary:
+ return None
+ if mode not in (MEMORY_MODE_AUTO, MEMORY_MODE_FAST):
+ return None
+ if explicit_offload and normalize_memory_mode(requested_mode) is None:
+ return None
+ if mode == MEMORY_MODE_FAST:
+ limit = _fast_device_budget_mib(device_memory)
+ else:
+ limit = int(budget * 0.85)
+ return plan_companion_split(
+ single_card_policy = policy,
+ model_dense_mib = model_dense_mib,
+ companion_dense_mib = companion_dense_mib,
+ runtime_headroom_mib = runtime_headroom_mib,
+ base_overhead_mib = base_overhead_mib,
+ primary_limit_mib = limit,
+ secondary_limit_mib = lambda ordinal: secondary_resident_limit_mib(ordinal, mode),
+ selection = selection,
+ text_encoder_dense_mib = text_encoder_dense_mib,
+ phase_headroom_mib = (
+ (
+ calibrated_activation.text_encoder_mib,
+ calibrated_activation.denoise_mib,
+ calibrated_activation.decode_mib,
+ )
+ if calibrated_activation is not None
+ else None
+ ),
+ )
def _calibrated_faster_tier(
@@ -3680,6 +3790,34 @@ def apply_memory_plan(
)
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
+ elif getattr(plan, "secondary_ordinal", None) is not None:
+ from .diffusion_multigpu import apply_split_plan
+ if not apply_split_plan(pipe, plan, placement = placement, logger = logger):
+ fallback = getattr(plan, "fallback_plan", None)
+ if fallback is not None and getattr(fallback, "secondary_ordinal", None) is None:
+ # the policy one card would have used (group offload keeps a torchao denoiser loadable)
+ result = apply_memory_plan(
+ pipe, fallback, device = device, placement_device = placement_device, logger = logger
+ )
+ # applied_plan() must report the fallback (or the lower re-plan it recorded), not the split
+ if getattr(pipe, "_unsloth_applied_plan", None) is None:
+ try:
+ pipe._unsloth_applied_plan = fallback
+ except Exception: # noqa: BLE001
+ pass
+ return result
+ _fallback_to_model_offload()
+ policy = OFFLOAD_MODEL
+ try:
+ pipe._unsloth_applied_plan = replace(
+ plan,
+ offload_policy = OFFLOAD_MODEL,
+ secondary_ordinal = None,
+ secondary_components = None,
+ fallback_plan = None,
+ )
+ except Exception: # noqa: BLE001
+ pass
else:
from .diffusion_fast_load import fast_upload
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
new file mode 100644
index 00000000000..81af0ca2b12
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -0,0 +1,2087 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Several selected CUDA / ROCm cards for one image or video pipeline, without NCCL.
+
+Three levers, all at a component or denoiser-call boundary, never inside a model forward:
+
+* Component split (planner): the DiT stays whole and resident on the primary card while the text encoders, VAE and
+ other companions sit on a secondary card, when that removes offload on the primary. Inputs move to a module's own
+ card at its entry points and outputs come back to the primary, so the pipeline still sees one device. Copies are
+ exact, so the output is bit-identical to a single-card resident run.
+* Second denoiser (after load): a two-DiT pipeline (an MoE ``transformer_2``) keeps one expert per card when both
+ experts do not fit one card.
+* CFG-parallel (generate): when guidance runs separate cond / uncond denoiser calls on the same latents, every odd
+ branch of a step runs on a replica of the denoiser on the secondary card, launched from a worker thread bound to
+ that card, while the primary computes the even branch. One output is copied per step; guidance and the scheduler
+ step stay on the primary. Branches are recognised from the call boundary alone (same latent tensor object in
+ consecutive calls), so true CFG, rescale and multi-branch guidance work unchanged and a guidance-distilled model
+ (one call per step) never builds a replica.
+
+Kill switches: ``UNSLOTH_DIFFUSION_MULTIGPU=0`` (everything here; one card as before), ``UNSLOTH_DIFFUSION_CFG_PARALLEL=0``
+(only the replica). ``UNSLOTH_DIFFUSION_MULTIGPU=force`` splits the companions even when one card would hold
+everything (for proving bit-identity). With one selected card nothing here engages.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import copy
+import dataclasses
+import os
+import queue
+import threading
+import time
+import types
+import weakref
+from collections import OrderedDict
+from typing import Any, Callable, Iterator, Optional, Sequence
+
+MULTIGPU_ENV = "UNSLOTH_DIFFUSION_MULTIGPU"
+CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL"
+
+DENOISER_ATTRS: tuple[str, ...] = (
+ "transformer",
+ "transformer_2",
+ "unconditional_transformer",
+ "unet",
+)
+_KEEP_WITH_DENOISER_PREFIXES: tuple[str, ...] = ("controlnet",)
+_ENTRY_METHODS: tuple[str, ...] = (
+ "forward",
+ "encode",
+ "decode",
+ "get_text_features",
+ "get_image_features",
+)
+# Sentinel for ``MemoryPlan.secondary_components``: only the text encoders move (the VAE decodes beside the denoiser).
+TEXT_ENCODERS_ONLY: tuple[str, ...] = ("<text_encoders>",)
+TEXT_ENCODER_HEADROOM_MIB = 1024
+_RESIDENT_FRACTION = 0.85
+
+
+def multigpu_mode() -> str:
+ """``off`` / ``auto`` (default) / ``force``."""
+ raw = (os.environ.get(MULTIGPU_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "disable", "disabled"):
+ return "off"
+ if raw == "force":
+ return "force"
+ return "auto"
+
+
+def cfg_parallel_mode() -> str:
+ """``off`` / ``auto`` (default) / ``force`` (also across different card models)."""
+ if multigpu_mode() == "off":
+ return "off"
+ raw = (os.environ.get(CFG_PARALLEL_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "disable", "disabled"):
+ return "off"
+ if raw == "force":
+ return "force"
+ return "auto"
+
+
[email protected](frozen = True)
+class MultiGpuSelection:
+ primary: int
+ secondaries: tuple[int, ...]
+ # No gpu_ids (the UI's Automatic): only the companion split runs, and only when one card would offload.
+ automatic: bool = False
+
+
+_SELECTION: contextvars.ContextVar[Optional[MultiGpuSelection]] = contextvars.ContextVar(
+ "unsloth_diffusion_multigpu_selection", default = None
+)
+
+
+AUTOMATIC_ORDINALS = "automatic"
+
+
+def make_selection(
+ primary: Optional[int],
+ ordinals: Optional[Sequence[int]],
+ *,
+ automatic: bool = False,
+) -> Optional[MultiGpuSelection]:
+ """A selection when at least two distinct cards were picked and the lever is on, else None (one card)."""
+ if primary is None or multigpu_mode() == "off":
+ return None
+ others = tuple(dict.fromkeys(int(o) for o in (ordinals or ()) if int(o) != int(primary)))
+ try:
+ import torch
+
+ # nvidia-smi can list a card this process's torch cannot address (a mask applied after the listing).
+ if torch.cuda.is_available():
+ visible = int(torch.cuda.device_count())
+ others = tuple(o for o in others if 0 <= o < visible)
+ except Exception: # noqa: BLE001 - unknown count: the capability filter below still applies
+ pass
+ if not others:
+ return None
+ # The load dtype and the device-specific patches are resolved for the primary (bf16 only from Ampere, FP8 kernels
+ # from Ada): a card of a lower compute capability may not run them, so it takes no component.
+ need = _capability(int(primary))
+ if need is not None:
+ others = tuple(o for o in others if (_capability(o) or need) >= need)
+ if not others:
+ return None
+ return MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)
+
+
+def models_kept_loaded() -> bool:
+ """Keep multiple models loaded is on (``fit`` / ``always``). Unreadable reads as off, like the arbiter."""
+ try:
+ from .gpu_residency import current_policy
+ return current_policy() != "off"
+ except Exception: # noqa: BLE001
+ return False
+
+
+def automatic_selection(primary: Optional[int]) -> Optional[MultiGpuSelection]:
+ """The selection of an automatic load: the ranked card (or the current one) plus every other parent-visible CUDA
+ card. None on one GPU, ROCm, XPU, MPS and CPU (the same cheap gate automatic ranking uses)."""
+ if multigpu_mode() == "off":
+ return None
+ try:
+ from .diffusion_device import auto_cuda_ranking_possible, resolve_diffusion_device_target
+
+ if not auto_cuda_ranking_possible() or resolve_diffusion_device_target().device != "cuda":
+ return None
+ import torch
+
+ if primary is None:
+ primary = int(torch.cuda.current_device())
+ count = int(torch.cuda.device_count())
+ except Exception: # noqa: BLE001 -- an unreadable host keeps one card
+ return None
+ return make_selection(primary, range(count), automatic = True)
+
+
+def _capability(ordinal: int) -> Optional[tuple[int, int]]:
+ try:
+ import torch
+ major, minor = torch.cuda.get_device_capability(int(ordinal))
+ return int(major), int(minor)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def selection_scope(
+ selection: Optional[MultiGpuSelection],
+) -> Iterator[Optional[MultiGpuSelection]]:
+ """Make ``selection`` visible to the planner and placement for the duration of one load (thread / task local)."""
+ if selection is None:
+ yield None
+ return
+ # Only a CFG replica reads the recorded compile arguments; automatic placement never builds one.
+ if not selection.automatic or cfg_parallel_mode() == "force":
+ install_compile_recorder()
+ token = _SELECTION.set(selection)
+ try:
+ yield selection
+ finally:
+ _SELECTION.reset(token)
+
+
+def with_load_selection(run_load: Callable[..., Any]) -> Callable[..., Any]:
+ """Decorate a backend's ``_run_load(self, **kwargs)``: pop ``_gpu_ordinals`` (every selected card, from
+ begin_load, or :data:`AUTOMATIC_ORDINALS`) and run the load inside :func:`selection_scope` with ``gpu_ordinal``
+ as the primary. Without a multi-card selection the load runs exactly as before."""
+ import functools
+
+ @functools.wraps(run_load)
+ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+ ordinals = kwargs.pop("_gpu_ordinals", None)
+ if ordinals and models_kept_loaded():
+ selection = None
+ elif ordinals == AUTOMATIC_ORDINALS:
+ selection = automatic_selection(kwargs.get("gpu_ordinal"))
+ else:
+ selection = make_selection(kwargs.get("gpu_ordinal"), ordinals)
+ with selection_scope(selection):
+ return run_load(self, *args, **kwargs)
+
+ return wrapper
+
+
+def current_selection() -> Optional[MultiGpuSelection]:
+ if multigpu_mode() == "off":
+ return None
+ return _SELECTION.get()
+
+
[email protected](frozen = True)
+class DeviceProbe:
+ ordinal: int
+ name: str
+ total_mib: Optional[int]
+ free_mib: Optional[int]
+
+
+_PROBE_LOCK = threading.Lock()
+_NAMES: dict[int, str] = {}
+
+
+def _free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
+ selection = _SELECTION.get()
+ if selection is not None and selection.automatic and int(ordinal) != selection.primary:
+ # Sized from nvidia-smi: an automatic load must not open a CUDA context on a card it may never use.
+ from .diffusion_device import smi_free_total_mib
+ return smi_free_total_mib(int(ordinal))
+ try:
+ import torch
+
+ from utils.hardware import trusted_mem_get_info
+
+ free, total = trusted_mem_get_info(int(ordinal), module = torch.cuda)
+ return int(free) // (1 << 20), int(total) // (1 << 20)
+ except Exception: # noqa: BLE001
+ return None, None
+
+
+def probe_devices(ordinals: Sequence[int]) -> dict[int, DeviceProbe]:
+ """Name (cached per process) and free / total memory of each card."""
+ import torch
+
+ ords = [int(o) for o in ordinals]
+ with _PROBE_LOCK:
+ for o in ords:
+ if o not in _NAMES:
+ try:
+ _NAMES[o] = str(torch.cuda.get_device_name(o))
+ except Exception: # noqa: BLE001
+ _NAMES[o] = ""
+ out: dict[int, DeviceProbe] = {}
+ for o in ords:
+ free, total = _free_total_mib(o)
+ out[o] = DeviceProbe(ordinal = o, name = _NAMES.get(o, ""), total_mib = total, free_mib = free)
+ return out
+
+
+def best_secondary(
+ selection: MultiGpuSelection, probes: Optional[dict[int, DeviceProbe]] = None
+) -> Optional[int]:
+ if not selection.secondaries:
+ return None
+ if probes is None and selection.automatic:
+ free = {o: _free_total_mib(o)[0] for o in selection.secondaries}
+ return max(
+ selection.secondaries, key = lambda o: (free[o] if free[o] is not None else -1, -o)
+ )
+ if probes is None:
+ probes = probe_devices((selection.primary, *selection.secondaries))
+
+ def key(o: int) -> tuple[int, int]:
+ p = probes.get(o)
+ return (p.free_mib if p is not None and p.free_mib is not None else -1, -o)
+
+ return max(selection.secondaries, key = key)
+
+
[email protected](frozen = True)
+class CompanionSplit:
+ secondary_ordinal: int
+ primary_required_mib: int
+ primary_limit_mib: int
+ secondary_required_mib: int
+ secondary_limit_mib: int
+ reason: str
+ # None = every companion moves; TEXT_ENCODERS_ONLY = only the encoders (the VAE stays with the denoiser)
+ components: Optional[tuple[str, ...]] = None
+
+
+def plan_companion_split(
+ *,
+ single_card_policy: str,
+ model_dense_mib: Optional[int],
+ companion_dense_mib: Optional[int],
+ runtime_headroom_mib: int,
+ base_overhead_mib: int,
+ primary_limit_mib: Optional[int],
+ secondary_limit_mib: Callable[[int], Optional[int]],
+ selection: Optional[MultiGpuSelection],
+ text_encoder_dense_mib: Optional[int] = None,
+ phase_headroom_mib: Optional[tuple[int, int, int]] = None,
+) -> Optional[CompanionSplit]:
+ """Pure decision: return a split when the single-card plan offloads (or ``force``), the denoiser fits the primary
+ under the resident rule, and the moved companions fit a secondary. Tried in order: every companion (text encoders
+ + VAE + the rest, the secondary priced with the full runtime headroom since it runs the decode), then the text
+ encoders alone (the VAE decodes on the primary; the secondary needs only an encoder forward's room).
+ ``primary_limit_mib`` / ``secondary_limit_mib(ordinal)`` are each card's resident limit under the caller's memory
+ mode rule (auto: 85% of the safe budget; fast: the fast budget). ``phase_headroom_mib`` = measured (encode,
+ denoise, decode) MiB above the weights for families Studio has calibrated: each card is then priced only for the
+ phases it runs instead of the flat runtime headroom."""
+ mode = multigpu_mode()
+ if selection is None or mode == "off":
+ return None
+ if single_card_policy == "none" and mode != "force":
+ return None
+ if model_dense_mib is None or not companion_dense_mib or primary_limit_mib is None:
+ return None
+ limits: dict[int, int] = {}
+ for ordinal in selection.secondaries:
+ limit = secondary_limit_mib(ordinal)
+ if limit is not None:
+ limits[ordinal] = int(limit)
+ if not limits:
+ return None
+ runtime = int(runtime_headroom_mib)
+ if phase_headroom_mib is not None:
+ encode, denoise, decode = (int(v) for v in phase_headroom_mib)
+ # (moved MiB, primary headroom, secondary headroom, components, label)
+ companions_split = (denoise, max(encode, decode))
+ encoders_split = (max(denoise, decode), encode)
+ else:
+ companions_split = (runtime, runtime)
+ encoders_split = (runtime, TEXT_ENCODER_HEADROOM_MIB)
+ candidates: list[tuple[int, int, int, Optional[tuple[str, ...]], str]] = [
+ (int(companion_dense_mib), *companions_split, None, "text encoders + VAE"),
+ ]
+ if text_encoder_dense_mib and 0 < int(text_encoder_dense_mib) <= int(companion_dense_mib):
+ candidates.append(
+ (int(text_encoder_dense_mib), *encoders_split, TEXT_ENCODERS_ONLY, "text encoders")
+ )
+ for moved_mib, pri_headroom, sec_headroom, components, label in candidates:
+ primary_required = (
+ max(0, int(model_dense_mib) - moved_mib) + int(pri_headroom) + int(base_overhead_mib)
+ )
+ if primary_required > int(primary_limit_mib):
+ continue
+ secondary_required = moved_mib + sec_headroom + int(base_overhead_mib)
+ fits = [o for o, limit in limits.items() if secondary_required <= limit]
+ if not fits:
+ continue
+ best = max(fits, key = lambda o: (limits[o], -o))
+ return CompanionSplit(
+ secondary_ordinal = best,
+ primary_required_mib = primary_required,
+ primary_limit_mib = int(primary_limit_mib),
+ secondary_required_mib = secondary_required,
+ secondary_limit_mib = limits[best],
+ components = components,
+ reason = (
+ f"multi-GPU: denoiser resident on cuda:{selection.primary} ({primary_required} of "
+ f"{int(primary_limit_mib)} MiB), {label} on cuda:{best} ({secondary_required} of {limits[best]} MiB); "
+ f"single card would use '{single_card_policy}'"
+ ),
+ )
+ return None
+
+
+def choose_component_assignment(
+ sizes: dict[str, tuple[int, str]],
+ *,
+ headroom_mib: int,
+ overhead_mib: int,
+ primary_limit_mib: int,
+ secondary_limit_mib: int,
+) -> Optional[tuple[str, ...]]:
+ """Names to place on the secondary so each card holds its share resident, or None. ``sizes`` maps component name
+ -> (MiB, role) with role 'dit' / 'text_encoder' / 'other'. A card running a denoiser or the VAE (any 'other'
+ component) is priced with the runtime headroom; a card holding only text encoders with an encoder forward's room.
+ Tried in order: every companion; the text encoders alone; the second denoiser; the second denoiser plus the
+ companions. The first denoiser always stays on the primary."""
+ dits = [n for n, (_m, r) in sizes.items() if r == "dit"]
+ if not dits:
+ return None
+ first = "transformer" if "transformer" in dits else dits[0]
+ seconds = [n for n in dits if n != first]
+ companions = [
+ n
+ for n, (_m, r) in sizes.items()
+ if r != "dit" and not n.startswith(_KEEP_WITH_DENOISER_PREFIXES)
+ ]
+ encoders = [n for n in companions if sizes[n][1] == "text_encoder"]
+ candidates: list[tuple[str, ...]] = []
+ if companions:
+ candidates.append(tuple(companions))
+ if encoders and len(encoders) < len(companions):
+ candidates.append(tuple(encoders))
+ if seconds:
+ candidates.append(tuple(seconds))
+ if companions:
+ candidates.append(tuple(seconds + companions))
+
+ def need(names: list) -> int:
+ weights = sum(sizes[n][0] for n in names)
+ busy = any(sizes[n][1] != "text_encoder" for n in names)
+ return weights + (headroom_mib if busy else TEXT_ENCODER_HEADROOM_MIB) + overhead_mib
+
+ for moved in candidates:
+ on_primary = [n for n in sizes if n not in moved]
+ if need(on_primary) <= primary_limit_mib and need(list(moved)) <= secondary_limit_mib:
+ return moved
+ return None
+
+
+def refine_plan_for_multigpu(
+ pipe: Any,
+ plan: Any,
+ logger: Any = None,
+) -> Any:
+ """Post-load: an offloading plan becomes a resident split when the LOADED components can be shared by the cards.
+
+ No-op without a selection, for explicit offload modes, a plan already resident (unless forced) or split, unified memory
+ or an unknown budget. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""
+ selection = current_selection()
+ if selection is None or plan is None:
+ return plan
+ if getattr(plan, "secondary_ordinal", None) is not None:
+ return plan
+ if plan.offload_policy == "none" and multigpu_mode() != "force":
+ return plan
+ if getattr(plan, "requested_mode", None) not in ("auto", "fast"):
+ return plan
+ if plan.estimates.get("explicit_cpu_offload"):
+ return plan
+ memory = plan.device_memory
+ if memory.is_unified or getattr(memory, "device", None) != "cuda":
+ return plan
+ budget = plan.estimates.get("safe_device_budget_mib")
+ if budget is None:
+ return plan
+ from .diffusion_memory import _loaded_component_mib
+
+ sizes = _loaded_component_mib(pipe)
+ if not sizes:
+ return plan
+ secondary = best_secondary(selection)
+ if secondary is None:
+ return plan
+ mode = getattr(plan, "requested_mode", None)
+ sec_limit = secondary_resident_limit_mib(secondary, mode)
+ if sec_limit is None:
+ return plan
+ from .diffusion_memory import _fast_device_budget_mib
+
+ primary_limit = (
+ _fast_device_budget_mib(memory) if mode == "fast" else int(int(budget) * _RESIDENT_FRACTION)
+ )
+ if primary_limit is None:
+ return plan
+ headroom = int(
+ plan.estimates.get("measured_runtime_headroom_mib")
+ or plan.estimates.get("calibrated_headroom_mib")
+ or plan.estimates.get("runtime_headroom_mib")
+ or 0
+ )
+ overhead = int(plan.estimates.get("base_overhead_mib") or 0)
+ moved = choose_component_assignment(
+ sizes,
+ headroom_mib = headroom,
+ overhead_mib = overhead,
+ primary_limit_mib = int(primary_limit),
+ secondary_limit_mib = int(sec_limit),
+ )
+ if moved is None:
+ return plan
+ reason = (
+ f"multi-GPU (loaded sizes): {', '.join(moved)} on cuda:{secondary}, the rest resident on "
+ f"cuda:{selection.primary}; single card would use '{plan.offload_policy}'"
+ )
+ if logger is not None:
+ logger.info("diffusion.multigpu: %s", reason)
+ return dataclasses.replace(
+ plan,
+ offload_policy = "none",
+ # Kept: the fit above may be priced with the tiled decode peak (calibrated_headroom_mib), so the savers the
+ # single-card plan chose must stay on.
+ vae_tiling = plan.vae_tiling,
+ vae_slicing = plan.vae_slicing,
+ stream_text_encoders = False,
+ stream_transformer = True,
+ resident_transformer_mib = None,
+ resident_text_encoder_mib = None,
+ reasons = tuple(plan.reasons) + (reason,),
+ secondary_ordinal = secondary,
+ secondary_components = tuple(moved),
+ fallback_plan = plan,
+ )
+
+
+def secondary_device_memory(ordinal: int) -> Any:
+ from .diffusion_memory import DeviceMemory
+
+ free, total = _free_total_mib(int(ordinal))
+ if free is None:
+ return None
+ return DeviceMemory(
+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
+ )
+
+
+def secondary_safe_budget_mib(ordinal: int) -> Optional[int]:
+ from .diffusion_memory import _safe_device_budget_mib
+ memory = secondary_device_memory(ordinal)
+ return None if memory is None else _safe_device_budget_mib(memory)
+
+
+def secondary_resident_limit_mib(ordinal: int, mode: str) -> Optional[int]:
+ from .diffusion_memory import _fast_device_budget_mib
+
+ if mode == "fast":
+ memory = secondary_device_memory(ordinal)
+ return None if memory is None else _fast_device_budget_mib(memory)
+ budget = secondary_safe_budget_mib(ordinal)
+ return None if budget is None else int(int(budget) * _RESIDENT_FRACTION)
+
+
+def _torch():
+ import torch
+ return torch
+
+
+def move_to_device(
+ obj: Any,
+ device: Any,
+ _depth: int = 0,
+) -> Any:
+ torch = _torch()
+ if isinstance(obj, torch.Tensor):
+ return obj if obj.device == device else obj.to(device)
+ if (
+ _depth > 6
+ or obj is None
+ or isinstance(obj, (str, bytes, int, float, bool, torch.device, torch.dtype))
+ ):
+ return obj
+ if isinstance(obj, dict):
+ # ModelOutput is an OrderedDict that also mirrors its keys as attributes; __setitem__ keeps both in sync.
+ try:
+ out = copy.copy(obj)
+ for key, value in obj.items():
+ moved = move_to_device(value, device, _depth + 1)
+ if moved is not value:
+ out[key] = moved
+ return out
+ except Exception: # noqa: BLE001
+ return type(obj)((k, move_to_device(v, device, _depth + 1)) for k, v in obj.items())
+ if isinstance(obj, tuple):
+ moved = [move_to_device(v, device, _depth + 1) for v in obj]
+ if all(a is b for a, b in zip(moved, obj)):
+ return obj
+ if hasattr(obj, "_fields"):
+ return type(obj)(*moved)
+ return type(obj)(moved) if type(obj) is not tuple else tuple(moved)
+ if isinstance(obj, list):
+ return [move_to_device(v, device, _depth + 1) for v in obj]
+ if isinstance(obj, torch.nn.Module):
+ return obj
+ if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
+ changes = {}
+ for f in dataclasses.fields(obj):
+ value = getattr(obj, f.name, None)
+ moved = move_to_device(value, device, _depth + 1)
+ if moved is not value:
+ changes[f.name] = moved
+ if not changes:
+ return obj
+ try:
+ return dataclasses.replace(obj, **changes)
+ except Exception: # noqa: BLE001 - init=False fields
+ out = copy.copy(obj)
+ for k, v in changes.items():
+ object.__setattr__(out, k, v)
+ return out
+ state = getattr(obj, "__dict__", None)
+ if (
+ isinstance(state, dict)
+ and state
+ and any(isinstance(v, torch.Tensor) for v in state.values())
+ ):
+ if all(
+ isinstance(
+ v, (torch.Tensor, bool, int, float, str, type(None), torch.dtype, torch.device)
+ )
+ for v in state.values()
+ ):
+ out = copy.copy(obj)
+ for k, v in state.items():
+ if isinstance(v, torch.Tensor) and v.device != device:
+ setattr(out, k, v.to(device))
+ return out
+ return obj
+
+
+def _module_device(module: Any) -> Any:
+ torch = _torch()
+ for tensor in module.parameters():
+ return tensor.device
+ for tensor in module.buffers():
+ return tensor.device
+ return None
+
+
+def _device_context(device: Any):
+ torch = _torch()
+ if getattr(device, "type", None) == "cuda" and device.index is not None:
+ return torch.cuda.device(device)
+ return contextlib.nullcontext()
+
+
+def _crosses(device: Any, home: Any) -> bool:
+ """A call must move tensors: the module sits on a different accelerator card than the pipeline's home. A module on
+ the CPU (offloaded, or between loads) is left to its own hooks."""
+ return device is not None and device != home and device.type == "cuda"
+
+
+def install_boundary(module: Any, home: Any) -> list[str]:
+ """Wrap ``module``'s entry points: arguments go to the module's CURRENT card, results come back to ``home``."""
+ torch = _torch()
+ home = torch.device(home)
+ wrapped_names: list[str] = []
+ for name in _ENTRY_METHODS:
+ bound = getattr(module, name, None)
+ if not callable(bound) or getattr(bound, "_unsloth_boundary", None) is not None:
+ continue
+
+ def _make(orig: Callable[..., Any]) -> Callable[..., Any]:
+ def boundary(*args: Any, **kwargs: Any) -> Any:
+ device = _module_device(module)
+ if not _crosses(device, home):
+ return orig(*args, **kwargs)
+ args, kwargs = move_to_device((args, kwargs), device)
+ with _device_context(device):
+ out = orig(*args, **kwargs)
+ return move_to_device(out, home)
+
+ boundary._unsloth_boundary = orig # type: ignore[attr-defined]
+ boundary.__wrapped__ = orig # type: ignore[attr-defined]
+ boundary._unsloth_had_override = name in module.__dict__ # type: ignore[attr-defined]
+ return boundary
+
+ setattr(module, name, _make(bound))
+ wrapped_names.append(name)
+ return wrapped_names
+
+
+def remove_boundary(module: Any) -> None:
+ for name in _ENTRY_METHODS:
+ current = module.__dict__.get(name) if hasattr(module, "__dict__") else None
+ if current is not None and getattr(current, "_unsloth_boundary", None) is not None:
+ if getattr(current, "_unsloth_had_override", False):
+ module.__dict__[name] = current._unsloth_boundary
+ continue
+ try:
+ delattr(module, name)
+ except AttributeError:
+ pass
+
+
+_EXEC_PATCH_LOCK = threading.Lock()
+
+
+# Set once this process places a split, so a one-card load never walks its components looking for one.
+_SPLIT_EVER_PLACED = False
+
+
+def _split_home(pipe: Any) -> Any:
+ if not _SPLIT_EVER_PLACED:
+ return None
+ try:
+ components = pipe.components
+ except Exception: # noqa: BLE001
+ return None
+ if not isinstance(components, dict):
+ return None
+ for module in components.values():
+ home = getattr(module, "_unsloth_split_home", None)
+ if home is not None:
+ return home
+ return None
+
+
+def install_execution_device_patch() -> bool:
+ """``DiffusionPipeline._execution_device`` / ``.device`` report the primary card of a split pipeline. diffusers
+ returns the first non-CPU component in signature order, which is the VAE (on the secondary) for most families.
+ Inert for any pipeline without a split denoiser; also covers ``from_pipe`` siblings, which share the modules."""
+ try:
+ from diffusers import DiffusionPipeline
+ except Exception: # noqa: BLE001
+ return False
+ with _EXEC_PATCH_LOCK:
+ if getattr(DiffusionPipeline, "_unsloth_split_exec_patch", False):
+ return True
+ orig_exec = DiffusionPipeline.__dict__.get("_execution_device")
+ orig_dev = DiffusionPipeline.__dict__.get("device")
+ if not isinstance(orig_exec, property) or not isinstance(orig_dev, property):
+ return False
+
+ def _exec(self: Any) -> Any:
+ home = _split_home(self)
+ return home if home is not None else orig_exec.fget(self)
+
+ def _dev(self: Any) -> Any:
+ home = _split_home(self)
+ return home if home is not None else orig_dev.fget(self)
+
+ DiffusionPipeline._execution_device = property(_exec, doc = orig_exec.__doc__)
+ DiffusionPipeline.device = property(_dev, doc = orig_dev.__doc__)
+ DiffusionPipeline._unsloth_split_exec_patch = True
+ return True
+
+
+def companion_names(pipe: Any) -> tuple[str, ...]:
+ torch = _torch()
+ names = []
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if not isinstance(module, torch.nn.Module):
+ continue
+ if name in DENOISER_ATTRS or str(name).startswith(_KEEP_WITH_DENOISER_PREFIXES):
+ continue
+ names.append(str(name))
+ return tuple(names)
+
+
+_PLACEMENTS: "weakref.WeakKeyDictionary[Any, dict]" = weakref.WeakKeyDictionary()
+
+
+def place_split(
+ pipe: Any,
+ *,
+ primary: Any,
+ secondary_ordinal: int,
+ names: Optional[Sequence[str]] = None,
+ logger: Any = None,
+) -> dict[str, str]:
+ """Place ``names`` (default: the companions) on ``cuda:<secondary_ordinal>`` and everything else on ``primary``;
+ returns component -> device. Raises on failure (the caller falls back to single-card offload)."""
+ torch = _torch()
+ from .diffusion_fast_load import fast_upload
+
+ primary_dev = torch.device(primary)
+ if primary_dev.type == "cuda" and primary_dev.index is None:
+ primary_dev = torch.device("cuda", torch.cuda.current_device())
+ secondary_dev = torch.device("cuda", int(secondary_ordinal))
+ if names and tuple(names) == TEXT_ENCODERS_ONLY:
+ moved = tuple(n for n in companion_names(pipe) if n.startswith("text_encoder"))
+ else:
+ moved = tuple(names) if names else companion_names(pipe)
+ components = {
+ str(n): m
+ for n, m in (getattr(pipe, "components", {}) or {}).items()
+ if isinstance(m, torch.nn.Module)
+ }
+ placement: dict[str, str] = {}
+ order = sorted(components, key = lambda n: 0 if n in moved else 1)
+ for name in order:
+ module = components[name]
+ device = secondary_dev if name in moved else primary_dev
+ with fast_upload([module], device, logger = logger):
+ module.to(device)
+ placement[name] = str(device)
+ for name in moved:
+ if name in components:
+ install_boundary(components[name], primary_dev)
+ global _SPLIT_EVER_PLACED
+ _SPLIT_EVER_PLACED = True
+ for name, module in components.items():
+ if name not in moved:
+ module._unsloth_split_home = primary_dev
+ install_execution_device_patch()
+ info = {
+ "placement": placement,
+ "primary": str(primary_dev),
+ "secondary": str(secondary_dev),
+ "moved": list(moved),
+ }
+ try:
+ _PLACEMENTS[pipe] = info
+ except TypeError:
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.multigpu: placed %s on %s, the rest on %s",
+ ", ".join(moved) or "nothing",
+ secondary_dev,
+ primary_dev,
+ )
+ return placement
+
+
+def undo_split(pipe: Any) -> None:
+ torch = _torch()
+ for module in (getattr(pipe, "components", {}) or {}).values():
+ if isinstance(module, torch.nn.Module):
+ remove_boundary(module)
+ if "_unsloth_split_home" in module.__dict__:
+ del module.__dict__["_unsloth_split_home"]
+ try:
+ _PLACEMENTS.pop(pipe, None)
+ except TypeError:
+ pass
+
+
+def apply_split_plan(
+ pipe: Any,
+ plan: Any,
+ *,
+ placement: str,
+ logger: Any = None,
+) -> bool:
+ """Place a split plan."""
+ secondary = getattr(plan, "secondary_ordinal", None)
+ if secondary is None:
+ return False
+ try:
+ place_split(
+ pipe,
+ primary = placement,
+ secondary_ordinal = int(secondary),
+ names = getattr(plan, "secondary_components", None),
+ logger = logger,
+ )
+ return True
+ except Exception as exc: # noqa: BLE001 - fall back to one card
+ if logger is not None:
+ logger.warning(
+ "diffusion.multigpu: split placement failed (%s); falling back to one card", exc
+ )
+ undo_split(pipe)
+ try:
+ pipe.to("cpu")
+ _torch().cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ return False
+
+
+def is_split(pipe: Any) -> bool:
+ if pipe is None:
+ return False
+ try:
+ return pipe in _PLACEMENTS
+ except TypeError:
+ return False
+
+
+def placement_status(pipe: Any) -> Optional[dict[str, Any]]:
+ if pipe is None:
+ return None
+ try:
+ info = _PLACEMENTS.get(pipe)
+ cfg = _CFG_ROUTERS.get(pipe)
+ declined = _CFG_DECLINES.get(pipe)
+ except TypeError:
+ return None
+ if info is None and cfg is None and declined is None:
+ return None
+ out: dict[str, Any] = dict(info or {})
+ if cfg is not None:
+ out["cfg_parallel"] = cfg.describe()
+ elif declined is not None:
+ out["cfg_parallel"] = {"armed": False, "declined": declined}
+ return out
+
+
+_COMPILE_RECORDER_LOCK = threading.Lock()
+
+
+def install_compile_recorder() -> bool:
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False
+ with _COMPILE_RECORDER_LOCK:
+ current = torch.nn.Module.compile
+ if getattr(current, "_unsloth_recorder", False):
+ return True
+
+ def compile(self: Any, *args: Any, **kwargs: Any) -> Any: # noqa: A001 - mirrors nn.Module.compile
+ self.__dict__["_unsloth_compile_args"] = (args, dict(kwargs))
+ return current(self, *args, **kwargs)
+
+ compile._unsloth_recorder = True # type: ignore[attr-defined]
+ compile.__wrapped__ = current # type: ignore[attr-defined]
+ torch.nn.Module.compile = compile
+ return True
+
+
+_ROOT_HOOK_DICTS = (
+ "_forward_pre_hooks",
+ "_forward_pre_hooks_with_kwargs",
+ "_forward_hooks",
+ "_forward_hooks_with_kwargs",
+ "_forward_hooks_always_called",
+)
+
+
+# Offload machinery moves weights between host and card without changing the math. A replica holds its weights
+# resident, so it drops these instead of inheriting them.
+_OFFLOAD_REGISTRY_HOOKS = frozenset(
+ {"group_offloading", "layer_execution_tracker", "lazy_prefetch_group_offloading"}
+)
+_OFFLOAD_MODULE_MARKERS = (
+ "diffusers.hooks.group_offloading",
+ "diffusers.hooks.hooks",
+ "accelerate.hooks",
+ "diffusion_offload_prefetch",
+ "diffusion_small_host",
+ "diffusion_pinned_arena",
+)
+
+
+def _origin_modules(fn: Any) -> list[str]:
+ out: list[str] = []
+ seen = 0
+ while fn is not None and seen < 6:
+ seen += 1
+ out.append(str(getattr(fn, "__module__", "") or ""))
+ out.append(str(getattr(type(fn), "__module__", "") or ""))
+ out.append(str(getattr(getattr(fn, "func", None), "__module__", "") or ""))
+ fn = (
+ getattr(fn, "__wrapped__", None)
+ or getattr(fn, "__func__", None)
+ or getattr(fn, "func", None)
+ )
+ return out
+
+
+def _is_offload_object(obj: Any) -> bool:
+ return any(marker in mod for mod in _origin_modules(obj) for marker in _OFFLOAD_MODULE_MARKERS)
+
+
+def _registry_kind(sub: Any) -> str:
+ registry = sub.__dict__.get("_diffusers_hook")
+ hooks = getattr(registry, "hooks", None) if registry is not None else None
+ if not hooks:
+ return "none"
+ return "offload" if set(hooks) <= _OFFLOAD_REGISTRY_HOOKS else "other"
+
+
+def _immutable_capture(value: Any, depth: int) -> bool:
+ """A closure cell a replica may share with its source. deepcopy keeps a function (and so its closure) as the same
+ object, and the two branches call it concurrently, so only values nobody can mutate qualify: scalars, strings,
+ dtypes / devices, tuples / frozensets of those, and plain functions whose own captures qualify. A list, dict, set,
+ tensor, bound method or arbitrary object (a counter, a cache) would be shared state."""
+ torch = _torch()
+ if depth > 3:
+ return False
+ if isinstance(
+ value, (str, bytes, int, float, complex, bool, type(None), torch.dtype, torch.device, type)
+ ):
+ return True
+ if isinstance(value, (tuple, frozenset)):
+ return all(_immutable_capture(v, depth + 1) for v in value)
+ if isinstance(value, types.FunctionType):
+ return _portable(value, depth)
+ return False
+
+
+def _portable(fn: Any, depth: int = 0) -> bool:
+ """True when a hook / forward override can be shared by a copy of its module: nothing it captures (closure cells,
+ bound ``self``, partial arguments, the attributes of a captured object) is a module or a CUDA tensor, which would
+ keep pointing at the source card."""
+ torch = _torch()
+ if depth > 3:
+ return False
+ if isinstance(fn, torch.nn.Module) or (
+ isinstance(fn, torch.Tensor) and fn.device.type != "cpu"
+ ):
+ return False
+ if isinstance(fn, (list, tuple, set)):
+ return all(_portable(v, depth + 1) for v in fn)
+ if isinstance(fn, dict):
+ return all(_portable(v, depth + 1) for v in fn.values())
+ if isinstance(
+ fn,
+ (str, bytes, int, float, bool, type(None), torch.Tensor, torch.dtype, torch.device, type),
+ ):
+ return True
+ if isinstance(fn, (types.FunctionType, types.MethodType)):
+ owner = getattr(fn, "__self__", None)
+ if owner is not None and not _portable(owner, depth + 1):
+ return False
+ func = getattr(fn, "__func__", fn)
+ for cell in getattr(func, "__closure__", None) or ():
+ try:
+ value = cell.cell_contents
+ except ValueError:
+ continue
+ if not _immutable_capture(value, depth + 1):
+ return False
+ return True
+ partial_func = getattr(fn, "func", None)
+ if partial_func is not None and hasattr(fn, "args"):
+ return _portable(partial_func, depth + 1) and all(
+ _portable(a, depth + 1)
+ for a in (*fn.args, *(getattr(fn, "keywords", None) or {}).values())
+ )
+ state = getattr(fn, "__dict__", None)
+ if isinstance(state, dict):
+ return all(_portable(v, depth + 1) for v in state.values())
+ return True
+
+
+# Where a hook shared by a replica may come from. Studio's own hooks keep their per-call state on the module (mirrored
+# by _mirror_hook_state); a foreign hook may keep it in a module global that both threads would then race on, and that
+# cannot be seen from the function, so it declines the replica.
+_TRUSTED_HOOK_MODULES = ("core.inference.", "studio.backend.core.inference.")
+
+
+def _hook_origin(fn: Any) -> str:
+ target = getattr(fn, "func", None) if hasattr(fn, "args") and hasattr(fn, "func") else fn
+ target = getattr(target, "__func__", target)
+ if isinstance(target, (types.FunctionType, types.BuiltinFunctionType)):
+ return str(getattr(target, "__module__", None) or "")
+ return str(getattr(type(target), "__module__", None) or "")
+
+
+def _trusted_hook(fn: Any) -> bool:
+ return _hook_origin(fn).startswith(_TRUSTED_HOOK_MODULES) and _portable(fn)
+
+
+def _hooks_present(module: Any) -> Optional[str]:
+ """Name of the first submodule carrying state a replica cannot inherit faithfully, else None: a non-offload hook
+ registry (step caches), an accelerate hook, or a forward hook / instance forward override below the root whose
+ captured state points at the source (deepcopy treats functions as atoms). Offload machinery (diffusers group
+ offloading, Studio's prefetchers) is dropped by :func:`build_replica`, which holds every weight resident. The
+ ROOT's torch forward hooks run on the source around the router for every branch and the copy does not carry them,
+ but whatever per-call state they write must reach the replica: only Studio's own root hooks are accepted, whose
+ state is the ``_unsloth_*`` scalars :func:`_mirror_hook_state` copies."""
+ for name, sub in module.named_modules():
+ kind = _registry_kind(sub)
+ if kind == "other":
+ return f"{name or '<root>'}._diffusers_hook ({', '.join(sorted(sub._diffusers_hook.hooks))})"
+ if getattr(sub, "_hf_hook", None) is not None:
+ return f"{name or '<root>'}._hf_hook"
+ for attr in ("_forward_hooks", "_forward_pre_hooks"):
+ for hook in (getattr(sub, attr, None) or {}).values():
+ if _is_offload_object(hook):
+ continue
+ if name and not _trusted_hook(hook):
+ return f"{name}.{attr}"
+ if not name and not _hook_origin(hook).startswith(_TRUSTED_HOOK_MODULES):
+ return f"<root>.{attr} (a hook outside Studio: its per-call state cannot be mirrored)"
+ override = sub.__dict__.get("forward")
+ if override is None or isinstance(override, (types.MethodType, CfgParallelRouter)):
+ continue
+ if kind == "offload" or _is_offload_object(override):
+ continue
+ if callable(override) and not _trusted_hook(override):
+ return f"{name or '<root>'}.forward (instance override)"
+ return None
+
+
+def _offload_attr_keys(sub: Any) -> list[str]:
+ torch = _torch()
+ keys = []
+ kind = _registry_kind(sub)
+ for key, value in sub.__dict__.items():
+ if key in ("_parameters", "_buffers", "_modules") or key.startswith("_forward_"):
+ continue
+ if key == "_diffusers_hook" and kind == "offload":
+ keys.append(key)
+ elif key == "forward" and (kind == "offload" or _is_offload_object(value)):
+ keys.append(key)
+ elif not isinstance(value, (torch.Tensor, torch.nn.Module)) and _is_offload_object(value):
+ keys.append(key)
+ return keys
+
+
+_DROP = object()
+
+
+def _rebind_root_forward(original: Any, source: Any, replica: Any) -> None:
+ """Give ``replica`` the root ``forward`` the source ran before the router took its slot: offload wrappers (math
+ neutral, dropped with the offload state) are peeled off, and a method bound to the source is rebound to the copy.
+ Anything else cannot be reproduced and raises (CFG-parallel then declines)."""
+ seen = 0
+ while original is not None and _is_offload_object(original) and seen < 8:
+ seen += 1
+ original = getattr(original, "__wrapped__", None)
+ if isinstance(original, types.MethodType) and original.__self__ is source:
+ if original.__func__ is getattr(type(source), "forward", None):
+ return
+ replica.__dict__["forward"] = types.MethodType(original.__func__, replica)
+ return
+ raise RuntimeError(
+ f"the denoiser's root forward override ({original!r}) cannot be reproduced on a replica"
+ )
+
+
+def build_replica(module: Any, device: Any) -> Any:
+ """A copy of ``module`` whose parameters and buffers are created directly on ``device`` (never staged on the
+ source card), with every recorded ``Module.compile`` re-applied to the copy. Raises when it cannot be built."""
+ torch = _torch()
+ device = torch.device(device)
+ memo: dict[int, Any] = {}
+ keep: list[Any] = []
+ for _name, param in module.named_parameters(remove_duplicate = True):
+ if id(param) in memo:
+ continue
+ copied = param.detach().to(device)
+ new = copied
+ if isinstance(param, torch.nn.Parameter):
+ try:
+ new = torch.nn.Parameter(copied, requires_grad = param.requires_grad)
+ except Exception: # noqa: BLE001 - a tensor subclass that refuses the wrapper stays a plain tensor
+ new = copied
+ memo[id(param)] = new
+ keep.append(param)
+ for _name, buffer in module.named_buffers(remove_duplicate = True):
+ if id(buffer) in memo:
+ continue
+ memo[id(buffer)] = buffer.detach().to(device)
+ keep.append(buffer)
+ for sub in module.modules():
+ offload_keys = set(_offload_attr_keys(sub))
+ for key, value in sub.__dict__.items():
+ if isinstance(value, CfgParallelRouter) or key in offload_keys:
+ # never traversed: offload state holds pinned host copies a deepcopy would duplicate
+ memo[id(value)] = _DROP
+ for attr in ("_forward_hooks", "_forward_pre_hooks"):
+ for hook in (getattr(sub, attr, None) or {}).values():
+ if _is_offload_object(hook):
+ memo[id(hook)] = _DROP
+ # Per-step precision state (the NVFP4 W4A16 protect schedule) is advanced by the source's scheduler: the copy
+ # must read the same controller, or the two CFG branches of one step run different GEMMs.
+ for value in sub.__dict__.values():
+ if type(value).__name__ in _SHARED_STATE_TYPES:
+ memo[id(value)] = value
+ guard = module.__dict__.get("_unsloth_compile_guard")
+ if guard is not None:
+ memo[id(guard)] = _DROP
+ memo[id(memo)] = keep
+ replica = copy.deepcopy(module, memo)
+ router = module.__dict__.get("forward")
+ for sub in replica.modules():
+ for key in [k for k, v in sub.__dict__.items() if v is _DROP]:
+ del sub.__dict__[key]
+ for attr in _ROOT_HOOK_DICTS:
+ hooks = sub.__dict__.get(attr)
+ if hooks:
+ for hook_id in [k for k, v in hooks.items() if v is _DROP]:
+ del hooks[hook_id]
+ for other in _ROOT_HOOK_DICTS:
+ sub.__dict__.get(other, {}).pop(hook_id, None)
+ if isinstance(router, CfgParallelRouter) and router.had_override:
+ _rebind_root_forward(router.orig_forward, module, replica)
+ # The source's own __call__ runs its root hooks around every branch; the copy must not run them a second time.
+ for attr in _ROOT_HOOK_DICTS:
+ hooks = getattr(replica, attr, None)
+ if hooks is not None:
+ hooks.clear()
+ compiled_subs = 0
+ for (name, src), (_rname, dst) in zip(module.named_modules(), replica.named_modules()):
+ for key, value in list(dst.__dict__.items()):
+ if (
+ isinstance(value, torch.Tensor)
+ and value.device != device
+ and value.device.type == "cuda"
+ ):
+ dst.__dict__[key] = value.to(device)
+ if getattr(src, "_compiled_call_impl", None) is not None:
+ recorded = src.__dict__.get("_unsloth_compile_args")
+ dst._compiled_call_impl = None
+ if recorded is None:
+ raise RuntimeError(f"compiled submodule '{name}' has no recorded compile arguments")
+ args, kwargs = recorded
+ dst.compile(*args, **kwargs)
+ compiled_subs += 1
+ elif "_compiled_call_impl" in dst.__dict__:
+ dst._compiled_call_impl = None
+ if compiled_subs:
+ _rewrap_compiled(module, replica)
+ replica._unsloth_replica_compiled = compiled_subs
+ replica.eval()
+ return replica
+
+
+_SHARED_STATE_TYPES = frozenset({"NVFP4StepController"})
+
+
+def _rewrap_compiled(source: Any, replica: Any) -> None:
+ """Re-apply the wrappers Studio's compile path puts around the source's compiled blocks (``dst.compile`` replaced
+ them): the compile guard (eager fallback on a compile-time failure, the background-compile eager gate, the compile
+ config on this thread) and the block-0 restride, in the order the source got them."""
+ if "_unsloth_compile_guard" in source.__dict__:
+ from .diffusion_speed import guard_compiled_blocks
+ guard_compiled_blocks(replica)
+ from . import diffusion_block_restride
+
+ if any(
+ diffusion_block_restride.is_wrapped(getattr(m, "_compiled_call_impl", None))
+ for m in source.modules()
+ ):
+ diffusion_block_restride.install(replica)
+
+
+_ADAPTER_ATTRS = (
+ "scaling",
+ # PEFT stores the selection here; ``active_adapters`` is a property over it (never in ``__dict__``)
+ "_active_adapter",
+ "active_adapters",
+ "merged_adapters",
+ "disable_adapters",
+ "_disable_adapters",
+)
+
+
+def weights_fingerprint(module: Any) -> int:
+ """Changes whenever the denoiser's weights or adapter state do: a parameter or buffer (NVFP4 / quantised layers
+ keep their packed weights and scales as buffers) replaced, edited in place (fused LoRA bumps the version counter),
+ added or removed (PEFT injection), or an adapter's scale / activation changed. A ``.data`` swap to different values
+ is not seen; diffusers' LoRA paths do not do that."""
+ parts: list = []
+ for name, param in (*module.named_parameters(), *module.named_buffers()):
+ try:
+ version = param._version
+ except Exception: # noqa: BLE001 - inference tensors carry no version
+ version = -1
+ # Identity, not the data pointer: an encode-time offload round trip moves the same weights and must not force
+ # a rebuild, while a replaced parameter (new object) or an in-place edit (version) must.
+ parts.append((name, id(param), version))
+ for name, sub in module.named_modules():
+ for attr in _ADAPTER_ATTRS:
+ value = sub.__dict__.get(attr)
+ if value is not None and not callable(value):
+ parts.append((name, attr, repr(value)))
+ return hash(tuple(parts))
+
+
+_LATENT_KEYS = ("hidden_states", "sample", "x", "latents", "img", "latent")
+
+
+def _latent_arg(args: tuple, kwargs: dict) -> Any:
+ torch = _torch()
+ for key in _LATENT_KEYS:
+ value = kwargs.get(key)
+ if isinstance(value, torch.Tensor):
+ return value
+ for value in args:
+ if isinstance(value, torch.Tensor):
+ return value
+ return None
+
+
+_PLAIN_TYPES = (type(None), bool, int, float, str, bytes)
+
+
+def _holds_opaque_tensors(obj: Any, depth: int = 0) -> bool:
+ """True when ``obj`` carries tensors inside something other than a plain tuple / list / dict (a prefix-KV cache
+ object, a namedtuple, a value holder). The replica walker cannot move those to its card, and a mutable cache must
+ not be shared across the two threads, so such a branch stays on the primary."""
+ torch = _torch()
+ if isinstance(obj, (torch.Tensor, *_PLAIN_TYPES, torch.device, torch.dtype, torch.nn.Module)):
+ return False
+ if depth < 4 and isinstance(obj, tuple) and not hasattr(obj, "_fields"):
+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj)
+ if depth < 4 and isinstance(obj, list):
+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj)
+ if depth < 4 and isinstance(obj, dict):
+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj.values())
+ if isinstance(obj, tuple):
+ values: Any = obj
+ else:
+ try:
+ values = vars(obj).values()
+ except TypeError:
+ return False
+ for value in values:
+ if isinstance(value, torch.Tensor) or (
+ isinstance(value, (list, tuple, dict)) and next(_iter_tensors(value), None) is not None
+ ):
+ return True
+ if not isinstance(value, (*_PLAIN_TYPES, list, tuple, dict)) and depth < 4:
+ if _holds_opaque_tensors(value, depth + 1):
+ return True
+ return False
+
+
+def _iter_tensors(obj: Any, depth: int = 0) -> Iterator[Any]:
+ torch = _torch()
+ if isinstance(obj, torch.Tensor):
+ yield obj
+ elif depth < 4 and isinstance(obj, (list, tuple)):
+ for v in obj:
+ yield from _iter_tensors(v, depth + 1)
+ elif depth < 4 and isinstance(obj, dict):
+ for v in obj.values():
+ yield from _iter_tensors(v, depth + 1)
+
+
+class _ReplicaWorker:
+ """One daemon thread bound to the replica's card: set_device, grad / inference / autocast mode and the caller's
+ context variables (inductor overrides live in one on torch >= 2.12) are applied per job."""
+
+ def __init__(self, device: Any):
+ self.device = device
+ self._jobs: "queue.Queue[Optional[tuple]]" = queue.Queue()
+ self._thread = threading.Thread(
+ target = self._loop, name = f"unsloth-cfg-replica-{device}", daemon = True
+ )
+ self._thread.start()
+
+ def _loop(self) -> None:
+ torch = _torch()
+ if getattr(self.device, "type", None) == "cuda":
+ try:
+ torch.cuda.set_device(self.device)
+ except Exception: # noqa: BLE001
+ pass
+ while True:
+ job = self._jobs.get()
+ if job is None:
+ return
+ fn, ctx, modes, done = job
+ try:
+ done["result"] = ctx.run(self._run, fn, modes)
+ except BaseException as exc: # noqa: BLE001 - surfaced on the caller's thread
+ done["error"] = exc
+ finally:
+ done["event"].set()
+
+ def _run(self, fn: Callable[[], Any], modes: dict) -> Any:
+ torch = _torch()
+ with contextlib.ExitStack() as stack:
+ if modes.get("inference"):
+ stack.enter_context(torch.inference_mode())
+ stack.enter_context(torch.set_grad_enabled(modes.get("grad", False)))
+ autocast_dtype = modes.get("autocast")
+ if autocast_dtype is not None:
+ stack.enter_context(torch.autocast(device_type = "cuda", dtype = autocast_dtype))
+ stack.enter_context(_device_context(self.device))
+ return fn()
+
+ def submit(self, fn: Callable[[], Any]) -> dict:
+ torch = _torch()
+ autocast = None
+ try:
+ if torch.is_autocast_enabled("cuda"):
+ autocast = torch.get_autocast_dtype("cuda")
+ except Exception: # noqa: BLE001 - older torch
+ pass
+ modes = {
+ "inference": torch.is_inference_mode_enabled(),
+ "grad": torch.is_grad_enabled(),
+ "autocast": autocast,
+ }
+ done: dict[str, Any] = {"event": threading.Event()}
+ self._jobs.put((fn, contextvars.copy_context(), modes, done))
+ return done
+
+ @staticmethod
+ def wait(done: dict) -> Any:
+ done["event"].wait()
+ if "error" in done:
+ raise done["error"]
+ return done.get("result")
+
+ def call(self, fn: Callable[[], Any]) -> Any:
+ return self.wait(self.submit(fn))
+
+ def close(self) -> None:
+ self._jobs.put(None)
+
+
+def _leaves(
+ obj: Any,
+ path: tuple = (),
+ depth: int = 0,
+) -> list[tuple[tuple, Any]]:
+ if depth < 4 and isinstance(obj, (tuple, list)) and not hasattr(obj, "_fields"):
+ out: list = []
+ for i, v in enumerate(obj):
+ out.extend(_leaves(v, path + (i,), depth + 1))
+ return out or [(path, obj)]
+ if depth < 4 and isinstance(obj, dict):
+ out = []
+ for k, v in obj.items():
+ out.extend(_leaves(v, path + (k,), depth + 1))
+ return out or [(path, obj)]
+ return [(path, obj)]
+
+
+def _rebuild(
+ template: Any,
+ values: dict,
+ path: tuple = (),
+ depth: int = 0,
+) -> Any:
+ if (
+ depth < 4
+ and isinstance(template, (tuple, list))
+ and not hasattr(template, "_fields")
+ and len(template)
+ ):
+ items = [_rebuild(v, values, path + (i,), depth + 1) for i, v in enumerate(template)]
+ return type(template)(items) if isinstance(template, tuple) else items
+ if depth < 4 and isinstance(template, dict) and template:
+ return type(template)(
+ (k, _rebuild(v, values, path + (k,), depth + 1)) for k, v in template.items()
+ )
+ return values[path]
+
+
+_SMALL_TENSOR_NUMEL = 4096
+
+
+def _bitwise_equal(a: Any, b: Any) -> bool:
+ torch = _torch()
+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):
+ return False
+ if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:
+ return False
+ if a is b:
+ return True
+ try:
+ return bool(
+ torch.equal(
+ a.reshape(-1).contiguous().view(torch.uint8),
+ b.reshape(-1).contiguous().view(torch.uint8),
+ )
+ )
+ except Exception: # noqa: BLE001 - a layout that cannot be viewed as bytes is never assumed equal
+ return False
+
+
+def _unchanged_since_launch(snapshots: list) -> bool:
+ """True when every (source, launch-time copy) pair still holds the same bytes. One host sync for all pairs, on
+ the replica's card; it waits for the primary branch (the copy is ordered after it), which the guidance combine
+ waits for anyway, while the replica keeps running."""
+ torch = _torch()
+ checks = []
+ for src, snap in snapshots:
+ if src.shape != snap.shape or src.dtype != snap.dtype:
+ return False
+ if src.numel() == 0:
+ continue
+ try:
+ now = src.to(snap.device).reshape(-1).contiguous().view(torch.uint8)
+ checks.append(
+ torch.equal(now, snap.reshape(-1).contiguous().view(torch.uint8))
+ if snap.device.type == "cpu"
+ else (now == snap.reshape(-1).contiguous().view(torch.uint8)).all()
+ )
+ except Exception: # noqa: BLE001 - an unverifiable tensor never accepts a prediction
+ return False
+ if not checks:
+ return True
+ host = [c for c in checks if isinstance(c, bool)]
+ device = [c for c in checks if not isinstance(c, bool)]
+ return all(host) and (not device or bool(torch.stack(device).all().item()))
+
+
+def _same_value(a: Any, b: Any) -> bool:
+ if a is b:
+ return True
+ torch = _torch()
+ if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor):
+ return False
+ try:
+ return bool(a == b) and type(a) is type(b)
+ except Exception: # noqa: BLE001 - incomparable objects must be the same object
+ return False
+
+
+# Per-call state Studio's root hooks write onto denoiser submodules (``_unsloth_*`` scalars, e.g. the HunyuanVideo-1.5
+# trim hook's null-mask flag). The root hooks run only on the source, around the router, so the replica's copy of that
+# state is mirrored from the source before every replica call and the prediction is verified against it.
+_STATE_PREFIX = "_unsloth_"
+_STATE_TYPES = (bool, int, float, str, type(None))
+_STATE_KEEP = frozenset({"_unsloth_replica_compiled"})
+
+
+def _hook_state(pairs: list) -> tuple:
+ out = []
+ for i, (src, _dst) in enumerate(pairs):
+ for key, value in src.__dict__.items():
+ if key.startswith(_STATE_PREFIX) and isinstance(value, _STATE_TYPES):
+ out.append((i, key, value))
+ return tuple(out)
+
+
+def _mirror_hook_state(pairs: list) -> tuple:
+ state = _hook_state(pairs)
+ wanted: dict = {}
+ for i, key, value in state:
+ wanted.setdefault(i, {})[key] = value
+ for i, (_src, dst) in enumerate(pairs):
+ mine = wanted.get(i, {})
+ for key in [
+ k
+ for k, v in dst.__dict__.items()
+ if k.startswith(_STATE_PREFIX)
+ and isinstance(v, _STATE_TYPES)
+ and k not in mine
+ and k not in _STATE_KEEP
+ ]:
+ del dst.__dict__[key]
+ for key, value in mine.items():
+ if dst.__dict__.get(key, _DROP) is not value and dst.__dict__.get(key, _DROP) != value:
+ dst.__dict__[key] = value
+ return state
+
+
+def _background_compile_call() -> bool:
+ try:
+ from .diffusion_bg_compile import capture_suppressed, eager_forced
+ except Exception: # noqa: BLE001
+ return False
+ return bool(eager_forced() or capture_suppressed())
+
+
+class CfgParallelRouter:
+ """Installed as the denoiser's ``forward``: routes odd branches of a step to the replica.
+
+ Once a step has shown two branches, the next step's first odd branch is predicted from the previous one (same
+ structure; the latent, and every tensor the previous odd call shared with its primary call, replaced by this
+ step's; the rest -- negative embeds, masks -- kept) and launched on the replica BEFORE the primary branch runs, so
+ the two cards overlap even when the primary forward blocks the host. The real call must then pass exactly the
+ predicted tensors (identity) and equal plain values, else the prediction is discarded and the branch runs
+ unpredicted."""
+
+ def __init__(
+ self,
+ module: Any,
+ *,
+ primary: Any,
+ secondary: Any,
+ replica_factory: Optional[Callable[[], Any]] = None,
+ logger: Any = None,
+ ):
+ torch = _torch()
+ self.module_ref = weakref.ref(module)
+ self.orig_forward = module.forward
+ self.had_override = "forward" in module.__dict__
+ self.primary = torch.device(primary)
+ self.secondary = torch.device(secondary)
+ self.logger = logger
+ self.replica: Any = None
+ self.replica_factory = replica_factory or (lambda: build_replica(module, self.secondary))
+ self.worker: Optional[_ReplicaWorker] = None
+ self.disabled_reason: Optional[str] = None
+ self.prev_latent: Any = None
+ self.branch = 0
+ self.cfg_seen = False
+ self.singles = 0
+ self.prefetched: dict[int, tuple[Any, Any]] = {}
+ # Tensors the odd branches of the previous step read (negative embeds, masks): copied again before the next
+ # step's primary branch is queued, so they are always fresh and never wait behind it.
+ self.carry: list[Any] = []
+ self.next_carry: list[Any] = []
+ self.stats = {
+ "replica_calls": 0,
+ "primary_calls": 0,
+ "steps_parallel": 0,
+ "replica_build_s": None,
+ "replica_rebuilds": 0,
+ "predicted_hits": 0,
+ "predicted_misses": 0,
+ "opaque_primary_calls": 0,
+ }
+ self.template: Optional[tuple[Any, Any, dict]] = None
+ self.pending: Optional[dict] = None
+ self.b0_leaves: dict = {}
+ self.fingerprint: Optional[int] = None
+ self.pairs: list = []
+
+ def describe(self) -> dict[str, Any]:
+ return {
+ "secondary": str(self.secondary),
+ "replica_built": self.replica is not None,
+ "replica_compiled_submodules": getattr(self.replica, "_unsloth_replica_compiled", None),
+ "disabled_reason": self.disabled_reason,
+ **self.stats,
+ }
+
+ def disable(self, reason: str) -> None:
+ if self.disabled_reason is None:
+ self.disabled_reason = reason
+ if self.logger is not None:
+ self.logger.warning(
+ "diffusion.multigpu: CFG-parallel disabled (%s); one card from here", reason
+ )
+ self.drop_replica()
+
+ def drop_replica(self) -> None:
+ if self.worker is not None:
+ self.worker.close()
+ self.worker = None
+ self._drop_pending()
+ self.replica = None
+ self.pairs = []
+ self.prefetched.clear()
+ self.carry = []
+ self.next_carry = []
+ self.prev_latent = None
+ self.template = None
+ self.b0_leaves = {}
+
+ def end_render(self) -> None:
+ self._drop_pending()
+ self.prefetched.clear()
+ self.carry = []
+ self.next_carry = []
+ self.prev_latent = None
+ self.template = None
+ self.b0_leaves = {}
+ self.branch = 0
+ # the next render shows its own guidance shape: no speculative branch before its first odd call
+ self.cfg_seen = False
+ self.singles = 0
+
+ def __call__(self, *args: Any, **kwargs: Any) -> Any:
+ if self.disabled_reason is not None or _background_compile_call():
+ # Disabled: no branch state to keep. A render that must run eager while its compile is in flight, or the
+ # background compile thread itself replaying recorded inputs: both branches stay on the source, which
+ # keeps that render bit-identical to an eager one and leaves the router's per-render state untouched.
+ self.stats["primary_calls"] += 1
+ return self.orig_forward(*args, **kwargs)
+ latent = _latent_arg(args, kwargs)
+ if latent is not None and self.prev_latent is not None and latent is self.prev_latent:
+ self.branch += 1
+ else:
+ if self.branch == 0 and self.prev_latent is not None:
+ self.singles += 1
+ if self.singles >= 3:
+ self.cfg_seen = False
+ self.branch = 0
+ self.prev_latent = latent
+ self.prefetched.clear()
+ self.carry, self.next_carry = self.next_carry, []
+ if self.replica is not None and self.disabled_reason is None:
+ self._drop_if_stale()
+ self._drop_pending()
+ self.b0_leaves = dict(_leaves((args, kwargs)))
+ if self.cfg_seen and self.replica is not None and self.disabled_reason is None:
+ # Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after
+ # it would wait for it on the source stream and serialise the two branches.
+ self._prefetch(args, kwargs)
+ self._launch_prediction(latent)
+ if (
+ self.branch % 2 == 1
+ and self.disabled_reason is None
+ and _holds_opaque_tensors((args, kwargs))
+ ):
+ self._drop_pending()
+ self.template = None
+ self.stats["opaque_primary_calls"] += 1
+ elif self.branch % 2 == 1 and self.disabled_reason is None:
+ self.singles = 0
+ if self.replica is None:
+ self.cfg_seen = True
+ self._build_replica()
+ if self.branch == 1:
+ self._record_template(args, kwargs, latent)
+ elif self.cfg_seen:
+ if self.branch == 1:
+ self._record_template(args, kwargs, latent)
+ try:
+ out = self._take_prediction(args, kwargs) if self.branch == 1 else None
+ if out is None:
+ out = self._run_replica(args, kwargs)
+ self.stats["replica_calls"] += 1
+ if self.branch == 1:
+ self.stats["steps_parallel"] += 1
+ return out
+ except Exception as exc: # noqa: BLE001 - this branch is recomputed on the primary below
+ self.disable(f"replica call failed: {type(exc).__name__}: {exc}")
+ self.cfg_seen = True
+ self.stats["primary_calls"] += 1
+ return self.orig_forward(*args, **kwargs)
+
+ def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:
+ torch = _torch()
+ b0_by_id = {id(v): p for p, v in self.b0_leaves.items() if isinstance(v, torch.Tensor)}
+ kinds: dict = {}
+ for path, leaf in _leaves((args, kwargs)):
+ if isinstance(leaf, torch.Tensor):
+ if leaf is latent:
+ kinds[path] = ("latent",)
+ elif id(leaf) in b0_by_id and self.b0_leaves.get(b0_by_id[id(leaf)]) is leaf:
+ kinds[path] = ("b0", b0_by_id[id(leaf)])
+ elif (
+ isinstance(self.b0_leaves.get(path), torch.Tensor)
+ and leaf.numel() <= _SMALL_TENSOR_NUMEL
+ and _bitwise_equal(leaf, self.b0_leaves[path])
+ ):
+ kinds[path] = ("b0eq", path)
+ else:
+ kinds[path] = ("const", leaf)
+ else:
+ kinds[path] = ("value", leaf)
+ self.template = ((args, kwargs), kinds)
+
+ def _predict(self, latent: Any) -> Optional[tuple]:
+ if self.template is None or latent is None:
+ return None
+ structure, kinds = self.template
+ values: dict = {}
+ for path, kind in kinds.items():
+ if kind[0] == "latent":
+ values[path] = latent
+ elif kind[0] in ("b0", "b0eq"):
+ if kind[1] not in self.b0_leaves:
+ return None
+ values[path] = self.b0_leaves[kind[1]]
+ else:
+ values[path] = kind[1]
+ try:
+ return _rebuild(structure, values)
+ except KeyError:
+ return None
+
+ def _launch_prediction(self, latent: Any) -> None:
+ predicted = self._predict(latent)
+ if predicted is None or self.worker is None:
+ return
+ p_args, p_kwargs = predicted
+ try:
+ r_args = self._to_replica(p_args)
+ r_kwargs = self._to_replica(p_kwargs)
+ except Exception: # noqa: BLE001 - no prediction this step; the branch runs unpredicted
+ return
+ torch = _torch()
+ # What the replica consumes, kept per tensor: at the real call the sources are compared bitwise against it, so
+ # an in-place edit between launch and the odd branch (same object, new values) rejects the prediction.
+ # Version counters cannot do this: inference tensors do not track one.
+ snapshots = []
+ for (_path, src), (_rpath, used) in zip(
+ _leaves((p_args, p_kwargs)), _leaves((r_args, r_kwargs))
+ ):
+ if isinstance(src, torch.Tensor):
+ snapshots.append((src, used.clone() if used is src else used))
+ replica = self.replica
+ _structure, kinds = self.template
+ hook_state = _mirror_hook_state(self.pairs)
+ self.pending = {
+ "hook_state": hook_state,
+ "leaves": _leaves((p_args, p_kwargs)),
+ "by_value": {path for path, kind in kinds.items() if kind[0] == "b0eq"},
+ "snapshots": snapshots,
+ "job": self.worker.submit(lambda: replica(*r_args, **r_kwargs)),
+ }
+
+ def _take_prediction(self, args: tuple, kwargs: dict) -> Any:
+ pending, self.pending = self.pending, None
+ if pending is None:
+ return None
+ actual = _leaves((args, kwargs))
+ predicted = pending["leaves"]
+ by_value = pending["by_value"]
+ ok = len(actual) == len(predicted) and all(
+ pa == pp and (_bitwise_equal(va, vp) if pa in by_value else _same_value(va, vp))
+ for (pa, va), (pp, vp) in zip(actual, predicted)
+ )
+ # the source's root hooks have now run for THIS call: the replica must have seen the same per-call state
+ ok = ok and _hook_state(self.pairs) == pending["hook_state"]
+ ok = ok and _unchanged_since_launch(pending["snapshots"])
+ try:
+ out = _ReplicaWorker.wait(pending["job"])
+ except Exception:
+ if ok:
+ raise
+ out = None
+ if not ok:
+ self.stats["predicted_misses"] += 1
+ self.prefetched.clear()
+ return None
+ self.stats["predicted_hits"] += 1
+ return move_to_device(out, self.primary)
+
+ def _drop_pending(self) -> None:
+ pending, self.pending = self.pending, None
+ if pending is not None:
+ with contextlib.suppress(Exception):
+ _ReplicaWorker.wait(pending["job"])
+ self.stats["predicted_misses"] += 1
+
+ def _drop_if_stale(self) -> None:
+ module = self.module_ref()
+ if module is None:
+ return
+ try:
+ current = weights_fingerprint(module)
+ except Exception as exc: # noqa: BLE001 - unverifiable weights: never risk a stale replica
+ self.disable(f"cannot fingerprint the denoiser: {exc}")
+ return
+ if current != self.fingerprint:
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.multigpu: denoiser weights changed; rebuilding the CFG-parallel replica"
+ )
+ self.stats["replica_rebuilds"] += 1
+ self.drop_replica()
+ return
+ # A step cache (transformer_cache=auto) may be installed on the source at generate time, after the replica
+ # was built: the copy would then run uncached while the source's cache saw only the even branches.
+ hooked = _hooks_present(module)
+ if hooked is not None:
+ self.disable(f"denoiser gained hooks after the replica was built ({hooked})")
+
+ def _build_replica(self) -> None:
+ module = self.module_ref()
+ if module is None:
+ return
+ t0 = time.perf_counter()
+ try:
+ hooked = _hooks_present(module)
+ if hooked is not None:
+ raise RuntimeError(f"denoiser carries hooks ({hooked})")
+ need_mib = sum(
+ t.numel() * t.element_size() for t in (*module.parameters(), *module.buffers())
+ ) // (1 << 20)
+ free, _total = _free_total_mib(self.secondary.index)
+ if free is not None:
+ with contextlib.suppress(Exception):
+ torch = _torch()
+ free += (
+ torch.cuda.memory_reserved(self.secondary)
+ - torch.cuda.memory_allocated(self.secondary)
+ ) // (1 << 20)
+ if free is not None and need_mib + 1024 > free:
+ raise RuntimeError(
+ f"replica needs {need_mib} MiB, {free} MiB free on {self.secondary}"
+ )
+ self.fingerprint = weights_fingerprint(module)
+ with _device_context(self.secondary):
+ self.replica = self.replica_factory()
+ self.pairs = list(zip(module.modules(), self.replica.modules()))
+ self.worker = _ReplicaWorker(self.secondary)
+ weakref.finalize(self, self.worker.close)
+ self.stats["replica_build_s"] = round(time.perf_counter() - t0, 3)
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.multigpu: CFG-parallel replica on %s built in %.2fs",
+ self.secondary,
+ time.perf_counter() - t0,
+ )
+ except Exception as exc: # noqa: BLE001
+ self.replica = None
+ with contextlib.suppress(Exception):
+ _torch().cuda.empty_cache()
+ self.disable(f"replica unavailable: {exc}")
+
+ def _prefetch(self, args: tuple, kwargs: dict) -> None:
+ for tensor in (*_iter_tensors((args, kwargs)), *self.carry):
+ if tensor.device == self.secondary or id(tensor) in self.prefetched:
+ continue
+ self.prefetched[id(tensor)] = (tensor, tensor.to(self.secondary))
+
+ def _to_replica(self, obj: Any) -> Any:
+ torch = _torch()
+
+ def one(tensor: Any) -> Any:
+ if tensor is not self.prev_latent and not any(tensor is t for t in self.next_carry):
+ self.next_carry.append(tensor)
+ if tensor.device == self.secondary:
+ return tensor
+ hit = self.prefetched.get(id(tensor))
+ if hit is not None and hit[0] is tensor:
+ return hit[1]
+ copied = tensor.to(self.secondary)
+ self.prefetched[id(tensor)] = (tensor, copied)
+ return copied
+
+ def walk(value: Any, depth: int = 0) -> Any:
+ if isinstance(value, torch.Tensor):
+ return one(value)
+ if depth < 4 and isinstance(value, tuple) and not hasattr(value, "_fields"):
+ return tuple(walk(v, depth + 1) for v in value)
+ if depth < 4 and isinstance(value, list):
+ return [walk(v, depth + 1) for v in value]
+ if depth < 4 and isinstance(value, dict):
+ return type(value)((k, walk(v, depth + 1)) for k, v in value.items())
+ return value
+
+ return walk(obj)
+
+ def _run_replica(self, args: tuple, kwargs: dict) -> Any:
+ # Unpredicted: nothing verified the copies made before the primary branch ran, and an input edited in place
+ # since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).
+ self.prefetched.clear()
+ _mirror_hook_state(self.pairs)
+ r_args = self._to_replica(args)
+ r_kwargs = self._to_replica(kwargs)
+ replica = self.replica
+ out = self.worker.call(lambda: replica(*r_args, **r_kwargs))
+ return move_to_device(out, self.primary)
+
+
+_CFG_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgParallelRouter]" = weakref.WeakKeyDictionary()
+_CFG_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()
+
+
+def cfg_parallel_decline(
+ pipe: Any, *, offload_policy: str, selection: Optional[MultiGpuSelection]
+) -> Optional[str]:
+ if selection is None:
+ return "one card selected"
+ mode = cfg_parallel_mode()
+ if mode == "off":
+ return f"{CFG_PARALLEL_ENV}=0"
+ # A replica takes a whole denoiser's worth of another card; automatic placement only borrows one to avoid offload.
+ if selection.automatic and mode != "force":
+ return f"automatic GPU placement (select several cards or set {CFG_PARALLEL_ENV}=force)"
+ from .diffusion_bg_compile import load_time_enabled as bg_compile_enabled
+
+ if bg_compile_enabled():
+ # its replays call the denoiser, which would enter the router and race a render's branch state
+ return "background compile replays through the denoiser (UNSLOTH_DIFFUSION_BG_COMPILE)"
+ denoiser = getattr(pipe, "transformer", None)
+ torch = _torch()
+ if not isinstance(denoiser, torch.nn.Module):
+ return "no DiT denoiser (a U-Net batches its guidance branches)"
+ # Whole-module / sequential offload page the denoiser per call; a group / streaming plan is fine only when it left
+ # the denoiser itself resident (encoders streamed), which shows as a denoiser without offload hooks.
+ if offload_policy in ("model", "sequential"):
+ return f"denoiser is offloaded ({offload_policy})"
+ hooked = _hooks_present(denoiser)
+ if hooked is not None:
+ return f"denoiser is offloaded or hooked ({offload_policy}: {hooked})"
+ if getattr(pipe, "transformer_2", None) is not None:
+ return "two-expert denoiser"
+ if getattr(denoiser, "_compiled_call_impl", None) is not None:
+ return "denoiser is compiled whole"
+ if _module_device(denoiser) is None or _module_device(denoiser).type != "cuda":
+ return "denoiser is not on a CUDA card"
+ return None
+
+
+def install_cfg_parallel(
+ pipe: Any,
+ *,
+ offload_policy: str,
+ logger: Any = None,
+) -> Optional[str]:
+ """Arm CFG-parallel on ``pipe.transformer`` when a secondary card can hold a replica. The replica is built on the
+ first step that shows two branches, so a guidance-distilled model never pays for it. Returns the decline reason or
+ None when armed."""
+ selection = current_selection()
+ reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)
+ if reason is None:
+ denoiser = pipe.transformer
+ primary = _module_device(denoiser)
+ probes = probe_devices((selection.primary, *selection.secondaries))
+ same = [
+ o
+ for o in selection.secondaries
+ if probes.get(o) is not None
+ and probes.get(primary.index) is not None
+ and probes[o].name == probes[primary.index].name
+ ]
+ candidates = same if cfg_parallel_mode() != "force" else list(selection.secondaries)
+ try:
+ placed = ((_PLACEMENTS.get(pipe) or {}).get("placement") or {}).values()
+ except TypeError:
+ placed = ()
+ occupied = {str(d) for d in placed} - {str(primary)}
+ if occupied:
+ candidates = [o for o in candidates if f"cuda:{o}" not in occupied]
+ if not candidates and occupied:
+ reason = f"the only replica card ({', '.join(sorted(occupied))}) holds the split's components"
+ elif not candidates:
+ reason = "secondary card is a different model (bit-identity needs the same kernels)"
+ else:
+ need_mib = sum(
+ t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers())
+ ) // (1 << 20)
+ best = max(candidates, key = lambda o: probes[o].free_mib or -1)
+ free = probes[best].free_mib
+ if free is not None and need_mib + 1024 > free:
+ reason = f"replica needs {need_mib} MiB, {free} MiB free on cuda:{best}"
+ else:
+ router = CfgParallelRouter(
+ denoiser, primary = primary, secondary = f"cuda:{best}", logger = logger
+ )
+ try:
+ _CFG_ROUTERS[pipe] = router
+ except TypeError:
+ return "pipeline cannot be tracked (not weak-referenceable)"
+ denoiser.forward = router
+ denoiser._unsloth_cfg_router = router
+ if logger is not None:
+ logger.info(
+ "diffusion.multigpu: CFG-parallel armed (replica on cuda:%s when guidance runs two branches)",
+ best,
+ )
+ return None
+ if selection is not None:
+ with contextlib.suppress(TypeError):
+ _CFG_DECLINES[pipe] = reason
+ if logger is not None:
+ logger.info("diffusion.multigpu: CFG-parallel not armed (%s)", reason)
+ return reason
+
+
+def invalidate_cfg_replica(
+ pipe: Any,
+ reason: str,
+ logger: Any = None,
+) -> bool:
+ """Drop the CFG-parallel replica of ``pipe`` after its denoiser was mutated in ways the per-step staleness check
+ does not see (attention backend, eager patches, compile); the next guided step rebuilds it from the source. True
+ when a replica was dropped."""
+ try:
+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
+ except TypeError:
+ return False
+ if router is None or router.replica is None:
+ return False
+ if logger is not None:
+ logger.info(
+ "diffusion.multigpu: CFG-parallel replica dropped (%s); rebuilt on the next guided step",
+ reason,
+ )
+ router.stats["replica_rebuilds"] += 1
+ router.drop_replica()
+ return True
+
+
+def drop_displaced_cfg_router(
+ pipe: Any,
+ reason: str,
+ logger: Any = None,
+) -> bool:
+ """Uninstall ``pipe``'s CFG-parallel router when a later layer (the deferred speed profile's graphs) replaced the
+ denoiser's forward and bypasses it, so status reports CFG-parallel as declined rather than armed. True if dropped."""
+ try:
+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
+ except TypeError:
+ return False
+ if router is None:
+ return False
+ module = router.module_ref()
+ if module is not None and module.__dict__.get("forward") is router:
+ return False
+ uninstall_cfg_parallel(pipe)
+ with contextlib.suppress(TypeError):
+ _CFG_DECLINES[pipe] = reason
+ if logger is not None:
+ logger.info("diffusion.multigpu: CFG-parallel dropped (%s)", reason)
+ return True
+
+
+def end_cfg_render(pipe: Any) -> None:
+ try:
+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
+ except TypeError:
+ return
+ if router is not None:
+ router.end_render()
+
+
+def uninstall_cfg_parallel(pipe: Any) -> None:
+ try:
+ router = _CFG_ROUTERS.pop(pipe, None) if pipe is not None else None
+ except TypeError:
+ return
+ if router is None:
+ return
+ if router.disabled_reason is None:
+ router.disabled_reason = "uninstalled"
+ router.drop_replica()
+ module = router.module_ref()
+ if module is not None:
+ if module.__dict__.get("forward") is router:
+ if router.had_override:
+ module.__dict__["forward"] = router.orig_forward
+ else:
+ del module.__dict__["forward"]
+ if module.__dict__.get("_unsloth_cfg_router") is router:
+ del module.__dict__["_unsloth_cfg_router"]
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index f04cd5068f5..e1e28a247df 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -104,6 +104,18 @@ from .diffusion_device import (
auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
+ resolve_selected_cuda_ordinals,
+)
+from .diffusion_multigpu import (
+ AUTOMATIC_ORDINALS,
+ end_cfg_render,
+ install_cfg_parallel,
+ multigpu_mode,
+ is_split as multigpu_is_split,
+ placement_status as multigpu_placement_status,
+ refine_plan_for_multigpu,
+ uninstall_cfg_parallel,
+ with_load_selection,
)
from .diffusion_memory import (
DEFAULT_BASE_OVERHEAD_MIB,
@@ -1427,6 +1439,7 @@ class _VideoLoadState:
vram_floor_mib: Optional[int] = None
# MiB still on the device during the VAE decode (the streamed DiT has offloaded by then).
vram_decode_floor_mib: Optional[int] = None
+ vram_split_cards: Optional[tuple] = None
# MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.
denoiser_host_copy: bool = False
# MiniMax-H3: conditioner streamed leaf by leaf (VRAM floor counts its streamed footprint).
@@ -3148,6 +3161,143 @@ def _video_offload_vram_floor_mib(
return None
+# Families whose text encoder runs outside ``pipe.components`` (a sidecar worker on the primary): a multi-GPU split
+# cannot move it, so the companion split the planner priced with it would leave it on the primary beside the DiT.
+_VIDEO_SIDECAR_ENCODER_FAMILIES = frozenset({"ltx-2.5"})
+
+
+def _video_single_card_for_sidecar(
+ fam: Any,
+ plan: Any,
+ logger: Any = None,
+) -> Any:
+ """The single-card plan in place of a companion split for a sidecar-encoder family; any other plan unchanged."""
+ if getattr(fam, "name", None) not in _VIDEO_SIDECAR_ENCODER_FAMILIES:
+ return plan
+ fallback = getattr(plan, "fallback_plan", None)
+ if getattr(plan, "secondary_ordinal", None) is None or fallback is None:
+ return plan
+ if logger is not None:
+ logger.info(
+ "video.multigpu: one card (the %s text encoder runs in a sidecar a split cannot move)",
+ fam.name,
+ )
+ return fallback
+
+
+def _video_refine_for_multigpu(
+ fam: Any,
+ pipe: Any,
+ plan: Any,
+ logger: Any = None,
+) -> Any:
+ """``refine_plan_for_multigpu``, declined for a sidecar-encoder family (its loaded sizes omit the encoder)."""
+ if getattr(fam, "name", None) in _VIDEO_SIDECAR_ENCODER_FAMILIES:
+ return plan
+ return refine_plan_for_multigpu(pipe, plan, logger)
+
+
+def _video_split_card_floors(pipe: Any) -> Optional[tuple]:
+ """``((ordinal, resident MiB, runs the decode), ...)`` for a pipeline split across cards, else None. A split plan
+ reports no offload policy, so the offload floor is None and the request guard would be skipped; this records what
+ each card holds so the guard can run per card."""
+ status = multigpu_placement_status(pipe) if multigpu_is_split(pipe) else None
+ placement = (status or {}).get("placement") or {}
+ if not placement:
+ return None
+ try:
+ from itertools import chain
+
+ import torch
+
+ from .diffusion_prequant import tensor_payload_bytes
+
+ components = getattr(pipe, "components", None) or {}
+ cards: dict[int, list] = {}
+ seen: set[int] = set()
+ for name, where in placement.items():
+ device = torch.device(where)
+ module = components.get(name)
+ if (
+ device.type != "cuda"
+ or device.index is None
+ or not isinstance(module, torch.nn.Module)
+ ):
+ continue
+ card = cards.setdefault(int(device.index), [0, False])
+ for tensor in chain(module.parameters(), module.buffers()):
+ if id(tensor) in seen or getattr(tensor, "is_meta", False):
+ continue
+ seen.add(id(tensor))
+ card[0] += tensor_payload_bytes(tensor)
+ card[1] = card[1] or str(name) == "vae"
+ if len(cards) < 2:
+ return None
+ home = placement.get("transformer")
+ home = torch.device(home).index if home else None
+ denoisers = {
+ torch.device(where).index
+ for name, where in placement.items()
+ if str(name).startswith("transformer") and torch.device(where).type == "cuda"
+ }
+ return tuple(
+ (o, int(b // (1024 * 1024)), bool(d), o == home, o in denoisers)
+ for o, (b, d) in sorted(cards.items())
+ )
+ except Exception: # noqa: BLE001 -- an unreadable placement keeps the unchecked behaviour
+ return None
+
+
+def video_split_shortfall_message(
+ cards: tuple,
+ *,
+ family: str,
+ available_mib: Callable[[int], int],
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ frames: Optional[int] = None,
+) -> Optional[str]:
+ """The offload guard run on every card of a split: each card's resident weights plus the base headroom, the
+ decoded-clip share on the card that runs the VAE decode, and on the denoiser's card, when the decode ran
+ elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. The first refusal,
+ else None."""
+ for card in cards:
+ ordinal, floor_mib, decodes = card[:3]
+ home = len(card) > 3 and bool(card[3])
+ denoises = bool(card[4]) if len(card) > 4 else home
+ shaped = bool(width and height and frames)
+ frames_mib = (
+ -(-int(width) * int(height) * int(frames) * 3 * 4 // (1 << 20)) if shaped else 0
+ )
+ if shaped and denoises:
+ phase_mib = max(
+ frames_mib if home and not decodes else 0, _VIDEO_DENOISE_ACTIVATION_MIB
+ )
+ shortfall = video_offload_shortfall_message(
+ family = family,
+ floor_mib = int(floor_mib) + phase_mib,
+ available_mib = int(available_mib(int(ordinal))),
+ placement = f"split across GPUs, cuda:{int(ordinal)}",
+ )
+ if shortfall is not None:
+ return shortfall
+ elif shaped and home and not decodes:
+ floor_mib = int(floor_mib) + frames_mib
+ shortfall = video_offload_shortfall_message(
+ family = family,
+ floor_mib = int(floor_mib),
+ available_mib = int(available_mib(int(ordinal))),
+ placement = f"split across GPUs, cuda:{int(ordinal)}",
+ width = width if decodes else None,
+ height = height if decodes else None,
+ frames = frames if decodes else None,
+ decode_floor_mib = int(floor_mib) if decodes else None,
+ )
+ if shortfall is not None:
+ return shortfall
+ return None
+
+
def video_offload_shortfall_message(
*,
family: str,
@@ -3929,6 +4079,16 @@ class VideoBackend:
if resolve_diffusion_device_target().device == "cuda"
else None
)
+ gpu_ordinals = (
+ resolve_selected_cuda_ordinals(gpu_ids)
+ if gpu_ids
+ and len(set(gpu_ids)) > 1
+ and multigpu_mode() != "off"
+ and resolve_diffusion_device_target().device == "cuda"
+ else AUTOMATIC_ORDINALS
+ if not gpu_ids
+ else []
+ )
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
@@ -4029,11 +4189,14 @@ class VideoBackend:
gpu_ordinal = gpu_ordinal,
_load_token = token,
_cancel_event = cancel_event,
+ _gpu_ordinals = gpu_ordinals,
),
daemon = True,
).start()
return self.status()
+ # The multi-GPU selection (begin_load's ``_gpu_ordinals``) is visible to this load's planner and placement only.
+ @with_load_selection
@scoped_local_files_only
def _run_load(self, **kwargs: Any) -> None:
token = kwargs.get("_load_token")
@@ -7864,7 +8027,12 @@ class VideoBackend:
)
planned = replanned
replanned_for_quant = True
- return planned, dense_plan, replanned_for_quant
+ # a sidecar-encoder family (LTX-2.5) keeps one card: a split cannot move its encoder
+ return (
+ _video_single_card_for_sidecar(fam, planned, logger if log else None),
+ _video_single_card_for_sidecar(fam, dense_plan),
+ replanned_for_quant,
+ )
denoiser_seed_scheme: Optional[str] = None
if kind == "pipeline" and _video_auto_denoiser_planned != DENOISER_SEED_DECLINED:
@@ -8900,6 +9068,7 @@ class VideoBackend:
if refined is not plan:
logger.info("video.memory: %s", refined.reasons[-1])
plan = refined
+ plan = _video_refine_for_multigpu(fam, pipe, plan, logger)
vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
free_mib = getattr(getattr(plan, "device_memory", None), "free_mib", None)
if vram_floor_mib is not None and free_mib is not None:
@@ -8928,7 +9097,10 @@ class VideoBackend:
logger = logger,
)
# a transient OOM during setup may have re-planned to lower residency
+ planned = plan
plan = applied_plan(pipe, plan)
+ if getattr(planned, "secondary_ordinal", None) is not None and plan is not planned:
+ vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)
if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:
# ~1 GB; resident with the rest, else render() moves it in for the upsample only.
@@ -9110,6 +9282,7 @@ class VideoBackend:
},
logger = logger,
)
+ install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)
# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.
from . import diffusion_bg_compile as bg_compile
@@ -9175,6 +9348,9 @@ class VideoBackend:
pipe, dataclasses.replace(plan, offload_policy = offload_policy)
)
),
+ vram_split_cards = (
+ _video_split_card_floors(pipe) if offload_policy == OFFLOAD_NONE else None
+ ),
vram_decode_floor_mib = _video_offload_vram_floor_mib(
pipe,
plan
@@ -11189,6 +11365,35 @@ class VideoBackend:
)
if shortfall is not None:
raise RuntimeError(shortfall)
+ elif (
+ state.engine == "diffusers"
+ and not fam.modular_workflow
+ and getattr(state, "vram_split_cards", None)
+ and hasattr(torch, "cuda")
+ and hasattr(torch.cuda, "mem_get_info")
+ ):
+ from utils.hardware import trusted_mem_get_info
+
+ def _available_mib(ordinal: int) -> int:
+ device_obj = torch.device("cuda", ordinal)
+ free_bytes, _ = trusted_mem_get_info(device_obj, module = torch.cuda)
+ reserved_bytes = (
+ torch.cuda.memory_reserved(device_obj)
+ if hasattr(torch.cuda, "memory_reserved")
+ else 0
+ )
+ return int((free_bytes + reserved_bytes) // (1024 * 1024))
+
+ shortfall = video_split_shortfall_message(
+ state.vram_split_cards,
+ family = fam.name,
+ available_mib = _available_mib,
+ width = width,
+ height = height,
+ frames = frames,
+ )
+ if shortfall is not None:
+ raise RuntimeError(shortfall)
elif (
state.engine == "diffusers"
and not fam.modular_workflow
@@ -11816,6 +12021,7 @@ class VideoBackend:
_log_failed_generation(request_shape, exc)
raise
finally:
+ end_cfg_render(getattr(state, "pipe", None))
try:
from . import video_stream_residency
video_stream_residency.discard_request(getattr(state, "pipe", None))
@@ -12576,6 +12782,7 @@ class VideoBackend:
close_encoder(getattr(state, "pipe", None))
# A pinner still copying would hold its chunks past release_pinned_host_memory().
stop_background_pins(getattr(state, "pipe", None))
+ uninstall_cfg_parallel(getattr(state, "pipe", None))
# Before clear_gpu_cache(), or the graph pool stays reserved.
diffusion_cuda_graph.uninstall_all(
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()
@@ -12770,6 +12977,7 @@ class VideoBackend:
"supports_audio_flow_shift": fam.default_audio_flow_shift is not None
and state.engine != "sd_cpp",
},
+ "multi_gpu": multigpu_placement_status(getattr(state, "pipe", None)),
"resolved": resolved,
}
diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.py
index 39856a6e615..2522f469663 100644
--- a/studio/backend/core/inference/video_vae_untiled.py
+++ b/studio/backend/core/inference/video_vae_untiled.py
@@ -84,6 +84,23 @@ def _free_bytes(device: Any) -> Optional[int]:
return None
+def _decode_device(vae: Any, z: Any) -> Any:
+ """Where the decode allocates: the latent's card, unless a multi-GPU split put the VAE on another CUDA card (its
+ boundary moves the latent there inside this wrapper). An offloaded VAE (weights on the host) keeps the latent's."""
+ device = getattr(z, "device", None)
+ try:
+ vae_device = next(vae.parameters()).device
+ except Exception: # noqa: BLE001
+ return device
+ if (
+ getattr(vae_device, "type", None) == "cuda"
+ and getattr(device, "type", None) == "cuda"
+ and vae_device != device
+ ):
+ return vae_device
+ return device
+
+
def install_untiled_decode(
pipe: Any,
family: str,
@@ -111,7 +128,7 @@ def install_untiled_decode(
need = untiled_decode_bytes(
family, tuple(getattr(z, "shape", ())), itemsize = _decoder_itemsize(vae)
)
- free = _free_bytes(getattr(z, "device", None))
+ free = _free_bytes(_decode_device(vae, z))
fits = (
need is not None
and free is not None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 331b32be408..6ae93ddddaf 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5518,6 +5518,12 @@ class DiffusionDownloadPlanResponse(BaseModel):
class DiffusionStatusResponse(BaseModel):
"""Current diffusion backend state."""
+ multi_gpu: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "Multi-GPU placement (component -> card, primary, secondary) and CFG-parallel state (armed, "
+ "call counts, or why it declined); null when the load had one card to use. A multi-card load that kept "
+ "everything on its primary card reports why it declined.",
+ )
loaded: bool = Field(False, description = "Whether a diffusion model is loaded")
evicted: List[str] = Field(
default_factory = list,
@@ -7070,6 +7076,12 @@ class VideoGenerationDefaults(BaseModel):
class VideoStatusResponse(BaseModel):
"""Current video backend state."""
+ multi_gpu: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "Multi-GPU placement (component -> card, primary, secondary) and CFG-parallel state (armed, "
+ "call counts, or why it declined); null when the load had one card to use. A multi-card load that kept "
+ "everything on its primary card reports why it declined.",
+ )
loaded: bool = Field(False, description = "Whether a video model is loaded")
evicted: List[str] = Field(
default_factory = list,
diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.py
new file mode 100644
index 00000000000..82653c12870
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_multigpu.py
@@ -0,0 +1,2335 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Multi-GPU diffusion layer (core/inference/diffusion_multigpu.py), CPU only.
+
+Cross-card behaviour is exercised with fake devices: the planner reads budgets through injected callables, placement
+and the boundary through a patched card predicate, and CFG-parallel with a replica on the same (CPU) device, which
+checks the routing, the prefetch / constant caches and every fallback without a second card."""
+
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import threading
+from collections import OrderedDict
+
+import pytest
+import torch
+
+from core.inference import diffusion_memory as dm
+from core.inference import diffusion_multigpu as mg
+
+
+def _from_diffusers(module: str, name: str):
+ # Not importorskip: importing diffusers can raise RuntimeError (bitsandbytes without CUDA), an environment fact.
+ # Only the import is skippable; a symbol missing from an importable diffusers is a failure.
+ import importlib
+ try:
+ loaded = importlib.import_module(module)
+ except (ImportError, RuntimeError) as exc:
+ pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")
+ return getattr(loaded, name)
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)
+ yield
+
+
+def test_one_card_or_the_kill_switch_means_no_selection(monkeypatch):
+ assert mg.make_selection(0, [0]) is None
+ assert mg.make_selection(None, [0, 1]) is None
+ assert mg.make_selection(1, [0, 1]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
+ assert mg.make_selection(1, [0, 1]) is None
+ assert mg.cfg_parallel_mode() == "off"
+
+
+def test_selection_is_scoped_to_the_load_thread():
+ sel = mg.make_selection(0, [0, 1])
+ seen = {}
+ with mg.selection_scope(sel):
+ assert mg.current_selection() == sel
+ t = threading.Thread(target = lambda: seen.setdefault("other", mg.current_selection()))
+ t.start()
+ t.join()
+ assert mg.current_selection() is None
+ assert seen["other"] is None
+
+
+def test_resolve_selected_cuda_ordinals_translates_the_mask(monkeypatch):
+ import utils.hardware.hardware as hw
+ from core.inference import diffusion_device as dd
+
+ monkeypatch.setattr(
+ hw,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": None, "numeric_ids": [4, 5], "supports_explicit_gpu_ids": True},
+ )
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 8)
+ assert dd.resolve_selected_cuda_ordinals([5, 4]) == [0, 1]
+ assert dd.resolve_selected_cuda_ordinals([]) == []
+ with pytest.raises(ValueError):
+ dd.resolve_selected_cuda_ordinals([0])
+
+
+def _split(**overrides):
+ kwargs = dict(
+ single_card_policy = "group",
+ model_dense_mib = 20000,
+ companion_dense_mib = 9000,
+ runtime_headroom_mib = 2000,
+ base_overhead_mib = 1000,
+ primary_limit_mib = 14000,
+ secondary_limit_mib = lambda o: 12750,
+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,)),
+ )
+ kwargs.update(overrides)
+ return mg.plan_companion_split(**kwargs)
+
+
+def test_split_when_the_denoiser_alone_fits_the_primary():
+ split = _split()
+ assert split is not None and split.secondary_ordinal == 1
+ assert split.primary_required_mib == 11000 + 2000 + 1000
+ assert split.secondary_required_mib == 9000 + 3000
+
+
+ "overrides",
+ [
+ dict(primary_limit_mib = 13000), # denoiser + headroom do not fit the primary either
+ dict(secondary_limit_mib = lambda o: 11999), # companions + headroom = 12000
+ dict(secondary_limit_mib = lambda o: None),
+ dict(single_card_policy = "none"),
+ dict(companion_dense_mib = None),
+ dict(selection = None),
+ ],
+)
+def test_no_split_keeps_single_card_behaviour(overrides):
+ assert _split(**overrides) is None
+
+
+def test_text_encoders_alone_when_the_decode_needs_the_runtime_room():
+ # Video-like: the secondary cannot also hold the decode headroom, but the encoders alone fit it, and the denoiser +
+ # VAE fit the primary.
+ kwargs = dict(
+ companion_dense_mib = 11500,
+ text_encoder_dense_mib = 11000,
+ model_dense_mib = 16000,
+ runtime_headroom_mib = 4000,
+ primary_limit_mib = 12000,
+ secondary_limit_mib = lambda o: 13600,
+ )
+ split = _split(**kwargs)
+ assert split is not None and split.components == mg.TEXT_ENCODERS_ONLY
+ assert split.secondary_required_mib == 11000 + mg.TEXT_ENCODER_HEADROOM_MIB + 1000
+ assert split.primary_required_mib == 5000 + 4000 + 1000
+ # without the encoder share the planner cannot price it and keeps one card
+ assert _split(**{**kwargs, "text_encoder_dense_mib": None}) is None
+
+
+def test_measured_phases_price_each_card_for_what_it_runs():
+ kwargs = dict(
+ model_dense_mib = 7630 + 7500 + 160,
+ companion_dense_mib = 7660,
+ text_encoder_dense_mib = 7500,
+ runtime_headroom_mib = 8192,
+ base_overhead_mib = 2048,
+ primary_limit_mib = 14400,
+ secondary_limit_mib = lambda o: 12495,
+ )
+ assert _split(**kwargs) is None
+ split = _split(**kwargs, phase_headroom_mib = (1516, 1160, 2645))
+ assert split is not None and split.components is None
+ assert split.primary_required_mib == 7630 + 1160 + 2048
+ assert split.secondary_required_mib == 7660 + 2645 + 2048
+
+
+def test_force_splits_even_when_one_card_fits(monkeypatch):
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "force")
+ assert _split(single_card_policy = "none") is not None
+
+
+def test_split_picks_the_secondary_with_the_most_room():
+ split = _split(
+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2)),
+ secondary_limit_mib = lambda o: {1: 12750, 2: 34000}[o],
+ )
+ assert split.secondary_ordinal == 2
+
+
+class _Target:
+ supports_model_cpu_offload: bool = True
+ ordinal: int = 0
+ device: str = "cuda"
+ backend: str = "cuda"
+ is_cuda_torch_device: bool = True
+
+
+def _memory(free = 24000, total = 24576):
+ return dm.DeviceMemory(
+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
+ )
+
+
+def _plan(**kw):
+ args = dict(
+ target = _Target(),
+ device_memory = _memory(),
+ model_dense_mib = 20000,
+ companion_dense_mib = 9000,
+ text_encoder_dense_mib = 8500,
+ runtime_headroom_mib = 2000,
+ )
+ args.update(kw)
+ return dm.plan_diffusion_memory(**args)
+
+
+def test_planner_unchanged_without_a_selection():
+ plan = _plan()
+ assert plan.offload_policy != dm.OFFLOAD_NONE
+ assert plan.secondary_ordinal is None
+
+
+def test_planner_splits_under_a_selection(monkeypatch):
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
+ single = _plan()
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ plan = _plan()
+ balanced = _plan(requested_mode = "balanced")
+ legacy = _plan(explicit_offload = True)
+ other = _plan(target = _Target(ordinal = 1))
+ assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1
+ assert plan.vae_tiling is False and plan.vae_slicing is False
+ assert plan.as_public_dict()["secondary_ordinal"] == 1
+ assert "multi-GPU" in plan.reasons[-1]
+ for p in (balanced, legacy, other):
+ assert p.secondary_ordinal is None
+ assert balanced.offload_policy == dm.OFFLOAD_GROUP
+ assert other.offload_policy == single.offload_policy
+
+
+def test_planner_split_declined_when_the_secondary_is_small(monkeypatch):
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 3400)
+ single = _plan()
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ plan = _plan()
+ assert plan == single
+
+
+def test_planner_split_off_on_unified_memory(monkeypatch):
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
+ unified = dm.DeviceMemory(
+ backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 16000, total_mib = 16384
+ )
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ plan = _plan(device_memory = unified)
+ assert plan.secondary_ordinal is None
+
+
+def test_assignment_prefers_moving_the_companions():
+ sizes = {
+ "transformer": (10000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ moved = mg.choose_component_assignment(
+ sizes,
+ headroom_mib = 2000,
+ overhead_mib = 1000,
+ primary_limit_mib = 13500,
+ secondary_limit_mib = 13000,
+ )
+ assert moved == ("text_encoder", "vae")
+
+
+def test_assignment_can_move_only_the_text_encoders():
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (11000, "text_encoder"),
+ "vae": (1400, "other"),
+ }
+ moved = mg.choose_component_assignment(
+ sizes,
+ headroom_mib = 4000,
+ overhead_mib = 1000,
+ primary_limit_mib = 16000,
+ secondary_limit_mib = 13500,
+ )
+ assert moved == ("text_encoder",)
+
+
+def test_place_split_resolves_text_encoders_only(monkeypatch):
+ class Pipe:
+ def __init__(self):
+ self.transformer = torch.nn.Linear(2, 2)
+ self.text_encoder = torch.nn.Linear(2, 2)
+ self.text_encoder_2 = torch.nn.Linear(2, 2)
+ self.vae = _Vae()
+
+ @property
+ def components(self):
+ return dict(vars(self))
+
+ monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)
+ pipe = Pipe()
+ real_device = torch.device
+ monkeypatch.setattr(
+ torch, "device", lambda *a: real_device("cpu") if a and a[0] == "cuda" else real_device(*a)
+ )
+ try:
+ mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1, names = mg.TEXT_ENCODERS_ONLY)
+ finally:
+ monkeypatch.setattr(torch, "device", real_device)
+ assert mg.placement_status(pipe)["moved"] == ["text_encoder", "text_encoder_2"]
+ assert "decode" not in pipe.vae.__dict__
+
+
+def test_assignment_puts_the_second_expert_on_the_secondary():
+ sizes = {
+ "transformer": (12000, "dit"),
+ "transformer_2": (12000, "dit"),
+ "text_encoder": (5000, "text_encoder"),
+ "vae": (500, "other"),
+ }
+ moved = mg.choose_component_assignment(
+ sizes,
+ headroom_mib = 2000,
+ overhead_mib = 1000,
+ primary_limit_mib = 21000,
+ secondary_limit_mib = 16000,
+ )
+ assert moved == ("transformer_2",)
+ moved = mg.choose_component_assignment(
+ sizes,
+ headroom_mib = 2000,
+ overhead_mib = 1000,
+ primary_limit_mib = 15500,
+ secondary_limit_mib = 21000,
+ )
+ assert moved == ("transformer_2", "text_encoder", "vae")
+ assert (
+ mg.choose_component_assignment(
+ sizes,
+ headroom_mib = 2000,
+ overhead_mib = 1000,
+ primary_limit_mib = 9000,
+ secondary_limit_mib = 9000,
+ )
+ is None
+ )
+
+
+def test_refine_plan_for_multigpu(monkeypatch):
+ plan = _plan()
+ assert plan.offload_policy != dm.OFFLOAD_NONE
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)
+ assert mg.refine_plan_for_multigpu(object(), plan) is plan
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ refined = mg.refine_plan_for_multigpu(object(), plan)
+ assert refined.offload_policy == dm.OFFLOAD_NONE
+ assert refined.secondary_ordinal == 1
+ assert refined.secondary_components == ("text_encoder", "vae")
+
+
+class _Holder:
+ def __init__(self, t):
+ self.parameters = t
+ self.mean = t * 2
+ self.deterministic = False
+
+
+class _DC:
+ sample: torch.Tensor
+ note: str = "x"
+
+
+def test_move_to_device_keeps_container_types():
+ t = torch.ones(2)
+ Named = __import__("collections").namedtuple("Named", "a b")
+ obj = {
+ "tuple": (t, 1),
+ "list": [t],
+ "named": Named(t, "s"),
+ "ordered": OrderedDict(x = t),
+ "dc": _DC(t),
+ "holder": _Holder(t),
+ "plain": "keep",
+ }
+ out = mg.move_to_device(obj, torch.device("meta"))
+ assert out["tuple"][0].device.type == "meta" and out["tuple"][1] == 1
+ assert out["list"][0].device.type == "meta"
+ assert (
+ type(out["named"]) is Named
+ and out["named"].a.device.type == "meta"
+ and out["named"].b == "s"
+ )
+ assert isinstance(out["ordered"], OrderedDict) and out["ordered"]["x"].device.type == "meta"
+ assert isinstance(out["dc"], _DC) and out["dc"].sample.device.type == "meta"
+ assert isinstance(out["holder"], _Holder) and out["holder"].mean.device.type == "meta"
+ assert out["holder"].deterministic is False and obj["holder"].parameters.device.type == "cpu"
+ assert out["plain"] == "keep"
+ assert mg.move_to_device(t, torch.device("cpu")) is t
+
+
+def test_model_output_moves_with_its_attributes():
+ BaseOutput = _from_diffusers("diffusers.utils", "BaseOutput")
+
+ @dataclasses.dataclass
+ class Out(BaseOutput):
+ sample: torch.Tensor = None
+
+ out = mg.move_to_device(Out(sample = torch.ones(1)), torch.device("meta"))
+ assert out.sample.device.type == "meta" and out[0].device.type == "meta"
+
+
+class _Vae(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(2, 2)
+ self.seen = []
+
+ def forward(self, x):
+ self.seen.append(("forward", x.device))
+ return self.lin(x)
+
+ def decode(
+ self,
+ z,
+ return_dict = False,
+ ):
+ self.seen.append(("decode", z.device))
+ return (self.lin(z),)
+
+
+def test_boundary_moves_inputs_in_and_outputs_home(monkeypatch):
+ vae = _Vae()
+ home = torch.device("meta")
+ moves = []
+
+ def fake_move(
+ obj,
+ device,
+ _depth = 0,
+ ):
+ moves.append(device)
+ return obj
+
+ monkeypatch.setattr(mg, "_crosses", lambda device, h: True)
+ monkeypatch.setattr(mg, "move_to_device", fake_move)
+ wrapped = mg.install_boundary(vae, home)
+ assert set(wrapped) == {"forward", "decode"}
+ vae.decode(torch.ones(1, 2))
+ assert moves == [torch.device("cpu"), home]
+ assert vae.seen[-1] == ("decode", torch.device("cpu"))
+ vae(torch.ones(1, 2))
+ assert moves[2:] == [torch.device("cpu"), home]
+ assert mg.install_boundary(vae, home) == []
+ mg.remove_boundary(vae)
+ assert "decode" not in vae.__dict__ and "forward" not in vae.__dict__
+
+
+def test_boundary_is_inert_on_the_same_card():
+ vae = _Vae()
+ mg.install_boundary(vae, torch.device("cpu"))
+ x = torch.ones(1, 2)
+ assert torch.equal(vae(x), vae.lin(x))
+
+
+def test_execution_device_reports_the_primary_of_a_split_pipeline(monkeypatch):
+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
+ DiffusionPipeline = _from_diffusers("diffusers", "DiffusionPipeline")
+
+ class Pipe(DiffusionPipeline):
+ def __init__(self, vae, transformer):
+ super().__init__()
+ self.register_modules(vae = vae, transformer = transformer)
+
+ class Mod(torch.nn.Linear):
+ @property
+ def device(self):
+ return self.weight.device
+
+ vae = Mod(1, 1).to("meta")
+ transformer = Mod(1, 1)
+ pipe = Pipe(vae, transformer)
+ assert mg.install_execution_device_patch()
+ before = pipe.device
+ vae._unsloth_split_home = torch.device("meta")
+ assert pipe.device == torch.device("meta")
+ del vae._unsloth_split_home
+ transformer._unsloth_split_home = torch.device("cpu")
+ assert pipe._execution_device == torch.device("cpu")
+ assert pipe.device == torch.device("cpu")
+ del transformer._unsloth_split_home
+ assert pipe.device == before
+
+
+def test_place_split_and_fallback(monkeypatch):
+ class Pipe:
+ def __init__(self):
+ self.transformer = torch.nn.Linear(2, 2)
+ self.text_encoder = torch.nn.Linear(2, 2)
+ self.vae = _Vae()
+ self.scheduler = object()
+
+ @property
+ def components(self):
+ return {
+ "transformer": self.transformer,
+ "text_encoder": self.text_encoder,
+ "vae": self.vae,
+ "scheduler": self.scheduler,
+ }
+
+ def to(self, device):
+ for m in (self.transformer, self.text_encoder, self.vae):
+ m.to(device)
+
+ pipe = Pipe()
+ assert mg.companion_names(pipe) == ("text_encoder", "vae")
+ moved_to = {}
+ real_device = torch.device
+
+ def fake_device(*args):
+ if args and args[0] == "cuda":
+ return real_device("cpu")
+ return real_device(*args)
+
+ monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)
+ monkeypatch.setattr(torch, "device", fake_device)
+ try:
+ placement = mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1)
+ finally:
+ monkeypatch.setattr(torch, "device", real_device)
+ assert set(placement) == {"transformer", "text_encoder", "vae"}
+ assert pipe.transformer._unsloth_split_home == torch.device("cpu")
+ assert "decode" in pipe.vae.__dict__
+ status = mg.placement_status(pipe)
+ assert status["moved"] == ["text_encoder", "vae"]
+ mg.undo_split(pipe)
+ assert (
+ "decode" not in pipe.vae.__dict__ and "_unsloth_split_home" not in pipe.transformer.__dict__
+ )
+ assert mg.placement_status(pipe) is None
+ del moved_to
+
+ @dataclasses.dataclass
+ class P:
+ secondary_ordinal: int = 1
+ secondary_components: tuple = None
+
+ def boom(*a, **k):
+ raise RuntimeError("CUDA out of memory")
+
+ monkeypatch.setattr(mg, "place_split", boom)
+ assert mg.apply_split_plan(pipe, P(), placement = "cpu") is False
+
+
+def test_apply_memory_plan_falls_back_to_model_offload(monkeypatch):
+ calls = []
+
+ class Pipe:
+ components: dict = {}
+
+ def enable_model_cpu_offload(self, device = None):
+ calls.append(("offload", device))
+
+ plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE, secondary_ordinal = 1)
+ monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)
+ monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger = None: 0)
+ pipe = Pipe()
+ policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
+ assert policy == dm.OFFLOAD_MODEL and calls == [("offload", "cuda:0")]
+ applied = dm.applied_plan(pipe, plan)
+ assert applied.offload_policy == dm.OFFLOAD_MODEL and applied.secondary_ordinal is None
+
+ placed = []
+ monkeypatch.setattr(
+ mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: placed.append(1) or True
+ )
+ policy, _tiling = dm.apply_memory_plan(Pipe(), plan, device = "cuda", placement_device = "cuda:0")
+ assert policy == dm.OFFLOAD_NONE and placed == [1]
+
+
+class _Dit(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(4, 4)
+ self.calls = 0
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ return_dict = False,
+ ):
+ self.calls += 1
+ return (self.proj(hidden_states) * timestep + encoder_hidden_states.sum(),)
+
+
+def _router(dit, **kw):
+ replicas = []
+
+ def factory():
+ r = mg.build_replica(dit, torch.device("cpu"))
+ replicas.append(r)
+ return r
+
+ router = mg.CfgParallelRouter(
+ dit, primary = "cpu", secondary = "cpu", replica_factory = factory, **kw
+ )
+ dit.forward = router
+ return router, replicas
+
+
+def _cfg_loop(
+ dit,
+ latents,
+ steps,
+ branches = 2,
+ scale = 4.0,
+):
+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))
+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))
+ extra = torch.randn(3, 4, generator = torch.Generator().manual_seed(3))
+ for i in range(steps):
+ t = torch.tensor(1.0 - i / steps)
+ outs = [
+ dit(hidden_states = latents, timestep = t, encoder_hidden_states = e)[0]
+ for e in (cond, uncond, extra)[:branches]
+ ]
+ if branches == 1:
+ pred = outs[0]
+ else:
+ pred = outs[1] + scale * (outs[0] - outs[1])
+ if branches == 3:
+ pred = pred + 0.5 * (outs[0] - outs[2])
+ latents = latents - 0.1 * pred
+ return latents
+
+
[email protected]("branches", [2, 3])
+def test_cfg_parallel_is_exact_and_routes_odd_branches(branches):
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)
+ router, replicas = _router(dit)
+ out = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)
+ assert torch.equal(out, reference)
+ assert len(replicas) == 1 and replicas[0] is not dit
+ assert router.stats["steps_parallel"] == 4
+ assert router.stats["replica_calls"] == 4 * (branches // 2)
+ assert router.describe()["disabled_reason"] is None
+ assert replicas[0].proj.weight is not dit.proj.weight
+ assert torch.equal(replicas[0].proj.weight, dit.proj.weight)
+
+
+def test_guidance_distilled_model_never_builds_a_replica():
+ dit = _Dit()
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 6, branches = 1)
+ assert replicas == [] and router.replica is None
+ assert router.stats["replica_calls"] == 0
+
+
+def test_replica_failure_recomputes_on_the_primary():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ router, replicas = _router(dit)
+
+ class Broken(torch.nn.Module):
+ def forward(self, *a, **k):
+ raise RuntimeError("replica kernel failed")
+
+ router.replica_factory = lambda: Broken()
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert "replica call failed" in router.describe()["disabled_reason"]
+ assert router.replica is None
+
+
+def test_replica_declined_when_a_submodule_hook_captures_the_source():
+ dit = _Dit()
+ source = dit.proj
+ dit.proj.register_forward_hook(lambda m, i, o: o + source.bias)
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert replicas == []
+ assert "proj._forward_hooks" in router.describe()["disabled_reason"]
+
+
+def test_portable_hooks_and_root_hooks_are_accepted():
+ torch.manual_seed(0)
+ dit = _Dit()
+ scale = 0.5
+ hook = lambda m, i, o: o * scale # noqa: E731 - captures only a float: shared safely
+ hook.__module__ = mg.__name__
+ dit.proj.register_forward_hook(hook)
+ calls = []
+ root_hook = lambda m, a: calls.append(1) # noqa: E731 - root: runs on the source around every branch
+ root_hook.__module__ = mg.__name__
+ dit.register_forward_pre_hook(root_hook)
+ assert mg._hooks_present(dit) is None
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ n_ref = len(calls)
+ router, replicas = _router(dit)
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert router.stats["replica_calls"] == 3
+ # the root hook ran exactly once per denoiser call, routed or not (stripped from the copy)
+ assert len(calls) == 2 * n_ref
+ assert not replicas[0]._forward_pre_hooks
+
+
+def test_closure_forward_override_is_declined_but_bound_methods_are_rebound():
+ import types
+
+ dit = _Dit()
+ source = dit.proj
+ dit.proj.forward = lambda x: source.weight @ x # a closure would keep reading the source module
+ assert "instance override" in mg._hooks_present(dit)
+ dit2 = _Dit()
+ dit2.proj.forward = types.MethodType(
+ lambda self, x: torch.nn.functional.linear(x, self.weight), dit2.proj
+ )
+ assert mg._hooks_present(dit2) is None
+ replica = mg.build_replica(dit2, torch.device("cpu"))
+ assert replica.proj.forward.__self__ is replica.proj
+
+
+def test_replica_reapplies_recorded_compile(monkeypatch):
+ mg.install_compile_recorder()
+ dit = _Dit()
+ dit.proj.compile(backend = "eager")
+ assert dit.proj.__dict__["_unsloth_compile_args"] == ((), {"backend": "eager"})
+ replica = mg.build_replica(dit, torch.device("cpu"))
+ assert replica._unsloth_replica_compiled == 1
+ assert replica.proj._compiled_call_impl is not None
+ assert replica.proj._compiled_call_impl is not dit.proj._compiled_call_impl
+ x = torch.randn(2, 4)
+ assert torch.equal(replica.proj(x), dit.proj(x))
+
+
+def test_compiled_without_recorded_arguments_is_refused():
+ dit = _Dit()
+ dit.proj._compiled_call_impl = lambda *a, **k: None
+ with pytest.raises(RuntimeError, match = "recorded compile"):
+ mg.build_replica(dit, torch.device("cpu"))
+
+
+def test_worker_propagates_grad_mode_and_context_variables():
+ var = __import__("contextvars").ContextVar("probe", default = "unset")
+ worker = mg._ReplicaWorker(torch.device("cpu"))
+ try:
+ var.set("caller")
+ with torch.inference_mode():
+ seen = worker.call(
+ lambda: (
+ torch.is_inference_mode_enabled(),
+ var.get(),
+ threading.current_thread().name,
+ )
+ )
+ assert seen[0] is True and seen[1] == "caller" and seen[2].startswith("unsloth-cfg-replica")
+ with pytest.raises(ValueError):
+ worker.call(lambda: (_ for _ in ()).throw(ValueError("boom")))
+ finally:
+ worker.close()
+
+
+def test_branch_tensors_are_prefetched_fresh_every_step():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ router, _replicas = _router(dit)
+ copies = []
+ real_prefetch = router._prefetch
+
+ def spy(args, kwargs):
+ real_prefetch(args, kwargs)
+ copies.append(len(router.carry))
+
+ router._prefetch = spy
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ # Prefetch runs before every primary branch from step 1. Step 1 has no carry yet (step 0 built the replica on its
+ # odd branch); from step 2 the previous step's odd-branch tensors except its latent (timestep, uncond embeds).
+ assert copies == [0, 2, 2]
+
+
+def test_cfg_parallel_decline_reasons(monkeypatch):
+ class Pipe:
+ transformer = _Dit()
+
+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ assert (
+ mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = None)
+ == "one card selected"
+ )
+ assert "offloaded" in mg.cfg_parallel_decline(Pipe(), offload_policy = "model", selection = sel)
+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel)
+ hooked = Pipe()
+ hooked.transformer = _Dit()
+ source = hooked.transformer.proj
+ hooked.transformer.proj.register_forward_pre_hook(lambda m, a: source)
+ assert "hooked" in mg.cfg_parallel_decline(hooked, offload_policy = "group", selection = sel)
+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = sel)
+
+ class Unet:
+ transformer = None
+
+ assert "U-Net" in mg.cfg_parallel_decline(Unet(), offload_policy = "none", selection = sel)
+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "0")
+ assert mg.CFG_PARALLEL_ENV in mg.cfg_parallel_decline(
+ Pipe(), offload_policy = "none", selection = sel
+ )
+
+
+def test_status_reports_a_declined_cfg_parallel():
+ class Pipe:
+ transformer = _Dit()
+
+ pipe = Pipe()
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ reason = mg.install_cfg_parallel(pipe, offload_policy = "model")
+ assert "offloaded" in reason
+ assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": reason}}
+ other = Pipe()
+ assert mg.install_cfg_parallel(other, offload_policy = "none") == "one card selected"
+ assert mg.placement_status(other) is None
+
+
+def test_replica_is_rebuilt_when_the_weights_change():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ router, replicas = _router(dit)
+ _cfg_loop(dit, latents.clone(), steps = 3)
+ assert len(replicas) == 1
+ with torch.no_grad():
+ dit.proj.weight.mul_(0.5)
+ reference_router = dit.__dict__.pop("forward")
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ dit.forward = reference_router
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert router.stats["replica_rebuilds"] == 1 and len(replicas) == 2
+ assert torch.equal(replicas[1].proj.weight, dit.proj.weight)
+
+
+def test_adapter_scale_changes_the_fingerprint():
+ dit = _Dit()
+ before = mg.weights_fingerprint(dit)
+ dit.proj.scaling = {"default": 1.0}
+ with_adapter = mg.weights_fingerprint(dit)
+ dit.proj.scaling = {"default": 0.5}
+ assert len({before, with_adapter, mg.weights_fingerprint(dit)}) == 3
+
+
+def test_with_load_selection_scopes_and_pops_the_ordinals():
+ seen = {}
+
+ class Backend:
+ @mg.with_load_selection
+ def _run_load(self, **kwargs):
+ """body"""
+ seen["kwargs"] = dict(kwargs)
+ seen["selection"] = mg.current_selection()
+
+ Backend()._run_load(gpu_ordinal = 1, _gpu_ordinals = [0, 1], repo_id = "x")
+ assert seen["kwargs"] == {"gpu_ordinal": 1, "repo_id": "x"}
+ assert seen["selection"] == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
+ assert mg.current_selection() is None
+ Backend()._run_load(gpu_ordinal = 1, repo_id = "x")
+ assert seen["selection"] is None
+ import inspect
+
+ assert '"""body"""' in inspect.getsource(Backend._run_load)
+
+
+def test_portability_looks_inside_captured_containers():
+ holder = {"m": torch.nn.Linear(1, 1)}
+ assert not mg._portable(lambda m, a: holder)
+ plain = {"scale": 2.0}
+ # even scalar-only containers are shared mutable state between the source and the replica
+ assert not mg._portable(lambda m, a: plain)
+ frozen = ("scale", 2.0)
+ assert mg._portable(lambda m, a: frozen)
+
+
+class _BlockDit(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(4, 4)
+ self.blocks = torch.nn.ModuleList([torch.nn.Linear(4, 4) for _ in range(3)])
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ return_dict = False,
+ ):
+ x = self.proj_in(hidden_states) * timestep
+ for block in self.blocks:
+ x = block(x) + encoder_hidden_states.sum()
+ return (x,)
+
+
+def test_replica_drops_group_offload_and_stays_exact():
+ apply_group_offloading = _from_diffusers("diffusers.hooks", "apply_group_offloading")
+
+ torch.manual_seed(0)
+ dit = _BlockDit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ apply_group_offloading(
+ dit,
+ onload_device = torch.device("cpu"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ )
+ assert mg._hooks_present(dit) is None
+ assert mg._registry_kind(dit) == "offload"
+ router, replicas = _router(dit)
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert router.stats["replica_calls"] == 3
+ replica = replicas[0]
+ for sub in replica.modules():
+ assert "_diffusers_hook" not in sub.__dict__
+ assert "forward" not in sub.__dict__
+
+
+def test_step_cache_registry_is_declined():
+ class Registry:
+ hooks = {"first_block_cache": object()}
+
+ dit = _Dit()
+ dit._diffusers_hook = Registry()
+ assert "first_block_cache" in mg._hooks_present(dit)
+
+
+def test_offload_torch_hooks_are_stripped_from_the_replica():
+ dit = _Dit()
+ source = dit.proj
+ calls = []
+
+ def prefetch_begin(module, args):
+ calls.append(source)
+
+ prefetch_begin.__module__ = "core.inference.diffusion_offload_prefetch"
+ dit.proj.register_forward_pre_hook(prefetch_begin)
+ assert mg._hooks_present(dit) is None
+ replica = mg.build_replica(dit, torch.device("cpu"))
+ assert not replica.proj._forward_pre_hooks
+ replica(
+ hidden_states = torch.ones(1, 4),
+ timestep = torch.tensor(1.0),
+ encoder_hidden_states = torch.ones(1, 4),
+ )
+ assert calls == []
+
+
+def test_predicted_branch_hits_and_stays_exact():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 6)
+ router, _replicas = _router(dit)
+ out = _cfg_loop(dit, latents.clone(), steps = 6)
+ assert torch.equal(out, reference)
+ assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0
+ assert router.stats["replica_calls"] == 5
+
+
+def test_mispredicted_branch_is_recomputed_exactly():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ uncond = torch.randn(3, 4)
+
+ def loop(steps):
+ x = latents.clone()
+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))
+ for i in range(steps):
+ t = torch.tensor(1.0 - i / steps)
+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
+ # a fresh negative tensor every step (equal values, new object): the prediction cannot be trusted
+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = uncond.clone())[0]
+ x = x - 0.1 * (b + 4.0 * (a - b))
+ return x
+
+ reference = loop(5)
+ router, _replicas = _router(dit)
+ out = loop(5)
+ assert torch.equal(out, reference)
+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4
+ assert router.stats["replica_calls"] == 4
+
+
+def test_inline_recomputed_scalars_are_predicted_by_value():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ cond = torch.randn(3, 4)
+ uncond = torch.randn(3, 4)
+
+ def loop(steps):
+ x = latents.clone()
+ for i in range(steps):
+ t = torch.tensor(1000.0 * (1.0 - i / steps))
+ a = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = cond)[0]
+ b = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = uncond)[0]
+ x = x - 0.1 * (b + 4.0 * (a - b))
+ return x
+
+ reference = loop(6)
+ router, _replicas = _router(dit)
+ out = loop(6)
+ assert torch.equal(out, reference)
+ assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0
+
+
+def test_bitwise_equal_distinguishes_signed_zero():
+ assert mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([0.0]))
+ assert mg._bitwise_equal(torch.tensor(0.5), torch.tensor(1.0) / 2)
+ assert not mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([-0.0]))
+ assert not mg._bitwise_equal(torch.tensor([1.0]), torch.tensor([1.0], dtype = torch.float64))
+
+
+def test_failed_split_applies_the_single_card_plan(monkeypatch):
+ calls = []
+
+ class Pipe:
+ components: dict = {}
+
+ def enable_model_cpu_offload(self, device = None):
+ calls.append(("model", device))
+
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ plan = _plan()
+ assert plan.secondary_ordinal == 1
+ assert plan.fallback_plan is not None and plan.fallback_plan.offload_policy == dm.OFFLOAD_GROUP
+ monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)
+ monkeypatch.setattr(
+ dm,
+ "_apply_group_offload",
+ lambda pipe, placement, logger, **kw: calls.append(("group", placement)) or True,
+ )
+ pipe = Pipe()
+ policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
+ assert policy == dm.OFFLOAD_GROUP and calls == [("group", "cuda:0")]
+ assert dm.applied_plan(pipe, plan) is plan.fallback_plan
+
+
+def test_post_load_split_keeps_the_original_plan_as_fallback(monkeypatch):
+ plan = _plan()
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ refined = mg.refine_plan_for_multigpu(object(), plan)
+ assert refined.fallback_plan is plan
+
+
+def test_hook_with_a_mutable_closure_counter_is_declined():
+ dit = _Dit()
+ counter = [0]
+
+ def count(module, args):
+ counter[0] += 1
+
+ dit.proj.register_forward_pre_hook(count)
+ assert "proj._forward_pre_hooks" in mg._hooks_present(dit)
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert replicas == [] and "hooks" in router.describe()["disabled_reason"]
+
+
+def test_replica_is_dropped_when_the_source_gains_a_cache_hook():
+ # transformer_cache=auto installs a stateful step cache on the SOURCE at generate time, after the replica exists
+ torch.manual_seed(0)
+ dit = _Dit()
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert len(replicas) == 1
+
+ class Registry:
+ hooks = {"fbc_leader": object()}
+
+ dit._diffusers_hook = Registry()
+ _cfg_loop(dit, torch.randn(2, 4), steps = 2)
+ assert router.replica is None
+ assert "fbc_leader" in router.describe()["disabled_reason"]
+
+
+def test_post_load_split_keeps_vae_tiling(monkeypatch):
+ plan = dataclasses.replace(_plan(), vae_tiling = True, vae_slicing = True)
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ refined = mg.refine_plan_for_multigpu(object(), plan)
+ assert refined.secondary_ordinal == 1
+ assert refined.vae_tiling is True and refined.vae_slicing is True
+
+
+def test_replica_keeps_a_bound_root_forward_override():
+ import types
+
+ torch.manual_seed(0)
+ dit = _Dit()
+
+ def doubled(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ return_dict = False,
+ ):
+ return (2 * type(self).forward(self, hidden_states, timestep, encoder_hidden_states)[0],)
+
+ dit.forward = types.MethodType(doubled, dit)
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ router, replicas = _router(dit)
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert router.stats["replica_calls"] == 3
+ assert replicas[0].forward.__self__ is replicas[0]
+
+
+def test_in_place_edit_after_launch_rejects_the_prediction():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ cond = torch.randn(3, 4)
+ uncond = torch.randn(3, 4)
+
+ def loop(steps):
+ x = latents.clone()
+ neg = uncond.clone()
+ for i in range(steps):
+ t = torch.tensor(1.0 - i / steps)
+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
+ neg.mul_(1.01) # same object, new values, after the prediction was launched
+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]
+ x = x - 0.1 * (b + 4.0 * (a - b))
+ return x
+
+ with torch.inference_mode():
+ reference = loop(5)
+ router, _replicas = _router(dit)
+ out = loop(5)
+ assert torch.equal(out, reference)
+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4
+ router.prefetched = {id(uncond): (uncond, uncond)}
+ router.pending = {
+ "leaves": [],
+ "by_value": set(),
+ "hook_state": None,
+ "snapshots": [],
+ "job": None,
+ }
+ assert router._take_prediction((), {"x": torch.zeros(1)}) is None
+ assert router.prefetched == {}
+
+
+def test_deferred_speed_profile_invalidates_the_replica(monkeypatch):
+ # The 3rd generation compiles / patches the source denoiser; a replica built during the eager images must go.
+ import types as _types
+
+ from core.inference.diffusion import DiffusionBackend
+
+ torch.manual_seed(0)
+ dit = _Dit()
+ router, replicas = _router(dit)
+
+ class Pipe:
+ transformer = dit
+
+ pipe = Pipe()
+ mg._CFG_ROUTERS[pipe] = router
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert router.replica is not None
+ state = _types.SimpleNamespace(
+ pipe = pipe,
+ speed_deferred = True,
+ deferred_static_plan = None,
+ resolved = {},
+ transformer_cache = None,
+ )
+ fake_self = _types.SimpleNamespace(_engage_deferred_profile = lambda state: None)
+ DiffusionBackend._engage_deferred_speed(fake_self, state)
+ assert router.replica is None and router.disabled_reason is None
+ _cfg_loop(dit, torch.randn(2, 4), steps = 2)
+ assert len(replicas) == 2
+ assert router.replica is not None
+
+ def broken_profile(state):
+ raise RuntimeError("compile failed")
+
+ with pytest.raises(RuntimeError, match = "compile failed"):
+ DiffusionBackend._engage_deferred_speed(
+ _types.SimpleNamespace(_engage_deferred_profile = broken_profile), state
+ )
+ assert router.replica is None
+
+
+class _FlagLinear(torch.nn.Linear):
+ def forward(self, x):
+ out = super().forward(x)
+ return -out if getattr(self, "_unsloth_flip", False) else out
+
+
+class _FlagDit(_Dit):
+ def __init__(self):
+ super().__init__()
+ self.proj = _FlagLinear(4, 4)
+
+
+def _flag_pre_hook(module, args, kwargs):
+ module.proj._unsloth_flip = bool(kwargs["encoder_hidden_states"].sum() < 0) != bool(
+ kwargs["timestep"] < 0.5
+ )
+
+
+def _flag_post_hook(module, args, output):
+ module.proj.__dict__.pop("_unsloth_flip", None)
+
+
+_flag_pre_hook.__module__ = mg.__name__
+_flag_post_hook.__module__ = mg.__name__
+
+
+def _plain_state_pre_hook(module, args, kwargs):
+ # a foreign root hook keeping per-call state in an ordinary attribute, which the replica would never see
+ module.proj.flag = bool(kwargs["encoder_hidden_states"].sum() < 0)
+
+
+def test_foreign_root_hook_declines_the_replica():
+ dit = _FlagDit()
+ dit.register_forward_pre_hook(_plain_state_pre_hook, with_kwargs = True)
+ assert "a hook outside Studio" in mg._hooks_present(dit)
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert replicas == [] and router.stats["replica_calls"] == 0
+
+
+def test_root_hook_state_on_submodules_reaches_the_replica():
+ torch.manual_seed(0)
+ dit = _FlagDit()
+ dit.register_forward_pre_hook(_flag_pre_hook, with_kwargs = True)
+ dit.register_forward_hook(_flag_post_hook)
+ latents = torch.randn(2, 4)
+ cond = torch.ones(3, 4)
+ neg = -torch.ones(3, 4)
+
+ def loop(steps):
+ x = latents.clone()
+ for i in range(steps):
+ t = torch.tensor(1.0 - i / steps)
+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]
+ x = x - 0.1 * (b + 4.0 * (a - b))
+ return x
+
+ reference = loop(5)
+ router, _replicas = _router(dit)
+ out = loop(5)
+ assert torch.equal(out, reference)
+ assert router.stats["replica_calls"] == 4
+
+
+def test_runtime_controlnet_streams_beside_a_split_pipeline(monkeypatch):
+ # A split reports policy "none", but its fit priced only the base pipeline: a ControlNet added at generate time
+ # must stream as the single-card offload plan would have, not land whole on the primary.
+ import types as _types
+
+ from core.inference.diffusion import _controlnet_streams
+
+ class Pipe:
+ pass
+
+ pipe = Pipe()
+ state = _types.SimpleNamespace(pipe = pipe, offload_policy = "none", calibrated_placement = False)
+ assert _controlnet_streams(state) is False # one card, resident: unchanged
+ mg._PLACEMENTS[pipe] = {"moved": ["text_encoder", "vae"]}
+ assert _controlnet_streams(state) is True
+ assert _controlnet_streams(_types.SimpleNamespace(pipe = Pipe(), offload_policy = "group")) is True
+
+
+def test_generate_guard_checks_the_vae_card(monkeypatch):
+ # The guard reads the primary's free memory; with the VAE on the secondary, the decode must be checked there.
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ seen = []
+
+ @contextlib.contextmanager
+ def scope(ordinal):
+ seen.append(("scope", ordinal))
+ yield
+
+ def shortfall(**kw):
+ seen.append(("guard", kw["device_memory"], kw.get("vae_tile_side")))
+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+
+ monkeypatch.setattr(D, "diffusion_device_scope", scope)
+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
+ monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: ("restore", True))
+
+ class Vae(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1))
+
+ pipe = _types.SimpleNamespace(vae = Vae())
+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
+ monkeypatch.setattr(mg, "is_split", lambda p: True)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
+ restore, tiled = D._guard_vae_card(
+ pipe, target, {"device_memory": "card0-free", "vae_tile_side": 512}, None, None
+ )
+ assert ("scope", 1) in seen and ("guard", "card1-free", 512) in seen
+ assert (restore, tiled) == ("restore", True)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
+ seen.clear()
+ assert D._guard_vae_card(pipe, target, {"device_memory": "card0-free"}, None, None) == (
+ None,
+ False,
+ )
+ assert seen == []
+
+
+def test_secondary_of_lower_compute_capability_takes_no_component(monkeypatch):
+ caps = {0: (8, 9), 1: (7, 5), 2: (8, 9), 3: (9, 0)}
+ monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))
+ assert mg.make_selection(0, [0, 1, 2, 3]) == mg.MultiGpuSelection(primary = 0, secondaries = (2, 3))
+ assert mg.make_selection(0, [0, 1]) is None
+ assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
+ assert mg.make_selection(0, [0, 7]) == mg.MultiGpuSelection(primary = 0, secondaries = (7,))
+
+
+class _Cache:
+ """A prefix-KV style cache: a value holder whose layer tensors the denoiser reads and writes."""
+
+ def __init__(self):
+ self.keys = [torch.ones(4)]
+ self.writers: list = []
+
+
+class _CacheDit(_Dit):
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ kv_cache = None,
+ return_dict = False,
+ ):
+ if kv_cache is not None:
+ kv_cache.writers.append(threading.current_thread())
+ hidden_states = hidden_states + kv_cache.keys[0]
+ return super().forward(hidden_states, timestep, encoder_hidden_states)
+
+
+def test_branch_with_a_cache_object_stays_on_the_primary():
+ torch.manual_seed(0)
+ dit = _CacheDit()
+ cond, uncond = torch.randn(3, 4), torch.randn(3, 4)
+
+ def loop(latents):
+ cache = _Cache()
+ for i in range(4):
+ t = torch.tensor(1.0 - i / 4)
+ outs = [
+ dit(hidden_states = latents, timestep = t, encoder_hidden_states = e, kv_cache = cache)[0]
+ for e in (cond, uncond)
+ ]
+ latents = latents - 0.1 * (outs[1] + 4.0 * (outs[0] - outs[1]))
+ return latents, cache
+
+ latents = torch.randn(2, 4)
+ reference, _ = loop(latents.clone())
+ router, replicas = _router(dit)
+ out, cache = loop(latents.clone())
+ assert torch.equal(out, reference)
+ # the mutable cache is never handed to the replica thread, and no replica is paid for
+ assert set(cache.writers) == {threading.current_thread()}
+ assert replicas == [] and router.stats["replica_calls"] == 0
+ assert router.stats["opaque_primary_calls"] == 4
+ assert router.describe()["disabled_reason"] is None
+ assert not mg._holds_opaque_tensors(((torch.ones(1),), {"a": [torch.ones(1)], "b": None}))
+ assert mg._holds_opaque_tensors({"kv_cache": _Cache()})
+
+
+def test_split_video_request_is_guarded_on_every_card():
+ from core.inference import video as vid
+
+ class Pipe:
+ def __init__(self):
+ self.components = {
+ "transformer": torch.nn.Linear(1024, 1024),
+ "text_encoder": torch.nn.Linear(1024, 512),
+ "vae": torch.nn.Linear(512, 512),
+ }
+
+ pipe = Pipe()
+ mg._PLACEMENTS[pipe] = {
+ "placement": {"transformer": "cuda:0", "text_encoder": "cuda:1", "vae": "cuda:1"},
+ "moved": ["text_encoder", "vae"],
+ }
+ cards = vid._video_split_card_floors(pipe)
+ assert cards == ((0, 4, False, True, True), (1, 3, True, False, False))
+ big = dict(width = 1280, height = 720, frames = 121)
+ clip = vid.estimate_video_runtime_mib(width = 1280, height = 720, num_frames = 121)
+ roomy = {0: 1 << 20, 1: 1 << 20}
+ assert (
+ vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **big) is None
+ )
+ # the decode card is priced for the decoded clip; the denoiser card for its weights, the base headroom and the
+ # decoded frames copied back to it
+ frames_mib = -(-1280 * 720 * 121 * 3 * 4 // (1 << 20))
+ home_mib = max(frames_mib, vid._VIDEO_DENOISE_ACTIVATION_MIB)
+ tight = {0: 4 + home_mib + dm.DEFAULT_BASE_OVERHEAD_MIB, 1: 3 + dm.DEFAULT_BASE_OVERHEAD_MIB}
+ message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)
+ assert clip > 0 and message is not None and "cuda:1" in message
+ tight[1] = 1 << 20
+ assert (
+ vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big) is None
+ )
+ tight[0] -= 1
+ message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)
+ assert "cuda:0" in message
+ tight[0] = 4 + dm.DEFAULT_BASE_OVERHEAD_MIB
+ assert vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get) is None
+ del mg._PLACEMENTS[pipe]
+ assert vid._video_split_card_floors(pipe) is None
+
+
+def test_primary_tile_verdict_does_not_tile_a_vae_decoding_elsewhere(monkeypatch):
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ engaged = []
+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
+ # the VAE's card decodes untiled; only the smaller primary asks for tiling
+ monkeypatch.setattr(
+ D,
+ "raise_on_image_activation_shortfall",
+ lambda **kw: _types.SimpleNamespace(action = "ok", overridden = False),
+ )
+ monkeypatch.setattr(
+ D, "engage_vae_tiling", lambda pipe, logger = None: engaged.append(1) or ("restore", True)
+ )
+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
+ monkeypatch.setattr(mg, "is_split", lambda p: True)
+ primary_verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+ kwargs = {"device_memory": "card0-free", "vae_tile_side": 512, "vae_sliced": False}
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
+ assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) is None
+ assert engaged == []
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
+ assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) == "restore"
+ assert engaged == [1]
+
+
+def test_secondary_free_memory_reads_the_trusted_probe(monkeypatch):
+ import utils.hardware as hw
+
+ seen = []
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *a: (80 << 30, 80 << 30))
+ monkeypatch.setattr(
+ hw,
+ "trusted_mem_get_info",
+ lambda device = None, module = None: seen.append(device) or (1 << 30, 80 << 30),
+ )
+ assert mg._free_total_mib(1) == (1024, 80 * 1024)
+ assert seen == [1]
+
+
+_GLOBAL_HOOK_CALLS = []
+
+
+def _global_state_hook(module, inputs, output):
+ _GLOBAL_HOOK_CALLS.append(1)
+ return output * (1.0 + len(_GLOBAL_HOOK_CALLS) % 2)
+
+
+def test_foreign_submodule_hook_declines_the_replica():
+ dit = _Dit()
+ dit.proj.register_forward_hook(_global_state_hook)
+ assert _global_state_hook.__closure__ is None
+ assert mg._hooks_present(dit) == "proj._forward_hooks"
+ router, replicas = _router(dit)
+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)
+ assert replicas == [] and router.stats["replica_calls"] == 0
+ _global_state_hook.__module__, saved = mg.__name__, _global_state_hook.__module__
+ try:
+ assert mg._hooks_present(_Dit()) is None
+ hooked = _Dit()
+ hooked.proj.register_forward_hook(_global_state_hook)
+ assert mg._hooks_present(hooked) is None
+ finally:
+ _global_state_hook.__module__ = saved
+
+
+def test_force_mode_refines_a_resident_plan(monkeypatch):
+ plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE)
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ assert mg.refine_plan_for_multigpu(object(), plan) is plan
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "force")
+ refined = mg.refine_plan_for_multigpu(object(), plan)
+ assert refined.secondary_ordinal == 1 and refined.offload_policy == dm.OFFLOAD_NONE
+ assert refined.secondary_components == ("text_encoder", "vae")
+ assert refined.fallback_plan is plan
+
+
+def test_ltx25_sidecar_encoder_keeps_the_single_card_plan(monkeypatch):
+ import types as _types
+
+ from core.inference import video as vid
+
+ single = _plan()
+ split = dataclasses.replace(
+ single, offload_policy = dm.OFFLOAD_NONE, secondary_ordinal = 1, fallback_plan = single
+ )
+ ltx25 = _types.SimpleNamespace(name = "ltx-2.5")
+ wan = _types.SimpleNamespace(name = "wan2.2")
+ assert vid._video_single_card_for_sidecar(ltx25, split) is single
+ assert vid._video_single_card_for_sidecar(wan, split) is split
+ assert vid._video_single_card_for_sidecar(ltx25, single) is single
+ refined = []
+ monkeypatch.setattr(
+ vid, "refine_plan_for_multigpu", lambda pipe, plan, logger = None: refined.append(1) or split
+ )
+ assert vid._video_refine_for_multigpu(ltx25, object(), single) is single
+ assert vid._video_refine_for_multigpu(wan, object(), single) is split and refined == [1]
+
+
+def _fake_cuda_host(
+ monkeypatch,
+ *,
+ count,
+ ranking_possible = True,
+ current = 0,
+):
+ from core.inference import diffusion_device as dd
+
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: ranking_possible)
+ monkeypatch.setattr(
+ dd, "resolve_diffusion_device_target", lambda **kw: _Target(ordinal = kw.get("ordinal"))
+ )
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: count)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: current)
+ monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))
+ from core.inference import gpu_residency
+
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: "off")
+
+
+def test_automatic_selection_uses_every_visible_card(monkeypatch):
+ _fake_cuda_host(monkeypatch, count = 3)
+ assert mg.automatic_selection(None) == mg.MultiGpuSelection(
+ primary = 0, secondaries = (1, 2), automatic = True
+ )
+ assert mg.automatic_selection(2) == mg.MultiGpuSelection(
+ primary = 2, secondaries = (0, 1), automatic = True
+ )
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
+ assert mg.automatic_selection(None) is None
+
+
+def test_automatic_selection_is_inert_on_ordinary_hosts(monkeypatch):
+ # one GPU, ROCm, XPU, MPS and CPU all fail the ranking gate; a gated host never touches torch.cuda
+ _fake_cuda_host(monkeypatch, count = 2, ranking_possible = False)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: pytest.fail("probed a gated host"))
+ assert mg.automatic_selection(None) is None
+ _fake_cuda_host(monkeypatch, count = 1)
+ assert mg.automatic_selection(None) is None
+
+
+def test_with_load_selection_resolves_the_automatic_marker(monkeypatch):
+ _fake_cuda_host(monkeypatch, count = 2)
+ seen = {}
+
+ class Backend:
+ @mg.with_load_selection
+ def _run_load(self, **kwargs):
+ seen["kwargs"] = dict(kwargs)
+ seen["selection"] = mg.current_selection()
+
+ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS, repo_id = "x")
+ assert seen["kwargs"] == {"gpu_ordinal": None, "repo_id": "x"}
+ assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
+
+
+def test_automatic_load_splits_only_when_one_card_would_offload(monkeypatch):
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
+ target = _Target(ordinal = None)
+ single = _plan(target = target)
+ with mg.selection_scope(selection):
+ plan = _plan(target = target)
+ fits = _plan(
+ target = target,
+ model_dense_mib = 4000,
+ companion_dense_mib = 2000,
+ text_encoder_dense_mib = 1500,
+ )
+ assert single.offload_policy != dm.OFFLOAD_NONE and single.secondary_ordinal is None
+ assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1
+ assert fits.secondary_ordinal is None
+
+
+def test_automatic_placement_does_not_arm_cfg_parallel(monkeypatch):
+ class Pipe:
+ transformer = _Dit()
+
+ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
+ assert "automatic" in mg.cfg_parallel_decline(
+ Pipe(), offload_policy = "none", selection = automatic
+ )
+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "force")
+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = automatic)
+
+
[email protected]("module_file", ["diffusion.py", "video.py"])
+def test_begin_load_marks_an_unpicked_load_automatic(module_file):
+ import ast
+ from pathlib import Path
+
+ path = Path(__file__).resolve().parents[1] / "core" / "inference" / module_file
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ begin_loads = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.FunctionDef) and node.name == "begin_load"
+ ]
+ assert len(begin_loads) == 1
+ assigned = [
+ node
+ for node in ast.walk(begin_loads[0])
+ if isinstance(node, ast.Assign)
+ and any(getattr(t, "id", None) == "gpu_ordinals" for t in node.targets)
+ ]
+ assert len(assigned) == 1
+ value = assigned[0].value
+ assert isinstance(value, ast.IfExp) and isinstance(value.orelse, ast.IfExp)
+ assert ast.unparse(value.orelse.body) == "AUTOMATIC_ORDINALS"
+ assert ast.unparse(value.orelse.test) == "not gpu_ids"
+ assert ast.unparse(value.orelse.orelse) == "[]"
+
+
+def test_automatic_selection_leaves_module_compile_alone(monkeypatch):
+ def original(self, *args, **kwargs):
+ return self
+
+ monkeypatch.setattr(torch.nn.Module, "compile", original)
+ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
+ with mg.selection_scope(automatic):
+ assert torch.nn.Module.compile is original
+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
+ assert getattr(torch.nn.Module.compile, "_unsloth_recorder", False)
+
+
+def test_automatic_secondaries_are_sized_without_a_cuda_context(monkeypatch):
+ from core.inference import diffusion_device as dd
+
+ monkeypatch.setattr(
+ torch.cuda, "mem_get_info", lambda *a: pytest.fail("opened a context on a secondary")
+ )
+ monkeypatch.setattr(
+ mg, "probe_devices", lambda *a, **k: pytest.fail("probed every card through torch")
+ )
+ readings = {1: (9000, 15360), 2: (14000, 15360)}
+ monkeypatch.setattr(dd, "smi_free_total_mib", lambda o: readings.get(o, (None, None)))
+ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2), automatic = True)
+ with mg.selection_scope(automatic):
+ assert mg.best_secondary(automatic) == 2
+ memory = mg.secondary_device_memory(2)
+ assert (memory.free_mib, memory.total_mib) == (14000, 15360)
+ assert mg.secondary_device_memory(3) is None
+
+
+def test_smi_reading_keys_ordinals_through_the_mask(monkeypatch):
+ import utils.hardware.hardware as hw
+ from core.inference import diffusion_device as dd
+
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [4, 5])
+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda visible: ({}, ""))
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": [4, 5]})
+ rows = [
+ {"index": 4, "vram_total_gb": 24.0, "vram_used_gb": 20.0},
+ {"index": 5, "vram_total_gb": 24.0, "vram_used_gb": 1.0},
+ ]
+ monkeypatch.setattr(hw, "_smi_visible_utilization", lambda spec: {"devices": rows})
+ assert dd.smi_free_total_mib(1) == (23 * 1024, 24 * 1024)
+ assert dd.smi_free_total_mib(2) == (None, None)
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {5})
+ assert dd.smi_free_total_mib(1) == (None, None)
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)
+ assert dd.smi_free_total_mib(1) == (None, None)
+
+
+def test_unsplit_automatic_load_has_no_other_vae_card(monkeypatch):
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
+ automatic = _types.SimpleNamespace(ordinal = None, is_cuda_torch_device = True)
+ assert D._vae_on_other_card(pipe, automatic) is None
+ monkeypatch.setattr(mg, "is_split", lambda p: True)
+ assert D._vae_on_other_card(pipe, automatic) is None
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
+ assert D._vae_on_other_card(pipe, automatic) == torch.device("cuda", 1)
+
+
+def test_secondary_vae_guard_refuses_a_batch_slicing_did_not_engage(monkeypatch):
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ calls = []
+
+ def shortfall(**kw):
+ calls.append(kw.get("vae_sliced"))
+ if kw.get("vae_sliced") is False:
+ raise ValueError("does not fit unsliced")
+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+
+ restored = []
+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
+ monkeypatch.setattr(
+ D, "engage_vae_tiling", lambda pipe, logger = None: (lambda: restored.append(1), True)
+ )
+ monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)
+ monkeypatch.setattr(mg, "is_split", lambda p: True)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
+ with pytest.raises(ValueError):
+ D._guard_vae_card(pipe, target, {"batch_size": 2, "vae_sliced": True}, None, None)
+ assert restored == [1] and calls == [True, False]
+
+
[email protected]("vae_card", [0, 1])
+def test_a_broken_recheck_keeps_the_tiling_it_engaged(monkeypatch, vae_card):
+ """A re-check that raises something other than the refusal fails open, as generate() always did: the decode
+ stays tiled (the guard said only a tiled decode fits) and the handle comes back for the caller to restore."""
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ def shortfall(**kw):
+ if kw.get("vae_sliced") is False:
+ raise RuntimeError("probe broke")
+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+
+ restored = []
+ restore = lambda: restored.append(1) # noqa: E731
+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "free")
+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
+ monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: (restore, True))
+ monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)
+ monkeypatch.setattr(mg, "is_split", lambda p: True)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", vae_card))
+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
+ kwargs = {"batch_size": 2, "vae_sliced": True, "vae_tile_side": 512, "device_memory": "free"}
+ verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+ assert D._tile_for_verdict(pipe, verdict, target, kwargs, 2, None, None) is restore
+ assert restored == []
+
+
+def test_no_split_ever_placed_skips_the_component_walk(monkeypatch):
+ reads = []
+
+ class Pipe:
+ @property
+ def components(self):
+ reads.append(1)
+ return {}
+
+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", False)
+ assert mg._split_home(Pipe()) is None and reads == []
+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
+ assert mg._split_home(Pipe()) is None and reads == [1]
+
+
+def test_a_card_torch_cannot_address_is_not_a_secondary(monkeypatch):
+ """nvidia-smi can list two cards while this process's torch addresses one: no second card, no selection."""
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))
+ assert mg.make_selection(0, [0, 1]) is None
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+
+
+def test_refinement_keeps_an_explicit_cpu_offload(monkeypatch):
+ sizes = {
+ "transformer": (9000, "dit"),
+ "text_encoder": (8000, "text_encoder"),
+ "vae": (300, "other"),
+ }
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)
+ legacy = _plan(explicit_offload = True)
+ assert legacy.estimates.get("explicit_cpu_offload")
+ moded = _plan(explicit_offload = True, requested_mode = "auto")
+ assert not moded.estimates.get("explicit_cpu_offload")
+ with mg.selection_scope(mg.make_selection(0, [0, 1])):
+ assert mg.refine_plan_for_multigpu(object(), legacy) is legacy
+ assert mg.refine_plan_for_multigpu(object(), moded).secondary_ordinal == 1
+
+
+def test_render_end_releases_the_branch_tensors():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ router, replicas = _router(dit)
+
+ class Pipe:
+ transformer = dit
+
+ pipe = Pipe()
+ mg._CFG_ROUTERS[pipe] = router
+ _cfg_loop(dit, latents.clone(), steps = 3)
+ assert router.template is not None and router.prev_latent is not None
+ mg.end_cfg_render(pipe)
+ assert router.template is None and router.prev_latent is None and router.pending is None
+ assert router.cfg_seen is False
+ assert router.prefetched == {} and router.carry == [] and router.b0_leaves == {}
+ assert router.replica is not None
+ reference_router = dit.__dict__.pop("forward")
+ reference = _cfg_loop(dit, latents.clone(), steps = 3)
+ dit.forward = reference_router
+ assert torch.equal(_cfg_loop(dit, latents.clone(), steps = 3), reference)
+ assert len(replicas) == 1
+ mg.end_cfg_render(object())
+
+
+def test_workflow_pipeline_of_a_split_still_guards_the_vae_card(monkeypatch):
+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
+ import types as _types
+
+ from core.inference import diffusion as D
+
+ dit = torch.nn.Linear(1, 1)
+ dit._unsloth_split_home = torch.device("cuda", 0)
+ sibling = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1), components = {"transformer": dit})
+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
+ assert not mg.is_split(sibling)
+ assert D._vae_on_other_card(sibling, target) == torch.device("cuda", 1)
+
+
+def test_cfg_replica_avoids_the_card_a_split_decodes_on(monkeypatch):
+ class Pipe:
+ transformer = _Dit()
+
+ probes = {
+ o: mg.DeviceProbe(ordinal = o, name = "B200", total_mib = 180000, free_mib = 170000)
+ for o in (0, 1, 2)
+ }
+ monkeypatch.setattr(mg, "cfg_parallel_decline", lambda pipe, **kw: None)
+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
+ monkeypatch.setattr(
+ mg, "probe_devices", lambda ordinals, **kw: {o: probes[o] for o in ordinals}
+ )
+ pipe = Pipe()
+ mg._PLACEMENTS[pipe] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}
+ try:
+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
+ reason = mg.install_cfg_parallel(pipe, offload_policy = "none")
+ assert reason is not None and "holds the split's components" in reason
+ te_only = Pipe()
+ mg._PLACEMENTS[te_only] = {
+ "placement": {"transformer": "cuda:0", "vae": "cuda:0", "text_encoder": "cuda:1"}
+ }
+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
+ assert "holds the split's components" in mg.install_cfg_parallel(
+ te_only, offload_policy = "none"
+ )
+ other = Pipe()
+ mg._PLACEMENTS[other] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}
+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1, 2))):
+ assert mg.install_cfg_parallel(other, offload_policy = "none") is None
+ assert str(mg._CFG_ROUTERS[other].secondary) == "cuda:2"
+ finally:
+ mg.uninstall_cfg_parallel(pipe)
+ mg.uninstall_cfg_parallel(other)
+
+
+def test_background_compile_declines_cfg_parallel(monkeypatch):
+ class Pipe:
+ transformer = _Dit()
+
+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_BG_COMPILE", "1")
+ assert "background compile" in mg.cfg_parallel_decline(
+ Pipe(), offload_policy = "none", selection = sel
+ )
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE")
+ assert "background compile" not in mg.cfg_parallel_decline(
+ Pipe(), offload_policy = "none", selection = sel
+ )
+
+
+def test_a_displaced_router_is_reported_declined():
+ dit = _Dit()
+ router, _replicas = _router(dit)
+
+ class Pipe:
+ transformer = dit
+
+ pipe = Pipe()
+ mg._CFG_ROUTERS[pipe] = router
+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is False
+ graphed = lambda *a, **k: None # noqa: E731
+ dit.__dict__["forward"] = graphed
+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is True
+ assert dit.__dict__["forward"] is graphed
+ assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": "graphs"}}
+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is False
+
+
+def test_kept_models_keep_every_media_load_on_one_card(monkeypatch):
+ from core.inference import gpu_residency
+
+ _fake_cuda_host(monkeypatch, count = 2)
+ seen = {}
+
+ class Backend:
+ @mg.with_load_selection
+ def _run_load(self, **kwargs):
+ seen["selection"] = mg.current_selection()
+
+ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)
+ assert seen["selection"] is not None
+ Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])
+ assert seen["selection"] is not None
+ for mode in ("fit", "always"):
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda mode = mode: mode)
+ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)
+ assert seen["selection"] is None
+ Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])
+ assert seen["selection"] is None
+
+
+def test_status_models_keep_the_multi_gpu_report():
+ from models.inference import DiffusionStatusResponse, VideoStatusResponse
+
+ report = {"primary": "cuda:0", "secondary": "cuda:1", "moved": ["text_encoder"]}
+ assert (
+ DiffusionStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report
+ )
+ assert VideoStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report
+
+
[email protected]("module_file", ["video.py"])
+def test_video_split_fallback_recomputes_the_floor(module_file):
+ import ast
+ from pathlib import Path
+
+ source = (Path(__file__).resolve().parents[1] / "core" / "inference" / module_file).read_text(
+ encoding = "utf-8"
+ )
+ tree = ast.parse(source)
+ hits = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ and "secondary_ordinal" in ast.unparse(node.test)
+ and "plan is not planned" in ast.unparse(node.test)
+ and any(
+ isinstance(b, ast.Assign) and getattr(b.targets[0], "id", None) == "vram_floor_mib"
+ for b in node.body
+ )
+ ]
+ assert len(hits) == 1
+
+
+def test_a_split_refuses_a_controlnet_it_cannot_stream():
+ # The resident fallback would put the whole ControlNet beside a denoiser one card could not hold with it.
+ import ast
+ from pathlib import Path
+
+ path = Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py"
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ tests = [
+ ast.unparse(node.test)
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)
+ ]
+ assert len(tests) == 1
+ branch = next(
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)
+ )
+ refusal = branch.orelse[0]
+ assert (
+ isinstance(refusal, ast.If)
+ and ast.unparse(refusal.test) == "calibrated or is_split(state.pipe)"
+ )
+ assert any(isinstance(n, ast.Raise) for n in ast.walk(refusal))
+
+
+def test_replica_rebuild_counts_the_allocator_cache_as_free(monkeypatch):
+ import types as _types
+
+ dit = _Dit()
+ router, _replicas = _router(dit)
+ router.secondary = torch.device("cuda", 1)
+ monkeypatch.setattr(mg, "_free_total_mib", lambda o: (100, 15360))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 4096 << 20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda d = None: 0)
+ monkeypatch.setattr(mg, "_device_context", lambda d: contextlib.nullcontext())
+ monkeypatch.setattr(
+ mg, "_ReplicaWorker", lambda device: _types.SimpleNamespace(close = lambda: None)
+ )
+ router._build_replica()
+ assert router.replica is not None and router.disabled_reason is None
+ router.replica = None
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 0)
+ router._build_replica()
+ assert router.replica is None and "MiB free" in (router.disabled_reason or "")
+
+
+def test_a_two_expert_video_split_prices_the_denoise_phase_on_both_cards():
+ from core.inference import video as vid
+
+ class Pipe:
+ def __init__(self):
+ self.components = {
+ "transformer": torch.nn.Linear(1024, 1024),
+ "transformer_2": torch.nn.Linear(1024, 1024),
+ "vae": torch.nn.Linear(512, 512),
+ }
+
+ pipe = Pipe()
+ mg._PLACEMENTS[pipe] = {
+ "placement": {"transformer": "cuda:0", "vae": "cuda:0", "transformer_2": "cuda:1"},
+ "moved": ["transformer_2"],
+ }
+ try:
+ cards = vid._video_split_card_floors(pipe)
+ assert cards == ((0, 5, True, True, True), (1, 4, False, False, True))
+ shape = dict(width = 832, height = 480, frames = 25)
+ roomy = {0: 1 << 20, 1: 1 << 20}
+ assert (
+ vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **shape)
+ is None
+ )
+ # the expert card has room for its weights but not the denoise activations
+ expert_short = {0: 1 << 20, 1: 4 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100}
+ message = vid.video_split_shortfall_message(
+ cards, family = "f", available_mib = expert_short.get, **shape
+ )
+ assert message is not None and "cuda:1" in message
+ primary_short = {0: 5 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100, 1: 1 << 20}
+ message = vid.video_split_shortfall_message(
+ cards, family = "f", available_mib = primary_short.get, **shape
+ )
+ assert message is not None and "cuda:0" in message
+ finally:
+ del mg._PLACEMENTS[pipe]
+
+
+def test_untiled_decode_probes_the_card_the_vae_lives_on(monkeypatch):
+ import types as _types
+
+ from core.inference import video_vae_untiled as vu
+
+ vae = torch.nn.Linear(1, 1)
+ latent = _types.SimpleNamespace(device = torch.device("cuda", 0))
+ monkeypatch.setattr(
+ torch.nn.Linear,
+ "parameters",
+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 1))]),
+ )
+ assert vu._decode_device(vae, latent) == torch.device("cuda", 1)
+ # same card, or an offloaded VAE whose weights sit on the host: the latent's card, as before
+ monkeypatch.setattr(
+ torch.nn.Linear,
+ "parameters",
+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 0))]),
+ )
+ assert vu._decode_device(vae, latent) == torch.device("cuda", 0)
+ monkeypatch.setattr(
+ torch.nn.Linear,
+ "parameters",
+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cpu"))]),
+ )
+ assert vu._decode_device(vae, latent) == torch.device("cuda", 0)
+
+
+def test_untiled_decode_sizes_the_card_the_vae_decodes_on(monkeypatch):
+ import types as _types
+
+ from core.inference import video_vae_untiled as vu
+
+ family = next(iter({**vu._BYTES_PER_LATENT_PIXEL, **vu._BYTES_PER_FRAME_LATENT_PIXEL}))
+ probed = []
+ monkeypatch.setattr(vu, "_free_bytes", lambda device: probed.append(device))
+
+ class Vae:
+ use_tiling = True
+
+ def decode(self, z, *args, **kwargs):
+ return "tiled"
+
+ def parameters(self):
+ return iter([_types.SimpleNamespace(device = torch.device("cuda", 1))])
+
+ pipe = _types.SimpleNamespace(vae = Vae())
+ assert vu.install_untiled_decode(pipe, family)
+ latent = _types.SimpleNamespace(device = torch.device("cuda", 0), shape = (1, 16, 3, 8, 8))
+ assert pipe.vae.decode(latent) == "tiled"
+ assert probed == [torch.device("cuda", 1)]
+
+
[email protected]("module_name", ["diffusion", "video"])
+def test_begin_load_hands_the_picked_cards_to_the_load_thread(monkeypatch, module_name):
+ import importlib
+ import types as _types
+
+ backend_mod = importlib.import_module(f"core.inference.{module_name}")
+ backend_cls = (
+ backend_mod.DiffusionBackend if module_name == "diffusion" else backend_mod.VideoBackend
+ )
+ seen = {}
+
+ def first_load_step(*args, **kwargs):
+ seen["selection"] = mg.current_selection()
+ raise RuntimeError("stop after the selection is read")
+
+ class InlineThread:
+ def __init__(
+ self,
+ target,
+ kwargs,
+ daemon = None,
+ ):
+ self.target, self.kwargs = target, kwargs
+
+ def start(self):
+ self.target(**self.kwargs)
+
+ monkeypatch.setattr(backend_mod, "account_thread", InlineThread)
+ monkeypatch.setattr(
+ backend_mod,
+ "resolve_diffusion_device_target",
+ lambda *a, **k: _types.SimpleNamespace(device = "cuda"),
+ )
+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinal", lambda ids: 0)
+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", lambda ids: [0, 1])
+ monkeypatch.setattr(mg, "_capability", lambda ordinal: None)
+ monkeypatch.setattr(mg, "models_kept_loaded", lambda: False)
+ fam = _types.SimpleNamespace(name = "ltx-2", base_repo = "base/repo", single_file_is_pipeline = False)
+ monkeypatch.setattr(backend_cls, "validate_load_request", lambda self, *a, **k: fam)
+ if module_name == "diffusion":
+ monkeypatch.setattr(backend_cls, "assert_precision_available", lambda self, *a, **k: None)
+ monkeypatch.setattr(backend_mod, "detect_family_for_pick", first_load_step)
+ else:
+ from core.inference import video_minimax_h3
+
+ monkeypatch.setattr(backend_mod, "assert_video_precision_available", lambda *a, **k: None)
+ monkeypatch.setattr(backend_mod, "_ltx23_prequant_serves_on_card", lambda *a, **k: False)
+ monkeypatch.setattr(video_minimax_h3, "is_h3_native", lambda *a, **k: False)
+ monkeypatch.setattr(backend_mod, "_detect_load_family", first_load_step)
+
+ backend_cls().begin_load("org/model", gpu_ids = [0, 1])
+ assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ # and only for that load
+ assert mg.current_selection() is None
+
+ # the kill switch: the extra cards are never even resolved (the single pick is, as before the multi-GPU layer)
+ def never(ids):
+ raise AssertionError("resolved several cards under UNSLOTH_DIFFUSION_MULTIGPU=0")
+
+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", never)
+ seen.clear()
+ backend_cls().begin_load("org/model", gpu_ids = [0, 1])
+ assert seen["selection"] is None
+
+
+def test_fingerprint_sees_buffers_and_the_peft_adapter_selection():
+ dit = _Dit()
+ dit.proj.register_buffer("w_scale", torch.ones(4))
+ base = mg.weights_fingerprint(dit)
+ dit.proj.w_scale.mul_(2.0)
+ edited = mg.weights_fingerprint(dit)
+ assert edited != base
+ dit.proj.w_scale = torch.ones(4)
+ assert mg.weights_fingerprint(dit) not in (base, edited)
+ before = mg.weights_fingerprint(dit)
+ dit.proj._active_adapter = ["a"]
+ with_a = mg.weights_fingerprint(dit)
+ dit.proj._active_adapter = ["b"]
+ assert len({before, with_a, mg.weights_fingerprint(dit)}) == 3
+
+
+def test_replica_shares_the_nvfp4_step_controller():
+ from core.inference.diffusion_nvfp4_protect import NVFP4StepController
+
+ dit = _Dit()
+ ctl = NVFP4StepController("0")
+ dit.proj.protect = ctl
+ replica = mg.build_replica(dit, torch.device("cpu"))
+ assert replica.proj.protect is ctl
+
+
+def test_boundary_and_router_removal_restore_an_instance_override():
+ import types
+
+ dit = _Dit()
+
+ def own_forward(self, *args, **kwargs):
+ return ("own",)
+
+ override = types.MethodType(own_forward, dit)
+ dit.forward = override
+ mg.install_boundary(dit, "cpu")
+ assert dit.__dict__["forward"] is not override
+ mg.remove_boundary(dit)
+ assert dit.__dict__["forward"] is override
+ plain = _Dit()
+ mg.install_boundary(plain, "cpu")
+ mg.remove_boundary(plain)
+ assert "forward" not in plain.__dict__
+
+ class Pipe:
+ pass
+
+ pipe = Pipe()
+ router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu")
+ dit.forward = router
+ mg._CFG_ROUTERS[pipe] = router
+ mg.uninstall_cfg_parallel(pipe)
+ assert dit.__dict__["forward"] is override
+
+
+def test_uninstalled_router_left_inside_a_wrapper_is_a_passthrough():
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ router, replicas = _router(dit)
+
+ class Skip:
+ def __init__(self, inner):
+ self.inner = inner
+
+ def __call__(self, *args, **kwargs):
+ return self.inner(*args, **kwargs)
+
+ dit.forward = Skip(router)
+
+ class Pipe:
+ pass
+
+ pipe = Pipe()
+ mg._CFG_ROUTERS[pipe] = router
+ assert mg.drop_displaced_cfg_router(pipe, "replaced")
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference)
+ assert replicas == [] and router.replica is None and router.prev_latent is None
+ assert router.stats["replica_calls"] == 0
+
+
+def test_router_stays_on_the_source_for_a_forced_eager_render_and_the_compile_thread():
+ from core.inference import diffusion_bg_compile as bg
+
+ torch.manual_seed(0)
+ dit = _Dit()
+ latents = torch.randn(2, 4)
+ reference = _cfg_loop(dit, latents.clone(), steps = 4)
+ router, replicas = _router(dit)
+ for var in (bg._FORCE_EAGER, bg._NO_CAPTURE):
+ token = var.set(True)
+ try:
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ finally:
+ var.reset(token)
+ assert torch.equal(out, reference)
+ assert replicas == [] and router.stats["replica_calls"] == 0 and router.prev_latent is None
+ out = _cfg_loop(dit, latents.clone(), steps = 4)
+ assert torch.equal(out, reference) and router.stats["replica_calls"] == 3
+
+
+def test_replica_blocks_get_their_own_compile_guard():
+ from core.inference.diffusion_speed import guard_compiled_blocks
+
+ mg.install_compile_recorder()
+ dit = _Dit()
+ dit.proj.compile(backend = "eager") # lazy: nothing is traced here
+ assert guard_compiled_blocks(dit) == 1
+ source_guard = dit._unsloth_compile_guard
+ replica = mg.build_replica(dit, torch.device("cpu"))
+ wrapped = replica.proj._compiled_call_impl
+ guard = getattr(wrapped, "_unsloth_compile_guard", None)
+ assert guard is not None and guard is not source_guard
+ assert replica.__dict__.get("_unsloth_compile_guard") is guard
diff --git a/studio/backend/tests/test_diffusion_multigpu_cuda.py b/studio/backend/tests/test_diffusion_multigpu_cuda.py
new file mode 100644
index 00000000000..ab25ed31e77
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_multigpu_cuda.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CFG-parallel on two real CUDA cards. On one device ``.to()`` aliases storage, so the CPU tests in
+test_diffusion_multigpu.py cannot see a replica that diverged from its source; these can. Skipped below two cards."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from core.inference import diffusion_multigpu as mg
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs two CUDA cards"
+)
+
+
+class _Dit(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(4, 4)
+ self.register_buffer("scale", torch.ones(4))
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ return_dict = False,
+ ):
+ return ((self.proj(hidden_states) * self.scale) * timestep + encoder_hidden_states.sum(),)
+
+
+def _render(
+ dit,
+ latents,
+ cond,
+ neg,
+ steps,
+ between = None,
+):
+ x = latents.clone()
+ for i in range(steps):
+ t = torch.tensor(1.0 - i / steps, device = x.device)
+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
+ if between is not None:
+ between(i, x)
+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]
+ x = x - 0.1 * (b + 4.0 * (a - b))
+ return x
+
+
+def _setup():
+ torch.manual_seed(0)
+ dit = _Dit().cuda(0)
+ latents = torch.randn(2, 4, device = "cuda:0")
+ cond = torch.randn(3, 4, device = "cuda:0")
+ neg = torch.randn(3, 4, device = "cuda:0")
+ return dit, latents, cond, neg
+
+
+def _arm(dit):
+ router = mg.CfgParallelRouter(dit, primary = "cuda:0", secondary = "cuda:1")
+ dit.forward = router
+ return router
+
+
+def test_an_input_edited_after_the_prefetch_reaches_an_unpredicted_replica_call():
+ dit, latents, cond, neg = _setup()
+
+ def edit(step, x):
+ if (
+ step == 1
+ ): # the first step of a render runs unpredicted (no template yet): only the prefetch was made
+ x.add_(1.0)
+
+ with torch.no_grad():
+ reference = [_render(dit, latents, cond, neg, 4), _render(dit, latents, cond, neg, 4, edit)]
+ router = _arm(dit)
+ first = _render(dit, latents, cond, neg, 4)
+ router.end_render()
+ second = _render(dit, latents, cond, neg, 4, edit)
+ torch.cuda.synchronize()
+ assert router.stats["replica_calls"] > 0 and router.replica is not None
+ assert torch.equal(first, reference[0])
+ assert torch.equal(second, reference[1])
+
+
+def test_an_edited_buffer_rebuilds_the_replica():
+ dit, latents, cond, neg = _setup()
+ with torch.no_grad():
+ router = _arm(dit)
+ _render(dit, latents, cond, neg, 3)
+ router.end_render()
+ assert router.replica is not None
+ dit.scale.mul_(3.0)
+ dit.forward = dit.__class__.forward.__get__(dit)
+ reference = _render(dit, latents, cond, neg, 4)
+ dit.forward = router
+ out = _render(dit, latents, cond, neg, 4)
+ torch.cuda.synchronize()
+ assert router.stats["replica_rebuilds"] >= 1
+ assert torch.equal(out, reference)
diff --git a/studio/backend/tests/test_diffusion_multigpu_wiring.py b/studio/backend/tests/test_diffusion_multigpu_wiring.py
new file mode 100644
index 00000000000..7248a4b10b9
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_multigpu_wiring.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The image backend's multi-GPU call sites, on the fake runtime of test_diffusion_backend, CPU only.
+
+The helpers are unit-tested in test_diffusion_multigpu.py; these pin that a real load / generate / status reaches them:
+the plan refinement and the CFG-parallel arming in load_pipeline, the guard's VAE-card tile decision and the CFG render
+release in generate, and the ``multi_gpu`` status key."""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+from core.inference import diffusion as D
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_backend import _loaded_backend, fake_runtime # noqa: E402,F401
+
+
+def test_image_load_generate_and_status_reach_the_multi_gpu_layer(
+ fake_runtime, tmp_path, monkeypatch
+):
+ calls = []
+ report = {"primary": "cuda:0", "secondary": "cuda:1", "placement": {"text_encoder": "cuda:1"}}
+
+ def refine(
+ pipe,
+ plan,
+ logger = None,
+ ):
+ calls.append("refine")
+ return plan
+
+ def tile_for_verdict(
+ pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger
+ ):
+ calls.append(("tile", verdict))
+ return restore_vae
+
+ monkeypatch.setattr(D, "refine_plan_for_multigpu", refine)
+ monkeypatch.setattr(
+ D,
+ "install_cfg_parallel",
+ lambda pipe, offload_policy = None, logger = None: calls.append("cfg"),
+ )
+ monkeypatch.setattr(D, "end_cfg_render", lambda pipe: calls.append(("end", pipe)))
+ monkeypatch.setattr(D, "multigpu_placement_status", lambda pipe: report)
+ verdict = types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", lambda **kw: verdict)
+ monkeypatch.setattr(D, "_tile_for_verdict", tile_for_verdict)
+
+ backend = _loaded_backend(tmp_path)
+ assert calls == ["refine", "cfg"]
+ assert backend.status()["multi_gpu"] == report
+ calls.clear()
+ backend.generate(prompt = "a sloth", width = 512, height = 512, steps = 2)
+ assert ("tile", verdict) in calls
+ assert calls[-1:] == [("end", backend._state.pipe)]
+ pipe = backend._state.pipe
+ monkeypatch.setattr(D, "uninstall_cfg_parallel", lambda p: calls.append(("uninstall", p)))
+ backend.unload()
+ assert ("uninstall", pipe) in calls
diff --git a/studio/backend/tests/test_video_multigpu_wiring.py b/studio/backend/tests/test_video_multigpu_wiring.py
new file mode 100644
index 00000000000..0bb83412a8e
--- /dev/null
+++ b/studio/backend/tests/test_video_multigpu_wiring.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The video backend's multi-GPU call sites, on the fake runtime of test_video_backend, CPU only.
+
+The helpers are unit-tested in test_diffusion_multigpu.py; these pin that a real load / generate / status reaches them:
+the plan refinement, the CFG-parallel arming and the per-card floors of a split in load_pipeline, the CFG render
+release in generate, and the ``multi_gpu`` status key."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from core.inference import video as V
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_video_backend import _load_gguf, fake_runtime # noqa: E402,F401
+
+
+def test_video_load_generate_and_status_reach_the_multi_gpu_layer(
+ fake_runtime, tmp_path, monkeypatch
+):
+ calls = []
+ report = {"primary": "cuda:0", "secondary": "cuda:1", "placement": {"text_encoder": "cuda:1"}}
+ floors = ((0, 9000, False), (1, 7000, True))
+
+ def refine(
+ pipe,
+ plan,
+ logger = None,
+ ):
+ calls.append("refine")
+ return plan
+
+ monkeypatch.setattr(V, "refine_plan_for_multigpu", refine)
+ monkeypatch.setattr(
+ V,
+ "install_cfg_parallel",
+ lambda pipe, offload_policy = None, logger = None: calls.append("cfg"),
+ )
+ monkeypatch.setattr(V, "end_cfg_render", lambda pipe: calls.append(("end", pipe)))
+ monkeypatch.setattr(V, "multigpu_placement_status", lambda pipe: report)
+ monkeypatch.setattr(V, "_video_split_card_floors", lambda pipe: floors)
+ sidecar_checked = []
+ single_card_for_sidecar = V._video_single_card_for_sidecar
+
+ def sidecar_check(
+ fam,
+ plan,
+ logger = None,
+ ):
+ sidecar_checked.append(fam.name)
+ return single_card_for_sidecar(fam, plan, logger)
+
+ monkeypatch.setattr(V, "_video_single_card_for_sidecar", sidecar_check)
+
+ backend = V.VideoBackend()
+ status = _load_gguf(backend, tmp_path)
+ assert calls == ["refine", "cfg"]
+ assert sidecar_checked == ["ltx-2", "ltx-2"]
+ assert status["multi_gpu"] == report
+ assert backend._state.vram_split_cards == floors
+ calls.clear()
+ backend.generate(prompt = "a sloth surfing", width = 512, height = 512, num_frames = 9, fps = 24)
+ assert calls[-1:] == [("end", backend._state.pipe)]
+ pipe = backend._state.pipe
+ monkeypatch.setattr(V, "uninstall_cfg_parallel", lambda p: calls.append(("uninstall", p)))
+ backend.unload()
+ assert ("uninstall", pipe) in calls