Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: multi-GPU component placement and CFG-parallel (#13725)
Daniel Han811c08e782cf
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 7c1978bf850..99094681eed 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -106,6 +106,19 @@ from .diffusion_device import (auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,+ resolve_selected_cuda_ordinals,+)+from .diffusion_multigpu import (+ AUTOMATIC_ORDINALS,+ end_cfg_render,+ drop_displaced_cfg_router,+ install_cfg_parallel,+ multigpu_mode,+ invalidate_cfg_replica,+ placement_status as multigpu_placement_status,+ refine_plan_for_multigpu,+ uninstall_cfg_parallel,+ with_load_selection,)from .diffusion_conditioning import (LocalizedEdit,@@ -2104,6 +2117,129 @@ def _plans_at_requested_speed(fn: Callable[..., Any]) -> Callable[..., Any]:return wrapper+def _vae_on_other_card(pipe: Any, guard_target: Any) -> Any:+ """The CUDA device of ``pipe.vae`` when a multi-GPU split put it on another card than ``guard_target``, else None."""+ from .diffusion_multigpu import _module_device, _split_home, is_split++ if not is_split(pipe) and _split_home(pipe) is None:+ return None+ vae = getattr(pipe, "vae", None)+ try:+ device = _module_device(vae) if vae is not None else None+ except Exception: # noqa: BLE001 - an unreadable VAE keeps the primary's verdict+ return None+ if device is None or getattr(device, "type", None) != "cuda":+ return None+ if device.index in (None, placed_cuda_ordinal(guard_target)):+ return None+ return device+++def _guard_vae_card(+ pipe: Any, guard_target: Any, guard_kwargs: dict, restore_vae: Any, logger: Any+) -> tuple[Any, bool]:+ """Re-run the generate-time activation guard on the VAE's card when a multi-GPU split put it on another card than+ the denoiser: the decode allocates there, and the primary's free memory says nothing about it. Engages VAE tiling+ when only a tiled decode fits that card (unless already tiled) and raises the guard's refusal when nothing fits.+ Returns ``(restore_vae, tiled)`` for a tiling it engaged, else ``(None, False)``. Conservative: the whole request's+ estimate is checked against the VAE's card."""+ device = _vae_on_other_card(pipe, guard_target)+ if device is None:+ return None, False+ with diffusion_device_scope(device.index):+ vae_kwargs = {+ **guard_kwargs,+ "device_memory": reclaimable_snapshot_device_memory(guard_target),+ }+ verdict = raise_on_image_activation_shortfall(**vae_kwargs)+ if verdict.action != ACTIVATION_TILE or restore_vae is not None:+ return None, False+ restore, tiled = engage_vae_tiling(pipe, logger = logger)+ try:+ if not tiled and not verdict.overridden:+ raise_on_image_activation_shortfall(**{**vae_kwargs, "vae_tile_side": None})+ elif (+ int(vae_kwargs.get("batch_size") or 1) > 1+ and vae_kwargs.get("vae_sliced")+ and not verdict.overridden+ and not vae_is_sliced(getattr(pipe, "vae", None))+ ):+ raise_on_image_activation_shortfall(**{**vae_kwargs, "vae_sliced": False})+ except ValueError:+ # The refusal: the caller never receives the handle, so undo the tiling here.+ if restore is not None:+ restore()+ raise+ except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe and keep the tiling, as the caller does+ (logger or get_logger(__name__)).warning(+ "diffusion.generate: VAE card headroom re-check skipped (%s)", exc+ )+ except BaseException:+ if restore is not None:+ restore()+ raise+ return restore, tiled+++def _tile_for_verdict(+ pipe: Any,+ verdict: Any,+ guard_target: Any,+ guard_kwargs: dict,+ guard_batch: int,+ restore_vae: Any,+ logger: Any,+) -> Any:+ """Engage VAE tiling when the guard says only a tiled decode fits; returns the VAE restore handle. A multi-GPU split+ that decodes on another card is decided by that card's verdict alone: the primary never runs the decode, so its+ tile verdict must not switch the secondary's VAE to the (not bit-identical) tiled path."""+ vae_restore, vae_card_tiled = _guard_vae_card(+ pipe, guard_target, guard_kwargs, restore_vae, logger+ )+ if vae_card_tiled:+ return vae_restore+ if verdict.action != ACTIVATION_TILE or _vae_on_other_card(pipe, guard_target):+ return restore_vae+ restore_vae, vae_tiled = engage_vae_tiling(pipe, logger = logger)+ try:+ if not vae_tiled and not verdict.overridden:+ # Passed only because it would be tiled; tiling failed, so refuse on the untiled figure.+ raise_on_image_activation_shortfall(**{**guard_kwargs, "vae_tile_side": None})+ elif (+ guard_batch > 1+ and guard_kwargs["vae_sliced"]+ and not verdict.overridden+ and not vae_is_sliced(getattr(pipe, "vae", None))+ ):+ raise_on_image_activation_shortfall(**{**guard_kwargs, "vae_sliced": False})+ except ValueError:+ # the caller never receives the handle on a refusal: undo the tiling here+ if restore_vae is not None:+ restore_vae()+ raise+ except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe and keep the tiling, as generate() does+ (logger or get_logger(__name__)).warning(+ "diffusion.generate: activation headroom re-check skipped (%s)", exc+ )+ except BaseException:+ if restore_vae is not None:+ restore_vae()+ raise+ return restore_vae+++def _controlnet_streams(state: Any) -> bool:+ """Whether a ControlNet added at generate time streams from host memory instead of loading whole beside the base+ model: under any offload policy, a calibrated placement (budgets only the base model), and a multi-GPU split+ (reports "none", but its fit priced only the base pipeline, which one card would have offloaded)."""+ from .diffusion_multigpu import is_split+ return (+ getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE+ or bool(getattr(state, "calibrated_placement", False))+ or is_split(getattr(state, "pipe", None))+ )++def _calibrated_activation(fam: Any, target: Any) -> Any:"""Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel."""if getattr(target, "backend", None) != "cuda" or getattr(target, "vendor", None) != "nvidia":@@ -4137,6 +4273,16 @@ class DiffusionBackend:if resolve_diffusion_device_target().device == "cuda"else None)+ gpu_ordinals = (+ resolve_selected_cuda_ordinals(gpu_ids)+ if gpu_ids+ and len(set(gpu_ids)) > 1+ and multigpu_mode() != "off"+ and resolve_diffusion_device_target().device == "cuda"+ else AUTOMATIC_ORDINALS+ if not gpu_ids+ else []+ )fam = self.validate_load_request(repo_id,gguf_filename = gguf_filename,@@ -4222,11 +4368,14 @@ class DiffusionBackend:vae_file = vae_file,_load_token = token,_cancel_event = cancel_event,+ _gpu_ordinals = gpu_ordinals,),daemon = True,).start()return self.status()+ # The multi-GPU selection (begin_load's ``_gpu_ordinals``) is visible to this load's planner and placement only.+ @with_load_selection@scoped_local_files_onlydef _run_load(self, **kwargs: Any) -> None:token = kwargs.get("_load_token")@@ -9236,6 +9385,7 @@ class DiffusionBackend:self._raise_if_load_cancelled(_load_token)# a refinement may have re-tiered the plan; routed encoders need group offloadplan = self._small_host_plan(pipe, plan, fam, logger)+ plan = refine_plan_for_multigpu(pipe, plan, logger)effective_policy, effective_tiling = apply_memory_plan(pipe,plan,@@ -9453,6 +9603,8 @@ class DiffusionBackend:load_bg_compile = (bg_compile.arm(bg_module, logger = logger) if bg_module is not None else None)+ # After every forward wrapper of the speed / graph layers, so the replica sees the final module.+ install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)te_releaser = te_release.maybe_install(pipe, plan, logger = logger)state = _LoadState(pipe = pipe,@@ -10997,11 +11149,14 @@ class DiffusionBackend:# Placement follows the base offload policy (resident base -> resident, offloaded -> group offload).# Calibrated tiers budget only the base model: ControlNet must stream, no resident fallback.calibrated = bool(getattr(state, "calibrated_placement", False))- if (getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE or calibrated) and (- _offload_controlnet_module(cn_model, state.device, logger)+ from .diffusion_multigpu import is_split++ # A split's resident fit also priced the base pipeline alone (one card would have offloaded it).+ if _controlnet_streams(state) and _offload_controlnet_module(+ cn_model, state.device, logger):pass- elif calibrated:+ elif calibrated or is_split(state.pipe):del cn_modelclear_gpu_cache()raise ValueError(@@ -11357,6 +11512,7 @@ class DiffusionBackend:try:self._engage_deferred_profile(state)except BaseException:+ invalidate_cfg_replica(state.pipe, "the deferred speed profile failed part way", logger)if installed:uninstall_static_step_skip(state.pipe)object.__setattr__(state, "transformer_cache", None)@@ -11364,6 +11520,13 @@ class DiffusionBackend:tc.clear()tc.update(tc_before)raise+ # the profile patched / compiled the source denoiser: a replica built during the eager images is stale+ invalidate_cfg_replica(+ state.pipe, "the deferred speed profile changed the denoiser", logger+ )+ drop_displaced_cfg_router(+ state.pipe, "the deferred speed profile replaced the denoiser's forward", logger+ )# Outside load_pipeline (3rd generation): its own probe scope, so one OOM is not re-probed by plan and apply.@rocm_probe_load_scoped@@ -12088,22 +12251,9 @@ class DiffusionBackend:logger = logger,)verdict = raise_on_image_activation_shortfall(**guard_kwargs)- if verdict.action == ACTIVATION_TILE:- restore_vae, vae_tiled = engage_vae_tiling(pipe, logger = logger)- if not vae_tiled and not verdict.overridden:- # Passed only because it would be tiled; tiling failed, so refuse on the untiled figure.- raise_on_image_activation_shortfall(- **{**guard_kwargs, "vae_tile_side": None}- )- elif (- guard_batch > 1- and guard_kwargs["vae_sliced"]- and not verdict.overridden- and not vae_is_sliced(getattr(pipe, "vae", None))- ):- raise_on_image_activation_shortfall(- **{**guard_kwargs, "vae_sliced": False}- )+ restore_vae = _tile_for_verdict(+ pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger+ )except ValueError:raise # the refusal itself: the route turns this into a 400 with the reasonexcept Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe@@ -12547,6 +12697,7 @@ class DiffusionBackend:logger.debug("diffusion.step_skip: reset failed: %s", exc)if restore_vae is not None:restore_vae()+ end_cfg_render(getattr(state, "pipe", None))if restore_resident is not None:try:restore_resident()@@ -12722,6 +12873,7 @@ class DiffusionBackend:if prebuild is not None:prebuild.cancel_all()compile_cache.restore(state.compile_cache_ctx, logger = logger)+ uninstall_cfg_parallel(state.pipe)# Before clear_gpu_cache(), or the graph pool stays reserved for the life of the process.cuda_graph.uninstall_all(state.cuda_graphs)uninstall_static_step_skip(state.pipe)@@ -12830,6 +12982,7 @@ class DiffusionBackend:"attention_backend": state.attention_backend,"transformer_cache": state.transformer_cache,"transformer_cache_stats": static_skip_stats(state.pipe),+ "multi_gpu": multigpu_placement_status(state.pipe),"resolved": resolved,"generation_defaults": _generation_defaults_for(state.repo_id,diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex d6ff36eb24e..664d6eb8215 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -338,8 +338,9 @@ def resolve_selected_cuda_ordinal(order (``4,5`` -> torch 0,1; ``1,0`` reverses them), hence going through the hardware layerthat owns the mask.- Neither engine shards a checkpoint, so several cards still resolve to one: most free VRAM- wins, as ``auto_select_gpu_ids`` already does for training, ties to the lowest ordinal. Taking+ Neither engine shards a checkpoint, so several cards resolve to one PRIMARY card (the+ multi-GPU layer, diffusion_multigpu, may place companions or a denoiser replica on the+ others, see :func:`resolve_selected_cuda_ordinals`): most free VRAM wins, as ``auto_select_gpu_ids`` already does for training, ties to the lowest ordinal. Takingthe FIRST id instead would land on ordinal 0 whenever everything is selected, i.e. the smallcard on the mixed boxes this exists for. Resolved ONCE per load and carried, never re-derived:free VRAM moves the moment the checkpoint lands.@@ -352,9 +353,36 @@ def resolve_selected_cuda_ordinal(still run -- they read the mask and nvidia-smi -- so the single card the UI sends resolves andonly a multi-card pick comes back None."""+ ordinals = resolve_selected_cuda_ordinals(gpu_ids)+ if not ordinals:+ return None+ if len(ordinals) == 1:+ return ordinals[0]+ if not allow_ranking:+ return None+ return rank_cuda_ordinals(ordinals)[0]+++def rank_cuda_ordinals(ordinals: list[int]) -> list[int]:+ """``ordinals`` by free VRAM, most first, ties to the lowest ordinal (an unreadable card sorts last)."""++ def _free_vram(ordinal: int) -> int:+ try:+ import torch+ return int(torch.cuda.mem_get_info(ordinal)[0])+ except Exception: # noqa: BLE001 -- an unreadable card sorts last rather than failing the load+ return -1++ return sorted(ordinals, key = lambda ordinal: (-_free_vram(ordinal), ordinal))+++def resolve_selected_cuda_ordinals(gpu_ids: Optional[list[int]]) -> list[int]:+ """Every torch ordinal of the selection, in ascending order (validated and translated through the mask exactly as+ :func:`resolve_selected_cuda_ordinal` does). Empty for no selection; raises ValueError for one this host cannot+ honour. The multi-GPU layer (diffusion_multigpu) uses the cards beyond the primary."""wanted = sorted({int(gpu_id) for gpu_id in gpu_ids or ()})if not wanted:- return None+ return []try:from utils.hardware.hardware import (get_parent_visible_gpu_ids,@@ -372,19 +400,7 @@ def resolve_selected_cuda_ordinal(f"Requested GPU {wanted} but none of them are visible to this process "f"(visible: {visible}). Clear the GPU selection to use the default device.")- if len(ordinals) == 1:- return ordinals[0]- if not allow_ranking:- return None-- def _free_vram(ordinal: int) -> int:- try:- import torch- return int(torch.cuda.mem_get_info(ordinal)[0])- except Exception: # noqa: BLE001 -- an unreadable card sorts last rather than failing the load- return -1-- return max(ordinals, key = lambda ordinal: (_free_vram(ordinal), -ordinal))+ return sorted(dict.fromkeys(ordinals))_UNPLANNED = object()@@ -486,6 +502,37 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:return 0+def smi_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:+ """(free, total) MiB of torch ordinal ``ordinal`` from nvidia-smi, so a card can be sized without a CUDA context+ on it. (None, None) wherever ``resolve_auto_cuda_ordinal`` would not trust the rows (ROCm, a CUDA order other+ than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). Never raises."""+ try:+ from utils.hardware import hardware as hw++ if hw.IS_ROCM or not hw._cuda_order_matches_smi():+ return None, None+ visible = hw.get_parent_visible_gpu_ids()+ if not 0 <= int(ordinal) < len(visible):+ return None, None+ index = visible[int(ordinal)]+ if hw._integrated_cuda_rows(visible)[0]:+ return None, None+ mig = hw._nvidia_mig_gpu_indices_or_none()+ if mig is None or index in mig:+ return None, None+ result = hw._smi_visible_utilization(hw._get_parent_visible_gpu_spec())+ for device in (result or {}).get("devices", []):+ if device.get("index") != index:+ continue+ total, used = device.get("vram_total_gb"), device.get("vram_used_gb")+ if total is None or used is None:+ return None, None+ return int(max(float(total) - float(used), 0.0) * 1024), int(float(total) * 1024)+ except Exception: # noqa: BLE001 -- an unreadable card is simply not used+ pass+ return None, None++def auto_cuda_ranking_possible_cached() -> Optional[bool]:"""``auto_cuda_ranking_possible`` when it can be answered from state already in memory, else None.diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex ece7a4317e8..3a294c9790c 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -19,7 +19,7 @@ from __future__ import annotationsimport functoolsimport osimport sys-from dataclasses import dataclass, replace+from dataclasses import dataclass, field, replacefrom typing import Any, Callable, Optional@@ -614,6 +614,11 @@ class MemoryPlan:# MiB of the streamed denoiser / encoders kept resident (whole groups); None streams every group.resident_transformer_mib: Optional[int] = Noneresident_text_encoder_mib: Optional[int] = None+ secondary_ordinal: Optional[int] = None+ secondary_components: Optional[tuple[str, ...]] = None+ # The single-card plan a split replaced, applied on the primary if the split placement fails (a torchao denoiser+ # cannot take the whole-module offload a blind fallback would choose).+ fallback_plan: Optional["MemoryPlan"] = field(default = None, compare = False, repr = False)@propertydef engages_offload(self) -> bool:@@ -632,6 +637,10 @@ class MemoryPlan:"stream_transformer": self.stream_transformer,"resident_transformer_mib": self.resident_transformer_mib,"resident_text_encoder_mib": self.resident_text_encoder_mib,+ "secondary_ordinal": self.secondary_ordinal,+ "secondary_components": (+ list(self.secondary_components) if self.secondary_components is not None else None+ ),}@@ -1965,6 +1974,8 @@ def plan_diffusion_memory(reasons.append("companions exceed budget; whole-module offload of every component")# The legacy cpu_offload flag applies only when no memory_mode was supplied, so an explicit `fast` stays resident.+ if explicit_offload and normalize_memory_mode(requested_mode) is None:+ estimates["explicit_cpu_offload"] = 1if (explicit_offloadand normalize_memory_mode(requested_mode) is None@@ -2003,18 +2014,117 @@ def plan_diffusion_memory(any_offload = policy != OFFLOAD_NONEreasons.append(reason)estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(tile)- return MemoryPlan(+ split = _multigpu_companion_split(+ target = target,+ device_memory = device_memory,+ mode = mode,+ requested_mode = requested_mode,+ explicit_offload = explicit_offload,+ can_offload = can_offload,+ policy = policy,+ budget = budget,+ model_dense_mib = model_dense_mib,+ companion_dense_mib = companion_dense_mib,+ runtime_headroom_mib = runtime_headroom_mib,+ base_overhead_mib = base_overhead_mib,+ text_encoder_dense_mib = text_encoder_dense_mib,+ calibrated_activation = calibrated_activation,+ )+ single_card = MemoryPlan(requested_mode = mode,offload_policy = policy,vae_tiling = tile,vae_slicing = any_offload,device_memory = device_memory,- estimates = estimates,+ estimates = dict(estimates),reasons = tuple(reasons),# only ever meaningful under group offload; every other tier already places the encodersstream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,stream_transformer = stream_transformer or policy != OFFLOAD_GROUP,)+ if split is not None:+ estimates["multigpu_primary_required_mib"] = split.primary_required_mib+ estimates["multigpu_secondary_required_mib"] = split.secondary_required_mib+ return MemoryPlan(+ requested_mode = mode,+ offload_policy = OFFLOAD_NONE,+ vae_tiling = False,+ vae_slicing = False,+ device_memory = device_memory,+ estimates = estimates,+ reasons = tuple(reasons) + (split.reason,),+ secondary_ordinal = split.secondary_ordinal,+ secondary_components = split.components,+ fallback_plan = single_card,+ )+ return single_card+++def _multigpu_companion_split(+ *,+ target: Any,+ device_memory: DeviceMemory,+ mode: str,+ requested_mode: Optional[str],+ explicit_offload: bool,+ can_offload: bool,+ policy: str,+ budget: Optional[int],+ model_dense_mib: Optional[int],+ companion_dense_mib: Optional[int],+ runtime_headroom_mib: int,+ base_overhead_mib: int,+ text_encoder_dense_mib: Optional[int] = None,+ calibrated_activation: Optional[CalibratedImageActivation] = None,+) -> Any:+ """The multi-GPU companion split for this plan, or None (one card: every existing decision unchanged). Only for+ ``auto`` / ``fast`` on a discrete CUDA card that is the primary of a multi-card selection; explicit offload modes+ and the legacy ``cpu_offload`` flag keep their meaning."""+ from .diffusion_multigpu import (+ current_selection,+ plan_companion_split,+ secondary_resident_limit_mib,+ )++ selection = current_selection()+ if selection is None or not can_offload or device_memory.is_unified:+ return None+ if getattr(device_memory, "device", None) != "cuda" or budget is None:+ return None+ ordinal = getattr(target, "ordinal", None)+ if ordinal is None and selection.automatic:+ from .diffusion_device import placed_cuda_ordinal+ ordinal = placed_cuda_ordinal(target)+ if ordinal != selection.primary:+ return None+ if mode not in (MEMORY_MODE_AUTO, MEMORY_MODE_FAST):+ return None+ if explicit_offload and normalize_memory_mode(requested_mode) is None:+ return None+ if mode == MEMORY_MODE_FAST:+ limit = _fast_device_budget_mib(device_memory)+ else:+ limit = int(budget * 0.85)+ return plan_companion_split(+ single_card_policy = policy,+ model_dense_mib = model_dense_mib,+ companion_dense_mib = companion_dense_mib,+ runtime_headroom_mib = runtime_headroom_mib,+ base_overhead_mib = base_overhead_mib,+ primary_limit_mib = limit,+ secondary_limit_mib = lambda ordinal: secondary_resident_limit_mib(ordinal, mode),+ selection = selection,+ text_encoder_dense_mib = text_encoder_dense_mib,+ phase_headroom_mib = (+ (+ calibrated_activation.text_encoder_mib,+ calibrated_activation.denoise_mib,+ calibrated_activation.decode_mib,+ )+ if calibrated_activation is not None+ else None+ ),+ )def _calibrated_faster_tier(@@ -3680,6 +3790,34 @@ def apply_memory_plan()_fallback_to_model_offload()policy = OFFLOAD_MODEL+ elif getattr(plan, "secondary_ordinal", None) is not None:+ from .diffusion_multigpu import apply_split_plan+ if not apply_split_plan(pipe, plan, placement = placement, logger = logger):+ fallback = getattr(plan, "fallback_plan", None)+ if fallback is not None and getattr(fallback, "secondary_ordinal", None) is None:+ # the policy one card would have used (group offload keeps a torchao denoiser loadable)+ result = apply_memory_plan(+ pipe, fallback, device = device, placement_device = placement_device, logger = logger+ )+ # applied_plan() must report the fallback (or the lower re-plan it recorded), not the split+ if getattr(pipe, "_unsloth_applied_plan", None) is None:+ try:+ pipe._unsloth_applied_plan = fallback+ except Exception: # noqa: BLE001+ pass+ return result+ _fallback_to_model_offload()+ policy = OFFLOAD_MODEL+ try:+ pipe._unsloth_applied_plan = replace(+ plan,+ offload_policy = OFFLOAD_MODEL,+ secondary_ordinal = None,+ secondary_components = None,+ fallback_plan = None,+ )+ except Exception: # noqa: BLE001+ passelse:from .diffusion_fast_load import fast_uploaddiff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pynew file mode 100644index 00000000000..81af0ca2b12--- /dev/null+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -0,0 +1,2087 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Several selected CUDA / ROCm cards for one image or video pipeline, without NCCL.++Three levers, all at a component or denoiser-call boundary, never inside a model forward:++* Component split (planner): the DiT stays whole and resident on the primary card while the text encoders, VAE and+ other companions sit on a secondary card, when that removes offload on the primary. Inputs move to a module's own+ card at its entry points and outputs come back to the primary, so the pipeline still sees one device. Copies are+ exact, so the output is bit-identical to a single-card resident run.+* Second denoiser (after load): a two-DiT pipeline (an MoE ``transformer_2``) keeps one expert per card when both+ experts do not fit one card.+* CFG-parallel (generate): when guidance runs separate cond / uncond denoiser calls on the same latents, every odd+ branch of a step runs on a replica of the denoiser on the secondary card, launched from a worker thread bound to+ that card, while the primary computes the even branch. One output is copied per step; guidance and the scheduler+ step stay on the primary. Branches are recognised from the call boundary alone (same latent tensor object in+ consecutive calls), so true CFG, rescale and multi-branch guidance work unchanged and a guidance-distilled model+ (one call per step) never builds a replica.++Kill switches: ``UNSLOTH_DIFFUSION_MULTIGPU=0`` (everything here; one card as before), ``UNSLOTH_DIFFUSION_CFG_PARALLEL=0``+(only the replica). ``UNSLOTH_DIFFUSION_MULTIGPU=force`` splits the companions even when one card would hold+everything (for proving bit-identity). With one selected card nothing here engages.+"""++from __future__ import annotations++import contextlib+import contextvars+import copy+import dataclasses+import os+import queue+import threading+import time+import types+import weakref+from collections import OrderedDict+from typing import Any, Callable, Iterator, Optional, Sequence++MULTIGPU_ENV = "UNSLOTH_DIFFUSION_MULTIGPU"+CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL"++DENOISER_ATTRS: tuple[str, ...] = (+ "transformer",+ "transformer_2",+ "unconditional_transformer",+ "unet",+)+_KEEP_WITH_DENOISER_PREFIXES: tuple[str, ...] = ("controlnet",)+_ENTRY_METHODS: tuple[str, ...] = (+ "forward",+ "encode",+ "decode",+ "get_text_features",+ "get_image_features",+)+# Sentinel for ``MemoryPlan.secondary_components``: only the text encoders move (the VAE decodes beside the denoiser).+TEXT_ENCODERS_ONLY: tuple[str, ...] = ("<text_encoders>",)+TEXT_ENCODER_HEADROOM_MIB = 1024+_RESIDENT_FRACTION = 0.85+++def multigpu_mode() -> str:+ """``off`` / ``auto`` (default) / ``force``."""+ raw = (os.environ.get(MULTIGPU_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no", "disable", "disabled"):+ return "off"+ if raw == "force":+ return "force"+ return "auto"+++def cfg_parallel_mode() -> str:+ """``off`` / ``auto`` (default) / ``force`` (also across different card models)."""+ if multigpu_mode() == "off":+ return "off"+ raw = (os.environ.get(CFG_PARALLEL_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no", "disable", "disabled"):+ return "off"+ if raw == "force":+ return "force"+ return "auto"++[email protected](frozen = True)+class MultiGpuSelection:+ primary: int+ secondaries: tuple[int, ...]+ # No gpu_ids (the UI's Automatic): only the companion split runs, and only when one card would offload.+ automatic: bool = False+++_SELECTION: contextvars.ContextVar[Optional[MultiGpuSelection]] = contextvars.ContextVar(+ "unsloth_diffusion_multigpu_selection", default = None+)+++AUTOMATIC_ORDINALS = "automatic"+++def make_selection(+ primary: Optional[int],+ ordinals: Optional[Sequence[int]],+ *,+ automatic: bool = False,+) -> Optional[MultiGpuSelection]:+ """A selection when at least two distinct cards were picked and the lever is on, else None (one card)."""+ if primary is None or multigpu_mode() == "off":+ return None+ others = tuple(dict.fromkeys(int(o) for o in (ordinals or ()) if int(o) != int(primary)))+ try:+ import torch++ # nvidia-smi can list a card this process's torch cannot address (a mask applied after the listing).+ if torch.cuda.is_available():+ visible = int(torch.cuda.device_count())+ others = tuple(o for o in others if 0 <= o < visible)+ except Exception: # noqa: BLE001 - unknown count: the capability filter below still applies+ pass+ if not others:+ return None+ # The load dtype and the device-specific patches are resolved for the primary (bf16 only from Ampere, FP8 kernels+ # from Ada): a card of a lower compute capability may not run them, so it takes no component.+ need = _capability(int(primary))+ if need is not None:+ others = tuple(o for o in others if (_capability(o) or need) >= need)+ if not others:+ return None+ return MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)+++def models_kept_loaded() -> bool:+ """Keep multiple models loaded is on (``fit`` / ``always``). Unreadable reads as off, like the arbiter."""+ try:+ from .gpu_residency import current_policy+ return current_policy() != "off"+ except Exception: # noqa: BLE001+ return False+++def automatic_selection(primary: Optional[int]) -> Optional[MultiGpuSelection]:+ """The selection of an automatic load: the ranked card (or the current one) plus every other parent-visible CUDA+ card. None on one GPU, ROCm, XPU, MPS and CPU (the same cheap gate automatic ranking uses)."""+ if multigpu_mode() == "off":+ return None+ try:+ from .diffusion_device import auto_cuda_ranking_possible, resolve_diffusion_device_target++ if not auto_cuda_ranking_possible() or resolve_diffusion_device_target().device != "cuda":+ return None+ import torch++ if primary is None:+ primary = int(torch.cuda.current_device())+ count = int(torch.cuda.device_count())+ except Exception: # noqa: BLE001 -- an unreadable host keeps one card+ return None+ return make_selection(primary, range(count), automatic = True)+++def _capability(ordinal: int) -> Optional[tuple[int, int]]:+ try:+ import torch+ major, minor = torch.cuda.get_device_capability(int(ordinal))+ return int(major), int(minor)+ except Exception: # noqa: BLE001+ return None+++def selection_scope(+ selection: Optional[MultiGpuSelection],+) -> Iterator[Optional[MultiGpuSelection]]:+ """Make ``selection`` visible to the planner and placement for the duration of one load (thread / task local)."""+ if selection is None:+ yield None+ return+ # Only a CFG replica reads the recorded compile arguments; automatic placement never builds one.+ if not selection.automatic or cfg_parallel_mode() == "force":+ install_compile_recorder()+ token = _SELECTION.set(selection)+ try:+ yield selection+ finally:+ _SELECTION.reset(token)+++def with_load_selection(run_load: Callable[..., Any]) -> Callable[..., Any]:+ """Decorate a backend's ``_run_load(self, **kwargs)``: pop ``_gpu_ordinals`` (every selected card, from+ begin_load, or :data:`AUTOMATIC_ORDINALS`) and run the load inside :func:`selection_scope` with ``gpu_ordinal``+ as the primary. Without a multi-card selection the load runs exactly as before."""+ import functools++ @functools.wraps(run_load)+ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:+ ordinals = kwargs.pop("_gpu_ordinals", None)+ if ordinals and models_kept_loaded():+ selection = None+ elif ordinals == AUTOMATIC_ORDINALS:+ selection = automatic_selection(kwargs.get("gpu_ordinal"))+ else:+ selection = make_selection(kwargs.get("gpu_ordinal"), ordinals)+ with selection_scope(selection):+ return run_load(self, *args, **kwargs)++ return wrapper+++def current_selection() -> Optional[MultiGpuSelection]:+ if multigpu_mode() == "off":+ return None+ return _SELECTION.get()++[email protected](frozen = True)+class DeviceProbe:+ ordinal: int+ name: str+ total_mib: Optional[int]+ free_mib: Optional[int]+++_PROBE_LOCK = threading.Lock()+_NAMES: dict[int, str] = {}+++def _free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:+ selection = _SELECTION.get()+ if selection is not None and selection.automatic and int(ordinal) != selection.primary:+ # Sized from nvidia-smi: an automatic load must not open a CUDA context on a card it may never use.+ from .diffusion_device import smi_free_total_mib+ return smi_free_total_mib(int(ordinal))+ try:+ import torch++ from utils.hardware import trusted_mem_get_info++ free, total = trusted_mem_get_info(int(ordinal), module = torch.cuda)+ return int(free) // (1 << 20), int(total) // (1 << 20)+ except Exception: # noqa: BLE001+ return None, None+++def probe_devices(ordinals: Sequence[int]) -> dict[int, DeviceProbe]:+ """Name (cached per process) and free / total memory of each card."""+ import torch++ ords = [int(o) for o in ordinals]+ with _PROBE_LOCK:+ for o in ords:+ if o not in _NAMES:+ try:+ _NAMES[o] = str(torch.cuda.get_device_name(o))+ except Exception: # noqa: BLE001+ _NAMES[o] = ""+ out: dict[int, DeviceProbe] = {}+ for o in ords:+ free, total = _free_total_mib(o)+ out[o] = DeviceProbe(ordinal = o, name = _NAMES.get(o, ""), total_mib = total, free_mib = free)+ return out+++def best_secondary(+ selection: MultiGpuSelection, probes: Optional[dict[int, DeviceProbe]] = None+) -> Optional[int]:+ if not selection.secondaries:+ return None+ if probes is None and selection.automatic:+ free = {o: _free_total_mib(o)[0] for o in selection.secondaries}+ return max(+ selection.secondaries, key = lambda o: (free[o] if free[o] is not None else -1, -o)+ )+ if probes is None:+ probes = probe_devices((selection.primary, *selection.secondaries))++ def key(o: int) -> tuple[int, int]:+ p = probes.get(o)+ return (p.free_mib if p is not None and p.free_mib is not None else -1, -o)++ return max(selection.secondaries, key = key)++[email protected](frozen = True)+class CompanionSplit:+ secondary_ordinal: int+ primary_required_mib: int+ primary_limit_mib: int+ secondary_required_mib: int+ secondary_limit_mib: int+ reason: str+ # None = every companion moves; TEXT_ENCODERS_ONLY = only the encoders (the VAE stays with the denoiser)+ components: Optional[tuple[str, ...]] = None+++def plan_companion_split(+ *,+ single_card_policy: str,+ model_dense_mib: Optional[int],+ companion_dense_mib: Optional[int],+ runtime_headroom_mib: int,+ base_overhead_mib: int,+ primary_limit_mib: Optional[int],+ secondary_limit_mib: Callable[[int], Optional[int]],+ selection: Optional[MultiGpuSelection],+ text_encoder_dense_mib: Optional[int] = None,+ phase_headroom_mib: Optional[tuple[int, int, int]] = None,+) -> Optional[CompanionSplit]:+ """Pure decision: return a split when the single-card plan offloads (or ``force``), the denoiser fits the primary+ under the resident rule, and the moved companions fit a secondary. Tried in order: every companion (text encoders+ + VAE + the rest, the secondary priced with the full runtime headroom since it runs the decode), then the text+ encoders alone (the VAE decodes on the primary; the secondary needs only an encoder forward's room).+ ``primary_limit_mib`` / ``secondary_limit_mib(ordinal)`` are each card's resident limit under the caller's memory+ mode rule (auto: 85% of the safe budget; fast: the fast budget). ``phase_headroom_mib`` = measured (encode,+ denoise, decode) MiB above the weights for families Studio has calibrated: each card is then priced only for the+ phases it runs instead of the flat runtime headroom."""+ mode = multigpu_mode()+ if selection is None or mode == "off":+ return None+ if single_card_policy == "none" and mode != "force":+ return None+ if model_dense_mib is None or not companion_dense_mib or primary_limit_mib is None:+ return None+ limits: dict[int, int] = {}+ for ordinal in selection.secondaries:+ limit = secondary_limit_mib(ordinal)+ if limit is not None:+ limits[ordinal] = int(limit)+ if not limits:+ return None+ runtime = int(runtime_headroom_mib)+ if phase_headroom_mib is not None:+ encode, denoise, decode = (int(v) for v in phase_headroom_mib)+ # (moved MiB, primary headroom, secondary headroom, components, label)+ companions_split = (denoise, max(encode, decode))+ encoders_split = (max(denoise, decode), encode)+ else:+ companions_split = (runtime, runtime)+ encoders_split = (runtime, TEXT_ENCODER_HEADROOM_MIB)+ candidates: list[tuple[int, int, int, Optional[tuple[str, ...]], str]] = [+ (int(companion_dense_mib), *companions_split, None, "text encoders + VAE"),+ ]+ if text_encoder_dense_mib and 0 < int(text_encoder_dense_mib) <= int(companion_dense_mib):+ candidates.append(+ (int(text_encoder_dense_mib), *encoders_split, TEXT_ENCODERS_ONLY, "text encoders")+ )+ for moved_mib, pri_headroom, sec_headroom, components, label in candidates:+ primary_required = (+ max(0, int(model_dense_mib) - moved_mib) + int(pri_headroom) + int(base_overhead_mib)+ )+ if primary_required > int(primary_limit_mib):+ continue+ secondary_required = moved_mib + sec_headroom + int(base_overhead_mib)+ fits = [o for o, limit in limits.items() if secondary_required <= limit]+ if not fits:+ continue+ best = max(fits, key = lambda o: (limits[o], -o))+ return CompanionSplit(+ secondary_ordinal = best,+ primary_required_mib = primary_required,+ primary_limit_mib = int(primary_limit_mib),+ secondary_required_mib = secondary_required,+ secondary_limit_mib = limits[best],+ components = components,+ reason = (+ f"multi-GPU: denoiser resident on cuda:{selection.primary} ({primary_required} of "+ f"{int(primary_limit_mib)} MiB), {label} on cuda:{best} ({secondary_required} of {limits[best]} MiB); "+ f"single card would use '{single_card_policy}'"+ ),+ )+ return None+++def choose_component_assignment(+ sizes: dict[str, tuple[int, str]],+ *,+ headroom_mib: int,+ overhead_mib: int,+ primary_limit_mib: int,+ secondary_limit_mib: int,+) -> Optional[tuple[str, ...]]:+ """Names to place on the secondary so each card holds its share resident, or None. ``sizes`` maps component name+ -> (MiB, role) with role 'dit' / 'text_encoder' / 'other'. A card running a denoiser or the VAE (any 'other'+ component) is priced with the runtime headroom; a card holding only text encoders with an encoder forward's room.+ Tried in order: every companion; the text encoders alone; the second denoiser; the second denoiser plus the+ companions. The first denoiser always stays on the primary."""+ dits = [n for n, (_m, r) in sizes.items() if r == "dit"]+ if not dits:+ return None+ first = "transformer" if "transformer" in dits else dits[0]+ seconds = [n for n in dits if n != first]+ companions = [+ n+ for n, (_m, r) in sizes.items()+ if r != "dit" and not n.startswith(_KEEP_WITH_DENOISER_PREFIXES)+ ]+ encoders = [n for n in companions if sizes[n][1] == "text_encoder"]+ candidates: list[tuple[str, ...]] = []+ if companions:+ candidates.append(tuple(companions))+ if encoders and len(encoders) < len(companions):+ candidates.append(tuple(encoders))+ if seconds:+ candidates.append(tuple(seconds))+ if companions:+ candidates.append(tuple(seconds + companions))++ def need(names: list) -> int:+ weights = sum(sizes[n][0] for n in names)+ busy = any(sizes[n][1] != "text_encoder" for n in names)+ return weights + (headroom_mib if busy else TEXT_ENCODER_HEADROOM_MIB) + overhead_mib++ for moved in candidates:+ on_primary = [n for n in sizes if n not in moved]+ if need(on_primary) <= primary_limit_mib and need(list(moved)) <= secondary_limit_mib:+ return moved+ return None+++def refine_plan_for_multigpu(+ pipe: Any,+ plan: Any,+ logger: Any = None,+) -> Any:+ """Post-load: an offloading plan becomes a resident split when the LOADED components can be shared by the cards.++ No-op without a selection, for explicit offload modes, a plan already resident (unless forced) or split, unified memory+ or an unknown budget. Returns the plan (a ``dataclasses.replace`` copy when it changed)."""+ selection = current_selection()+ if selection is None or plan is None:+ return plan+ if getattr(plan, "secondary_ordinal", None) is not None:+ return plan+ if plan.offload_policy == "none" and multigpu_mode() != "force":+ return plan+ if getattr(plan, "requested_mode", None) not in ("auto", "fast"):+ return plan+ if plan.estimates.get("explicit_cpu_offload"):+ return plan+ memory = plan.device_memory+ if memory.is_unified or getattr(memory, "device", None) != "cuda":+ return plan+ budget = plan.estimates.get("safe_device_budget_mib")+ if budget is None:+ return plan+ from .diffusion_memory import _loaded_component_mib++ sizes = _loaded_component_mib(pipe)+ if not sizes:+ return plan+ secondary = best_secondary(selection)+ if secondary is None:+ return plan+ mode = getattr(plan, "requested_mode", None)+ sec_limit = secondary_resident_limit_mib(secondary, mode)+ if sec_limit is None:+ return plan+ from .diffusion_memory import _fast_device_budget_mib++ primary_limit = (+ _fast_device_budget_mib(memory) if mode == "fast" else int(int(budget) * _RESIDENT_FRACTION)+ )+ if primary_limit is None:+ return plan+ headroom = int(+ plan.estimates.get("measured_runtime_headroom_mib")+ or plan.estimates.get("calibrated_headroom_mib")+ or plan.estimates.get("runtime_headroom_mib")+ or 0+ )+ overhead = int(plan.estimates.get("base_overhead_mib") or 0)+ moved = choose_component_assignment(+ sizes,+ headroom_mib = headroom,+ overhead_mib = overhead,+ primary_limit_mib = int(primary_limit),+ secondary_limit_mib = int(sec_limit),+ )+ if moved is None:+ return plan+ reason = (+ f"multi-GPU (loaded sizes): {', '.join(moved)} on cuda:{secondary}, the rest resident on "+ f"cuda:{selection.primary}; single card would use '{plan.offload_policy}'"+ )+ if logger is not None:+ logger.info("diffusion.multigpu: %s", reason)+ return dataclasses.replace(+ plan,+ offload_policy = "none",+ # Kept: the fit above may be priced with the tiled decode peak (calibrated_headroom_mib), so the savers the+ # single-card plan chose must stay on.+ vae_tiling = plan.vae_tiling,+ vae_slicing = plan.vae_slicing,+ stream_text_encoders = False,+ stream_transformer = True,+ resident_transformer_mib = None,+ resident_text_encoder_mib = None,+ reasons = tuple(plan.reasons) + (reason,),+ secondary_ordinal = secondary,+ secondary_components = tuple(moved),+ fallback_plan = plan,+ )+++def secondary_device_memory(ordinal: int) -> Any:+ from .diffusion_memory import DeviceMemory++ free, total = _free_total_mib(int(ordinal))+ if free is None:+ return None+ return DeviceMemory(+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total+ )+++def secondary_safe_budget_mib(ordinal: int) -> Optional[int]:+ from .diffusion_memory import _safe_device_budget_mib+ memory = secondary_device_memory(ordinal)+ return None if memory is None else _safe_device_budget_mib(memory)+++def secondary_resident_limit_mib(ordinal: int, mode: str) -> Optional[int]:+ from .diffusion_memory import _fast_device_budget_mib++ if mode == "fast":+ memory = secondary_device_memory(ordinal)+ return None if memory is None else _fast_device_budget_mib(memory)+ budget = secondary_safe_budget_mib(ordinal)+ return None if budget is None else int(int(budget) * _RESIDENT_FRACTION)+++def _torch():+ import torch+ return torch+++def move_to_device(+ obj: Any,+ device: Any,+ _depth: int = 0,+) -> Any:+ torch = _torch()+ if isinstance(obj, torch.Tensor):+ return obj if obj.device == device else obj.to(device)+ if (+ _depth > 6+ or obj is None+ or isinstance(obj, (str, bytes, int, float, bool, torch.device, torch.dtype))+ ):+ return obj+ if isinstance(obj, dict):+ # ModelOutput is an OrderedDict that also mirrors its keys as attributes; __setitem__ keeps both in sync.+ try:+ out = copy.copy(obj)+ for key, value in obj.items():+ moved = move_to_device(value, device, _depth + 1)+ if moved is not value:+ out[key] = moved+ return out+ except Exception: # noqa: BLE001+ return type(obj)((k, move_to_device(v, device, _depth + 1)) for k, v in obj.items())+ if isinstance(obj, tuple):+ moved = [move_to_device(v, device, _depth + 1) for v in obj]+ if all(a is b for a, b in zip(moved, obj)):+ return obj+ if hasattr(obj, "_fields"):+ return type(obj)(*moved)+ return type(obj)(moved) if type(obj) is not tuple else tuple(moved)+ if isinstance(obj, list):+ return [move_to_device(v, device, _depth + 1) for v in obj]+ if isinstance(obj, torch.nn.Module):+ return obj+ if dataclasses.is_dataclass(obj) and not isinstance(obj, type):+ changes = {}+ for f in dataclasses.fields(obj):+ value = getattr(obj, f.name, None)+ moved = move_to_device(value, device, _depth + 1)+ if moved is not value:+ changes[f.name] = moved+ if not changes:+ return obj+ try:+ return dataclasses.replace(obj, **changes)+ except Exception: # noqa: BLE001 - init=False fields+ out = copy.copy(obj)+ for k, v in changes.items():+ object.__setattr__(out, k, v)+ return out+ state = getattr(obj, "__dict__", None)+ if (+ isinstance(state, dict)+ and state+ and any(isinstance(v, torch.Tensor) for v in state.values())+ ):+ if all(+ isinstance(+ v, (torch.Tensor, bool, int, float, str, type(None), torch.dtype, torch.device)+ )+ for v in state.values()+ ):+ out = copy.copy(obj)+ for k, v in state.items():+ if isinstance(v, torch.Tensor) and v.device != device:+ setattr(out, k, v.to(device))+ return out+ return obj+++def _module_device(module: Any) -> Any:+ torch = _torch()+ for tensor in module.parameters():+ return tensor.device+ for tensor in module.buffers():+ return tensor.device+ return None+++def _device_context(device: Any):+ torch = _torch()+ if getattr(device, "type", None) == "cuda" and device.index is not None:+ return torch.cuda.device(device)+ return contextlib.nullcontext()+++def _crosses(device: Any, home: Any) -> bool:+ """A call must move tensors: the module sits on a different accelerator card than the pipeline's home. A module on+ the CPU (offloaded, or between loads) is left to its own hooks."""+ return device is not None and device != home and device.type == "cuda"+++def install_boundary(module: Any, home: Any) -> list[str]:+ """Wrap ``module``'s entry points: arguments go to the module's CURRENT card, results come back to ``home``."""+ torch = _torch()+ home = torch.device(home)+ wrapped_names: list[str] = []+ for name in _ENTRY_METHODS:+ bound = getattr(module, name, None)+ if not callable(bound) or getattr(bound, "_unsloth_boundary", None) is not None:+ continue++ def _make(orig: Callable[..., Any]) -> Callable[..., Any]:+ def boundary(*args: Any, **kwargs: Any) -> Any:+ device = _module_device(module)+ if not _crosses(device, home):+ return orig(*args, **kwargs)+ args, kwargs = move_to_device((args, kwargs), device)+ with _device_context(device):+ out = orig(*args, **kwargs)+ return move_to_device(out, home)++ boundary._unsloth_boundary = orig # type: ignore[attr-defined]+ boundary.__wrapped__ = orig # type: ignore[attr-defined]+ boundary._unsloth_had_override = name in module.__dict__ # type: ignore[attr-defined]+ return boundary++ setattr(module, name, _make(bound))+ wrapped_names.append(name)+ return wrapped_names+++def remove_boundary(module: Any) -> None:+ for name in _ENTRY_METHODS:+ current = module.__dict__.get(name) if hasattr(module, "__dict__") else None+ if current is not None and getattr(current, "_unsloth_boundary", None) is not None:+ if getattr(current, "_unsloth_had_override", False):+ module.__dict__[name] = current._unsloth_boundary+ continue+ try:+ delattr(module, name)+ except AttributeError:+ pass+++_EXEC_PATCH_LOCK = threading.Lock()+++# Set once this process places a split, so a one-card load never walks its components looking for one.+_SPLIT_EVER_PLACED = False+++def _split_home(pipe: Any) -> Any:+ if not _SPLIT_EVER_PLACED:+ return None+ try:+ components = pipe.components+ except Exception: # noqa: BLE001+ return None+ if not isinstance(components, dict):+ return None+ for module in components.values():+ home = getattr(module, "_unsloth_split_home", None)+ if home is not None:+ return home+ return None+++def install_execution_device_patch() -> bool:+ """``DiffusionPipeline._execution_device`` / ``.device`` report the primary card of a split pipeline. diffusers+ returns the first non-CPU component in signature order, which is the VAE (on the secondary) for most families.+ Inert for any pipeline without a split denoiser; also covers ``from_pipe`` siblings, which share the modules."""+ try:+ from diffusers import DiffusionPipeline+ except Exception: # noqa: BLE001+ return False+ with _EXEC_PATCH_LOCK:+ if getattr(DiffusionPipeline, "_unsloth_split_exec_patch", False):+ return True+ orig_exec = DiffusionPipeline.__dict__.get("_execution_device")+ orig_dev = DiffusionPipeline.__dict__.get("device")+ if not isinstance(orig_exec, property) or not isinstance(orig_dev, property):+ return False++ def _exec(self: Any) -> Any:+ home = _split_home(self)+ return home if home is not None else orig_exec.fget(self)++ def _dev(self: Any) -> Any:+ home = _split_home(self)+ return home if home is not None else orig_dev.fget(self)++ DiffusionPipeline._execution_device = property(_exec, doc = orig_exec.__doc__)+ DiffusionPipeline.device = property(_dev, doc = orig_dev.__doc__)+ DiffusionPipeline._unsloth_split_exec_patch = True+ return True+++def companion_names(pipe: Any) -> tuple[str, ...]:+ torch = _torch()+ names = []+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if not isinstance(module, torch.nn.Module):+ continue+ if name in DENOISER_ATTRS or str(name).startswith(_KEEP_WITH_DENOISER_PREFIXES):+ continue+ names.append(str(name))+ return tuple(names)+++_PLACEMENTS: "weakref.WeakKeyDictionary[Any, dict]" = weakref.WeakKeyDictionary()+++def place_split(+ pipe: Any,+ *,+ primary: Any,+ secondary_ordinal: int,+ names: Optional[Sequence[str]] = None,+ logger: Any = None,+) -> dict[str, str]:+ """Place ``names`` (default: the companions) on ``cuda:<secondary_ordinal>`` and everything else on ``primary``;+ returns component -> device. Raises on failure (the caller falls back to single-card offload)."""+ torch = _torch()+ from .diffusion_fast_load import fast_upload++ primary_dev = torch.device(primary)+ if primary_dev.type == "cuda" and primary_dev.index is None:+ primary_dev = torch.device("cuda", torch.cuda.current_device())+ secondary_dev = torch.device("cuda", int(secondary_ordinal))+ if names and tuple(names) == TEXT_ENCODERS_ONLY:+ moved = tuple(n for n in companion_names(pipe) if n.startswith("text_encoder"))+ else:+ moved = tuple(names) if names else companion_names(pipe)+ components = {+ str(n): m+ for n, m in (getattr(pipe, "components", {}) or {}).items()+ if isinstance(m, torch.nn.Module)+ }+ placement: dict[str, str] = {}+ order = sorted(components, key = lambda n: 0 if n in moved else 1)+ for name in order:+ module = components[name]+ device = secondary_dev if name in moved else primary_dev+ with fast_upload([module], device, logger = logger):+ module.to(device)+ placement[name] = str(device)+ for name in moved:+ if name in components:+ install_boundary(components[name], primary_dev)+ global _SPLIT_EVER_PLACED+ _SPLIT_EVER_PLACED = True+ for name, module in components.items():+ if name not in moved:+ module._unsloth_split_home = primary_dev+ install_execution_device_patch()+ info = {+ "placement": placement,+ "primary": str(primary_dev),+ "secondary": str(secondary_dev),+ "moved": list(moved),+ }+ try:+ _PLACEMENTS[pipe] = info+ except TypeError:+ pass+ if logger is not None:+ logger.info(+ "diffusion.multigpu: placed %s on %s, the rest on %s",+ ", ".join(moved) or "nothing",+ secondary_dev,+ primary_dev,+ )+ return placement+++def undo_split(pipe: Any) -> None:+ torch = _torch()+ for module in (getattr(pipe, "components", {}) or {}).values():+ if isinstance(module, torch.nn.Module):+ remove_boundary(module)+ if "_unsloth_split_home" in module.__dict__:+ del module.__dict__["_unsloth_split_home"]+ try:+ _PLACEMENTS.pop(pipe, None)+ except TypeError:+ pass+++def apply_split_plan(+ pipe: Any,+ plan: Any,+ *,+ placement: str,+ logger: Any = None,+) -> bool:+ """Place a split plan."""+ secondary = getattr(plan, "secondary_ordinal", None)+ if secondary is None:+ return False+ try:+ place_split(+ pipe,+ primary = placement,+ secondary_ordinal = int(secondary),+ names = getattr(plan, "secondary_components", None),+ logger = logger,+ )+ return True+ except Exception as exc: # noqa: BLE001 - fall back to one card+ if logger is not None:+ logger.warning(+ "diffusion.multigpu: split placement failed (%s); falling back to one card", exc+ )+ undo_split(pipe)+ try:+ pipe.to("cpu")+ _torch().cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ return False+++def is_split(pipe: Any) -> bool:+ if pipe is None:+ return False+ try:+ return pipe in _PLACEMENTS+ except TypeError:+ return False+++def placement_status(pipe: Any) -> Optional[dict[str, Any]]:+ if pipe is None:+ return None+ try:+ info = _PLACEMENTS.get(pipe)+ cfg = _CFG_ROUTERS.get(pipe)+ declined = _CFG_DECLINES.get(pipe)+ except TypeError:+ return None+ if info is None and cfg is None and declined is None:+ return None+ out: dict[str, Any] = dict(info or {})+ if cfg is not None:+ out["cfg_parallel"] = cfg.describe()+ elif declined is not None:+ out["cfg_parallel"] = {"armed": False, "declined": declined}+ return out+++_COMPILE_RECORDER_LOCK = threading.Lock()+++def install_compile_recorder() -> bool:+ try:+ import torch+ except Exception: # noqa: BLE001+ return False+ with _COMPILE_RECORDER_LOCK:+ current = torch.nn.Module.compile+ if getattr(current, "_unsloth_recorder", False):+ return True++ def compile(self: Any, *args: Any, **kwargs: Any) -> Any: # noqa: A001 - mirrors nn.Module.compile+ self.__dict__["_unsloth_compile_args"] = (args, dict(kwargs))+ return current(self, *args, **kwargs)++ compile._unsloth_recorder = True # type: ignore[attr-defined]+ compile.__wrapped__ = current # type: ignore[attr-defined]+ torch.nn.Module.compile = compile+ return True+++_ROOT_HOOK_DICTS = (+ "_forward_pre_hooks",+ "_forward_pre_hooks_with_kwargs",+ "_forward_hooks",+ "_forward_hooks_with_kwargs",+ "_forward_hooks_always_called",+)+++# Offload machinery moves weights between host and card without changing the math. A replica holds its weights+# resident, so it drops these instead of inheriting them.+_OFFLOAD_REGISTRY_HOOKS = frozenset(+ {"group_offloading", "layer_execution_tracker", "lazy_prefetch_group_offloading"}+)+_OFFLOAD_MODULE_MARKERS = (+ "diffusers.hooks.group_offloading",+ "diffusers.hooks.hooks",+ "accelerate.hooks",+ "diffusion_offload_prefetch",+ "diffusion_small_host",+ "diffusion_pinned_arena",+)+++def _origin_modules(fn: Any) -> list[str]:+ out: list[str] = []+ seen = 0+ while fn is not None and seen < 6:+ seen += 1+ out.append(str(getattr(fn, "__module__", "") or ""))+ out.append(str(getattr(type(fn), "__module__", "") or ""))+ out.append(str(getattr(getattr(fn, "func", None), "__module__", "") or ""))+ fn = (+ getattr(fn, "__wrapped__", None)+ or getattr(fn, "__func__", None)+ or getattr(fn, "func", None)+ )+ return out+++def _is_offload_object(obj: Any) -> bool:+ return any(marker in mod for mod in _origin_modules(obj) for marker in _OFFLOAD_MODULE_MARKERS)+++def _registry_kind(sub: Any) -> str:+ registry = sub.__dict__.get("_diffusers_hook")+ hooks = getattr(registry, "hooks", None) if registry is not None else None+ if not hooks:+ return "none"+ return "offload" if set(hooks) <= _OFFLOAD_REGISTRY_HOOKS else "other"+++def _immutable_capture(value: Any, depth: int) -> bool:+ """A closure cell a replica may share with its source. deepcopy keeps a function (and so its closure) as the same+ object, and the two branches call it concurrently, so only values nobody can mutate qualify: scalars, strings,+ dtypes / devices, tuples / frozensets of those, and plain functions whose own captures qualify. A list, dict, set,+ tensor, bound method or arbitrary object (a counter, a cache) would be shared state."""+ torch = _torch()+ if depth > 3:+ return False+ if isinstance(+ value, (str, bytes, int, float, complex, bool, type(None), torch.dtype, torch.device, type)+ ):+ return True+ if isinstance(value, (tuple, frozenset)):+ return all(_immutable_capture(v, depth + 1) for v in value)+ if isinstance(value, types.FunctionType):+ return _portable(value, depth)+ return False+++def _portable(fn: Any, depth: int = 0) -> bool:+ """True when a hook / forward override can be shared by a copy of its module: nothing it captures (closure cells,+ bound ``self``, partial arguments, the attributes of a captured object) is a module or a CUDA tensor, which would+ keep pointing at the source card."""+ torch = _torch()+ if depth > 3:+ return False+ if isinstance(fn, torch.nn.Module) or (+ isinstance(fn, torch.Tensor) and fn.device.type != "cpu"+ ):+ return False+ if isinstance(fn, (list, tuple, set)):+ return all(_portable(v, depth + 1) for v in fn)+ if isinstance(fn, dict):+ return all(_portable(v, depth + 1) for v in fn.values())+ if isinstance(+ fn,+ (str, bytes, int, float, bool, type(None), torch.Tensor, torch.dtype, torch.device, type),+ ):+ return True+ if isinstance(fn, (types.FunctionType, types.MethodType)):+ owner = getattr(fn, "__self__", None)+ if owner is not None and not _portable(owner, depth + 1):+ return False+ func = getattr(fn, "__func__", fn)+ for cell in getattr(func, "__closure__", None) or ():+ try:+ value = cell.cell_contents+ except ValueError:+ continue+ if not _immutable_capture(value, depth + 1):+ return False+ return True+ partial_func = getattr(fn, "func", None)+ if partial_func is not None and hasattr(fn, "args"):+ return _portable(partial_func, depth + 1) and all(+ _portable(a, depth + 1)+ for a in (*fn.args, *(getattr(fn, "keywords", None) or {}).values())+ )+ state = getattr(fn, "__dict__", None)+ if isinstance(state, dict):+ return all(_portable(v, depth + 1) for v in state.values())+ return True+++# Where a hook shared by a replica may come from. Studio's own hooks keep their per-call state on the module (mirrored+# by _mirror_hook_state); a foreign hook may keep it in a module global that both threads would then race on, and that+# cannot be seen from the function, so it declines the replica.+_TRUSTED_HOOK_MODULES = ("core.inference.", "studio.backend.core.inference.")+++def _hook_origin(fn: Any) -> str:+ target = getattr(fn, "func", None) if hasattr(fn, "args") and hasattr(fn, "func") else fn+ target = getattr(target, "__func__", target)+ if isinstance(target, (types.FunctionType, types.BuiltinFunctionType)):+ return str(getattr(target, "__module__", None) or "")+ return str(getattr(type(target), "__module__", None) or "")+++def _trusted_hook(fn: Any) -> bool:+ return _hook_origin(fn).startswith(_TRUSTED_HOOK_MODULES) and _portable(fn)+++def _hooks_present(module: Any) -> Optional[str]:+ """Name of the first submodule carrying state a replica cannot inherit faithfully, else None: a non-offload hook+ registry (step caches), an accelerate hook, or a forward hook / instance forward override below the root whose+ captured state points at the source (deepcopy treats functions as atoms). Offload machinery (diffusers group+ offloading, Studio's prefetchers) is dropped by :func:`build_replica`, which holds every weight resident. The+ ROOT's torch forward hooks run on the source around the router for every branch and the copy does not carry them,+ but whatever per-call state they write must reach the replica: only Studio's own root hooks are accepted, whose+ state is the ``_unsloth_*`` scalars :func:`_mirror_hook_state` copies."""+ for name, sub in module.named_modules():+ kind = _registry_kind(sub)+ if kind == "other":+ return f"{name or '<root>'}._diffusers_hook ({', '.join(sorted(sub._diffusers_hook.hooks))})"+ if getattr(sub, "_hf_hook", None) is not None:+ return f"{name or '<root>'}._hf_hook"+ for attr in ("_forward_hooks", "_forward_pre_hooks"):+ for hook in (getattr(sub, attr, None) or {}).values():+ if _is_offload_object(hook):+ continue+ if name and not _trusted_hook(hook):+ return f"{name}.{attr}"+ if not name and not _hook_origin(hook).startswith(_TRUSTED_HOOK_MODULES):+ return f"<root>.{attr} (a hook outside Studio: its per-call state cannot be mirrored)"+ override = sub.__dict__.get("forward")+ if override is None or isinstance(override, (types.MethodType, CfgParallelRouter)):+ continue+ if kind == "offload" or _is_offload_object(override):+ continue+ if callable(override) and not _trusted_hook(override):+ return f"{name or '<root>'}.forward (instance override)"+ return None+++def _offload_attr_keys(sub: Any) -> list[str]:+ torch = _torch()+ keys = []+ kind = _registry_kind(sub)+ for key, value in sub.__dict__.items():+ if key in ("_parameters", "_buffers", "_modules") or key.startswith("_forward_"):+ continue+ if key == "_diffusers_hook" and kind == "offload":+ keys.append(key)+ elif key == "forward" and (kind == "offload" or _is_offload_object(value)):+ keys.append(key)+ elif not isinstance(value, (torch.Tensor, torch.nn.Module)) and _is_offload_object(value):+ keys.append(key)+ return keys+++_DROP = object()+++def _rebind_root_forward(original: Any, source: Any, replica: Any) -> None:+ """Give ``replica`` the root ``forward`` the source ran before the router took its slot: offload wrappers (math+ neutral, dropped with the offload state) are peeled off, and a method bound to the source is rebound to the copy.+ Anything else cannot be reproduced and raises (CFG-parallel then declines)."""+ seen = 0+ while original is not None and _is_offload_object(original) and seen < 8:+ seen += 1+ original = getattr(original, "__wrapped__", None)+ if isinstance(original, types.MethodType) and original.__self__ is source:+ if original.__func__ is getattr(type(source), "forward", None):+ return+ replica.__dict__["forward"] = types.MethodType(original.__func__, replica)+ return+ raise RuntimeError(+ f"the denoiser's root forward override ({original!r}) cannot be reproduced on a replica"+ )+++def build_replica(module: Any, device: Any) -> Any:+ """A copy of ``module`` whose parameters and buffers are created directly on ``device`` (never staged on the+ source card), with every recorded ``Module.compile`` re-applied to the copy. Raises when it cannot be built."""+ torch = _torch()+ device = torch.device(device)+ memo: dict[int, Any] = {}+ keep: list[Any] = []+ for _name, param in module.named_parameters(remove_duplicate = True):+ if id(param) in memo:+ continue+ copied = param.detach().to(device)+ new = copied+ if isinstance(param, torch.nn.Parameter):+ try:+ new = torch.nn.Parameter(copied, requires_grad = param.requires_grad)+ except Exception: # noqa: BLE001 - a tensor subclass that refuses the wrapper stays a plain tensor+ new = copied+ memo[id(param)] = new+ keep.append(param)+ for _name, buffer in module.named_buffers(remove_duplicate = True):+ if id(buffer) in memo:+ continue+ memo[id(buffer)] = buffer.detach().to(device)+ keep.append(buffer)+ for sub in module.modules():+ offload_keys = set(_offload_attr_keys(sub))+ for key, value in sub.__dict__.items():+ if isinstance(value, CfgParallelRouter) or key in offload_keys:+ # never traversed: offload state holds pinned host copies a deepcopy would duplicate+ memo[id(value)] = _DROP+ for attr in ("_forward_hooks", "_forward_pre_hooks"):+ for hook in (getattr(sub, attr, None) or {}).values():+ if _is_offload_object(hook):+ memo[id(hook)] = _DROP+ # Per-step precision state (the NVFP4 W4A16 protect schedule) is advanced by the source's scheduler: the copy+ # must read the same controller, or the two CFG branches of one step run different GEMMs.+ for value in sub.__dict__.values():+ if type(value).__name__ in _SHARED_STATE_TYPES:+ memo[id(value)] = value+ guard = module.__dict__.get("_unsloth_compile_guard")+ if guard is not None:+ memo[id(guard)] = _DROP+ memo[id(memo)] = keep+ replica = copy.deepcopy(module, memo)+ router = module.__dict__.get("forward")+ for sub in replica.modules():+ for key in [k for k, v in sub.__dict__.items() if v is _DROP]:+ del sub.__dict__[key]+ for attr in _ROOT_HOOK_DICTS:+ hooks = sub.__dict__.get(attr)+ if hooks:+ for hook_id in [k for k, v in hooks.items() if v is _DROP]:+ del hooks[hook_id]+ for other in _ROOT_HOOK_DICTS:+ sub.__dict__.get(other, {}).pop(hook_id, None)+ if isinstance(router, CfgParallelRouter) and router.had_override:+ _rebind_root_forward(router.orig_forward, module, replica)+ # The source's own __call__ runs its root hooks around every branch; the copy must not run them a second time.+ for attr in _ROOT_HOOK_DICTS:+ hooks = getattr(replica, attr, None)+ if hooks is not None:+ hooks.clear()+ compiled_subs = 0+ for (name, src), (_rname, dst) in zip(module.named_modules(), replica.named_modules()):+ for key, value in list(dst.__dict__.items()):+ if (+ isinstance(value, torch.Tensor)+ and value.device != device+ and value.device.type == "cuda"+ ):+ dst.__dict__[key] = value.to(device)+ if getattr(src, "_compiled_call_impl", None) is not None:+ recorded = src.__dict__.get("_unsloth_compile_args")+ dst._compiled_call_impl = None+ if recorded is None:+ raise RuntimeError(f"compiled submodule '{name}' has no recorded compile arguments")+ args, kwargs = recorded+ dst.compile(*args, **kwargs)+ compiled_subs += 1+ elif "_compiled_call_impl" in dst.__dict__:+ dst._compiled_call_impl = None+ if compiled_subs:+ _rewrap_compiled(module, replica)+ replica._unsloth_replica_compiled = compiled_subs+ replica.eval()+ return replica+++_SHARED_STATE_TYPES = frozenset({"NVFP4StepController"})+++def _rewrap_compiled(source: Any, replica: Any) -> None:+ """Re-apply the wrappers Studio's compile path puts around the source's compiled blocks (``dst.compile`` replaced+ them): the compile guard (eager fallback on a compile-time failure, the background-compile eager gate, the compile+ config on this thread) and the block-0 restride, in the order the source got them."""+ if "_unsloth_compile_guard" in source.__dict__:+ from .diffusion_speed import guard_compiled_blocks+ guard_compiled_blocks(replica)+ from . import diffusion_block_restride++ if any(+ diffusion_block_restride.is_wrapped(getattr(m, "_compiled_call_impl", None))+ for m in source.modules()+ ):+ diffusion_block_restride.install(replica)+++_ADAPTER_ATTRS = (+ "scaling",+ # PEFT stores the selection here; ``active_adapters`` is a property over it (never in ``__dict__``)+ "_active_adapter",+ "active_adapters",+ "merged_adapters",+ "disable_adapters",+ "_disable_adapters",+)+++def weights_fingerprint(module: Any) -> int:+ """Changes whenever the denoiser's weights or adapter state do: a parameter or buffer (NVFP4 / quantised layers+ keep their packed weights and scales as buffers) replaced, edited in place (fused LoRA bumps the version counter),+ added or removed (PEFT injection), or an adapter's scale / activation changed. A ``.data`` swap to different values+ is not seen; diffusers' LoRA paths do not do that."""+ parts: list = []+ for name, param in (*module.named_parameters(), *module.named_buffers()):+ try:+ version = param._version+ except Exception: # noqa: BLE001 - inference tensors carry no version+ version = -1+ # Identity, not the data pointer: an encode-time offload round trip moves the same weights and must not force+ # a rebuild, while a replaced parameter (new object) or an in-place edit (version) must.+ parts.append((name, id(param), version))+ for name, sub in module.named_modules():+ for attr in _ADAPTER_ATTRS:+ value = sub.__dict__.get(attr)+ if value is not None and not callable(value):+ parts.append((name, attr, repr(value)))+ return hash(tuple(parts))+++_LATENT_KEYS = ("hidden_states", "sample", "x", "latents", "img", "latent")+++def _latent_arg(args: tuple, kwargs: dict) -> Any:+ torch = _torch()+ for key in _LATENT_KEYS:+ value = kwargs.get(key)+ if isinstance(value, torch.Tensor):+ return value+ for value in args:+ if isinstance(value, torch.Tensor):+ return value+ return None+++_PLAIN_TYPES = (type(None), bool, int, float, str, bytes)+++def _holds_opaque_tensors(obj: Any, depth: int = 0) -> bool:+ """True when ``obj`` carries tensors inside something other than a plain tuple / list / dict (a prefix-KV cache+ object, a namedtuple, a value holder). The replica walker cannot move those to its card, and a mutable cache must+ not be shared across the two threads, so such a branch stays on the primary."""+ torch = _torch()+ if isinstance(obj, (torch.Tensor, *_PLAIN_TYPES, torch.device, torch.dtype, torch.nn.Module)):+ return False+ if depth < 4 and isinstance(obj, tuple) and not hasattr(obj, "_fields"):+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj)+ if depth < 4 and isinstance(obj, list):+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj)+ if depth < 4 and isinstance(obj, dict):+ return any(_holds_opaque_tensors(v, depth + 1) for v in obj.values())+ if isinstance(obj, tuple):+ values: Any = obj+ else:+ try:+ values = vars(obj).values()+ except TypeError:+ return False+ for value in values:+ if isinstance(value, torch.Tensor) or (+ isinstance(value, (list, tuple, dict)) and next(_iter_tensors(value), None) is not None+ ):+ return True+ if not isinstance(value, (*_PLAIN_TYPES, list, tuple, dict)) and depth < 4:+ if _holds_opaque_tensors(value, depth + 1):+ return True+ return False+++def _iter_tensors(obj: Any, depth: int = 0) -> Iterator[Any]:+ torch = _torch()+ if isinstance(obj, torch.Tensor):+ yield obj+ elif depth < 4 and isinstance(obj, (list, tuple)):+ for v in obj:+ yield from _iter_tensors(v, depth + 1)+ elif depth < 4 and isinstance(obj, dict):+ for v in obj.values():+ yield from _iter_tensors(v, depth + 1)+++class _ReplicaWorker:+ """One daemon thread bound to the replica's card: set_device, grad / inference / autocast mode and the caller's+ context variables (inductor overrides live in one on torch >= 2.12) are applied per job."""++ def __init__(self, device: Any):+ self.device = device+ self._jobs: "queue.Queue[Optional[tuple]]" = queue.Queue()+ self._thread = threading.Thread(+ target = self._loop, name = f"unsloth-cfg-replica-{device}", daemon = True+ )+ self._thread.start()++ def _loop(self) -> None:+ torch = _torch()+ if getattr(self.device, "type", None) == "cuda":+ try:+ torch.cuda.set_device(self.device)+ except Exception: # noqa: BLE001+ pass+ while True:+ job = self._jobs.get()+ if job is None:+ return+ fn, ctx, modes, done = job+ try:+ done["result"] = ctx.run(self._run, fn, modes)+ except BaseException as exc: # noqa: BLE001 - surfaced on the caller's thread+ done["error"] = exc+ finally:+ done["event"].set()++ def _run(self, fn: Callable[[], Any], modes: dict) -> Any:+ torch = _torch()+ with contextlib.ExitStack() as stack:+ if modes.get("inference"):+ stack.enter_context(torch.inference_mode())+ stack.enter_context(torch.set_grad_enabled(modes.get("grad", False)))+ autocast_dtype = modes.get("autocast")+ if autocast_dtype is not None:+ stack.enter_context(torch.autocast(device_type = "cuda", dtype = autocast_dtype))+ stack.enter_context(_device_context(self.device))+ return fn()++ def submit(self, fn: Callable[[], Any]) -> dict:+ torch = _torch()+ autocast = None+ try:+ if torch.is_autocast_enabled("cuda"):+ autocast = torch.get_autocast_dtype("cuda")+ except Exception: # noqa: BLE001 - older torch+ pass+ modes = {+ "inference": torch.is_inference_mode_enabled(),+ "grad": torch.is_grad_enabled(),+ "autocast": autocast,+ }+ done: dict[str, Any] = {"event": threading.Event()}+ self._jobs.put((fn, contextvars.copy_context(), modes, done))+ return done++ @staticmethod+ def wait(done: dict) -> Any:+ done["event"].wait()+ if "error" in done:+ raise done["error"]+ return done.get("result")++ def call(self, fn: Callable[[], Any]) -> Any:+ return self.wait(self.submit(fn))++ def close(self) -> None:+ self._jobs.put(None)+++def _leaves(+ obj: Any,+ path: tuple = (),+ depth: int = 0,+) -> list[tuple[tuple, Any]]:+ if depth < 4 and isinstance(obj, (tuple, list)) and not hasattr(obj, "_fields"):+ out: list = []+ for i, v in enumerate(obj):+ out.extend(_leaves(v, path + (i,), depth + 1))+ return out or [(path, obj)]+ if depth < 4 and isinstance(obj, dict):+ out = []+ for k, v in obj.items():+ out.extend(_leaves(v, path + (k,), depth + 1))+ return out or [(path, obj)]+ return [(path, obj)]+++def _rebuild(+ template: Any,+ values: dict,+ path: tuple = (),+ depth: int = 0,+) -> Any:+ if (+ depth < 4+ and isinstance(template, (tuple, list))+ and not hasattr(template, "_fields")+ and len(template)+ ):+ items = [_rebuild(v, values, path + (i,), depth + 1) for i, v in enumerate(template)]+ return type(template)(items) if isinstance(template, tuple) else items+ if depth < 4 and isinstance(template, dict) and template:+ return type(template)(+ (k, _rebuild(v, values, path + (k,), depth + 1)) for k, v in template.items()+ )+ return values[path]+++_SMALL_TENSOR_NUMEL = 4096+++def _bitwise_equal(a: Any, b: Any) -> bool:+ torch = _torch()+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):+ return False+ if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:+ return False+ if a is b:+ return True+ try:+ return bool(+ torch.equal(+ a.reshape(-1).contiguous().view(torch.uint8),+ b.reshape(-1).contiguous().view(torch.uint8),+ )+ )+ except Exception: # noqa: BLE001 - a layout that cannot be viewed as bytes is never assumed equal+ return False+++def _unchanged_since_launch(snapshots: list) -> bool:+ """True when every (source, launch-time copy) pair still holds the same bytes. One host sync for all pairs, on+ the replica's card; it waits for the primary branch (the copy is ordered after it), which the guidance combine+ waits for anyway, while the replica keeps running."""+ torch = _torch()+ checks = []+ for src, snap in snapshots:+ if src.shape != snap.shape or src.dtype != snap.dtype:+ return False+ if src.numel() == 0:+ continue+ try:+ now = src.to(snap.device).reshape(-1).contiguous().view(torch.uint8)+ checks.append(+ torch.equal(now, snap.reshape(-1).contiguous().view(torch.uint8))+ if snap.device.type == "cpu"+ else (now == snap.reshape(-1).contiguous().view(torch.uint8)).all()+ )+ except Exception: # noqa: BLE001 - an unverifiable tensor never accepts a prediction+ return False+ if not checks:+ return True+ host = [c for c in checks if isinstance(c, bool)]+ device = [c for c in checks if not isinstance(c, bool)]+ return all(host) and (not device or bool(torch.stack(device).all().item()))+++def _same_value(a: Any, b: Any) -> bool:+ if a is b:+ return True+ torch = _torch()+ if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor):+ return False+ try:+ return bool(a == b) and type(a) is type(b)+ except Exception: # noqa: BLE001 - incomparable objects must be the same object+ return False+++# Per-call state Studio's root hooks write onto denoiser submodules (``_unsloth_*`` scalars, e.g. the HunyuanVideo-1.5+# trim hook's null-mask flag). The root hooks run only on the source, around the router, so the replica's copy of that+# state is mirrored from the source before every replica call and the prediction is verified against it.+_STATE_PREFIX = "_unsloth_"+_STATE_TYPES = (bool, int, float, str, type(None))+_STATE_KEEP = frozenset({"_unsloth_replica_compiled"})+++def _hook_state(pairs: list) -> tuple:+ out = []+ for i, (src, _dst) in enumerate(pairs):+ for key, value in src.__dict__.items():+ if key.startswith(_STATE_PREFIX) and isinstance(value, _STATE_TYPES):+ out.append((i, key, value))+ return tuple(out)+++def _mirror_hook_state(pairs: list) -> tuple:+ state = _hook_state(pairs)+ wanted: dict = {}+ for i, key, value in state:+ wanted.setdefault(i, {})[key] = value+ for i, (_src, dst) in enumerate(pairs):+ mine = wanted.get(i, {})+ for key in [+ k+ for k, v in dst.__dict__.items()+ if k.startswith(_STATE_PREFIX)+ and isinstance(v, _STATE_TYPES)+ and k not in mine+ and k not in _STATE_KEEP+ ]:+ del dst.__dict__[key]+ for key, value in mine.items():+ if dst.__dict__.get(key, _DROP) is not value and dst.__dict__.get(key, _DROP) != value:+ dst.__dict__[key] = value+ return state+++def _background_compile_call() -> bool:+ try:+ from .diffusion_bg_compile import capture_suppressed, eager_forced+ except Exception: # noqa: BLE001+ return False+ return bool(eager_forced() or capture_suppressed())+++class CfgParallelRouter:+ """Installed as the denoiser's ``forward``: routes odd branches of a step to the replica.++ Once a step has shown two branches, the next step's first odd branch is predicted from the previous one (same+ structure; the latent, and every tensor the previous odd call shared with its primary call, replaced by this+ step's; the rest -- negative embeds, masks -- kept) and launched on the replica BEFORE the primary branch runs, so+ the two cards overlap even when the primary forward blocks the host. The real call must then pass exactly the+ predicted tensors (identity) and equal plain values, else the prediction is discarded and the branch runs+ unpredicted."""++ def __init__(+ self,+ module: Any,+ *,+ primary: Any,+ secondary: Any,+ replica_factory: Optional[Callable[[], Any]] = None,+ logger: Any = None,+ ):+ torch = _torch()+ self.module_ref = weakref.ref(module)+ self.orig_forward = module.forward+ self.had_override = "forward" in module.__dict__+ self.primary = torch.device(primary)+ self.secondary = torch.device(secondary)+ self.logger = logger+ self.replica: Any = None+ self.replica_factory = replica_factory or (lambda: build_replica(module, self.secondary))+ self.worker: Optional[_ReplicaWorker] = None+ self.disabled_reason: Optional[str] = None+ self.prev_latent: Any = None+ self.branch = 0+ self.cfg_seen = False+ self.singles = 0+ self.prefetched: dict[int, tuple[Any, Any]] = {}+ # Tensors the odd branches of the previous step read (negative embeds, masks): copied again before the next+ # step's primary branch is queued, so they are always fresh and never wait behind it.+ self.carry: list[Any] = []+ self.next_carry: list[Any] = []+ self.stats = {+ "replica_calls": 0,+ "primary_calls": 0,+ "steps_parallel": 0,+ "replica_build_s": None,+ "replica_rebuilds": 0,+ "predicted_hits": 0,+ "predicted_misses": 0,+ "opaque_primary_calls": 0,+ }+ self.template: Optional[tuple[Any, Any, dict]] = None+ self.pending: Optional[dict] = None+ self.b0_leaves: dict = {}+ self.fingerprint: Optional[int] = None+ self.pairs: list = []++ def describe(self) -> dict[str, Any]:+ return {+ "secondary": str(self.secondary),+ "replica_built": self.replica is not None,+ "replica_compiled_submodules": getattr(self.replica, "_unsloth_replica_compiled", None),+ "disabled_reason": self.disabled_reason,+ **self.stats,+ }++ def disable(self, reason: str) -> None:+ if self.disabled_reason is None:+ self.disabled_reason = reason+ if self.logger is not None:+ self.logger.warning(+ "diffusion.multigpu: CFG-parallel disabled (%s); one card from here", reason+ )+ self.drop_replica()++ def drop_replica(self) -> None:+ if self.worker is not None:+ self.worker.close()+ self.worker = None+ self._drop_pending()+ self.replica = None+ self.pairs = []+ self.prefetched.clear()+ self.carry = []+ self.next_carry = []+ self.prev_latent = None+ self.template = None+ self.b0_leaves = {}++ def end_render(self) -> None:+ self._drop_pending()+ self.prefetched.clear()+ self.carry = []+ self.next_carry = []+ self.prev_latent = None+ self.template = None+ self.b0_leaves = {}+ self.branch = 0+ # the next render shows its own guidance shape: no speculative branch before its first odd call+ self.cfg_seen = False+ self.singles = 0++ def __call__(self, *args: Any, **kwargs: Any) -> Any:+ if self.disabled_reason is not None or _background_compile_call():+ # Disabled: no branch state to keep. A render that must run eager while its compile is in flight, or the+ # background compile thread itself replaying recorded inputs: both branches stay on the source, which+ # keeps that render bit-identical to an eager one and leaves the router's per-render state untouched.+ self.stats["primary_calls"] += 1+ return self.orig_forward(*args, **kwargs)+ latent = _latent_arg(args, kwargs)+ if latent is not None and self.prev_latent is not None and latent is self.prev_latent:+ self.branch += 1+ else:+ if self.branch == 0 and self.prev_latent is not None:+ self.singles += 1+ if self.singles >= 3:+ self.cfg_seen = False+ self.branch = 0+ self.prev_latent = latent+ self.prefetched.clear()+ self.carry, self.next_carry = self.next_carry, []+ if self.replica is not None and self.disabled_reason is None:+ self._drop_if_stale()+ self._drop_pending()+ self.b0_leaves = dict(_leaves((args, kwargs)))+ if self.cfg_seen and self.replica is not None and self.disabled_reason is None:+ # Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after+ # it would wait for it on the source stream and serialise the two branches.+ self._prefetch(args, kwargs)+ self._launch_prediction(latent)+ if (+ self.branch % 2 == 1+ and self.disabled_reason is None+ and _holds_opaque_tensors((args, kwargs))+ ):+ self._drop_pending()+ self.template = None+ self.stats["opaque_primary_calls"] += 1+ elif self.branch % 2 == 1 and self.disabled_reason is None:+ self.singles = 0+ if self.replica is None:+ self.cfg_seen = True+ self._build_replica()+ if self.branch == 1:+ self._record_template(args, kwargs, latent)+ elif self.cfg_seen:+ if self.branch == 1:+ self._record_template(args, kwargs, latent)+ try:+ out = self._take_prediction(args, kwargs) if self.branch == 1 else None+ if out is None:+ out = self._run_replica(args, kwargs)+ self.stats["replica_calls"] += 1+ if self.branch == 1:+ self.stats["steps_parallel"] += 1+ return out+ except Exception as exc: # noqa: BLE001 - this branch is recomputed on the primary below+ self.disable(f"replica call failed: {type(exc).__name__}: {exc}")+ self.cfg_seen = True+ self.stats["primary_calls"] += 1+ return self.orig_forward(*args, **kwargs)++ def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:+ torch = _torch()+ b0_by_id = {id(v): p for p, v in self.b0_leaves.items() if isinstance(v, torch.Tensor)}+ kinds: dict = {}+ for path, leaf in _leaves((args, kwargs)):+ if isinstance(leaf, torch.Tensor):+ if leaf is latent:+ kinds[path] = ("latent",)+ elif id(leaf) in b0_by_id and self.b0_leaves.get(b0_by_id[id(leaf)]) is leaf:+ kinds[path] = ("b0", b0_by_id[id(leaf)])+ elif (+ isinstance(self.b0_leaves.get(path), torch.Tensor)+ and leaf.numel() <= _SMALL_TENSOR_NUMEL+ and _bitwise_equal(leaf, self.b0_leaves[path])+ ):+ kinds[path] = ("b0eq", path)+ else:+ kinds[path] = ("const", leaf)+ else:+ kinds[path] = ("value", leaf)+ self.template = ((args, kwargs), kinds)++ def _predict(self, latent: Any) -> Optional[tuple]:+ if self.template is None or latent is None:+ return None+ structure, kinds = self.template+ values: dict = {}+ for path, kind in kinds.items():+ if kind[0] == "latent":+ values[path] = latent+ elif kind[0] in ("b0", "b0eq"):+ if kind[1] not in self.b0_leaves:+ return None+ values[path] = self.b0_leaves[kind[1]]+ else:+ values[path] = kind[1]+ try:+ return _rebuild(structure, values)+ except KeyError:+ return None++ def _launch_prediction(self, latent: Any) -> None:+ predicted = self._predict(latent)+ if predicted is None or self.worker is None:+ return+ p_args, p_kwargs = predicted+ try:+ r_args = self._to_replica(p_args)+ r_kwargs = self._to_replica(p_kwargs)+ except Exception: # noqa: BLE001 - no prediction this step; the branch runs unpredicted+ return+ torch = _torch()+ # What the replica consumes, kept per tensor: at the real call the sources are compared bitwise against it, so+ # an in-place edit between launch and the odd branch (same object, new values) rejects the prediction.+ # Version counters cannot do this: inference tensors do not track one.+ snapshots = []+ for (_path, src), (_rpath, used) in zip(+ _leaves((p_args, p_kwargs)), _leaves((r_args, r_kwargs))+ ):+ if isinstance(src, torch.Tensor):+ snapshots.append((src, used.clone() if used is src else used))+ replica = self.replica+ _structure, kinds = self.template+ hook_state = _mirror_hook_state(self.pairs)+ self.pending = {+ "hook_state": hook_state,+ "leaves": _leaves((p_args, p_kwargs)),+ "by_value": {path for path, kind in kinds.items() if kind[0] == "b0eq"},+ "snapshots": snapshots,+ "job": self.worker.submit(lambda: replica(*r_args, **r_kwargs)),+ }++ def _take_prediction(self, args: tuple, kwargs: dict) -> Any:+ pending, self.pending = self.pending, None+ if pending is None:+ return None+ actual = _leaves((args, kwargs))+ predicted = pending["leaves"]+ by_value = pending["by_value"]+ ok = len(actual) == len(predicted) and all(+ pa == pp and (_bitwise_equal(va, vp) if pa in by_value else _same_value(va, vp))+ for (pa, va), (pp, vp) in zip(actual, predicted)+ )+ # the source's root hooks have now run for THIS call: the replica must have seen the same per-call state+ ok = ok and _hook_state(self.pairs) == pending["hook_state"]+ ok = ok and _unchanged_since_launch(pending["snapshots"])+ try:+ out = _ReplicaWorker.wait(pending["job"])+ except Exception:+ if ok:+ raise+ out = None+ if not ok:+ self.stats["predicted_misses"] += 1+ self.prefetched.clear()+ return None+ self.stats["predicted_hits"] += 1+ return move_to_device(out, self.primary)++ def _drop_pending(self) -> None:+ pending, self.pending = self.pending, None+ if pending is not None:+ with contextlib.suppress(Exception):+ _ReplicaWorker.wait(pending["job"])+ self.stats["predicted_misses"] += 1++ def _drop_if_stale(self) -> None:+ module = self.module_ref()+ if module is None:+ return+ try:+ current = weights_fingerprint(module)+ except Exception as exc: # noqa: BLE001 - unverifiable weights: never risk a stale replica+ self.disable(f"cannot fingerprint the denoiser: {exc}")+ return+ if current != self.fingerprint:+ if self.logger is not None:+ self.logger.info(+ "diffusion.multigpu: denoiser weights changed; rebuilding the CFG-parallel replica"+ )+ self.stats["replica_rebuilds"] += 1+ self.drop_replica()+ return+ # A step cache (transformer_cache=auto) may be installed on the source at generate time, after the replica+ # was built: the copy would then run uncached while the source's cache saw only the even branches.+ hooked = _hooks_present(module)+ if hooked is not None:+ self.disable(f"denoiser gained hooks after the replica was built ({hooked})")++ def _build_replica(self) -> None:+ module = self.module_ref()+ if module is None:+ return+ t0 = time.perf_counter()+ try:+ hooked = _hooks_present(module)+ if hooked is not None:+ raise RuntimeError(f"denoiser carries hooks ({hooked})")+ need_mib = sum(+ t.numel() * t.element_size() for t in (*module.parameters(), *module.buffers())+ ) // (1 << 20)+ free, _total = _free_total_mib(self.secondary.index)+ if free is not None:+ with contextlib.suppress(Exception):+ torch = _torch()+ free += (+ torch.cuda.memory_reserved(self.secondary)+ - torch.cuda.memory_allocated(self.secondary)+ ) // (1 << 20)+ if free is not None and need_mib + 1024 > free:+ raise RuntimeError(+ f"replica needs {need_mib} MiB, {free} MiB free on {self.secondary}"+ )+ self.fingerprint = weights_fingerprint(module)+ with _device_context(self.secondary):+ self.replica = self.replica_factory()+ self.pairs = list(zip(module.modules(), self.replica.modules()))+ self.worker = _ReplicaWorker(self.secondary)+ weakref.finalize(self, self.worker.close)+ self.stats["replica_build_s"] = round(time.perf_counter() - t0, 3)+ if self.logger is not None:+ self.logger.info(+ "diffusion.multigpu: CFG-parallel replica on %s built in %.2fs",+ self.secondary,+ time.perf_counter() - t0,+ )+ except Exception as exc: # noqa: BLE001+ self.replica = None+ with contextlib.suppress(Exception):+ _torch().cuda.empty_cache()+ self.disable(f"replica unavailable: {exc}")++ def _prefetch(self, args: tuple, kwargs: dict) -> None:+ for tensor in (*_iter_tensors((args, kwargs)), *self.carry):+ if tensor.device == self.secondary or id(tensor) in self.prefetched:+ continue+ self.prefetched[id(tensor)] = (tensor, tensor.to(self.secondary))++ def _to_replica(self, obj: Any) -> Any:+ torch = _torch()++ def one(tensor: Any) -> Any:+ if tensor is not self.prev_latent and not any(tensor is t for t in self.next_carry):+ self.next_carry.append(tensor)+ if tensor.device == self.secondary:+ return tensor+ hit = self.prefetched.get(id(tensor))+ if hit is not None and hit[0] is tensor:+ return hit[1]+ copied = tensor.to(self.secondary)+ self.prefetched[id(tensor)] = (tensor, copied)+ return copied++ def walk(value: Any, depth: int = 0) -> Any:+ if isinstance(value, torch.Tensor):+ return one(value)+ if depth < 4 and isinstance(value, tuple) and not hasattr(value, "_fields"):+ return tuple(walk(v, depth + 1) for v in value)+ if depth < 4 and isinstance(value, list):+ return [walk(v, depth + 1) for v in value]+ if depth < 4 and isinstance(value, dict):+ return type(value)((k, walk(v, depth + 1)) for k, v in value.items())+ return value++ return walk(obj)++ def _run_replica(self, args: tuple, kwargs: dict) -> Any:+ # Unpredicted: nothing verified the copies made before the primary branch ran, and an input edited in place+ # since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).+ self.prefetched.clear()+ _mirror_hook_state(self.pairs)+ r_args = self._to_replica(args)+ r_kwargs = self._to_replica(kwargs)+ replica = self.replica+ out = self.worker.call(lambda: replica(*r_args, **r_kwargs))+ return move_to_device(out, self.primary)+++_CFG_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgParallelRouter]" = weakref.WeakKeyDictionary()+_CFG_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()+++def cfg_parallel_decline(+ pipe: Any, *, offload_policy: str, selection: Optional[MultiGpuSelection]+) -> Optional[str]:+ if selection is None:+ return "one card selected"+ mode = cfg_parallel_mode()+ if mode == "off":+ return f"{CFG_PARALLEL_ENV}=0"+ # A replica takes a whole denoiser's worth of another card; automatic placement only borrows one to avoid offload.+ if selection.automatic and mode != "force":+ return f"automatic GPU placement (select several cards or set {CFG_PARALLEL_ENV}=force)"+ from .diffusion_bg_compile import load_time_enabled as bg_compile_enabled++ if bg_compile_enabled():+ # its replays call the denoiser, which would enter the router and race a render's branch state+ return "background compile replays through the denoiser (UNSLOTH_DIFFUSION_BG_COMPILE)"+ denoiser = getattr(pipe, "transformer", None)+ torch = _torch()+ if not isinstance(denoiser, torch.nn.Module):+ return "no DiT denoiser (a U-Net batches its guidance branches)"+ # Whole-module / sequential offload page the denoiser per call; a group / streaming plan is fine only when it left+ # the denoiser itself resident (encoders streamed), which shows as a denoiser without offload hooks.+ if offload_policy in ("model", "sequential"):+ return f"denoiser is offloaded ({offload_policy})"+ hooked = _hooks_present(denoiser)+ if hooked is not None:+ return f"denoiser is offloaded or hooked ({offload_policy}: {hooked})"+ if getattr(pipe, "transformer_2", None) is not None:+ return "two-expert denoiser"+ if getattr(denoiser, "_compiled_call_impl", None) is not None:+ return "denoiser is compiled whole"+ if _module_device(denoiser) is None or _module_device(denoiser).type != "cuda":+ return "denoiser is not on a CUDA card"+ return None+++def install_cfg_parallel(+ pipe: Any,+ *,+ offload_policy: str,+ logger: Any = None,+) -> Optional[str]:+ """Arm CFG-parallel on ``pipe.transformer`` when a secondary card can hold a replica. The replica is built on the+ first step that shows two branches, so a guidance-distilled model never pays for it. Returns the decline reason or+ None when armed."""+ selection = current_selection()+ reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)+ if reason is None:+ denoiser = pipe.transformer+ primary = _module_device(denoiser)+ probes = probe_devices((selection.primary, *selection.secondaries))+ same = [+ o+ for o in selection.secondaries+ if probes.get(o) is not None+ and probes.get(primary.index) is not None+ and probes[o].name == probes[primary.index].name+ ]+ candidates = same if cfg_parallel_mode() != "force" else list(selection.secondaries)+ try:+ placed = ((_PLACEMENTS.get(pipe) or {}).get("placement") or {}).values()+ except TypeError:+ placed = ()+ occupied = {str(d) for d in placed} - {str(primary)}+ if occupied:+ candidates = [o for o in candidates if f"cuda:{o}" not in occupied]+ if not candidates and occupied:+ reason = f"the only replica card ({', '.join(sorted(occupied))}) holds the split's components"+ elif not candidates:+ reason = "secondary card is a different model (bit-identity needs the same kernels)"+ else:+ need_mib = sum(+ t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers())+ ) // (1 << 20)+ best = max(candidates, key = lambda o: probes[o].free_mib or -1)+ free = probes[best].free_mib+ if free is not None and need_mib + 1024 > free:+ reason = f"replica needs {need_mib} MiB, {free} MiB free on cuda:{best}"+ else:+ router = CfgParallelRouter(+ denoiser, primary = primary, secondary = f"cuda:{best}", logger = logger+ )+ try:+ _CFG_ROUTERS[pipe] = router+ except TypeError:+ return "pipeline cannot be tracked (not weak-referenceable)"+ denoiser.forward = router+ denoiser._unsloth_cfg_router = router+ if logger is not None:+ logger.info(+ "diffusion.multigpu: CFG-parallel armed (replica on cuda:%s when guidance runs two branches)",+ best,+ )+ return None+ if selection is not None:+ with contextlib.suppress(TypeError):+ _CFG_DECLINES[pipe] = reason+ if logger is not None:+ logger.info("diffusion.multigpu: CFG-parallel not armed (%s)", reason)+ return reason+++def invalidate_cfg_replica(+ pipe: Any,+ reason: str,+ logger: Any = None,+) -> bool:+ """Drop the CFG-parallel replica of ``pipe`` after its denoiser was mutated in ways the per-step staleness check+ does not see (attention backend, eager patches, compile); the next guided step rebuilds it from the source. True+ when a replica was dropped."""+ try:+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None+ except TypeError:+ return False+ if router is None or router.replica is None:+ return False+ if logger is not None:+ logger.info(+ "diffusion.multigpu: CFG-parallel replica dropped (%s); rebuilt on the next guided step",+ reason,+ )+ router.stats["replica_rebuilds"] += 1+ router.drop_replica()+ return True+++def drop_displaced_cfg_router(+ pipe: Any,+ reason: str,+ logger: Any = None,+) -> bool:+ """Uninstall ``pipe``'s CFG-parallel router when a later layer (the deferred speed profile's graphs) replaced the+ denoiser's forward and bypasses it, so status reports CFG-parallel as declined rather than armed. True if dropped."""+ try:+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None+ except TypeError:+ return False+ if router is None:+ return False+ module = router.module_ref()+ if module is not None and module.__dict__.get("forward") is router:+ return False+ uninstall_cfg_parallel(pipe)+ with contextlib.suppress(TypeError):+ _CFG_DECLINES[pipe] = reason+ if logger is not None:+ logger.info("diffusion.multigpu: CFG-parallel dropped (%s)", reason)+ return True+++def end_cfg_render(pipe: Any) -> None:+ try:+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None+ except TypeError:+ return+ if router is not None:+ router.end_render()+++def uninstall_cfg_parallel(pipe: Any) -> None:+ try:+ router = _CFG_ROUTERS.pop(pipe, None) if pipe is not None else None+ except TypeError:+ return+ if router is None:+ return+ if router.disabled_reason is None:+ router.disabled_reason = "uninstalled"+ router.drop_replica()+ module = router.module_ref()+ if module is not None:+ if module.__dict__.get("forward") is router:+ if router.had_override:+ module.__dict__["forward"] = router.orig_forward+ else:+ del module.__dict__["forward"]+ if module.__dict__.get("_unsloth_cfg_router") is router:+ del module.__dict__["_unsloth_cfg_router"]diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex f04cd5068f5..e1e28a247df 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -104,6 +104,18 @@ from .diffusion_device import (auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,+ resolve_selected_cuda_ordinals,+)+from .diffusion_multigpu import (+ AUTOMATIC_ORDINALS,+ end_cfg_render,+ install_cfg_parallel,+ multigpu_mode,+ is_split as multigpu_is_split,+ placement_status as multigpu_placement_status,+ refine_plan_for_multigpu,+ uninstall_cfg_parallel,+ with_load_selection,)from .diffusion_memory import (DEFAULT_BASE_OVERHEAD_MIB,@@ -1427,6 +1439,7 @@ class _VideoLoadState:vram_floor_mib: Optional[int] = None# MiB still on the device during the VAE decode (the streamed DiT has offloaded by then).vram_decode_floor_mib: Optional[int] = None+ vram_split_cards: Optional[tuple] = None# MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.denoiser_host_copy: bool = False# MiniMax-H3: conditioner streamed leaf by leaf (VRAM floor counts its streamed footprint).@@ -3148,6 +3161,143 @@ def _video_offload_vram_floor_mib(return None+# Families whose text encoder runs outside ``pipe.components`` (a sidecar worker on the primary): a multi-GPU split+# cannot move it, so the companion split the planner priced with it would leave it on the primary beside the DiT.+_VIDEO_SIDECAR_ENCODER_FAMILIES = frozenset({"ltx-2.5"})+++def _video_single_card_for_sidecar(+ fam: Any,+ plan: Any,+ logger: Any = None,+) -> Any:+ """The single-card plan in place of a companion split for a sidecar-encoder family; any other plan unchanged."""+ if getattr(fam, "name", None) not in _VIDEO_SIDECAR_ENCODER_FAMILIES:+ return plan+ fallback = getattr(plan, "fallback_plan", None)+ if getattr(plan, "secondary_ordinal", None) is None or fallback is None:+ return plan+ if logger is not None:+ logger.info(+ "video.multigpu: one card (the %s text encoder runs in a sidecar a split cannot move)",+ fam.name,+ )+ return fallback+++def _video_refine_for_multigpu(+ fam: Any,+ pipe: Any,+ plan: Any,+ logger: Any = None,+) -> Any:+ """``refine_plan_for_multigpu``, declined for a sidecar-encoder family (its loaded sizes omit the encoder)."""+ if getattr(fam, "name", None) in _VIDEO_SIDECAR_ENCODER_FAMILIES:+ return plan+ return refine_plan_for_multigpu(pipe, plan, logger)+++def _video_split_card_floors(pipe: Any) -> Optional[tuple]:+ """``((ordinal, resident MiB, runs the decode), ...)`` for a pipeline split across cards, else None. A split plan+ reports no offload policy, so the offload floor is None and the request guard would be skipped; this records what+ each card holds so the guard can run per card."""+ status = multigpu_placement_status(pipe) if multigpu_is_split(pipe) else None+ placement = (status or {}).get("placement") or {}+ if not placement:+ return None+ try:+ from itertools import chain++ import torch++ from .diffusion_prequant import tensor_payload_bytes++ components = getattr(pipe, "components", None) or {}+ cards: dict[int, list] = {}+ seen: set[int] = set()+ for name, where in placement.items():+ device = torch.device(where)+ module = components.get(name)+ if (+ device.type != "cuda"+ or device.index is None+ or not isinstance(module, torch.nn.Module)+ ):+ continue+ card = cards.setdefault(int(device.index), [0, False])+ for tensor in chain(module.parameters(), module.buffers()):+ if id(tensor) in seen or getattr(tensor, "is_meta", False):+ continue+ seen.add(id(tensor))+ card[0] += tensor_payload_bytes(tensor)+ card[1] = card[1] or str(name) == "vae"+ if len(cards) < 2:+ return None+ home = placement.get("transformer")+ home = torch.device(home).index if home else None+ denoisers = {+ torch.device(where).index+ for name, where in placement.items()+ if str(name).startswith("transformer") and torch.device(where).type == "cuda"+ }+ return tuple(+ (o, int(b // (1024 * 1024)), bool(d), o == home, o in denoisers)+ for o, (b, d) in sorted(cards.items())+ )+ except Exception: # noqa: BLE001 -- an unreadable placement keeps the unchecked behaviour+ return None+++def video_split_shortfall_message(+ cards: tuple,+ *,+ family: str,+ available_mib: Callable[[int], int],+ width: Optional[int] = None,+ height: Optional[int] = None,+ frames: Optional[int] = None,+) -> Optional[str]:+ """The offload guard run on every card of a split: each card's resident weights plus the base headroom, the+ decoded-clip share on the card that runs the VAE decode, and on the denoiser's card, when the decode ran+ elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. The first refusal,+ else None."""+ for card in cards:+ ordinal, floor_mib, decodes = card[:3]+ home = len(card) > 3 and bool(card[3])+ denoises = bool(card[4]) if len(card) > 4 else home+ shaped = bool(width and height and frames)+ frames_mib = (+ -(-int(width) * int(height) * int(frames) * 3 * 4 // (1 << 20)) if shaped else 0+ )+ if shaped and denoises:+ phase_mib = max(+ frames_mib if home and not decodes else 0, _VIDEO_DENOISE_ACTIVATION_MIB+ )+ shortfall = video_offload_shortfall_message(+ family = family,+ floor_mib = int(floor_mib) + phase_mib,+ available_mib = int(available_mib(int(ordinal))),+ placement = f"split across GPUs, cuda:{int(ordinal)}",+ )+ if shortfall is not None:+ return shortfall+ elif shaped and home and not decodes:+ floor_mib = int(floor_mib) + frames_mib+ shortfall = video_offload_shortfall_message(+ family = family,+ floor_mib = int(floor_mib),+ available_mib = int(available_mib(int(ordinal))),+ placement = f"split across GPUs, cuda:{int(ordinal)}",+ width = width if decodes else None,+ height = height if decodes else None,+ frames = frames if decodes else None,+ decode_floor_mib = int(floor_mib) if decodes else None,+ )+ if shortfall is not None:+ return shortfall+ return None++def video_offload_shortfall_message(*,family: str,@@ -3929,6 +4079,16 @@ class VideoBackend:if resolve_diffusion_device_target().device == "cuda"else None)+ gpu_ordinals = (+ resolve_selected_cuda_ordinals(gpu_ids)+ if gpu_ids+ and len(set(gpu_ids)) > 1+ and multigpu_mode() != "off"+ and resolve_diffusion_device_target().device == "cuda"+ else AUTOMATIC_ORDINALS+ if not gpu_ids+ else []+ )fam = self.validate_load_request(repo_id,gguf_filename = gguf_filename,@@ -4029,11 +4189,14 @@ class VideoBackend:gpu_ordinal = gpu_ordinal,_load_token = token,_cancel_event = cancel_event,+ _gpu_ordinals = gpu_ordinals,),daemon = True,).start()return self.status()+ # The multi-GPU selection (begin_load's ``_gpu_ordinals``) is visible to this load's planner and placement only.+ @with_load_selection@scoped_local_files_onlydef _run_load(self, **kwargs: Any) -> None:token = kwargs.get("_load_token")@@ -7864,7 +8027,12 @@ class VideoBackend:)planned = replannedreplanned_for_quant = True- return planned, dense_plan, replanned_for_quant+ # a sidecar-encoder family (LTX-2.5) keeps one card: a split cannot move its encoder+ return (+ _video_single_card_for_sidecar(fam, planned, logger if log else None),+ _video_single_card_for_sidecar(fam, dense_plan),+ replanned_for_quant,+ )denoiser_seed_scheme: Optional[str] = Noneif kind == "pipeline" and _video_auto_denoiser_planned != DENOISER_SEED_DECLINED:@@ -8900,6 +9068,7 @@ class VideoBackend:if refined is not plan:logger.info("video.memory: %s", refined.reasons[-1])plan = refined+ plan = _video_refine_for_multigpu(fam, pipe, plan, logger)vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)free_mib = getattr(getattr(plan, "device_memory", None), "free_mib", None)if vram_floor_mib is not None and free_mib is not None:@@ -8928,7 +9097,10 @@ class VideoBackend:logger = logger,)# a transient OOM during setup may have re-planned to lower residency+ planned = planplan = applied_plan(pipe, plan)+ if getattr(planned, "secondary_ordinal", None) is not None and plan is not planned:+ vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:# ~1 GB; resident with the rest, else render() moves it in for the upsample only.@@ -9110,6 +9282,7 @@ class VideoBackend:},logger = logger,)+ install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.from . import diffusion_bg_compile as bg_compile@@ -9175,6 +9348,9 @@ class VideoBackend:pipe, dataclasses.replace(plan, offload_policy = offload_policy))),+ vram_split_cards = (+ _video_split_card_floors(pipe) if offload_policy == OFFLOAD_NONE else None+ ),vram_decode_floor_mib = _video_offload_vram_floor_mib(pipe,plan@@ -11189,6 +11365,35 @@ class VideoBackend:)if shortfall is not None:raise RuntimeError(shortfall)+ elif (+ state.engine == "diffusers"+ and not fam.modular_workflow+ and getattr(state, "vram_split_cards", None)+ and hasattr(torch, "cuda")+ and hasattr(torch.cuda, "mem_get_info")+ ):+ from utils.hardware import trusted_mem_get_info++ def _available_mib(ordinal: int) -> int:+ device_obj = torch.device("cuda", ordinal)+ free_bytes, _ = trusted_mem_get_info(device_obj, module = torch.cuda)+ reserved_bytes = (+ torch.cuda.memory_reserved(device_obj)+ if hasattr(torch.cuda, "memory_reserved")+ else 0+ )+ return int((free_bytes + reserved_bytes) // (1024 * 1024))++ shortfall = video_split_shortfall_message(+ state.vram_split_cards,+ family = fam.name,+ available_mib = _available_mib,+ width = width,+ height = height,+ frames = frames,+ )+ if shortfall is not None:+ raise RuntimeError(shortfall)elif (state.engine == "diffusers"and not fam.modular_workflow@@ -11816,6 +12021,7 @@ class VideoBackend:_log_failed_generation(request_shape, exc)raisefinally:+ end_cfg_render(getattr(state, "pipe", None))try:from . import video_stream_residencyvideo_stream_residency.discard_request(getattr(state, "pipe", None))@@ -12576,6 +12782,7 @@ class VideoBackend:close_encoder(getattr(state, "pipe", None))# A pinner still copying would hold its chunks past release_pinned_host_memory().stop_background_pins(getattr(state, "pipe", None))+ uninstall_cfg_parallel(getattr(state, "pipe", None))# Before clear_gpu_cache(), or the graph pool stays reserved.diffusion_cuda_graph.uninstall_all(getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()@@ -12770,6 +12977,7 @@ class VideoBackend:"supports_audio_flow_shift": fam.default_audio_flow_shift is not Noneand state.engine != "sd_cpp",},+ "multi_gpu": multigpu_placement_status(getattr(state, "pipe", None)),"resolved": resolved,}diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.pyindex 39856a6e615..2522f469663 100644--- a/studio/backend/core/inference/video_vae_untiled.py+++ b/studio/backend/core/inference/video_vae_untiled.py@@ -84,6 +84,23 @@ def _free_bytes(device: Any) -> Optional[int]:return None+def _decode_device(vae: Any, z: Any) -> Any:+ """Where the decode allocates: the latent's card, unless a multi-GPU split put the VAE on another CUDA card (its+ boundary moves the latent there inside this wrapper). An offloaded VAE (weights on the host) keeps the latent's."""+ device = getattr(z, "device", None)+ try:+ vae_device = next(vae.parameters()).device+ except Exception: # noqa: BLE001+ return device+ if (+ getattr(vae_device, "type", None) == "cuda"+ and getattr(device, "type", None) == "cuda"+ and vae_device != device+ ):+ return vae_device+ return device++def install_untiled_decode(pipe: Any,family: str,@@ -111,7 +128,7 @@ def install_untiled_decode(need = untiled_decode_bytes(family, tuple(getattr(z, "shape", ())), itemsize = _decoder_itemsize(vae))- free = _free_bytes(getattr(z, "device", None))+ free = _free_bytes(_decode_device(vae, z))fits = (need is not Noneand free is not Nonediff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 331b32be408..6ae93ddddaf 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -5518,6 +5518,12 @@ class DiffusionDownloadPlanResponse(BaseModel):class DiffusionStatusResponse(BaseModel):"""Current diffusion backend state."""+ multi_gpu: Optional[Dict[str, Any]] = Field(+ None,+ description = "Multi-GPU placement (component -> card, primary, secondary) and CFG-parallel state (armed, "+ "call counts, or why it declined); null when the load had one card to use. A multi-card load that kept "+ "everything on its primary card reports why it declined.",+ )loaded: bool = Field(False, description = "Whether a diffusion model is loaded")evicted: List[str] = Field(default_factory = list,@@ -7070,6 +7076,12 @@ class VideoGenerationDefaults(BaseModel):class VideoStatusResponse(BaseModel):"""Current video backend state."""+ multi_gpu: Optional[Dict[str, Any]] = Field(+ None,+ description = "Multi-GPU placement (component -> card, primary, secondary) and CFG-parallel state (armed, "+ "call counts, or why it declined); null when the load had one card to use. A multi-card load that kept "+ "everything on its primary card reports why it declined.",+ )loaded: bool = Field(False, description = "Whether a video model is loaded")evicted: List[str] = Field(default_factory = list,diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.pynew file mode 100644index 00000000000..82653c12870--- /dev/null+++ b/studio/backend/tests/test_diffusion_multigpu.py@@ -0,0 +1,2335 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Multi-GPU diffusion layer (core/inference/diffusion_multigpu.py), CPU only.++Cross-card behaviour is exercised with fake devices: the planner reads budgets through injected callables, placement+and the boundary through a patched card predicate, and CFG-parallel with a replica on the same (CPU) device, which+checks the routing, the prefetch / constant caches and every fallback without a second card."""++from __future__ import annotations++import contextlib+import dataclasses+import threading+from collections import OrderedDict++import pytest+import torch++from core.inference import diffusion_memory as dm+from core.inference import diffusion_multigpu as mg+++def _from_diffusers(module: str, name: str):+ # Not importorskip: importing diffusers can raise RuntimeError (bitsandbytes without CUDA), an environment fact.+ # Only the import is skippable; a symbol missing from an importable diffusers is a failure.+ import importlib+ try:+ loaded = importlib.import_module(module)+ except (ImportError, RuntimeError) as exc:+ pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")+ return getattr(loaded, name)++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)+ yield+++def test_one_card_or_the_kill_switch_means_no_selection(monkeypatch):+ assert mg.make_selection(0, [0]) is None+ assert mg.make_selection(None, [0, 1]) is None+ assert mg.make_selection(1, [0, 1]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")+ assert mg.make_selection(1, [0, 1]) is None+ assert mg.cfg_parallel_mode() == "off"+++def test_selection_is_scoped_to_the_load_thread():+ sel = mg.make_selection(0, [0, 1])+ seen = {}+ with mg.selection_scope(sel):+ assert mg.current_selection() == sel+ t = threading.Thread(target = lambda: seen.setdefault("other", mg.current_selection()))+ t.start()+ t.join()+ assert mg.current_selection() is None+ assert seen["other"] is None+++def test_resolve_selected_cuda_ordinals_translates_the_mask(monkeypatch):+ import utils.hardware.hardware as hw+ from core.inference import diffusion_device as dd++ monkeypatch.setattr(+ hw,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": None, "numeric_ids": [4, 5], "supports_explicit_gpu_ids": True},+ )+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 8)+ assert dd.resolve_selected_cuda_ordinals([5, 4]) == [0, 1]+ assert dd.resolve_selected_cuda_ordinals([]) == []+ with pytest.raises(ValueError):+ dd.resolve_selected_cuda_ordinals([0])+++def _split(**overrides):+ kwargs = dict(+ single_card_policy = "group",+ model_dense_mib = 20000,+ companion_dense_mib = 9000,+ runtime_headroom_mib = 2000,+ base_overhead_mib = 1000,+ primary_limit_mib = 14000,+ secondary_limit_mib = lambda o: 12750,+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,)),+ )+ kwargs.update(overrides)+ return mg.plan_companion_split(**kwargs)+++def test_split_when_the_denoiser_alone_fits_the_primary():+ split = _split()+ assert split is not None and split.secondary_ordinal == 1+ assert split.primary_required_mib == 11000 + 2000 + 1000+ assert split.secondary_required_mib == 9000 + 3000+++ "overrides",+ [+ dict(primary_limit_mib = 13000), # denoiser + headroom do not fit the primary either+ dict(secondary_limit_mib = lambda o: 11999), # companions + headroom = 12000+ dict(secondary_limit_mib = lambda o: None),+ dict(single_card_policy = "none"),+ dict(companion_dense_mib = None),+ dict(selection = None),+ ],+)+def test_no_split_keeps_single_card_behaviour(overrides):+ assert _split(**overrides) is None+++def test_text_encoders_alone_when_the_decode_needs_the_runtime_room():+ # Video-like: the secondary cannot also hold the decode headroom, but the encoders alone fit it, and the denoiser ++ # VAE fit the primary.+ kwargs = dict(+ companion_dense_mib = 11500,+ text_encoder_dense_mib = 11000,+ model_dense_mib = 16000,+ runtime_headroom_mib = 4000,+ primary_limit_mib = 12000,+ secondary_limit_mib = lambda o: 13600,+ )+ split = _split(**kwargs)+ assert split is not None and split.components == mg.TEXT_ENCODERS_ONLY+ assert split.secondary_required_mib == 11000 + mg.TEXT_ENCODER_HEADROOM_MIB + 1000+ assert split.primary_required_mib == 5000 + 4000 + 1000+ # without the encoder share the planner cannot price it and keeps one card+ assert _split(**{**kwargs, "text_encoder_dense_mib": None}) is None+++def test_measured_phases_price_each_card_for_what_it_runs():+ kwargs = dict(+ model_dense_mib = 7630 + 7500 + 160,+ companion_dense_mib = 7660,+ text_encoder_dense_mib = 7500,+ runtime_headroom_mib = 8192,+ base_overhead_mib = 2048,+ primary_limit_mib = 14400,+ secondary_limit_mib = lambda o: 12495,+ )+ assert _split(**kwargs) is None+ split = _split(**kwargs, phase_headroom_mib = (1516, 1160, 2645))+ assert split is not None and split.components is None+ assert split.primary_required_mib == 7630 + 1160 + 2048+ assert split.secondary_required_mib == 7660 + 2645 + 2048+++def test_force_splits_even_when_one_card_fits(monkeypatch):+ monkeypatch.setenv(mg.MULTIGPU_ENV, "force")+ assert _split(single_card_policy = "none") is not None+++def test_split_picks_the_secondary_with_the_most_room():+ split = _split(+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2)),+ secondary_limit_mib = lambda o: {1: 12750, 2: 34000}[o],+ )+ assert split.secondary_ordinal == 2+++class _Target:+ supports_model_cpu_offload: bool = True+ ordinal: int = 0+ device: str = "cuda"+ backend: str = "cuda"+ is_cuda_torch_device: bool = True+++def _memory(free = 24000, total = 24576):+ return dm.DeviceMemory(+ backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total+ )+++def _plan(**kw):+ args = dict(+ target = _Target(),+ device_memory = _memory(),+ model_dense_mib = 20000,+ companion_dense_mib = 9000,+ text_encoder_dense_mib = 8500,+ runtime_headroom_mib = 2000,+ )+ args.update(kw)+ return dm.plan_diffusion_memory(**args)+++def test_planner_unchanged_without_a_selection():+ plan = _plan()+ assert plan.offload_policy != dm.OFFLOAD_NONE+ assert plan.secondary_ordinal is None+++def test_planner_splits_under_a_selection(monkeypatch):+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)+ single = _plan()+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ plan = _plan()+ balanced = _plan(requested_mode = "balanced")+ legacy = _plan(explicit_offload = True)+ other = _plan(target = _Target(ordinal = 1))+ assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1+ assert plan.vae_tiling is False and plan.vae_slicing is False+ assert plan.as_public_dict()["secondary_ordinal"] == 1+ assert "multi-GPU" in plan.reasons[-1]+ for p in (balanced, legacy, other):+ assert p.secondary_ordinal is None+ assert balanced.offload_policy == dm.OFFLOAD_GROUP+ assert other.offload_policy == single.offload_policy+++def test_planner_split_declined_when_the_secondary_is_small(monkeypatch):+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 3400)+ single = _plan()+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ plan = _plan()+ assert plan == single+++def test_planner_split_off_on_unified_memory(monkeypatch):+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)+ unified = dm.DeviceMemory(+ backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 16000, total_mib = 16384+ )+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ plan = _plan(device_memory = unified)+ assert plan.secondary_ordinal is None+++def test_assignment_prefers_moving_the_companions():+ sizes = {+ "transformer": (10000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ moved = mg.choose_component_assignment(+ sizes,+ headroom_mib = 2000,+ overhead_mib = 1000,+ primary_limit_mib = 13500,+ secondary_limit_mib = 13000,+ )+ assert moved == ("text_encoder", "vae")+++def test_assignment_can_move_only_the_text_encoders():+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (11000, "text_encoder"),+ "vae": (1400, "other"),+ }+ moved = mg.choose_component_assignment(+ sizes,+ headroom_mib = 4000,+ overhead_mib = 1000,+ primary_limit_mib = 16000,+ secondary_limit_mib = 13500,+ )+ assert moved == ("text_encoder",)+++def test_place_split_resolves_text_encoders_only(monkeypatch):+ class Pipe:+ def __init__(self):+ self.transformer = torch.nn.Linear(2, 2)+ self.text_encoder = torch.nn.Linear(2, 2)+ self.text_encoder_2 = torch.nn.Linear(2, 2)+ self.vae = _Vae()++ @property+ def components(self):+ return dict(vars(self))++ monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)+ pipe = Pipe()+ real_device = torch.device+ monkeypatch.setattr(+ torch, "device", lambda *a: real_device("cpu") if a and a[0] == "cuda" else real_device(*a)+ )+ try:+ mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1, names = mg.TEXT_ENCODERS_ONLY)+ finally:+ monkeypatch.setattr(torch, "device", real_device)+ assert mg.placement_status(pipe)["moved"] == ["text_encoder", "text_encoder_2"]+ assert "decode" not in pipe.vae.__dict__+++def test_assignment_puts_the_second_expert_on_the_secondary():+ sizes = {+ "transformer": (12000, "dit"),+ "transformer_2": (12000, "dit"),+ "text_encoder": (5000, "text_encoder"),+ "vae": (500, "other"),+ }+ moved = mg.choose_component_assignment(+ sizes,+ headroom_mib = 2000,+ overhead_mib = 1000,+ primary_limit_mib = 21000,+ secondary_limit_mib = 16000,+ )+ assert moved == ("transformer_2",)+ moved = mg.choose_component_assignment(+ sizes,+ headroom_mib = 2000,+ overhead_mib = 1000,+ primary_limit_mib = 15500,+ secondary_limit_mib = 21000,+ )+ assert moved == ("transformer_2", "text_encoder", "vae")+ assert (+ mg.choose_component_assignment(+ sizes,+ headroom_mib = 2000,+ overhead_mib = 1000,+ primary_limit_mib = 9000,+ secondary_limit_mib = 9000,+ )+ is None+ )+++def test_refine_plan_for_multigpu(monkeypatch):+ plan = _plan()+ assert plan.offload_policy != dm.OFFLOAD_NONE+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)+ assert mg.refine_plan_for_multigpu(object(), plan) is plan+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ refined = mg.refine_plan_for_multigpu(object(), plan)+ assert refined.offload_policy == dm.OFFLOAD_NONE+ assert refined.secondary_ordinal == 1+ assert refined.secondary_components == ("text_encoder", "vae")+++class _Holder:+ def __init__(self, t):+ self.parameters = t+ self.mean = t * 2+ self.deterministic = False+++class _DC:+ sample: torch.Tensor+ note: str = "x"+++def test_move_to_device_keeps_container_types():+ t = torch.ones(2)+ Named = __import__("collections").namedtuple("Named", "a b")+ obj = {+ "tuple": (t, 1),+ "list": [t],+ "named": Named(t, "s"),+ "ordered": OrderedDict(x = t),+ "dc": _DC(t),+ "holder": _Holder(t),+ "plain": "keep",+ }+ out = mg.move_to_device(obj, torch.device("meta"))+ assert out["tuple"][0].device.type == "meta" and out["tuple"][1] == 1+ assert out["list"][0].device.type == "meta"+ assert (+ type(out["named"]) is Named+ and out["named"].a.device.type == "meta"+ and out["named"].b == "s"+ )+ assert isinstance(out["ordered"], OrderedDict) and out["ordered"]["x"].device.type == "meta"+ assert isinstance(out["dc"], _DC) and out["dc"].sample.device.type == "meta"+ assert isinstance(out["holder"], _Holder) and out["holder"].mean.device.type == "meta"+ assert out["holder"].deterministic is False and obj["holder"].parameters.device.type == "cpu"+ assert out["plain"] == "keep"+ assert mg.move_to_device(t, torch.device("cpu")) is t+++def test_model_output_moves_with_its_attributes():+ BaseOutput = _from_diffusers("diffusers.utils", "BaseOutput")++ @dataclasses.dataclass+ class Out(BaseOutput):+ sample: torch.Tensor = None++ out = mg.move_to_device(Out(sample = torch.ones(1)), torch.device("meta"))+ assert out.sample.device.type == "meta" and out[0].device.type == "meta"+++class _Vae(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(2, 2)+ self.seen = []++ def forward(self, x):+ self.seen.append(("forward", x.device))+ return self.lin(x)++ def decode(+ self,+ z,+ return_dict = False,+ ):+ self.seen.append(("decode", z.device))+ return (self.lin(z),)+++def test_boundary_moves_inputs_in_and_outputs_home(monkeypatch):+ vae = _Vae()+ home = torch.device("meta")+ moves = []++ def fake_move(+ obj,+ device,+ _depth = 0,+ ):+ moves.append(device)+ return obj++ monkeypatch.setattr(mg, "_crosses", lambda device, h: True)+ monkeypatch.setattr(mg, "move_to_device", fake_move)+ wrapped = mg.install_boundary(vae, home)+ assert set(wrapped) == {"forward", "decode"}+ vae.decode(torch.ones(1, 2))+ assert moves == [torch.device("cpu"), home]+ assert vae.seen[-1] == ("decode", torch.device("cpu"))+ vae(torch.ones(1, 2))+ assert moves[2:] == [torch.device("cpu"), home]+ assert mg.install_boundary(vae, home) == []+ mg.remove_boundary(vae)+ assert "decode" not in vae.__dict__ and "forward" not in vae.__dict__+++def test_boundary_is_inert_on_the_same_card():+ vae = _Vae()+ mg.install_boundary(vae, torch.device("cpu"))+ x = torch.ones(1, 2)+ assert torch.equal(vae(x), vae.lin(x))+++def test_execution_device_reports_the_primary_of_a_split_pipeline(monkeypatch):+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)+ DiffusionPipeline = _from_diffusers("diffusers", "DiffusionPipeline")++ class Pipe(DiffusionPipeline):+ def __init__(self, vae, transformer):+ super().__init__()+ self.register_modules(vae = vae, transformer = transformer)++ class Mod(torch.nn.Linear):+ @property+ def device(self):+ return self.weight.device++ vae = Mod(1, 1).to("meta")+ transformer = Mod(1, 1)+ pipe = Pipe(vae, transformer)+ assert mg.install_execution_device_patch()+ before = pipe.device+ vae._unsloth_split_home = torch.device("meta")+ assert pipe.device == torch.device("meta")+ del vae._unsloth_split_home+ transformer._unsloth_split_home = torch.device("cpu")+ assert pipe._execution_device == torch.device("cpu")+ assert pipe.device == torch.device("cpu")+ del transformer._unsloth_split_home+ assert pipe.device == before+++def test_place_split_and_fallback(monkeypatch):+ class Pipe:+ def __init__(self):+ self.transformer = torch.nn.Linear(2, 2)+ self.text_encoder = torch.nn.Linear(2, 2)+ self.vae = _Vae()+ self.scheduler = object()++ @property+ def components(self):+ return {+ "transformer": self.transformer,+ "text_encoder": self.text_encoder,+ "vae": self.vae,+ "scheduler": self.scheduler,+ }++ def to(self, device):+ for m in (self.transformer, self.text_encoder, self.vae):+ m.to(device)++ pipe = Pipe()+ assert mg.companion_names(pipe) == ("text_encoder", "vae")+ moved_to = {}+ real_device = torch.device++ def fake_device(*args):+ if args and args[0] == "cuda":+ return real_device("cpu")+ return real_device(*args)++ monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)+ monkeypatch.setattr(torch, "device", fake_device)+ try:+ placement = mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1)+ finally:+ monkeypatch.setattr(torch, "device", real_device)+ assert set(placement) == {"transformer", "text_encoder", "vae"}+ assert pipe.transformer._unsloth_split_home == torch.device("cpu")+ assert "decode" in pipe.vae.__dict__+ status = mg.placement_status(pipe)+ assert status["moved"] == ["text_encoder", "vae"]+ mg.undo_split(pipe)+ assert (+ "decode" not in pipe.vae.__dict__ and "_unsloth_split_home" not in pipe.transformer.__dict__+ )+ assert mg.placement_status(pipe) is None+ del moved_to++ @dataclasses.dataclass+ class P:+ secondary_ordinal: int = 1+ secondary_components: tuple = None++ def boom(*a, **k):+ raise RuntimeError("CUDA out of memory")++ monkeypatch.setattr(mg, "place_split", boom)+ assert mg.apply_split_plan(pipe, P(), placement = "cpu") is False+++def test_apply_memory_plan_falls_back_to_model_offload(monkeypatch):+ calls = []++ class Pipe:+ components: dict = {}++ def enable_model_cpu_offload(self, device = None):+ calls.append(("offload", device))++ plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE, secondary_ordinal = 1)+ monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)+ monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger = None: 0)+ pipe = Pipe()+ policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")+ assert policy == dm.OFFLOAD_MODEL and calls == [("offload", "cuda:0")]+ applied = dm.applied_plan(pipe, plan)+ assert applied.offload_policy == dm.OFFLOAD_MODEL and applied.secondary_ordinal is None++ placed = []+ monkeypatch.setattr(+ mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: placed.append(1) or True+ )+ policy, _tiling = dm.apply_memory_plan(Pipe(), plan, device = "cuda", placement_device = "cuda:0")+ assert policy == dm.OFFLOAD_NONE and placed == [1]+++class _Dit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4)+ self.calls = 0++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ self.calls += 1+ return (self.proj(hidden_states) * timestep + encoder_hidden_states.sum(),)+++def _router(dit, **kw):+ replicas = []++ def factory():+ r = mg.build_replica(dit, torch.device("cpu"))+ replicas.append(r)+ return r++ router = mg.CfgParallelRouter(+ dit, primary = "cpu", secondary = "cpu", replica_factory = factory, **kw+ )+ dit.forward = router+ return router, replicas+++def _cfg_loop(+ dit,+ latents,+ steps,+ branches = 2,+ scale = 4.0,+):+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))+ extra = torch.randn(3, 4, generator = torch.Generator().manual_seed(3))+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ outs = [+ dit(hidden_states = latents, timestep = t, encoder_hidden_states = e)[0]+ for e in (cond, uncond, extra)[:branches]+ ]+ if branches == 1:+ pred = outs[0]+ else:+ pred = outs[1] + scale * (outs[0] - outs[1])+ if branches == 3:+ pred = pred + 0.5 * (outs[0] - outs[2])+ latents = latents - 0.1 * pred+ return latents++[email protected]("branches", [2, 3])+def test_cfg_parallel_is_exact_and_routes_odd_branches(branches):+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)+ router, replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)+ assert torch.equal(out, reference)+ assert len(replicas) == 1 and replicas[0] is not dit+ assert router.stats["steps_parallel"] == 4+ assert router.stats["replica_calls"] == 4 * (branches // 2)+ assert router.describe()["disabled_reason"] is None+ assert replicas[0].proj.weight is not dit.proj.weight+ assert torch.equal(replicas[0].proj.weight, dit.proj.weight)+++def test_guidance_distilled_model_never_builds_a_replica():+ dit = _Dit()+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 6, branches = 1)+ assert replicas == [] and router.replica is None+ assert router.stats["replica_calls"] == 0+++def test_replica_failure_recomputes_on_the_primary():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, replicas = _router(dit)++ class Broken(torch.nn.Module):+ def forward(self, *a, **k):+ raise RuntimeError("replica kernel failed")++ router.replica_factory = lambda: Broken()+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert "replica call failed" in router.describe()["disabled_reason"]+ assert router.replica is None+++def test_replica_declined_when_a_submodule_hook_captures_the_source():+ dit = _Dit()+ source = dit.proj+ dit.proj.register_forward_hook(lambda m, i, o: o + source.bias)+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert replicas == []+ assert "proj._forward_hooks" in router.describe()["disabled_reason"]+++def test_portable_hooks_and_root_hooks_are_accepted():+ torch.manual_seed(0)+ dit = _Dit()+ scale = 0.5+ hook = lambda m, i, o: o * scale # noqa: E731 - captures only a float: shared safely+ hook.__module__ = mg.__name__+ dit.proj.register_forward_hook(hook)+ calls = []+ root_hook = lambda m, a: calls.append(1) # noqa: E731 - root: runs on the source around every branch+ root_hook.__module__ = mg.__name__+ dit.register_forward_pre_hook(root_hook)+ assert mg._hooks_present(dit) is None+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ n_ref = len(calls)+ router, replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert router.stats["replica_calls"] == 3+ # the root hook ran exactly once per denoiser call, routed or not (stripped from the copy)+ assert len(calls) == 2 * n_ref+ assert not replicas[0]._forward_pre_hooks+++def test_closure_forward_override_is_declined_but_bound_methods_are_rebound():+ import types++ dit = _Dit()+ source = dit.proj+ dit.proj.forward = lambda x: source.weight @ x # a closure would keep reading the source module+ assert "instance override" in mg._hooks_present(dit)+ dit2 = _Dit()+ dit2.proj.forward = types.MethodType(+ lambda self, x: torch.nn.functional.linear(x, self.weight), dit2.proj+ )+ assert mg._hooks_present(dit2) is None+ replica = mg.build_replica(dit2, torch.device("cpu"))+ assert replica.proj.forward.__self__ is replica.proj+++def test_replica_reapplies_recorded_compile(monkeypatch):+ mg.install_compile_recorder()+ dit = _Dit()+ dit.proj.compile(backend = "eager")+ assert dit.proj.__dict__["_unsloth_compile_args"] == ((), {"backend": "eager"})+ replica = mg.build_replica(dit, torch.device("cpu"))+ assert replica._unsloth_replica_compiled == 1+ assert replica.proj._compiled_call_impl is not None+ assert replica.proj._compiled_call_impl is not dit.proj._compiled_call_impl+ x = torch.randn(2, 4)+ assert torch.equal(replica.proj(x), dit.proj(x))+++def test_compiled_without_recorded_arguments_is_refused():+ dit = _Dit()+ dit.proj._compiled_call_impl = lambda *a, **k: None+ with pytest.raises(RuntimeError, match = "recorded compile"):+ mg.build_replica(dit, torch.device("cpu"))+++def test_worker_propagates_grad_mode_and_context_variables():+ var = __import__("contextvars").ContextVar("probe", default = "unset")+ worker = mg._ReplicaWorker(torch.device("cpu"))+ try:+ var.set("caller")+ with torch.inference_mode():+ seen = worker.call(+ lambda: (+ torch.is_inference_mode_enabled(),+ var.get(),+ threading.current_thread().name,+ )+ )+ assert seen[0] is True and seen[1] == "caller" and seen[2].startswith("unsloth-cfg-replica")+ with pytest.raises(ValueError):+ worker.call(lambda: (_ for _ in ()).throw(ValueError("boom")))+ finally:+ worker.close()+++def test_branch_tensors_are_prefetched_fresh_every_step():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, _replicas = _router(dit)+ copies = []+ real_prefetch = router._prefetch++ def spy(args, kwargs):+ real_prefetch(args, kwargs)+ copies.append(len(router.carry))++ router._prefetch = spy+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ # Prefetch runs before every primary branch from step 1. Step 1 has no carry yet (step 0 built the replica on its+ # odd branch); from step 2 the previous step's odd-branch tensors except its latent (timestep, uncond embeds).+ assert copies == [0, 2, 2]+++def test_cfg_parallel_decline_reasons(monkeypatch):+ class Pipe:+ transformer = _Dit()++ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ assert (+ mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = None)+ == "one card selected"+ )+ assert "offloaded" in mg.cfg_parallel_decline(Pipe(), offload_policy = "model", selection = sel)+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel)+ hooked = Pipe()+ hooked.transformer = _Dit()+ source = hooked.transformer.proj+ hooked.transformer.proj.register_forward_pre_hook(lambda m, a: source)+ assert "hooked" in mg.cfg_parallel_decline(hooked, offload_policy = "group", selection = sel)+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = sel)++ class Unet:+ transformer = None++ assert "U-Net" in mg.cfg_parallel_decline(Unet(), offload_policy = "none", selection = sel)+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "0")+ assert mg.CFG_PARALLEL_ENV in mg.cfg_parallel_decline(+ Pipe(), offload_policy = "none", selection = sel+ )+++def test_status_reports_a_declined_cfg_parallel():+ class Pipe:+ transformer = _Dit()++ pipe = Pipe()+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ reason = mg.install_cfg_parallel(pipe, offload_policy = "model")+ assert "offloaded" in reason+ assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": reason}}+ other = Pipe()+ assert mg.install_cfg_parallel(other, offload_policy = "none") == "one card selected"+ assert mg.placement_status(other) is None+++def test_replica_is_rebuilt_when_the_weights_change():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ router, replicas = _router(dit)+ _cfg_loop(dit, latents.clone(), steps = 3)+ assert len(replicas) == 1+ with torch.no_grad():+ dit.proj.weight.mul_(0.5)+ reference_router = dit.__dict__.pop("forward")+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ dit.forward = reference_router+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert router.stats["replica_rebuilds"] == 1 and len(replicas) == 2+ assert torch.equal(replicas[1].proj.weight, dit.proj.weight)+++def test_adapter_scale_changes_the_fingerprint():+ dit = _Dit()+ before = mg.weights_fingerprint(dit)+ dit.proj.scaling = {"default": 1.0}+ with_adapter = mg.weights_fingerprint(dit)+ dit.proj.scaling = {"default": 0.5}+ assert len({before, with_adapter, mg.weights_fingerprint(dit)}) == 3+++def test_with_load_selection_scopes_and_pops_the_ordinals():+ seen = {}++ class Backend:+ @mg.with_load_selection+ def _run_load(self, **kwargs):+ """body"""+ seen["kwargs"] = dict(kwargs)+ seen["selection"] = mg.current_selection()++ Backend()._run_load(gpu_ordinal = 1, _gpu_ordinals = [0, 1], repo_id = "x")+ assert seen["kwargs"] == {"gpu_ordinal": 1, "repo_id": "x"}+ assert seen["selection"] == mg.MultiGpuSelection(primary = 1, secondaries = (0,))+ assert mg.current_selection() is None+ Backend()._run_load(gpu_ordinal = 1, repo_id = "x")+ assert seen["selection"] is None+ import inspect++ assert '"""body"""' in inspect.getsource(Backend._run_load)+++def test_portability_looks_inside_captured_containers():+ holder = {"m": torch.nn.Linear(1, 1)}+ assert not mg._portable(lambda m, a: holder)+ plain = {"scale": 2.0}+ # even scalar-only containers are shared mutable state between the source and the replica+ assert not mg._portable(lambda m, a: plain)+ frozen = ("scale", 2.0)+ assert mg._portable(lambda m, a: frozen)+++class _BlockDit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(4, 4)+ self.blocks = torch.nn.ModuleList([torch.nn.Linear(4, 4) for _ in range(3)])++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ x = self.proj_in(hidden_states) * timestep+ for block in self.blocks:+ x = block(x) + encoder_hidden_states.sum()+ return (x,)+++def test_replica_drops_group_offload_and_stays_exact():+ apply_group_offloading = _from_diffusers("diffusers.hooks", "apply_group_offloading")++ torch.manual_seed(0)+ dit = _BlockDit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ apply_group_offloading(+ dit,+ onload_device = torch.device("cpu"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ )+ assert mg._hooks_present(dit) is None+ assert mg._registry_kind(dit) == "offload"+ router, replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert router.stats["replica_calls"] == 3+ replica = replicas[0]+ for sub in replica.modules():+ assert "_diffusers_hook" not in sub.__dict__+ assert "forward" not in sub.__dict__+++def test_step_cache_registry_is_declined():+ class Registry:+ hooks = {"first_block_cache": object()}++ dit = _Dit()+ dit._diffusers_hook = Registry()+ assert "first_block_cache" in mg._hooks_present(dit)+++def test_offload_torch_hooks_are_stripped_from_the_replica():+ dit = _Dit()+ source = dit.proj+ calls = []++ def prefetch_begin(module, args):+ calls.append(source)++ prefetch_begin.__module__ = "core.inference.diffusion_offload_prefetch"+ dit.proj.register_forward_pre_hook(prefetch_begin)+ assert mg._hooks_present(dit) is None+ replica = mg.build_replica(dit, torch.device("cpu"))+ assert not replica.proj._forward_pre_hooks+ replica(+ hidden_states = torch.ones(1, 4),+ timestep = torch.tensor(1.0),+ encoder_hidden_states = torch.ones(1, 4),+ )+ assert calls == []+++def test_predicted_branch_hits_and_stays_exact():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 6)+ router, _replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 6)+ assert torch.equal(out, reference)+ assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0+ assert router.stats["replica_calls"] == 5+++def test_mispredicted_branch_is_recomputed_exactly():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ uncond = torch.randn(3, 4)++ def loop(steps):+ x = latents.clone()+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]+ # a fresh negative tensor every step (equal values, new object): the prediction cannot be trusted+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = uncond.clone())[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ return x++ reference = loop(5)+ router, _replicas = _router(dit)+ out = loop(5)+ assert torch.equal(out, reference)+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4+ assert router.stats["replica_calls"] == 4+++def test_inline_recomputed_scalars_are_predicted_by_value():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ cond = torch.randn(3, 4)+ uncond = torch.randn(3, 4)++ def loop(steps):+ x = latents.clone()+ for i in range(steps):+ t = torch.tensor(1000.0 * (1.0 - i / steps))+ a = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = cond)[0]+ b = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = uncond)[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ return x++ reference = loop(6)+ router, _replicas = _router(dit)+ out = loop(6)+ assert torch.equal(out, reference)+ assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0+++def test_bitwise_equal_distinguishes_signed_zero():+ assert mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([0.0]))+ assert mg._bitwise_equal(torch.tensor(0.5), torch.tensor(1.0) / 2)+ assert not mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([-0.0]))+ assert not mg._bitwise_equal(torch.tensor([1.0]), torch.tensor([1.0], dtype = torch.float64))+++def test_failed_split_applies_the_single_card_plan(monkeypatch):+ calls = []++ class Pipe:+ components: dict = {}++ def enable_model_cpu_offload(self, device = None):+ calls.append(("model", device))++ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ plan = _plan()+ assert plan.secondary_ordinal == 1+ assert plan.fallback_plan is not None and plan.fallback_plan.offload_policy == dm.OFFLOAD_GROUP+ monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)+ monkeypatch.setattr(+ dm,+ "_apply_group_offload",+ lambda pipe, placement, logger, **kw: calls.append(("group", placement)) or True,+ )+ pipe = Pipe()+ policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")+ assert policy == dm.OFFLOAD_GROUP and calls == [("group", "cuda:0")]+ assert dm.applied_plan(pipe, plan) is plan.fallback_plan+++def test_post_load_split_keeps_the_original_plan_as_fallback(monkeypatch):+ plan = _plan()+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ refined = mg.refine_plan_for_multigpu(object(), plan)+ assert refined.fallback_plan is plan+++def test_hook_with_a_mutable_closure_counter_is_declined():+ dit = _Dit()+ counter = [0]++ def count(module, args):+ counter[0] += 1++ dit.proj.register_forward_pre_hook(count)+ assert "proj._forward_pre_hooks" in mg._hooks_present(dit)+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert replicas == [] and "hooks" in router.describe()["disabled_reason"]+++def test_replica_is_dropped_when_the_source_gains_a_cache_hook():+ # transformer_cache=auto installs a stateful step cache on the SOURCE at generate time, after the replica exists+ torch.manual_seed(0)+ dit = _Dit()+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert len(replicas) == 1++ class Registry:+ hooks = {"fbc_leader": object()}++ dit._diffusers_hook = Registry()+ _cfg_loop(dit, torch.randn(2, 4), steps = 2)+ assert router.replica is None+ assert "fbc_leader" in router.describe()["disabled_reason"]+++def test_post_load_split_keeps_vae_tiling(monkeypatch):+ plan = dataclasses.replace(_plan(), vae_tiling = True, vae_slicing = True)+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ refined = mg.refine_plan_for_multigpu(object(), plan)+ assert refined.secondary_ordinal == 1+ assert refined.vae_tiling is True and refined.vae_slicing is True+++def test_replica_keeps_a_bound_root_forward_override():+ import types++ torch.manual_seed(0)+ dit = _Dit()++ def doubled(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ return (2 * type(self).forward(self, hidden_states, timestep, encoder_hidden_states)[0],)++ dit.forward = types.MethodType(doubled, dit)+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert router.stats["replica_calls"] == 3+ assert replicas[0].forward.__self__ is replicas[0]+++def test_in_place_edit_after_launch_rejects_the_prediction():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ cond = torch.randn(3, 4)+ uncond = torch.randn(3, 4)++ def loop(steps):+ x = latents.clone()+ neg = uncond.clone()+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]+ neg.mul_(1.01) # same object, new values, after the prediction was launched+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ return x++ with torch.inference_mode():+ reference = loop(5)+ router, _replicas = _router(dit)+ out = loop(5)+ assert torch.equal(out, reference)+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4+ router.prefetched = {id(uncond): (uncond, uncond)}+ router.pending = {+ "leaves": [],+ "by_value": set(),+ "hook_state": None,+ "snapshots": [],+ "job": None,+ }+ assert router._take_prediction((), {"x": torch.zeros(1)}) is None+ assert router.prefetched == {}+++def test_deferred_speed_profile_invalidates_the_replica(monkeypatch):+ # The 3rd generation compiles / patches the source denoiser; a replica built during the eager images must go.+ import types as _types++ from core.inference.diffusion import DiffusionBackend++ torch.manual_seed(0)+ dit = _Dit()+ router, replicas = _router(dit)++ class Pipe:+ transformer = dit++ pipe = Pipe()+ mg._CFG_ROUTERS[pipe] = router+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert router.replica is not None+ state = _types.SimpleNamespace(+ pipe = pipe,+ speed_deferred = True,+ deferred_static_plan = None,+ resolved = {},+ transformer_cache = None,+ )+ fake_self = _types.SimpleNamespace(_engage_deferred_profile = lambda state: None)+ DiffusionBackend._engage_deferred_speed(fake_self, state)+ assert router.replica is None and router.disabled_reason is None+ _cfg_loop(dit, torch.randn(2, 4), steps = 2)+ assert len(replicas) == 2+ assert router.replica is not None++ def broken_profile(state):+ raise RuntimeError("compile failed")++ with pytest.raises(RuntimeError, match = "compile failed"):+ DiffusionBackend._engage_deferred_speed(+ _types.SimpleNamespace(_engage_deferred_profile = broken_profile), state+ )+ assert router.replica is None+++class _FlagLinear(torch.nn.Linear):+ def forward(self, x):+ out = super().forward(x)+ return -out if getattr(self, "_unsloth_flip", False) else out+++class _FlagDit(_Dit):+ def __init__(self):+ super().__init__()+ self.proj = _FlagLinear(4, 4)+++def _flag_pre_hook(module, args, kwargs):+ module.proj._unsloth_flip = bool(kwargs["encoder_hidden_states"].sum() < 0) != bool(+ kwargs["timestep"] < 0.5+ )+++def _flag_post_hook(module, args, output):+ module.proj.__dict__.pop("_unsloth_flip", None)+++_flag_pre_hook.__module__ = mg.__name__+_flag_post_hook.__module__ = mg.__name__+++def _plain_state_pre_hook(module, args, kwargs):+ # a foreign root hook keeping per-call state in an ordinary attribute, which the replica would never see+ module.proj.flag = bool(kwargs["encoder_hidden_states"].sum() < 0)+++def test_foreign_root_hook_declines_the_replica():+ dit = _FlagDit()+ dit.register_forward_pre_hook(_plain_state_pre_hook, with_kwargs = True)+ assert "a hook outside Studio" in mg._hooks_present(dit)+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert replicas == [] and router.stats["replica_calls"] == 0+++def test_root_hook_state_on_submodules_reaches_the_replica():+ torch.manual_seed(0)+ dit = _FlagDit()+ dit.register_forward_pre_hook(_flag_pre_hook, with_kwargs = True)+ dit.register_forward_hook(_flag_post_hook)+ latents = torch.randn(2, 4)+ cond = torch.ones(3, 4)+ neg = -torch.ones(3, 4)++ def loop(steps):+ x = latents.clone()+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ return x++ reference = loop(5)+ router, _replicas = _router(dit)+ out = loop(5)+ assert torch.equal(out, reference)+ assert router.stats["replica_calls"] == 4+++def test_runtime_controlnet_streams_beside_a_split_pipeline(monkeypatch):+ # A split reports policy "none", but its fit priced only the base pipeline: a ControlNet added at generate time+ # must stream as the single-card offload plan would have, not land whole on the primary.+ import types as _types++ from core.inference.diffusion import _controlnet_streams++ class Pipe:+ pass++ pipe = Pipe()+ state = _types.SimpleNamespace(pipe = pipe, offload_policy = "none", calibrated_placement = False)+ assert _controlnet_streams(state) is False # one card, resident: unchanged+ mg._PLACEMENTS[pipe] = {"moved": ["text_encoder", "vae"]}+ assert _controlnet_streams(state) is True+ assert _controlnet_streams(_types.SimpleNamespace(pipe = Pipe(), offload_policy = "group")) is True+++def test_generate_guard_checks_the_vae_card(monkeypatch):+ # The guard reads the primary's free memory; with the VAE on the secondary, the decode must be checked there.+ import types as _types++ from core.inference import diffusion as D++ seen = []++ @contextlib.contextmanager+ def scope(ordinal):+ seen.append(("scope", ordinal))+ yield++ def shortfall(**kw):+ seen.append(("guard", kw["device_memory"], kw.get("vae_tile_side")))+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)++ monkeypatch.setattr(D, "diffusion_device_scope", scope)+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)+ monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: ("restore", True))++ class Vae(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1))++ pipe = _types.SimpleNamespace(vae = Vae())+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)+ monkeypatch.setattr(mg, "is_split", lambda p: True)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))+ restore, tiled = D._guard_vae_card(+ pipe, target, {"device_memory": "card0-free", "vae_tile_side": 512}, None, None+ )+ assert ("scope", 1) in seen and ("guard", "card1-free", 512) in seen+ assert (restore, tiled) == ("restore", True)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))+ seen.clear()+ assert D._guard_vae_card(pipe, target, {"device_memory": "card0-free"}, None, None) == (+ None,+ False,+ )+ assert seen == []+++def test_secondary_of_lower_compute_capability_takes_no_component(monkeypatch):+ caps = {0: (8, 9), 1: (7, 5), 2: (8, 9), 3: (9, 0)}+ monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))+ assert mg.make_selection(0, [0, 1, 2, 3]) == mg.MultiGpuSelection(primary = 0, secondaries = (2, 3))+ assert mg.make_selection(0, [0, 1]) is None+ assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))+ assert mg.make_selection(0, [0, 7]) == mg.MultiGpuSelection(primary = 0, secondaries = (7,))+++class _Cache:+ """A prefix-KV style cache: a value holder whose layer tensors the denoiser reads and writes."""++ def __init__(self):+ self.keys = [torch.ones(4)]+ self.writers: list = []+++class _CacheDit(_Dit):+ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ kv_cache = None,+ return_dict = False,+ ):+ if kv_cache is not None:+ kv_cache.writers.append(threading.current_thread())+ hidden_states = hidden_states + kv_cache.keys[0]+ return super().forward(hidden_states, timestep, encoder_hidden_states)+++def test_branch_with_a_cache_object_stays_on_the_primary():+ torch.manual_seed(0)+ dit = _CacheDit()+ cond, uncond = torch.randn(3, 4), torch.randn(3, 4)++ def loop(latents):+ cache = _Cache()+ for i in range(4):+ t = torch.tensor(1.0 - i / 4)+ outs = [+ dit(hidden_states = latents, timestep = t, encoder_hidden_states = e, kv_cache = cache)[0]+ for e in (cond, uncond)+ ]+ latents = latents - 0.1 * (outs[1] + 4.0 * (outs[0] - outs[1]))+ return latents, cache++ latents = torch.randn(2, 4)+ reference, _ = loop(latents.clone())+ router, replicas = _router(dit)+ out, cache = loop(latents.clone())+ assert torch.equal(out, reference)+ # the mutable cache is never handed to the replica thread, and no replica is paid for+ assert set(cache.writers) == {threading.current_thread()}+ assert replicas == [] and router.stats["replica_calls"] == 0+ assert router.stats["opaque_primary_calls"] == 4+ assert router.describe()["disabled_reason"] is None+ assert not mg._holds_opaque_tensors(((torch.ones(1),), {"a": [torch.ones(1)], "b": None}))+ assert mg._holds_opaque_tensors({"kv_cache": _Cache()})+++def test_split_video_request_is_guarded_on_every_card():+ from core.inference import video as vid++ class Pipe:+ def __init__(self):+ self.components = {+ "transformer": torch.nn.Linear(1024, 1024),+ "text_encoder": torch.nn.Linear(1024, 512),+ "vae": torch.nn.Linear(512, 512),+ }++ pipe = Pipe()+ mg._PLACEMENTS[pipe] = {+ "placement": {"transformer": "cuda:0", "text_encoder": "cuda:1", "vae": "cuda:1"},+ "moved": ["text_encoder", "vae"],+ }+ cards = vid._video_split_card_floors(pipe)+ assert cards == ((0, 4, False, True, True), (1, 3, True, False, False))+ big = dict(width = 1280, height = 720, frames = 121)+ clip = vid.estimate_video_runtime_mib(width = 1280, height = 720, num_frames = 121)+ roomy = {0: 1 << 20, 1: 1 << 20}+ assert (+ vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **big) is None+ )+ # the decode card is priced for the decoded clip; the denoiser card for its weights, the base headroom and the+ # decoded frames copied back to it+ frames_mib = -(-1280 * 720 * 121 * 3 * 4 // (1 << 20))+ home_mib = max(frames_mib, vid._VIDEO_DENOISE_ACTIVATION_MIB)+ tight = {0: 4 + home_mib + dm.DEFAULT_BASE_OVERHEAD_MIB, 1: 3 + dm.DEFAULT_BASE_OVERHEAD_MIB}+ message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)+ assert clip > 0 and message is not None and "cuda:1" in message+ tight[1] = 1 << 20+ assert (+ vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big) is None+ )+ tight[0] -= 1+ message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)+ assert "cuda:0" in message+ tight[0] = 4 + dm.DEFAULT_BASE_OVERHEAD_MIB+ assert vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get) is None+ del mg._PLACEMENTS[pipe]+ assert vid._video_split_card_floors(pipe) is None+++def test_primary_tile_verdict_does_not_tile_a_vae_decoding_elsewhere(monkeypatch):+ import types as _types++ from core.inference import diffusion as D++ engaged = []+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")+ # the VAE's card decodes untiled; only the smaller primary asks for tiling+ monkeypatch.setattr(+ D,+ "raise_on_image_activation_shortfall",+ lambda **kw: _types.SimpleNamespace(action = "ok", overridden = False),+ )+ monkeypatch.setattr(+ D, "engage_vae_tiling", lambda pipe, logger = None: engaged.append(1) or ("restore", True)+ )+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)+ monkeypatch.setattr(mg, "is_split", lambda p: True)+ primary_verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)+ kwargs = {"device_memory": "card0-free", "vae_tile_side": 512, "vae_sliced": False}+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))+ assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) is None+ assert engaged == []+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))+ assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) == "restore"+ assert engaged == [1]+++def test_secondary_free_memory_reads_the_trusted_probe(monkeypatch):+ import utils.hardware as hw++ seen = []+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *a: (80 << 30, 80 << 30))+ monkeypatch.setattr(+ hw,+ "trusted_mem_get_info",+ lambda device = None, module = None: seen.append(device) or (1 << 30, 80 << 30),+ )+ assert mg._free_total_mib(1) == (1024, 80 * 1024)+ assert seen == [1]+++_GLOBAL_HOOK_CALLS = []+++def _global_state_hook(module, inputs, output):+ _GLOBAL_HOOK_CALLS.append(1)+ return output * (1.0 + len(_GLOBAL_HOOK_CALLS) % 2)+++def test_foreign_submodule_hook_declines_the_replica():+ dit = _Dit()+ dit.proj.register_forward_hook(_global_state_hook)+ assert _global_state_hook.__closure__ is None+ assert mg._hooks_present(dit) == "proj._forward_hooks"+ router, replicas = _router(dit)+ _cfg_loop(dit, torch.randn(2, 4), steps = 3)+ assert replicas == [] and router.stats["replica_calls"] == 0+ _global_state_hook.__module__, saved = mg.__name__, _global_state_hook.__module__+ try:+ assert mg._hooks_present(_Dit()) is None+ hooked = _Dit()+ hooked.proj.register_forward_hook(_global_state_hook)+ assert mg._hooks_present(hooked) is None+ finally:+ _global_state_hook.__module__ = saved+++def test_force_mode_refines_a_resident_plan(monkeypatch):+ plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE)+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ assert mg.refine_plan_for_multigpu(object(), plan) is plan+ monkeypatch.setenv(mg.MULTIGPU_ENV, "force")+ refined = mg.refine_plan_for_multigpu(object(), plan)+ assert refined.secondary_ordinal == 1 and refined.offload_policy == dm.OFFLOAD_NONE+ assert refined.secondary_components == ("text_encoder", "vae")+ assert refined.fallback_plan is plan+++def test_ltx25_sidecar_encoder_keeps_the_single_card_plan(monkeypatch):+ import types as _types++ from core.inference import video as vid++ single = _plan()+ split = dataclasses.replace(+ single, offload_policy = dm.OFFLOAD_NONE, secondary_ordinal = 1, fallback_plan = single+ )+ ltx25 = _types.SimpleNamespace(name = "ltx-2.5")+ wan = _types.SimpleNamespace(name = "wan2.2")+ assert vid._video_single_card_for_sidecar(ltx25, split) is single+ assert vid._video_single_card_for_sidecar(wan, split) is split+ assert vid._video_single_card_for_sidecar(ltx25, single) is single+ refined = []+ monkeypatch.setattr(+ vid, "refine_plan_for_multigpu", lambda pipe, plan, logger = None: refined.append(1) or split+ )+ assert vid._video_refine_for_multigpu(ltx25, object(), single) is single+ assert vid._video_refine_for_multigpu(wan, object(), single) is split and refined == [1]+++def _fake_cuda_host(+ monkeypatch,+ *,+ count,+ ranking_possible = True,+ current = 0,+):+ from core.inference import diffusion_device as dd++ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: ranking_possible)+ monkeypatch.setattr(+ dd, "resolve_diffusion_device_target", lambda **kw: _Target(ordinal = kw.get("ordinal"))+ )+ monkeypatch.setattr(torch.cuda, "device_count", lambda: count)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: current)+ monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))+ from core.inference import gpu_residency++ monkeypatch.setattr(gpu_residency, "current_policy", lambda: "off")+++def test_automatic_selection_uses_every_visible_card(monkeypatch):+ _fake_cuda_host(monkeypatch, count = 3)+ assert mg.automatic_selection(None) == mg.MultiGpuSelection(+ primary = 0, secondaries = (1, 2), automatic = True+ )+ assert mg.automatic_selection(2) == mg.MultiGpuSelection(+ primary = 2, secondaries = (0, 1), automatic = True+ )+ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")+ assert mg.automatic_selection(None) is None+++def test_automatic_selection_is_inert_on_ordinary_hosts(monkeypatch):+ # one GPU, ROCm, XPU, MPS and CPU all fail the ranking gate; a gated host never touches torch.cuda+ _fake_cuda_host(monkeypatch, count = 2, ranking_possible = False)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: pytest.fail("probed a gated host"))+ assert mg.automatic_selection(None) is None+ _fake_cuda_host(monkeypatch, count = 1)+ assert mg.automatic_selection(None) is None+++def test_with_load_selection_resolves_the_automatic_marker(monkeypatch):+ _fake_cuda_host(monkeypatch, count = 2)+ seen = {}++ class Backend:+ @mg.with_load_selection+ def _run_load(self, **kwargs):+ seen["kwargs"] = dict(kwargs)+ seen["selection"] = mg.current_selection()++ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS, repo_id = "x")+ assert seen["kwargs"] == {"gpu_ordinal": None, "repo_id": "x"}+ assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)+++def test_automatic_load_splits_only_when_one_card_would_offload(monkeypatch):+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)+ target = _Target(ordinal = None)+ single = _plan(target = target)+ with mg.selection_scope(selection):+ plan = _plan(target = target)+ fits = _plan(+ target = target,+ model_dense_mib = 4000,+ companion_dense_mib = 2000,+ text_encoder_dense_mib = 1500,+ )+ assert single.offload_policy != dm.OFFLOAD_NONE and single.secondary_ordinal is None+ assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1+ assert fits.secondary_ordinal is None+++def test_automatic_placement_does_not_arm_cfg_parallel(monkeypatch):+ class Pipe:+ transformer = _Dit()++ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)+ assert "automatic" in mg.cfg_parallel_decline(+ Pipe(), offload_policy = "none", selection = automatic+ )+ monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "force")+ assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = automatic)++[email protected]("module_file", ["diffusion.py", "video.py"])+def test_begin_load_marks_an_unpicked_load_automatic(module_file):+ import ast+ from pathlib import Path++ path = Path(__file__).resolve().parents[1] / "core" / "inference" / module_file+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ begin_loads = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.FunctionDef) and node.name == "begin_load"+ ]+ assert len(begin_loads) == 1+ assigned = [+ node+ for node in ast.walk(begin_loads[0])+ if isinstance(node, ast.Assign)+ and any(getattr(t, "id", None) == "gpu_ordinals" for t in node.targets)+ ]+ assert len(assigned) == 1+ value = assigned[0].value+ assert isinstance(value, ast.IfExp) and isinstance(value.orelse, ast.IfExp)+ assert ast.unparse(value.orelse.body) == "AUTOMATIC_ORDINALS"+ assert ast.unparse(value.orelse.test) == "not gpu_ids"+ assert ast.unparse(value.orelse.orelse) == "[]"+++def test_automatic_selection_leaves_module_compile_alone(monkeypatch):+ def original(self, *args, **kwargs):+ return self++ monkeypatch.setattr(torch.nn.Module, "compile", original)+ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)+ with mg.selection_scope(automatic):+ assert torch.nn.Module.compile is original+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):+ assert getattr(torch.nn.Module.compile, "_unsloth_recorder", False)+++def test_automatic_secondaries_are_sized_without_a_cuda_context(monkeypatch):+ from core.inference import diffusion_device as dd++ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda *a: pytest.fail("opened a context on a secondary")+ )+ monkeypatch.setattr(+ mg, "probe_devices", lambda *a, **k: pytest.fail("probed every card through torch")+ )+ readings = {1: (9000, 15360), 2: (14000, 15360)}+ monkeypatch.setattr(dd, "smi_free_total_mib", lambda o: readings.get(o, (None, None)))+ automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2), automatic = True)+ with mg.selection_scope(automatic):+ assert mg.best_secondary(automatic) == 2+ memory = mg.secondary_device_memory(2)+ assert (memory.free_mib, memory.total_mib) == (14000, 15360)+ assert mg.secondary_device_memory(3) is None+++def test_smi_reading_keys_ordinals_through_the_mask(monkeypatch):+ import utils.hardware.hardware as hw+ from core.inference import diffusion_device as dd++ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [4, 5])+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda visible: ({}, ""))+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": [4, 5]})+ rows = [+ {"index": 4, "vram_total_gb": 24.0, "vram_used_gb": 20.0},+ {"index": 5, "vram_total_gb": 24.0, "vram_used_gb": 1.0},+ ]+ monkeypatch.setattr(hw, "_smi_visible_utilization", lambda spec: {"devices": rows})+ assert dd.smi_free_total_mib(1) == (23 * 1024, 24 * 1024)+ assert dd.smi_free_total_mib(2) == (None, None)+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {5})+ assert dd.smi_free_total_mib(1) == (None, None)+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)+ assert dd.smi_free_total_mib(1) == (None, None)+++def test_unsplit_automatic_load_has_no_other_vae_card(monkeypatch):+ import types as _types++ from core.inference import diffusion as D++ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))+ automatic = _types.SimpleNamespace(ordinal = None, is_cuda_torch_device = True)+ assert D._vae_on_other_card(pipe, automatic) is None+ monkeypatch.setattr(mg, "is_split", lambda p: True)+ assert D._vae_on_other_card(pipe, automatic) is None+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))+ assert D._vae_on_other_card(pipe, automatic) == torch.device("cuda", 1)+++def test_secondary_vae_guard_refuses_a_batch_slicing_did_not_engage(monkeypatch):+ import types as _types++ from core.inference import diffusion as D++ calls = []++ def shortfall(**kw):+ calls.append(kw.get("vae_sliced"))+ if kw.get("vae_sliced") is False:+ raise ValueError("does not fit unsliced")+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)++ restored = []+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)+ monkeypatch.setattr(+ D, "engage_vae_tiling", lambda pipe, logger = None: (lambda: restored.append(1), True)+ )+ monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)+ monkeypatch.setattr(mg, "is_split", lambda p: True)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)+ with pytest.raises(ValueError):+ D._guard_vae_card(pipe, target, {"batch_size": 2, "vae_sliced": True}, None, None)+ assert restored == [1] and calls == [True, False]++[email protected]("vae_card", [0, 1])+def test_a_broken_recheck_keeps_the_tiling_it_engaged(monkeypatch, vae_card):+ """A re-check that raises something other than the refusal fails open, as generate() always did: the decode+ stays tiled (the guard said only a tiled decode fits) and the handle comes back for the caller to restore."""+ import types as _types++ from core.inference import diffusion as D++ def shortfall(**kw):+ if kw.get("vae_sliced") is False:+ raise RuntimeError("probe broke")+ return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)++ restored = []+ restore = lambda: restored.append(1) # noqa: E731+ monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())+ monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "free")+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)+ monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: (restore, True))+ monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)+ monkeypatch.setattr(mg, "is_split", lambda p: True)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", vae_card))+ pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)+ kwargs = {"batch_size": 2, "vae_sliced": True, "vae_tile_side": 512, "device_memory": "free"}+ verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)+ assert D._tile_for_verdict(pipe, verdict, target, kwargs, 2, None, None) is restore+ assert restored == []+++def test_no_split_ever_placed_skips_the_component_walk(monkeypatch):+ reads = []++ class Pipe:+ @property+ def components(self):+ reads.append(1)+ return {}++ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", False)+ assert mg._split_home(Pipe()) is None and reads == []+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)+ assert mg._split_home(Pipe()) is None and reads == [1]+++def test_a_card_torch_cannot_address_is_not_a_secondary(monkeypatch):+ """nvidia-smi can list two cards while this process's torch addresses one: no second card, no selection."""+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))+ assert mg.make_selection(0, [0, 1]) is None+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))+++def test_refinement_keeps_an_explicit_cpu_offload(monkeypatch):+ sizes = {+ "transformer": (9000, "dit"),+ "text_encoder": (8000, "text_encoder"),+ "vae": (300, "other"),+ }+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)+ monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)+ monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None: 1)+ legacy = _plan(explicit_offload = True)+ assert legacy.estimates.get("explicit_cpu_offload")+ moded = _plan(explicit_offload = True, requested_mode = "auto")+ assert not moded.estimates.get("explicit_cpu_offload")+ with mg.selection_scope(mg.make_selection(0, [0, 1])):+ assert mg.refine_plan_for_multigpu(object(), legacy) is legacy+ assert mg.refine_plan_for_multigpu(object(), moded).secondary_ordinal == 1+++def test_render_end_releases_the_branch_tensors():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ router, replicas = _router(dit)++ class Pipe:+ transformer = dit++ pipe = Pipe()+ mg._CFG_ROUTERS[pipe] = router+ _cfg_loop(dit, latents.clone(), steps = 3)+ assert router.template is not None and router.prev_latent is not None+ mg.end_cfg_render(pipe)+ assert router.template is None and router.prev_latent is None and router.pending is None+ assert router.cfg_seen is False+ assert router.prefetched == {} and router.carry == [] and router.b0_leaves == {}+ assert router.replica is not None+ reference_router = dit.__dict__.pop("forward")+ reference = _cfg_loop(dit, latents.clone(), steps = 3)+ dit.forward = reference_router+ assert torch.equal(_cfg_loop(dit, latents.clone(), steps = 3), reference)+ assert len(replicas) == 1+ mg.end_cfg_render(object())+++def test_workflow_pipeline_of_a_split_still_guards_the_vae_card(monkeypatch):+ monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)+ import types as _types++ from core.inference import diffusion as D++ dit = torch.nn.Linear(1, 1)+ dit._unsloth_split_home = torch.device("cuda", 0)+ sibling = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1), components = {"transformer": dit})+ target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))+ assert not mg.is_split(sibling)+ assert D._vae_on_other_card(sibling, target) == torch.device("cuda", 1)+++def test_cfg_replica_avoids_the_card_a_split_decodes_on(monkeypatch):+ class Pipe:+ transformer = _Dit()++ probes = {+ o: mg.DeviceProbe(ordinal = o, name = "B200", total_mib = 180000, free_mib = 170000)+ for o in (0, 1, 2)+ }+ monkeypatch.setattr(mg, "cfg_parallel_decline", lambda pipe, **kw: None)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))+ monkeypatch.setattr(+ mg, "probe_devices", lambda ordinals, **kw: {o: probes[o] for o in ordinals}+ )+ pipe = Pipe()+ mg._PLACEMENTS[pipe] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}+ try:+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):+ reason = mg.install_cfg_parallel(pipe, offload_policy = "none")+ assert reason is not None and "holds the split's components" in reason+ te_only = Pipe()+ mg._PLACEMENTS[te_only] = {+ "placement": {"transformer": "cuda:0", "vae": "cuda:0", "text_encoder": "cuda:1"}+ }+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):+ assert "holds the split's components" in mg.install_cfg_parallel(+ te_only, offload_policy = "none"+ )+ other = Pipe()+ mg._PLACEMENTS[other] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1, 2))):+ assert mg.install_cfg_parallel(other, offload_policy = "none") is None+ assert str(mg._CFG_ROUTERS[other].secondary) == "cuda:2"+ finally:+ mg.uninstall_cfg_parallel(pipe)+ mg.uninstall_cfg_parallel(other)+++def test_background_compile_declines_cfg_parallel(monkeypatch):+ class Pipe:+ transformer = _Dit()++ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ monkeypatch.setenv("UNSLOTH_DIFFUSION_BG_COMPILE", "1")+ assert "background compile" in mg.cfg_parallel_decline(+ Pipe(), offload_policy = "none", selection = sel+ )+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE")+ assert "background compile" not in mg.cfg_parallel_decline(+ Pipe(), offload_policy = "none", selection = sel+ )+++def test_a_displaced_router_is_reported_declined():+ dit = _Dit()+ router, _replicas = _router(dit)++ class Pipe:+ transformer = dit++ pipe = Pipe()+ mg._CFG_ROUTERS[pipe] = router+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is False+ graphed = lambda *a, **k: None # noqa: E731+ dit.__dict__["forward"] = graphed+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is True+ assert dit.__dict__["forward"] is graphed+ assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": "graphs"}}+ assert mg.drop_displaced_cfg_router(pipe, "graphs") is False+++def test_kept_models_keep_every_media_load_on_one_card(monkeypatch):+ from core.inference import gpu_residency++ _fake_cuda_host(monkeypatch, count = 2)+ seen = {}++ class Backend:+ @mg.with_load_selection+ def _run_load(self, **kwargs):+ seen["selection"] = mg.current_selection()++ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)+ assert seen["selection"] is not None+ Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])+ assert seen["selection"] is not None+ for mode in ("fit", "always"):+ monkeypatch.setattr(gpu_residency, "current_policy", lambda mode = mode: mode)+ Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)+ assert seen["selection"] is None+ Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])+ assert seen["selection"] is None+++def test_status_models_keep_the_multi_gpu_report():+ from models.inference import DiffusionStatusResponse, VideoStatusResponse++ report = {"primary": "cuda:0", "secondary": "cuda:1", "moved": ["text_encoder"]}+ assert (+ DiffusionStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report+ )+ assert VideoStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report++[email protected]("module_file", ["video.py"])+def test_video_split_fallback_recomputes_the_floor(module_file):+ import ast+ from pathlib import Path++ source = (Path(__file__).resolve().parents[1] / "core" / "inference" / module_file).read_text(+ encoding = "utf-8"+ )+ tree = ast.parse(source)+ hits = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.If)+ and "secondary_ordinal" in ast.unparse(node.test)+ and "plan is not planned" in ast.unparse(node.test)+ and any(+ isinstance(b, ast.Assign) and getattr(b.targets[0], "id", None) == "vram_floor_mib"+ for b in node.body+ )+ ]+ assert len(hits) == 1+++def test_a_split_refuses_a_controlnet_it_cannot_stream():+ # The resident fallback would put the whole ControlNet beside a denoiser one card could not hold with it.+ import ast+ from pathlib import Path++ path = Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py"+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ tests = [+ ast.unparse(node.test)+ for node in ast.walk(tree)+ if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)+ ]+ assert len(tests) == 1+ branch = next(+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)+ )+ refusal = branch.orelse[0]+ assert (+ isinstance(refusal, ast.If)+ and ast.unparse(refusal.test) == "calibrated or is_split(state.pipe)"+ )+ assert any(isinstance(n, ast.Raise) for n in ast.walk(refusal))+++def test_replica_rebuild_counts_the_allocator_cache_as_free(monkeypatch):+ import types as _types++ dit = _Dit()+ router, _replicas = _router(dit)+ router.secondary = torch.device("cuda", 1)+ monkeypatch.setattr(mg, "_free_total_mib", lambda o: (100, 15360))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 4096 << 20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda d = None: 0)+ monkeypatch.setattr(mg, "_device_context", lambda d: contextlib.nullcontext())+ monkeypatch.setattr(+ mg, "_ReplicaWorker", lambda device: _types.SimpleNamespace(close = lambda: None)+ )+ router._build_replica()+ assert router.replica is not None and router.disabled_reason is None+ router.replica = None+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 0)+ router._build_replica()+ assert router.replica is None and "MiB free" in (router.disabled_reason or "")+++def test_a_two_expert_video_split_prices_the_denoise_phase_on_both_cards():+ from core.inference import video as vid++ class Pipe:+ def __init__(self):+ self.components = {+ "transformer": torch.nn.Linear(1024, 1024),+ "transformer_2": torch.nn.Linear(1024, 1024),+ "vae": torch.nn.Linear(512, 512),+ }++ pipe = Pipe()+ mg._PLACEMENTS[pipe] = {+ "placement": {"transformer": "cuda:0", "vae": "cuda:0", "transformer_2": "cuda:1"},+ "moved": ["transformer_2"],+ }+ try:+ cards = vid._video_split_card_floors(pipe)+ assert cards == ((0, 5, True, True, True), (1, 4, False, False, True))+ shape = dict(width = 832, height = 480, frames = 25)+ roomy = {0: 1 << 20, 1: 1 << 20}+ assert (+ vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **shape)+ is None+ )+ # the expert card has room for its weights but not the denoise activations+ expert_short = {0: 1 << 20, 1: 4 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100}+ message = vid.video_split_shortfall_message(+ cards, family = "f", available_mib = expert_short.get, **shape+ )+ assert message is not None and "cuda:1" in message+ primary_short = {0: 5 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100, 1: 1 << 20}+ message = vid.video_split_shortfall_message(+ cards, family = "f", available_mib = primary_short.get, **shape+ )+ assert message is not None and "cuda:0" in message+ finally:+ del mg._PLACEMENTS[pipe]+++def test_untiled_decode_probes_the_card_the_vae_lives_on(monkeypatch):+ import types as _types++ from core.inference import video_vae_untiled as vu++ vae = torch.nn.Linear(1, 1)+ latent = _types.SimpleNamespace(device = torch.device("cuda", 0))+ monkeypatch.setattr(+ torch.nn.Linear,+ "parameters",+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 1))]),+ )+ assert vu._decode_device(vae, latent) == torch.device("cuda", 1)+ # same card, or an offloaded VAE whose weights sit on the host: the latent's card, as before+ monkeypatch.setattr(+ torch.nn.Linear,+ "parameters",+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 0))]),+ )+ assert vu._decode_device(vae, latent) == torch.device("cuda", 0)+ monkeypatch.setattr(+ torch.nn.Linear,+ "parameters",+ lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cpu"))]),+ )+ assert vu._decode_device(vae, latent) == torch.device("cuda", 0)+++def test_untiled_decode_sizes_the_card_the_vae_decodes_on(monkeypatch):+ import types as _types++ from core.inference import video_vae_untiled as vu++ family = next(iter({**vu._BYTES_PER_LATENT_PIXEL, **vu._BYTES_PER_FRAME_LATENT_PIXEL}))+ probed = []+ monkeypatch.setattr(vu, "_free_bytes", lambda device: probed.append(device))++ class Vae:+ use_tiling = True++ def decode(self, z, *args, **kwargs):+ return "tiled"++ def parameters(self):+ return iter([_types.SimpleNamespace(device = torch.device("cuda", 1))])++ pipe = _types.SimpleNamespace(vae = Vae())+ assert vu.install_untiled_decode(pipe, family)+ latent = _types.SimpleNamespace(device = torch.device("cuda", 0), shape = (1, 16, 3, 8, 8))+ assert pipe.vae.decode(latent) == "tiled"+ assert probed == [torch.device("cuda", 1)]++[email protected]("module_name", ["diffusion", "video"])+def test_begin_load_hands_the_picked_cards_to_the_load_thread(monkeypatch, module_name):+ import importlib+ import types as _types++ backend_mod = importlib.import_module(f"core.inference.{module_name}")+ backend_cls = (+ backend_mod.DiffusionBackend if module_name == "diffusion" else backend_mod.VideoBackend+ )+ seen = {}++ def first_load_step(*args, **kwargs):+ seen["selection"] = mg.current_selection()+ raise RuntimeError("stop after the selection is read")++ class InlineThread:+ def __init__(+ self,+ target,+ kwargs,+ daemon = None,+ ):+ self.target, self.kwargs = target, kwargs++ def start(self):+ self.target(**self.kwargs)++ monkeypatch.setattr(backend_mod, "account_thread", InlineThread)+ monkeypatch.setattr(+ backend_mod,+ "resolve_diffusion_device_target",+ lambda *a, **k: _types.SimpleNamespace(device = "cuda"),+ )+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinal", lambda ids: 0)+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", lambda ids: [0, 1])+ monkeypatch.setattr(mg, "_capability", lambda ordinal: None)+ monkeypatch.setattr(mg, "models_kept_loaded", lambda: False)+ fam = _types.SimpleNamespace(name = "ltx-2", base_repo = "base/repo", single_file_is_pipeline = False)+ monkeypatch.setattr(backend_cls, "validate_load_request", lambda self, *a, **k: fam)+ if module_name == "diffusion":+ monkeypatch.setattr(backend_cls, "assert_precision_available", lambda self, *a, **k: None)+ monkeypatch.setattr(backend_mod, "detect_family_for_pick", first_load_step)+ else:+ from core.inference import video_minimax_h3++ monkeypatch.setattr(backend_mod, "assert_video_precision_available", lambda *a, **k: None)+ monkeypatch.setattr(backend_mod, "_ltx23_prequant_serves_on_card", lambda *a, **k: False)+ monkeypatch.setattr(video_minimax_h3, "is_h3_native", lambda *a, **k: False)+ monkeypatch.setattr(backend_mod, "_detect_load_family", first_load_step)++ backend_cls().begin_load("org/model", gpu_ids = [0, 1])+ assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ # and only for that load+ assert mg.current_selection() is None++ # the kill switch: the extra cards are never even resolved (the single pick is, as before the multi-GPU layer)+ def never(ids):+ raise AssertionError("resolved several cards under UNSLOTH_DIFFUSION_MULTIGPU=0")++ monkeypatch.setenv(mg.MULTIGPU_ENV, "0")+ monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", never)+ seen.clear()+ backend_cls().begin_load("org/model", gpu_ids = [0, 1])+ assert seen["selection"] is None+++def test_fingerprint_sees_buffers_and_the_peft_adapter_selection():+ dit = _Dit()+ dit.proj.register_buffer("w_scale", torch.ones(4))+ base = mg.weights_fingerprint(dit)+ dit.proj.w_scale.mul_(2.0)+ edited = mg.weights_fingerprint(dit)+ assert edited != base+ dit.proj.w_scale = torch.ones(4)+ assert mg.weights_fingerprint(dit) not in (base, edited)+ before = mg.weights_fingerprint(dit)+ dit.proj._active_adapter = ["a"]+ with_a = mg.weights_fingerprint(dit)+ dit.proj._active_adapter = ["b"]+ assert len({before, with_a, mg.weights_fingerprint(dit)}) == 3+++def test_replica_shares_the_nvfp4_step_controller():+ from core.inference.diffusion_nvfp4_protect import NVFP4StepController++ dit = _Dit()+ ctl = NVFP4StepController("0")+ dit.proj.protect = ctl+ replica = mg.build_replica(dit, torch.device("cpu"))+ assert replica.proj.protect is ctl+++def test_boundary_and_router_removal_restore_an_instance_override():+ import types++ dit = _Dit()++ def own_forward(self, *args, **kwargs):+ return ("own",)++ override = types.MethodType(own_forward, dit)+ dit.forward = override+ mg.install_boundary(dit, "cpu")+ assert dit.__dict__["forward"] is not override+ mg.remove_boundary(dit)+ assert dit.__dict__["forward"] is override+ plain = _Dit()+ mg.install_boundary(plain, "cpu")+ mg.remove_boundary(plain)+ assert "forward" not in plain.__dict__++ class Pipe:+ pass++ pipe = Pipe()+ router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu")+ dit.forward = router+ mg._CFG_ROUTERS[pipe] = router+ mg.uninstall_cfg_parallel(pipe)+ assert dit.__dict__["forward"] is override+++def test_uninstalled_router_left_inside_a_wrapper_is_a_passthrough():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, replicas = _router(dit)++ class Skip:+ def __init__(self, inner):+ self.inner = inner++ def __call__(self, *args, **kwargs):+ return self.inner(*args, **kwargs)++ dit.forward = Skip(router)++ class Pipe:+ pass++ pipe = Pipe()+ mg._CFG_ROUTERS[pipe] = router+ assert mg.drop_displaced_cfg_router(pipe, "replaced")+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert replicas == [] and router.replica is None and router.prev_latent is None+ assert router.stats["replica_calls"] == 0+++def test_router_stays_on_the_source_for_a_forced_eager_render_and_the_compile_thread():+ from core.inference import diffusion_bg_compile as bg++ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, replicas = _router(dit)+ for var in (bg._FORCE_EAGER, bg._NO_CAPTURE):+ token = var.set(True)+ try:+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ finally:+ var.reset(token)+ assert torch.equal(out, reference)+ assert replicas == [] and router.stats["replica_calls"] == 0 and router.prev_latent is None+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference) and router.stats["replica_calls"] == 3+++def test_replica_blocks_get_their_own_compile_guard():+ from core.inference.diffusion_speed import guard_compiled_blocks++ mg.install_compile_recorder()+ dit = _Dit()+ dit.proj.compile(backend = "eager") # lazy: nothing is traced here+ assert guard_compiled_blocks(dit) == 1+ source_guard = dit._unsloth_compile_guard+ replica = mg.build_replica(dit, torch.device("cpu"))+ wrapped = replica.proj._compiled_call_impl+ guard = getattr(wrapped, "_unsloth_compile_guard", None)+ assert guard is not None and guard is not source_guard+ assert replica.__dict__.get("_unsloth_compile_guard") is guarddiff --git a/studio/backend/tests/test_diffusion_multigpu_cuda.py b/studio/backend/tests/test_diffusion_multigpu_cuda.pynew file mode 100644index 00000000000..ab25ed31e77--- /dev/null+++ b/studio/backend/tests/test_diffusion_multigpu_cuda.py@@ -0,0 +1,104 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CFG-parallel on two real CUDA cards. On one device ``.to()`` aliases storage, so the CPU tests in+test_diffusion_multigpu.py cannot see a replica that diverged from its source; these can. Skipped below two cards."""++from __future__ import annotations++import pytest+import torch++from core.inference import diffusion_multigpu as mg++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs two CUDA cards"+)+++class _Dit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4)+ self.register_buffer("scale", torch.ones(4))++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ return ((self.proj(hidden_states) * self.scale) * timestep + encoder_hidden_states.sum(),)+++def _render(+ dit,+ latents,+ cond,+ neg,+ steps,+ between = None,+):+ x = latents.clone()+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps, device = x.device)+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]+ if between is not None:+ between(i, x)+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ return x+++def _setup():+ torch.manual_seed(0)+ dit = _Dit().cuda(0)+ latents = torch.randn(2, 4, device = "cuda:0")+ cond = torch.randn(3, 4, device = "cuda:0")+ neg = torch.randn(3, 4, device = "cuda:0")+ return dit, latents, cond, neg+++def _arm(dit):+ router = mg.CfgParallelRouter(dit, primary = "cuda:0", secondary = "cuda:1")+ dit.forward = router+ return router+++def test_an_input_edited_after_the_prefetch_reaches_an_unpredicted_replica_call():+ dit, latents, cond, neg = _setup()++ def edit(step, x):+ if (+ step == 1+ ): # the first step of a render runs unpredicted (no template yet): only the prefetch was made+ x.add_(1.0)++ with torch.no_grad():+ reference = [_render(dit, latents, cond, neg, 4), _render(dit, latents, cond, neg, 4, edit)]+ router = _arm(dit)+ first = _render(dit, latents, cond, neg, 4)+ router.end_render()+ second = _render(dit, latents, cond, neg, 4, edit)+ torch.cuda.synchronize()+ assert router.stats["replica_calls"] > 0 and router.replica is not None+ assert torch.equal(first, reference[0])+ assert torch.equal(second, reference[1])+++def test_an_edited_buffer_rebuilds_the_replica():+ dit, latents, cond, neg = _setup()+ with torch.no_grad():+ router = _arm(dit)+ _render(dit, latents, cond, neg, 3)+ router.end_render()+ assert router.replica is not None+ dit.scale.mul_(3.0)+ dit.forward = dit.__class__.forward.__get__(dit)+ reference = _render(dit, latents, cond, neg, 4)+ dit.forward = router+ out = _render(dit, latents, cond, neg, 4)+ torch.cuda.synchronize()+ assert router.stats["replica_rebuilds"] >= 1+ assert torch.equal(out, reference)diff --git a/studio/backend/tests/test_diffusion_multigpu_wiring.py b/studio/backend/tests/test_diffusion_multigpu_wiring.pynew file mode 100644index 00000000000..7248a4b10b9--- /dev/null+++ b/studio/backend/tests/test_diffusion_multigpu_wiring.py@@ -0,0 +1,67 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The image backend's multi-GPU call sites, on the fake runtime of test_diffusion_backend, CPU only.++The helpers are unit-tested in test_diffusion_multigpu.py; these pin that a real load / generate / status reaches them:+the plan refinement and the CFG-parallel arming in load_pipeline, the guard's VAE-card tile decision and the CFG render+release in generate, and the ``multi_gpu`` status key."""++from __future__ import annotations++import sys+import types+from pathlib import Path++from core.inference import diffusion as D++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_backend import _loaded_backend, fake_runtime # noqa: E402,F401+++def test_image_load_generate_and_status_reach_the_multi_gpu_layer(+ fake_runtime, tmp_path, monkeypatch+):+ calls = []+ report = {"primary": "cuda:0", "secondary": "cuda:1", "placement": {"text_encoder": "cuda:1"}}++ def refine(+ pipe,+ plan,+ logger = None,+ ):+ calls.append("refine")+ return plan++ def tile_for_verdict(+ pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger+ ):+ calls.append(("tile", verdict))+ return restore_vae++ monkeypatch.setattr(D, "refine_plan_for_multigpu", refine)+ monkeypatch.setattr(+ D,+ "install_cfg_parallel",+ lambda pipe, offload_policy = None, logger = None: calls.append("cfg"),+ )+ monkeypatch.setattr(D, "end_cfg_render", lambda pipe: calls.append(("end", pipe)))+ monkeypatch.setattr(D, "multigpu_placement_status", lambda pipe: report)+ verdict = types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)+ monkeypatch.setattr(D, "raise_on_image_activation_shortfall", lambda **kw: verdict)+ monkeypatch.setattr(D, "_tile_for_verdict", tile_for_verdict)++ backend = _loaded_backend(tmp_path)+ assert calls == ["refine", "cfg"]+ assert backend.status()["multi_gpu"] == report+ calls.clear()+ backend.generate(prompt = "a sloth", width = 512, height = 512, steps = 2)+ assert ("tile", verdict) in calls+ assert calls[-1:] == [("end", backend._state.pipe)]+ pipe = backend._state.pipe+ monkeypatch.setattr(D, "uninstall_cfg_parallel", lambda p: calls.append(("uninstall", p)))+ backend.unload()+ assert ("uninstall", pipe) in callsdiff --git a/studio/backend/tests/test_video_multigpu_wiring.py b/studio/backend/tests/test_video_multigpu_wiring.pynew file mode 100644index 00000000000..0bb83412a8e--- /dev/null+++ b/studio/backend/tests/test_video_multigpu_wiring.py@@ -0,0 +1,73 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The video backend's multi-GPU call sites, on the fake runtime of test_video_backend, CPU only.++The helpers are unit-tested in test_diffusion_multigpu.py; these pin that a real load / generate / status reaches them:+the plan refinement, the CFG-parallel arming and the per-card floors of a split in load_pipeline, the CFG render+release in generate, and the ``multi_gpu`` status key."""++from __future__ import annotations++import sys+from pathlib import Path++from core.inference import video as V++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_video_backend import _load_gguf, fake_runtime # noqa: E402,F401+++def test_video_load_generate_and_status_reach_the_multi_gpu_layer(+ fake_runtime, tmp_path, monkeypatch+):+ calls = []+ report = {"primary": "cuda:0", "secondary": "cuda:1", "placement": {"text_encoder": "cuda:1"}}+ floors = ((0, 9000, False), (1, 7000, True))++ def refine(+ pipe,+ plan,+ logger = None,+ ):+ calls.append("refine")+ return plan++ monkeypatch.setattr(V, "refine_plan_for_multigpu", refine)+ monkeypatch.setattr(+ V,+ "install_cfg_parallel",+ lambda pipe, offload_policy = None, logger = None: calls.append("cfg"),+ )+ monkeypatch.setattr(V, "end_cfg_render", lambda pipe: calls.append(("end", pipe)))+ monkeypatch.setattr(V, "multigpu_placement_status", lambda pipe: report)+ monkeypatch.setattr(V, "_video_split_card_floors", lambda pipe: floors)+ sidecar_checked = []+ single_card_for_sidecar = V._video_single_card_for_sidecar++ def sidecar_check(+ fam,+ plan,+ logger = None,+ ):+ sidecar_checked.append(fam.name)+ return single_card_for_sidecar(fam, plan, logger)++ monkeypatch.setattr(V, "_video_single_card_for_sidecar", sidecar_check)++ backend = V.VideoBackend()+ status = _load_gguf(backend, tmp_path)+ assert calls == ["refine", "cfg"]+ assert sidecar_checked == ["ltx-2", "ltx-2"]+ assert status["multi_gpu"] == report+ assert backend._state.vram_split_cards == floors+ calls.clear()+ backend.generate(prompt = "a sloth surfing", width = 512, height = 512, num_frames = 9, fps = 24)+ assert calls[-1:] == [("end", backend._state.pipe)]+ pipe = backend._state.pipe+ monkeypatch.setattr(V, "uninstall_cfg_parallel", lambda p: calls.append(("uninstall", p)))+ backend.unload()+ assert ("uninstall", pipe) in calls