Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: CFG-parallel for a group-offloaded denoiser (#16681)

Daniel Han4ddd527ad163
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index d8e6a48c8af..2226e640c00 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -1061,6 +1061,28 @@ def _module_device(module: Any) -> Any:
return None
+def _execution_device(module: Any) -> Any:
+ """The card ``module`` computes on: its first tensor's device, or, for a group-offloaded denoiser whose first
+ tensors sit in host memory between calls, the CUDA card its offload groups load onto."""
+ device = _module_device(module)
+ if device is not None and getattr(device, "type", None) == "cuda":
+ return device
+ try:
+ from .diffusion_memory import _offload_groups
+ for group in _offload_groups(module):
+ onload = getattr(group, "onload_device", None)
+ if onload is not None and getattr(onload, "type", None) == "cuda":
+ onload = _torch().device(onload)
+ if onload.index is None:
+ # an automatic load on the current card onloads to bare "cuda": name the card, so the replica
+ # search compares it with the secondaries by ordinal
+ onload = _torch().device("cuda", _torch().cuda.current_device())
+ return onload
+ except Exception: # noqa: BLE001 - unknown: the parameter device stands
+ pass
+ return device
+
+
def _device_context(device: Any):
torch = _torch()
if getattr(device, "type", None) == "cuda" and device.index is not None:
@@ -3232,7 +3254,8 @@ def cfg_parallel_decline(
return "two-expert denoiser"
if getattr(denoiser, "_compiled_call_impl", None) is not None:
return "denoiser is compiled whole"
- if _module_device(denoiser) is None or _module_device(denoiser).type != "cuda":
+ device = _execution_device(denoiser)
+ if device is None or device.type != "cuda":
return "denoiser is not on a CUDA card"
return None
@@ -3265,7 +3288,7 @@ def install_cfg_parallel(
reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)
if reason is None:
denoiser = _cfg_denoiser(pipe)
- primary = _module_device(denoiser)
+ primary = _execution_device(denoiser)
probes = probe_devices((selection.primary, *selection.secondaries))
same = [
o
diff --git a/studio/backend/tests/test_diffusion_unet_cfg_parallel.py b/studio/backend/tests/test_diffusion_unet_cfg_parallel.py
index a5ab5c0a36f..f762c39c416 100644
--- a/studio/backend/tests/test_diffusion_unet_cfg_parallel.py
+++ b/studio/backend/tests/test_diffusion_unet_cfg_parallel.py
@@ -310,3 +310,36 @@ def test_automatic_replica_is_not_built_on_a_card_that_got_busy(monkeypatch):
_sdxl_loop(unet, lat, steps = 2)
assert replicas == [] and "no longer idle" in (router.disabled_reason or "")
assert unet.calls[-1] == 2 # whole on the source from then on
+
+
+def test_group_offloaded_denoiser_runs_cfg_parallel_from_its_onload_card(monkeypatch):
+ from core.inference import diffusion_memory as dm
+
+ class Group:
+ onload_device = torch.device("cuda", 0)
+
+ class Pipe:
+ transformer = _Dit()
+
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [Group()])
+ monkeypatch.setattr(mg, "_hooks_present", lambda m: None)
+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ # parameters in host memory between calls, groups load onto cuda:0: a CFG-parallel candidate
+ assert mg._execution_device(Pipe.transformer) == torch.device("cuda", 0)
+ assert mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel) is None
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [])
+ assert (
+ mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel)
+ == "denoiser is not on a CUDA card"
+ )
+
+
+def test_group_offloaded_onto_bare_cuda_names_the_current_card(monkeypatch):
+ from core.inference import diffusion_memory as dm
+
+ class Group:
+ onload_device = torch.device("cuda")
+
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [Group()])
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ assert mg._execution_device(_Dit()) == torch.device("cuda", 0)