Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: CFG-parallel for a group-offloaded denoiser (#16681)
Daniel Han4ddd527ad163
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex d8e6a48c8af..2226e640c00 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -1061,6 +1061,28 @@ def _module_device(module: Any) -> Any:return None+def _execution_device(module: Any) -> Any:+ """The card ``module`` computes on: its first tensor's device, or, for a group-offloaded denoiser whose first+ tensors sit in host memory between calls, the CUDA card its offload groups load onto."""+ device = _module_device(module)+ if device is not None and getattr(device, "type", None) == "cuda":+ return device+ try:+ from .diffusion_memory import _offload_groups+ for group in _offload_groups(module):+ onload = getattr(group, "onload_device", None)+ if onload is not None and getattr(onload, "type", None) == "cuda":+ onload = _torch().device(onload)+ if onload.index is None:+ # an automatic load on the current card onloads to bare "cuda": name the card, so the replica+ # search compares it with the secondaries by ordinal+ onload = _torch().device("cuda", _torch().cuda.current_device())+ return onload+ except Exception: # noqa: BLE001 - unknown: the parameter device stands+ pass+ return device++def _device_context(device: Any):torch = _torch()if getattr(device, "type", None) == "cuda" and device.index is not None:@@ -3232,7 +3254,8 @@ def cfg_parallel_decline(return "two-expert denoiser"if getattr(denoiser, "_compiled_call_impl", None) is not None:return "denoiser is compiled whole"- if _module_device(denoiser) is None or _module_device(denoiser).type != "cuda":+ device = _execution_device(denoiser)+ if device is None or device.type != "cuda":return "denoiser is not on a CUDA card"return None@@ -3265,7 +3288,7 @@ def install_cfg_parallel(reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)if reason is None:denoiser = _cfg_denoiser(pipe)- primary = _module_device(denoiser)+ primary = _execution_device(denoiser)probes = probe_devices((selection.primary, *selection.secondaries))same = [odiff --git a/studio/backend/tests/test_diffusion_unet_cfg_parallel.py b/studio/backend/tests/test_diffusion_unet_cfg_parallel.pyindex a5ab5c0a36f..f762c39c416 100644--- a/studio/backend/tests/test_diffusion_unet_cfg_parallel.py+++ b/studio/backend/tests/test_diffusion_unet_cfg_parallel.py@@ -310,3 +310,36 @@ def test_automatic_replica_is_not_built_on_a_card_that_got_busy(monkeypatch):_sdxl_loop(unet, lat, steps = 2)assert replicas == [] and "no longer idle" in (router.disabled_reason or "")assert unet.calls[-1] == 2 # whole on the source from then on+++def test_group_offloaded_denoiser_runs_cfg_parallel_from_its_onload_card(monkeypatch):+ from core.inference import diffusion_memory as dm++ class Group:+ onload_device = torch.device("cuda", 0)++ class Pipe:+ transformer = _Dit()++ monkeypatch.setattr(dm, "_offload_groups", lambda module: [Group()])+ monkeypatch.setattr(mg, "_hooks_present", lambda m: None)+ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ # parameters in host memory between calls, groups load onto cuda:0: a CFG-parallel candidate+ assert mg._execution_device(Pipe.transformer) == torch.device("cuda", 0)+ assert mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel) is None+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [])+ assert (+ mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel)+ == "denoiser is not on a CUDA card"+ )+++def test_group_offloaded_onto_bare_cuda_names_the_current_card(monkeypatch):+ from core.inference import diffusion_memory as dm++ class Group:+ onload_device = torch.device("cuda")++ monkeypatch.setattr(dm, "_offload_groups", lambda module: [Group()])+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ assert mg._execution_device(_Dit()) == torch.device("cuda", 0)