Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stream an offloaded ControlNet through the group prefetcher (#15081)

Daniel Han1aba35cb3753
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 9a40a9d868f..6de7e1de12d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14856,22 +14856,47 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
base model was loaded with an offload policy: forcing the ControlNet fully resident with
``.to(device)`` would defeat that low-VRAM placement and can OOM. Group offloading is applied
to this single module, so it is isolated and reversible. Returns True on success; on any
- failure the caller falls back to a resident placement (a calibrated tier refuses instead)."""
+ failure the caller falls back to a resident placement (a calibrated tier refuses instead).
+
+ On CUDA: ``record_stream`` (without it diffusers synchronizes the compute stream on every group offload),
+ and the event-fenced ``GroupPrefetcher`` replaces diffusers' host-synced prefetch chain."""
try:
+ import inspect
+
import torch
from diffusers.hooks import apply_group_offloading
from .host_pinning import host_pinning_allowed
onload = torch.device(device)
- apply_group_offloading(
- cn_model,
- onload_device = onload,
- offload_device = torch.device("cpu"),
- offload_type = "block_level",
- num_blocks_per_group = 1,
- use_stream = onload.type == "cuda" and host_pinning_allowed(onload),
- )
+ use_stream = onload.type == "cuda" and host_pinning_allowed(onload)
+ kwargs: dict[str, Any] = {
+ "onload_device": onload,
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": 1,
+ "use_stream": use_stream,
+ }
+ params = inspect.signature(apply_group_offloading).parameters
+ prefetch = use_stream and "record_stream" in params and "non_blocking" in params
+ if prefetch:
+ from .diffusion_memory import (
+ _pin_top_level_group,
+ _skip_top_level_copy_back,
+ install_group_offload_buffer_restore,
+ install_group_offload_hooks_eager,
+ install_group_prefetch,
+ )
+
+ install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
+ kwargs["non_blocking"] = True
+ kwargs["record_stream"] = True
+ apply_group_offloading(cn_model, **kwargs)
+ if prefetch:
+ if not _pin_top_level_group(cn_model, logger):
+ _skip_top_level_copy_back(cn_model, logger)
+ install_group_prefetch(cn_model, onload, logger)
return True
except Exception as exc: # noqa: BLE001 - offload is best-effort; resident is the fallback
if logger is not None:
diff --git a/studio/backend/tests/test_diffusion_controlnet.py b/studio/backend/tests/test_diffusion_controlnet.py
index fdb936c6f76..b020bd34f57 100644
--- a/studio/backend/tests/test_diffusion_controlnet.py
+++ b/studio/backend/tests/test_diffusion_controlnet.py
@@ -422,3 +422,105 @@ def test_a_calibrated_tier_refuses_a_controlnet_it_cannot_stream(monkeypatch):
st.offload_policy = "group"
p = b._controlnet_pipe(st, resolved, threading.Event())
assert hasattr(p.controlnet, "device")
+
+
+def _fake_group_offload(monkeypatch, with_stream_kwargs = True):
+ """Record _offload_controlnet_module's apply_group_offloading kwargs and the denoiser streaming helpers it calls."""
+ from core.inference import diffusion_memory as dm
+
+ calls = {"apply": [], "helpers": []}
+ if with_stream_kwargs:
+
+ def apply_group_offloading(
+ module,
+ onload_device,
+ offload_device = None,
+ offload_type = "block_level",
+ num_blocks_per_group = None,
+ non_blocking = False,
+ use_stream = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ ):
+ calls["apply"].append(
+ dict(
+ onload_device = onload_device,
+ offload_type = offload_type,
+ num_blocks_per_group = num_blocks_per_group,
+ non_blocking = non_blocking,
+ use_stream = use_stream,
+ record_stream = record_stream,
+ )
+ )
+
+ else:
+
+ def apply_group_offloading(
+ module,
+ onload_device,
+ offload_device = None,
+ offload_type = "block_level",
+ num_blocks_per_group = None,
+ use_stream = False,
+ ):
+ calls["apply"].append(dict(onload_device = onload_device, use_stream = use_stream))
+
+ hooks = types.ModuleType("diffusers.hooks")
+ hooks.apply_group_offloading = apply_group_offloading
+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
+ for name, ret in (
+ ("install_group_offload_buffer_restore", True),
+ ("install_group_offload_hooks_eager", True),
+ ("_pin_top_level_group", True),
+ ("_skip_top_level_copy_back", True),
+ ("install_group_prefetch", 3),
+ ):
+ monkeypatch.setattr(
+ dm,
+ name,
+ lambda *a, _n = name, _r = ret, **k: calls["helpers"].append(_n) or _r,
+ )
+ return calls
+
+
+def test_streamed_controlnet_uses_the_event_fenced_prefetcher_on_cuda(monkeypatch):
+ from core.inference import diffusion as d
+
+ calls = _fake_group_offload(monkeypatch)
+ module = object()
+ assert d._offload_controlnet_module(module, "cuda", None) is True
+ (kwargs,) = calls["apply"]
+ assert kwargs["use_stream"] and kwargs["record_stream"] and kwargs["non_blocking"]
+ assert kwargs["offload_type"] == "block_level" and kwargs["num_blocks_per_group"] == 1
+ assert calls["helpers"] == [
+ "install_group_offload_buffer_restore",
+ "install_group_offload_hooks_eager",
+ "_pin_top_level_group",
+ "install_group_prefetch",
+ ]
+
+
+def test_streamed_controlnet_falls_back_to_no_copy_back_when_the_top_group_cannot_pin(monkeypatch):
+ from core.inference import diffusion as d
+ from core.inference import diffusion_memory as dm
+
+ calls = _fake_group_offload(monkeypatch)
+ monkeypatch.setattr(
+ dm, "_pin_top_level_group", lambda *a, **k: calls["helpers"].append("pin") and False
+ )
+ assert d._offload_controlnet_module(object(), "cuda", None) is True
+ assert calls["helpers"][-3:] == ["pin", "_skip_top_level_copy_back", "install_group_prefetch"]
+
+
[email protected]("device, with_stream_kwargs", [("cpu", True), ("cuda", False)])
+def test_controlnet_offload_unchanged_without_a_cuda_stream_path(
+ monkeypatch, device, with_stream_kwargs
+):
+ from core.inference import diffusion as d
+
+ calls = _fake_group_offload(monkeypatch, with_stream_kwargs)
+ assert d._offload_controlnet_module(object(), device, None) is True
+ (kwargs,) = calls["apply"]
+ assert kwargs["use_stream"] is (device == "cuda")
+ assert not kwargs.get("record_stream") and not kwargs.get("non_blocking")
+ assert calls["helpers"] == []