Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stream an offloaded ControlNet through the group prefetcher (#15081)
Daniel Han1aba35cb3753
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 9a40a9d868f..6de7e1de12d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -14856,22 +14856,47 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:base model was loaded with an offload policy: forcing the ControlNet fully resident with``.to(device)`` would defeat that low-VRAM placement and can OOM. Group offloading is appliedto this single module, so it is isolated and reversible. Returns True on success; on any- failure the caller falls back to a resident placement (a calibrated tier refuses instead)."""+ failure the caller falls back to a resident placement (a calibrated tier refuses instead).++ On CUDA: ``record_stream`` (without it diffusers synchronizes the compute stream on every group offload),+ and the event-fenced ``GroupPrefetcher`` replaces diffusers' host-synced prefetch chain."""try:+ import inspect+import torchfrom diffusers.hooks import apply_group_offloadingfrom .host_pinning import host_pinning_allowedonload = torch.device(device)- apply_group_offloading(- cn_model,- onload_device = onload,- offload_device = torch.device("cpu"),- offload_type = "block_level",- num_blocks_per_group = 1,- use_stream = onload.type == "cuda" and host_pinning_allowed(onload),- )+ use_stream = onload.type == "cuda" and host_pinning_allowed(onload)+ kwargs: dict[str, Any] = {+ "onload_device": onload,+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": 1,+ "use_stream": use_stream,+ }+ params = inspect.signature(apply_group_offloading).parameters+ prefetch = use_stream and "record_stream" in params and "non_blocking" in params+ if prefetch:+ from .diffusion_memory import (+ _pin_top_level_group,+ _skip_top_level_copy_back,+ install_group_offload_buffer_restore,+ install_group_offload_hooks_eager,+ install_group_prefetch,+ )++ install_group_offload_buffer_restore()+ install_group_offload_hooks_eager()+ kwargs["non_blocking"] = True+ kwargs["record_stream"] = True+ apply_group_offloading(cn_model, **kwargs)+ if prefetch:+ if not _pin_top_level_group(cn_model, logger):+ _skip_top_level_copy_back(cn_model, logger)+ install_group_prefetch(cn_model, onload, logger)return Trueexcept Exception as exc: # noqa: BLE001 - offload is best-effort; resident is the fallbackif logger is not None:diff --git a/studio/backend/tests/test_diffusion_controlnet.py b/studio/backend/tests/test_diffusion_controlnet.pyindex fdb936c6f76..b020bd34f57 100644--- a/studio/backend/tests/test_diffusion_controlnet.py+++ b/studio/backend/tests/test_diffusion_controlnet.py@@ -422,3 +422,105 @@ def test_a_calibrated_tier_refuses_a_controlnet_it_cannot_stream(monkeypatch):st.offload_policy = "group"p = b._controlnet_pipe(st, resolved, threading.Event())assert hasattr(p.controlnet, "device")+++def _fake_group_offload(monkeypatch, with_stream_kwargs = True):+ """Record _offload_controlnet_module's apply_group_offloading kwargs and the denoiser streaming helpers it calls."""+ from core.inference import diffusion_memory as dm++ calls = {"apply": [], "helpers": []}+ if with_stream_kwargs:++ def apply_group_offloading(+ module,+ onload_device,+ offload_device = None,+ offload_type = "block_level",+ num_blocks_per_group = None,+ non_blocking = False,+ use_stream = False,+ record_stream = False,+ low_cpu_mem_usage = False,+ ):+ calls["apply"].append(+ dict(+ onload_device = onload_device,+ offload_type = offload_type,+ num_blocks_per_group = num_blocks_per_group,+ non_blocking = non_blocking,+ use_stream = use_stream,+ record_stream = record_stream,+ )+ )++ else:++ def apply_group_offloading(+ module,+ onload_device,+ offload_device = None,+ offload_type = "block_level",+ num_blocks_per_group = None,+ use_stream = False,+ ):+ calls["apply"].append(dict(onload_device = onload_device, use_stream = use_stream))++ hooks = types.ModuleType("diffusers.hooks")+ hooks.apply_group_offloading = apply_group_offloading+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ for name, ret in (+ ("install_group_offload_buffer_restore", True),+ ("install_group_offload_hooks_eager", True),+ ("_pin_top_level_group", True),+ ("_skip_top_level_copy_back", True),+ ("install_group_prefetch", 3),+ ):+ monkeypatch.setattr(+ dm,+ name,+ lambda *a, _n = name, _r = ret, **k: calls["helpers"].append(_n) or _r,+ )+ return calls+++def test_streamed_controlnet_uses_the_event_fenced_prefetcher_on_cuda(monkeypatch):+ from core.inference import diffusion as d++ calls = _fake_group_offload(monkeypatch)+ module = object()+ assert d._offload_controlnet_module(module, "cuda", None) is True+ (kwargs,) = calls["apply"]+ assert kwargs["use_stream"] and kwargs["record_stream"] and kwargs["non_blocking"]+ assert kwargs["offload_type"] == "block_level" and kwargs["num_blocks_per_group"] == 1+ assert calls["helpers"] == [+ "install_group_offload_buffer_restore",+ "install_group_offload_hooks_eager",+ "_pin_top_level_group",+ "install_group_prefetch",+ ]+++def test_streamed_controlnet_falls_back_to_no_copy_back_when_the_top_group_cannot_pin(monkeypatch):+ from core.inference import diffusion as d+ from core.inference import diffusion_memory as dm++ calls = _fake_group_offload(monkeypatch)+ monkeypatch.setattr(+ dm, "_pin_top_level_group", lambda *a, **k: calls["helpers"].append("pin") and False+ )+ assert d._offload_controlnet_module(object(), "cuda", None) is True+ assert calls["helpers"][-3:] == ["pin", "_skip_top_level_copy_back", "install_group_prefetch"]++[email protected]("device, with_stream_kwargs", [("cpu", True), ("cuda", False)])+def test_controlnet_offload_unchanged_without_a_cuda_stream_path(+ monkeypatch, device, with_stream_kwargs+):+ from core.inference import diffusion as d++ calls = _fake_group_offload(monkeypatch, with_stream_kwargs)+ assert d._offload_controlnet_module(object(), device, None) is True+ (kwargs,) = calls["apply"]+ assert kwargs["use_stream"] is (device == "cuda")+ assert not kwargs.get("record_stream") and not kwargs.get("non_blocking")+ assert calls["helpers"] == []