Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster first renders and resident text encoder under group offload (#12519)

Daniel Hane700a335dae2
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 7777579fbf1..1641a775d17 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2440,10 +2440,19 @@ def _keep_groups_resident(
def _noop(*args: Any, **kwargs: Any) -> None:
return None
+ # Groups of this module still on the copy stream; with none, nothing can be in flight to wait for.
+ state = getattr(module, "_unsloth_stream_state", None)
+ if not isinstance(state, dict):
+ state = {"streamed": 1}
+ try:
+ module._unsloth_stream_state = state
+ except AttributeError:
+ pass
+
def _resident_onload(stream: Any) -> Callable[[], None]:
# a prefetching predecessor skips its own copy-stream wait and relies on this onload_ to do it
def onload_(*args: Any, **kwargs: Any) -> None:
- if stream is not None:
+ if stream is not None and state["streamed"]:
stream.synchronize()
return disable(onload_) if callable(disable) else onload_
@@ -2486,6 +2495,7 @@ def _keep_groups_resident(
group._unsloth_resident_bytes = need
left -= need
kept += need
+ state["streamed"] = _streamed_group_count(ordered)
if kept and onload.type == "cuda":
torch.cuda.synchronize(onload)
if logger is not None and kept:
@@ -2503,6 +2513,14 @@ def _keep_groups_resident(
return 0
+def _streamed_group_count(groups: list) -> int:
+ return sum(
+ 1
+ for g in groups
+ if getattr(g, "stream", None) is not None and not getattr(g, "_unsloth_resident", False)
+ )
+
+
def _release_group(group: Any) -> None:
onload_, offload_ = getattr(group, "_unsloth_streamed_hooks", (None, None))
for name, fn in (("onload_", onload_), ("offload_", offload_)):
@@ -2541,12 +2559,20 @@ def release_resident_groups(
break
if not getattr(group, "_unsloth_resident", False):
continue
+ # Before demoting: a release that fails part way must still wait on the copy stream.
+ state = getattr(module, "_unsloth_stream_state", None)
+ if isinstance(state, dict):
+ state["streamed"] = 1
_release_group(group)
left -= int(getattr(group, "_unsloth_resident_bytes", 0))
if module not in released:
released.append(module)
if not released:
return None
+ for module in released:
+ state = getattr(module, "_unsloth_stream_state", None)
+ if isinstance(state, dict):
+ state["streamed"] = _streamed_group_count(_offload_groups(module))
device = getattr(released[0], "_unsloth_resident_device", None)
if device is not None and torch.device(device).type == "cuda":
torch.cuda.synchronize(device)
@@ -3284,6 +3310,41 @@ def _defer_pinning(pipe: Any, module: Any, device: Any, logger: Any) -> bool:
return True
+EAGER_OFFLOAD_HOOKS_ENV = "UNSLOTH_DIFFUSION_EAGER_OFFLOAD_HOOKS"
+
+
+def install_group_offload_hooks_eager() -> bool:
+ """Keep diffusers' group-offload hooks out of compiled blocks: traced, the first-forward layer tracker guards on the
+ block name and recompiles per block (44 recompiles on Qwen-Image-2.1 at 16 GB, 5 eager). Must run before the hooks register."""
+ if _env_off(EAGER_OFFLOAD_HOOKS_ENV):
+ return False
+ try:
+ import torch
+ from diffusers.hooks import group_offloading as go
+ except Exception: # noqa: BLE001 - no group offload in this diffusers
+ return False
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ if not callable(disable):
+ return False
+ patched = False
+ for cls_name, methods in (
+ ("GroupOffloadingHook", ("pre_forward", "post_forward")),
+ ("LayerExecutionTrackerHook", ("pre_forward",)),
+ ("LazyPrefetchGroupOffloadingHook", ("post_forward",)),
+ ):
+ cls = getattr(go, cls_name, None)
+ for name in methods:
+ fn = getattr(cls, "__dict__", {}).get(name) if cls is not None else None
+ if fn is None or getattr(fn, "_unsloth_eager", False):
+ continue
+ eager = disable(fn)
+ eager._unsloth_eager = True
+ eager._unsloth_orig = fn
+ setattr(cls, name, eager)
+ patched = True
+ return patched
+
+
def install_group_offload_buffer_restore() -> bool:
"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""
try:
@@ -3463,6 +3524,7 @@ def _apply_group_offload(
from diffusers.hooks import apply_group_offloading
install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep
# only smaller companions resident.
@@ -4147,6 +4209,7 @@ def _apply_streaming_offload(
from diffusers.hooks import apply_group_offloading
install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
components = getattr(pipe, "components", {})
if not isinstance(components, dict):
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 2ccb68f8a08..ec44fb4a856 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -2051,6 +2051,7 @@ def stream_prequantized_module(
_remove_group_offload_hooks,
_streamed_pin_plan,
install_group_offload_buffer_restore,
+ install_group_offload_hooks_eager,
)
onload = torch.device(device)
@@ -2061,6 +2062,7 @@ def stream_prequantized_module(
# each weight's AccumulateGrad, and swap_tensors on onload hits Int8Tensor's missing aten.view.
module.requires_grad_(False)
install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
use_stream = onload.type == "cuda" and _weights_pinnable(module)
if onload.type == "cuda" and not use_stream:
# Sync copies measured 16.4 s/step vs 0.8 s resident (B200): rebuild v1 int8 as pinnable Int8Tensor.
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index d18ef95f6bb..e6c792b03c9 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -651,6 +651,7 @@ def stream_h3_text_encoder(
_pin_vision_embedding_device,
_remove_group_offload_hooks,
install_group_offload_buffer_restore,
+ install_group_offload_hooks_eager,
)
from .diffusion_prequant import _evict_rotation_hook, _unhook_from_manager
@@ -683,6 +684,7 @@ def stream_h3_text_encoder(
text_encoder.requires_grad_(False)
# The int8 weights are buffers, which stock diffusers leaves on the device after offload.
install_group_offload_buffer_restore()
+ install_group_offload_hooks_eager()
apply_group_offloading(target, **kwargs)
# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5).
_pin_vision_embedding_device(target)
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index ad9859b2fd8..2cf84153e36 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -1358,3 +1358,21 @@ def _clear_github_rate_limit_lockout():
freshness_flow._api_rate_limited_until = 0.0
yield
freshness_flow._api_rate_limited_until = 0.0
+
+
+def traced_offload_hooks(monkeypatch):
+ """diffusers' own group-offload hook methods for one test (install_group_offload_hooks_eager is process-wide)."""
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ from core.inference.diffusion_memory import install_group_offload_hooks_eager
+
+ install_group_offload_hooks_eager()
+ for cls in (
+ go.GroupOffloadingHook,
+ go.LayerExecutionTrackerHook,
+ go.LazyPrefetchGroupOffloadingHook,
+ ):
+ for name, fn in list(vars(cls).items()):
+ orig = getattr(fn, "_unsloth_orig", None)
+ if orig is not None:
+ monkeypatch.setattr(cls, name, orig)
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index fc368480711..734c2b070e6 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -377,7 +377,7 @@ def test_resident_group_keeps_copy_stream_wait(monkeypatch):
def synchronize(self):
Stream.waits += 1
- module = object()
+ module = types.SimpleNamespace()
stream = Stream()
groups = [
types.SimpleNamespace(
@@ -394,6 +394,13 @@ def test_resident_group_keeps_copy_stream_wait(monkeypatch):
monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)
dm._keep_groups_resident(module, 1, "cpu")
assert all(getattr(g, "_unsloth_resident", False) for g in groups)
+ # every group resident: nothing is ever queued on the copy stream, so there is nothing to wait for
+ for g in groups:
+ g.onload_()
+ g.offload_()
+ assert Stream.waits == 0
+ # one group of the module streams again (an oversized request): the resident ones wait for it again
+ module._unsloth_stream_state["streamed"] = 1
for g in groups:
g.onload_()
g.offload_()
@@ -487,3 +494,96 @@ def test_generate_releases_and_restores_resident_groups():
finally_at = src.index("if restore_resident is not None:")
assert "restore_resident()" in src[finally_at : finally_at + 200]
assert "pipe._unsloth_measured_reserve = (" in src
+
+
+def test_eager_offload_hooks_install_is_idempotent():
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ dm.install_group_offload_hooks_eager()
+ hooks = (
+ (go.GroupOffloadingHook, "pre_forward"),
+ (go.GroupOffloadingHook, "post_forward"),
+ (go.LayerExecutionTrackerHook, "pre_forward"),
+ (go.LazyPrefetchGroupOffloadingHook, "post_forward"),
+ )
+ patched = [cls.__dict__[name] for cls, name in hooks]
+ assert all(getattr(fn, "_unsloth_eager", False) for fn in patched)
+ assert dm.install_group_offload_hooks_eager() is False
+ assert [cls.__dict__[name] for cls, name in hooks] == patched
+
+
+def test_eager_offload_hooks_kill_switch(monkeypatch, traced_offload_hooks):
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ monkeypatch.setenv(dm.EAGER_OFFLOAD_HOOKS_ENV, "0")
+ assert dm.install_group_offload_hooks_eager() is False
+ assert not getattr(go.GroupOffloadingHook.__dict__["pre_forward"], "_unsloth_eager", False)
+
+
+def test_failed_release_keeps_the_copy_stream_wait(monkeypatch):
+ groups = [
+ types.SimpleNamespace(
+ stream = object(), _unsloth_resident = True, _unsloth_resident_bytes = 1 << 20
+ )
+ for _ in range(2)
+ ]
+ module = types.SimpleNamespace(_unsloth_resident_room = 1, _unsloth_stream_state = {"streamed": 0})
+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)
+
+ def boom(group):
+ raise RuntimeError("offload callback failed")
+
+ monkeypatch.setattr(dm, "_release_group", boom)
+ pipe = types.SimpleNamespace(components = {"transformer": module})
+ assert dm.release_resident_groups(pipe, 4) is None
+ assert module._unsloth_stream_state["streamed"] >= 1
+
+
+ not __import__("torch").cuda.is_available(), reason = "stream group offload needs a CUDA device"
+)
+def test_compiled_blocks_under_group_offload_skip_hook_tracing():
+ import torch
+
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ from diffusers.hooks import apply_group_offloading
+ from torch._dynamo.utils import counters
+
+ dm.install_group_offload_hooks_eager()
+ torch.manual_seed(0)
+ make = lambda: torch.nn.Sequential(torch.nn.Linear(256, 256), torch.nn.GELU())
+ blocks = torch.nn.ModuleList([make() for _ in range(6)])
+ plain = [make().cuda() for _ in blocks]
+ for ref, block in zip(plain, blocks):
+ ref.load_state_dict(block.state_dict())
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = blocks
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x)
+ return x
+
+ model = Model()
+ apply_group_offloading(
+ model,
+ onload_device = torch.device("cuda"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ )
+ x = torch.randn(64, 256, device = "cuda")
+ with torch.no_grad():
+ for block in [*model.blocks, *plain]:
+ block.compile()
+ expected = x
+ for ref in plain:
+ expected = ref(expected)
+ torch._dynamo.reset()
+ counters.clear()
+ outs = [model(x) for _ in range(3)]
+ for out in outs:
+ torch.testing.assert_close(out, expected, rtol = 0, atol = 1e-6)
+ # Traced hooks add a graph per hook variant; eager hooks leave only the block graph.
+ assert counters["stats"]["unique_graphs"] <= 1
diff --git a/studio/backend/tests/test_h3_stream_compile_and_top_group.py b/studio/backend/tests/test_h3_stream_compile_and_top_group.py
index c7c16c9baf0..846b8baaba7 100644
--- a/studio/backend/tests/test_h3_stream_compile_and_top_group.py
+++ b/studio/backend/tests/test_h3_stream_compile_and_top_group.py
@@ -91,7 +91,7 @@ def test_compiling_below_the_hooks_does_not_recompile_when_the_hook_state_change
assert _graphs() == before, "a hook-state change recompiled the block"
-def test_the_old_placement_recompiles_on_the_same_change(monkeypatch):
+def test_the_old_placement_recompiles_on_the_same_change(monkeypatch, traced_offload_hooks):
"""Control: Module.compile traces the hooks, so the same flip compiles new graphs (the 9 s first steps)."""
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
torch._dynamo.reset()