Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: faster first renders and resident text encoder under group offload (#12519)
Daniel Hane700a335dae2
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 7777579fbf1..1641a775d17 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2440,10 +2440,19 @@ def _keep_groups_resident(def _noop(*args: Any, **kwargs: Any) -> None:return None+ # Groups of this module still on the copy stream; with none, nothing can be in flight to wait for.+ state = getattr(module, "_unsloth_stream_state", None)+ if not isinstance(state, dict):+ state = {"streamed": 1}+ try:+ module._unsloth_stream_state = state+ except AttributeError:+ pass+def _resident_onload(stream: Any) -> Callable[[], None]:# a prefetching predecessor skips its own copy-stream wait and relies on this onload_ to do itdef onload_(*args: Any, **kwargs: Any) -> None:- if stream is not None:+ if stream is not None and state["streamed"]:stream.synchronize()return disable(onload_) if callable(disable) else onload_@@ -2486,6 +2495,7 @@ def _keep_groups_resident(group._unsloth_resident_bytes = needleft -= needkept += need+ state["streamed"] = _streamed_group_count(ordered)if kept and onload.type == "cuda":torch.cuda.synchronize(onload)if logger is not None and kept:@@ -2503,6 +2513,14 @@ def _keep_groups_resident(return 0+def _streamed_group_count(groups: list) -> int:+ return sum(+ 1+ for g in groups+ if getattr(g, "stream", None) is not None and not getattr(g, "_unsloth_resident", False)+ )++def _release_group(group: Any) -> None:onload_, offload_ = getattr(group, "_unsloth_streamed_hooks", (None, None))for name, fn in (("onload_", onload_), ("offload_", offload_)):@@ -2541,12 +2559,20 @@ def release_resident_groups(breakif not getattr(group, "_unsloth_resident", False):continue+ # Before demoting: a release that fails part way must still wait on the copy stream.+ state = getattr(module, "_unsloth_stream_state", None)+ if isinstance(state, dict):+ state["streamed"] = 1_release_group(group)left -= int(getattr(group, "_unsloth_resident_bytes", 0))if module not in released:released.append(module)if not released:return None+ for module in released:+ state = getattr(module, "_unsloth_stream_state", None)+ if isinstance(state, dict):+ state["streamed"] = _streamed_group_count(_offload_groups(module))device = getattr(released[0], "_unsloth_resident_device", None)if device is not None and torch.device(device).type == "cuda":torch.cuda.synchronize(device)@@ -3284,6 +3310,41 @@ def _defer_pinning(pipe: Any, module: Any, device: Any, logger: Any) -> bool:return True+EAGER_OFFLOAD_HOOKS_ENV = "UNSLOTH_DIFFUSION_EAGER_OFFLOAD_HOOKS"+++def install_group_offload_hooks_eager() -> bool:+ """Keep diffusers' group-offload hooks out of compiled blocks: traced, the first-forward layer tracker guards on the+ block name and recompiles per block (44 recompiles on Qwen-Image-2.1 at 16 GB, 5 eager). Must run before the hooks register."""+ if _env_off(EAGER_OFFLOAD_HOOKS_ENV):+ return False+ try:+ import torch+ from diffusers.hooks import group_offloading as go+ except Exception: # noqa: BLE001 - no group offload in this diffusers+ return False+ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ if not callable(disable):+ return False+ patched = False+ for cls_name, methods in (+ ("GroupOffloadingHook", ("pre_forward", "post_forward")),+ ("LayerExecutionTrackerHook", ("pre_forward",)),+ ("LazyPrefetchGroupOffloadingHook", ("post_forward",)),+ ):+ cls = getattr(go, cls_name, None)+ for name in methods:+ fn = getattr(cls, "__dict__", {}).get(name) if cls is not None else None+ if fn is None or getattr(fn, "_unsloth_eager", False):+ continue+ eager = disable(fn)+ eager._unsloth_eager = True+ eager._unsloth_orig = fn+ setattr(cls, name, eager)+ patched = True+ return patched++def install_group_offload_buffer_restore() -> bool:"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""try:@@ -3463,6 +3524,7 @@ def _apply_group_offload(from diffusers.hooks import apply_group_offloadinginstall_group_offload_buffer_restore()+ install_group_offload_hooks_eager()# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep# only smaller companions resident.@@ -4147,6 +4209,7 @@ def _apply_streaming_offload(from diffusers.hooks import apply_group_offloadinginstall_group_offload_buffer_restore()+ install_group_offload_hooks_eager()components = getattr(pipe, "components", {})if not isinstance(components, dict):diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex 2ccb68f8a08..ec44fb4a856 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -2051,6 +2051,7 @@ def stream_prequantized_module(_remove_group_offload_hooks,_streamed_pin_plan,install_group_offload_buffer_restore,+ install_group_offload_hooks_eager,)onload = torch.device(device)@@ -2061,6 +2062,7 @@ def stream_prequantized_module(# each weight's AccumulateGrad, and swap_tensors on onload hits Int8Tensor's missing aten.view.module.requires_grad_(False)install_group_offload_buffer_restore()+ install_group_offload_hooks_eager()use_stream = onload.type == "cuda" and _weights_pinnable(module)if onload.type == "cuda" and not use_stream:# Sync copies measured 16.4 s/step vs 0.8 s resident (B200): rebuild v1 int8 as pinnable Int8Tensor.diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex d18ef95f6bb..e6c792b03c9 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -651,6 +651,7 @@ def stream_h3_text_encoder(_pin_vision_embedding_device,_remove_group_offload_hooks,install_group_offload_buffer_restore,+ install_group_offload_hooks_eager,)from .diffusion_prequant import _evict_rotation_hook, _unhook_from_manager@@ -683,6 +684,7 @@ def stream_h3_text_encoder(text_encoder.requires_grad_(False)# The int8 weights are buffers, which stock diffusers leaves on the device after offload.install_group_offload_buffer_restore()+ install_group_offload_hooks_eager()apply_group_offloading(target, **kwargs)# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5)._pin_vision_embedding_device(target)diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex ad9859b2fd8..2cf84153e36 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -1358,3 +1358,21 @@ def _clear_github_rate_limit_lockout():freshness_flow._api_rate_limited_until = 0.0yieldfreshness_flow._api_rate_limited_until = 0.0+++def traced_offload_hooks(monkeypatch):+ """diffusers' own group-offload hook methods for one test (install_group_offload_hooks_eager is process-wide)."""+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ from core.inference.diffusion_memory import install_group_offload_hooks_eager++ install_group_offload_hooks_eager()+ for cls in (+ go.GroupOffloadingHook,+ go.LayerExecutionTrackerHook,+ go.LazyPrefetchGroupOffloadingHook,+ ):+ for name, fn in list(vars(cls).items()):+ orig = getattr(fn, "_unsloth_orig", None)+ if orig is not None:+ monkeypatch.setattr(cls, name, orig)diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex fc368480711..734c2b070e6 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -377,7 +377,7 @@ def test_resident_group_keeps_copy_stream_wait(monkeypatch):def synchronize(self):Stream.waits += 1- module = object()+ module = types.SimpleNamespace()stream = Stream()groups = [types.SimpleNamespace(@@ -394,6 +394,13 @@ def test_resident_group_keeps_copy_stream_wait(monkeypatch):monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)dm._keep_groups_resident(module, 1, "cpu")assert all(getattr(g, "_unsloth_resident", False) for g in groups)+ # every group resident: nothing is ever queued on the copy stream, so there is nothing to wait for+ for g in groups:+ g.onload_()+ g.offload_()+ assert Stream.waits == 0+ # one group of the module streams again (an oversized request): the resident ones wait for it again+ module._unsloth_stream_state["streamed"] = 1for g in groups:g.onload_()g.offload_()@@ -487,3 +494,96 @@ def test_generate_releases_and_restores_resident_groups():finally_at = src.index("if restore_resident is not None:")assert "restore_resident()" in src[finally_at : finally_at + 200]assert "pipe._unsloth_measured_reserve = (" in src+++def test_eager_offload_hooks_install_is_idempotent():+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ dm.install_group_offload_hooks_eager()+ hooks = (+ (go.GroupOffloadingHook, "pre_forward"),+ (go.GroupOffloadingHook, "post_forward"),+ (go.LayerExecutionTrackerHook, "pre_forward"),+ (go.LazyPrefetchGroupOffloadingHook, "post_forward"),+ )+ patched = [cls.__dict__[name] for cls, name in hooks]+ assert all(getattr(fn, "_unsloth_eager", False) for fn in patched)+ assert dm.install_group_offload_hooks_eager() is False+ assert [cls.__dict__[name] for cls, name in hooks] == patched+++def test_eager_offload_hooks_kill_switch(monkeypatch, traced_offload_hooks):+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ monkeypatch.setenv(dm.EAGER_OFFLOAD_HOOKS_ENV, "0")+ assert dm.install_group_offload_hooks_eager() is False+ assert not getattr(go.GroupOffloadingHook.__dict__["pre_forward"], "_unsloth_eager", False)+++def test_failed_release_keeps_the_copy_stream_wait(monkeypatch):+ groups = [+ types.SimpleNamespace(+ stream = object(), _unsloth_resident = True, _unsloth_resident_bytes = 1 << 20+ )+ for _ in range(2)+ ]+ module = types.SimpleNamespace(_unsloth_resident_room = 1, _unsloth_stream_state = {"streamed": 0})+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)++ def boom(group):+ raise RuntimeError("offload callback failed")++ monkeypatch.setattr(dm, "_release_group", boom)+ pipe = types.SimpleNamespace(components = {"transformer": module})+ assert dm.release_resident_groups(pipe, 4) is None+ assert module._unsloth_stream_state["streamed"] >= 1+++ not __import__("torch").cuda.is_available(), reason = "stream group offload needs a CUDA device"+)+def test_compiled_blocks_under_group_offload_skip_hook_tracing():+ import torch++ pytest.importorskip("diffusers.hooks.group_offloading")+ from diffusers.hooks import apply_group_offloading+ from torch._dynamo.utils import counters++ dm.install_group_offload_hooks_eager()+ torch.manual_seed(0)+ make = lambda: torch.nn.Sequential(torch.nn.Linear(256, 256), torch.nn.GELU())+ blocks = torch.nn.ModuleList([make() for _ in range(6)])+ plain = [make().cuda() for _ in blocks]+ for ref, block in zip(plain, blocks):+ ref.load_state_dict(block.state_dict())++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = blocks++ def forward(self, x):+ for block in self.blocks:+ x = block(x)+ return x++ model = Model()+ apply_group_offloading(+ model,+ onload_device = torch.device("cuda"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ )+ x = torch.randn(64, 256, device = "cuda")+ with torch.no_grad():+ for block in [*model.blocks, *plain]:+ block.compile()+ expected = x+ for ref in plain:+ expected = ref(expected)+ torch._dynamo.reset()+ counters.clear()+ outs = [model(x) for _ in range(3)]+ for out in outs:+ torch.testing.assert_close(out, expected, rtol = 0, atol = 1e-6)+ # Traced hooks add a graph per hook variant; eager hooks leave only the block graph.+ assert counters["stats"]["unique_graphs"] <= 1diff --git a/studio/backend/tests/test_h3_stream_compile_and_top_group.py b/studio/backend/tests/test_h3_stream_compile_and_top_group.pyindex c7c16c9baf0..846b8baaba7 100644--- a/studio/backend/tests/test_h3_stream_compile_and_top_group.py+++ b/studio/backend/tests/test_h3_stream_compile_and_top_group.py@@ -91,7 +91,7 @@ def test_compiling_below_the_hooks_does_not_recompile_when_the_hook_state_changeassert _graphs() == before, "a hook-state change recompiled the block"-def test_the_old_placement_recompiles_on_the_same_change(monkeypatch):+def test_the_old_placement_recompiles_on_the_same_change(monkeypatch, traced_offload_hooks):"""Control: Module.compile traces the hooks, so the same flip compiles new graphs (the 9 s first steps)."""monkeypatch.setattr(torch.cuda, "is_available", lambda: False)torch._dynamo.reset()