Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: give FLUX.3's per-mode block stacks their own offload groups (5 GiB L4 int8: -17.6% per image) (#17122)

Daniel Hand17d054b0b8a
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 7fdd96b978d..bda70345373 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4097,6 +4097,33 @@ def _resident_dit_fits(memory: Any, dit_mib: int, headroom_mib: int, other_mib:
STREAMED_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_STREAMED_RESIDENCY"
[email protected]_cache(maxsize = 1)
+def _diffusers_takes_block_modules() -> bool:
+ """Whether the installed diffusers' ``apply_group_offloading`` accepts ``block_modules``."""
+ try:
+ import inspect
+
+ from diffusers.hooks import apply_group_offloading
+ return "block_modules" in inspect.signature(apply_group_offloading).parameters
+ except Exception: # noqa: BLE001 - no diffusers / unreadable: the stock grouping
+ return False
+
+
+def _block_stacks(module: Any) -> list:
+ """The block stacks block-level group offload streams: ``ModuleList`` / ``Sequential`` children, plus the stacks of
+ the ModuleDicts ``_dict_block_modules`` hands to diffusers as ``block_modules``."""
+ import torch
+
+ stacks = []
+ dicts = set(_dict_block_modules(module)) if _diffusers_takes_block_modules() else set()
+ for name, child in module.named_children():
+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
+ stacks.append(child)
+ elif name in dicts:
+ stacks.extend(child.values())
+ return stacks
+
+
def _stream_window_mib(pipe: Any) -> int:
"""Device MiB the block prefetcher holds in flight: ``depth + 1`` of the largest denoiser block, or -1."""
try:
@@ -4110,9 +4137,7 @@ def _stream_window_mib(pipe: Any) -> int:
continue
if not isinstance(module, torch.nn.Module):
continue
- for child in module.children():
- if not isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
- continue
+ for child in _block_stacks(module):
for block in child:
seen: set[int] = set()
nbytes = 0
@@ -4168,9 +4193,8 @@ def _top_level_group_mib(pipe: Any) -> Optional[int]:
if not isinstance(module, torch.nn.Module):
continue
blocks: set[int] = set()
- for child in module.children():
- if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
- blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))
+ for child in _block_stacks(module):
+ blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))
seen: set[int] = set()
nbytes = 0
for t in (*module.parameters(), *module.buffers()):
@@ -6854,6 +6878,51 @@ def _module_pin_mib(module: Any) -> int:
return _module_host_mib(module)
+# Block-level offload groups only direct ModuleList children; split FLUX.3's ModuleDict of block stacks too ("0": off).
+GROUP_OFFLOAD_DICT_BLOCKS_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_DICT_BLOCKS"
+_DICT_BLOCK_CLASSES = frozenset({"Flux3Transformer2DModel"})
+
+
+def _dict_block_modules(module: Any) -> list[str]:
+ """Direct children of ``module`` that are ModuleDicts of block stacks."""
+ if _env_off(GROUP_OFFLOAD_DICT_BLOCKS_ENV) or type(module).__name__ not in _DICT_BLOCK_CLASSES:
+ return []
+ try:
+ import torch
+
+ names = []
+ for name, child in module.named_children():
+ if (
+ isinstance(child, torch.nn.ModuleDict)
+ and len(child) > 0
+ and all(
+ isinstance(v, (torch.nn.ModuleList, torch.nn.Sequential))
+ for v in child.values()
+ )
+ ):
+ names.append(name)
+ return names
+ except Exception: # noqa: BLE001 - unreadable: the stock grouping
+ return []
+
+
+def _with_dict_block_modules(
+ apply: Callable[..., Any], module: Any, kwargs: dict[str, Any]
+) -> dict[str, Any]:
+ if kwargs.get("offload_type", "block_level") != "block_level" or "block_modules" in kwargs:
+ return kwargs
+ names = _dict_block_modules(module)
+ if not names:
+ return kwargs
+ try:
+ import inspect
+ if "block_modules" not in inspect.signature(apply).parameters:
+ return kwargs
+ except (TypeError, ValueError):
+ return kwargs
+ return {**kwargs, "block_modules": names}
+
+
def _apply_group_offloading_packed(
apply: Callable[..., Any],
module: Any,
@@ -6867,6 +6936,7 @@ def _apply_group_offloading_packed(
pages, so overlap is unchanged. Lazy (``low_cpu_mem_usage``) and unstreamed applies run as before."""
from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload
+ kwargs = _with_dict_block_modules(apply, module, kwargs)
pack = (
bool(kwargs.get("use_stream"))
and not kwargs.get("low_cpu_mem_usage", False)
@@ -8322,7 +8392,12 @@ def _apply_group_offload(
and not gkwargs.get("low_cpu_mem_usage", False)
and not _torchao_weight_classes(module)
):
- apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})
+ apply_group_offloading(
+ module,
+ **_with_dict_block_modules(
+ apply_group_offloading, module, {**gkwargs, "low_cpu_mem_usage": True}
+ ),
+ )
_defer_pinning(pipe, module, onload, logger)
else:
_apply_group_offloading_packed(
@@ -8396,7 +8471,12 @@ def _apply_group_offload(
if "low_cpu_mem_usage" in _params and use_stream:
dkwargs["low_cpu_mem_usage"] = True
apply_group_offloading(
- dit, **_torchao_group_offload_kwargs(dit, dkwargs, pinned_mib)
+ dit,
+ **_with_dict_block_modules(
+ apply_group_offloading,
+ dit,
+ _torchao_group_offload_kwargs(dit, dkwargs, pinned_mib),
+ ),
)
installed += 1
transformer_demoted = True
diff --git a/studio/backend/tests/test_diffusion_flux3_block_groups.py b/studio/backend/tests/test_diffusion_flux3_block_groups.py
new file mode 100644
index 00000000000..8c300754e81
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_block_groups.py
@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3's per-mode block stacks (ModuleDicts of ModuleLists) become their own block-level offload groups instead of
+riding in the top-level group, which a streamed load onloads whole for every forward. CPU only."""
+
+from __future__ import annotations
+
+import pytest
+
+import core.inference.diffusion_memory as dm
+
+
+def _flux3_like():
+ import torch
+ class Flux3Transformer2DModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.x_embedder = torch.nn.Linear(4, 8)
+ self.mode_blocks = torch.nn.ModuleDict(
+ {
+ name: torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(3)])
+ for name in ("image", "image_cond")
+ }
+ )
+ self.modulations = torch.nn.ModuleDict({"image": torch.nn.Linear(8, 8)})
+ self.txt_mode_blocks = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
+
+ def forward(self, x):
+ x = self.x_embedder(x)
+ for block in self.txt_mode_blocks:
+ x = block(x)
+ for block in self.mode_blocks["image"]:
+ x = block(x)
+ return x
+
+ return Flux3Transformer2DModel()
+
+
+def test_flux3_mode_block_dicts_become_block_groups(monkeypatch):
+ import torch
+
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)
+ model = _flux3_like()
+ assert dm._dict_block_modules(model) == ["mode_blocks"]
+
+ def apply(
+ module,
+ *,
+ offload_type = "block_level",
+ block_modules = None,
+ **kw,
+ ):
+ return None
+
+ def old_apply(
+ module,
+ *,
+ offload_type = "block_level",
+ **kw,
+ ):
+ return None
+
+ assert dm._with_dict_block_modules(apply, model, {"offload_type": "block_level"})[
+ "block_modules"
+ ] == ["mode_blocks"]
+ # leaf-level, an older diffusers without the parameter, and other classes keep the stock grouping
+ assert "block_modules" not in dm._with_dict_block_modules(
+ apply, model, {"offload_type": "leaf_level"}
+ )
+ assert "block_modules" not in dm._with_dict_block_modules(
+ old_apply, model, {"offload_type": "block_level"}
+ )
+ assert (
+ dm._dict_block_modules(
+ torch.nn.ModuleDict({"a": torch.nn.ModuleList([torch.nn.Linear(2, 2)])})
+ )
+ == []
+ )
+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, "0")
+ assert dm._dict_block_modules(model) == []
+
+
+def test_flux3_mode_blocks_get_their_own_diffusers_groups(monkeypatch):
+ import inspect
+
+ import torch
+
+ from diffusers.hooks import apply_group_offloading
+
+ # an older diffusers without block_modules keeps the stock grouping
+ supported = "block_modules" in inspect.signature(apply_group_offloading).parameters
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)
+ counts = {}
+ for env in ("1", "0"):
+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, env)
+ model = _flux3_like()
+ x = torch.randn(2, 4)
+ want = model(x)
+ kwargs = dict(
+ onload_device = torch.device("cpu"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ )
+ dm._apply_group_offloading_packed(apply_group_offloading, model, kwargs)
+ groups = dm._offload_groups(model)
+ counts[env] = len(groups)
+ mode_groups = [
+ g for g in groups if any(m is b for b in model.mode_blocks["image"] for m in g.modules)
+ ]
+ assert (len(mode_groups) == 3) is (env == "1" and supported)
+ torch.testing.assert_close(model(x), want)
+ # 6 mode blocks become groups of their own; the stock grouping folds them into the top-level group
+ assert counts["1"] == counts["0"] + (6 if supported else 0)
+
+
+def _mib_model(monkeypatch):
+ """A FLUX.3-like model whose parameter sizes are read from fake storage sizes (MiB per tensor)."""
+ model = _flux3_like()
+ sizes = {}
+ for name, p in model.named_parameters():
+ sizes[id(p)] = 64 if ".mode_blocks." in "." + name else 4
+ monkeypatch.setattr(dm, "_storage_nbytes", lambda t: [sizes.get(id(t), 0) * 1024 * 1024])
+ return model
+
+
[email protected]("supported", [True, False])
+def test_top_level_group_leaves_the_mode_blocks_out(monkeypatch, supported):
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)
+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: supported)
+ model = _mib_model(monkeypatch)
+ pipe = type("P", (), {"components": {"transformer": model}})()
+ top = dm._top_level_group_mib(pipe)
+ # x_embedder + modulations (16 MiB) stay top level; the 6 mode blocks move out only if diffusers can split them
+ assert top == (16 if supported else 16 + 6 * 128)
+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, "0")
+ assert dm._top_level_group_mib(pipe) == 16 + 6 * 128
+
+
+def test_stream_window_counts_the_mode_blocks(monkeypatch):
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)
+ model = _mib_model(monkeypatch)
+ pipe = type("P", (), {"components": {"transformer": model}})()
+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: True)
+ with_dict = dm._stream_window_mib(pipe)
+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: False)
+ without = dm._stream_window_mib(pipe)
+ # the largest block is now a mode block (128 MiB) instead of a txt block (8 MiB)
+ assert with_dict > without > 0