Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: give FLUX.3's per-mode block stacks their own offload groups (5 GiB L4 int8: -17.6% per image) (#17122)
Daniel Hand17d054b0b8a
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 7fdd96b978d..bda70345373 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4097,6 +4097,33 @@ def _resident_dit_fits(memory: Any, dit_mib: int, headroom_mib: int, other_mib:STREAMED_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_STREAMED_RESIDENCY"[email protected]_cache(maxsize = 1)+def _diffusers_takes_block_modules() -> bool:+ """Whether the installed diffusers' ``apply_group_offloading`` accepts ``block_modules``."""+ try:+ import inspect++ from diffusers.hooks import apply_group_offloading+ return "block_modules" in inspect.signature(apply_group_offloading).parameters+ except Exception: # noqa: BLE001 - no diffusers / unreadable: the stock grouping+ return False+++def _block_stacks(module: Any) -> list:+ """The block stacks block-level group offload streams: ``ModuleList`` / ``Sequential`` children, plus the stacks of+ the ModuleDicts ``_dict_block_modules`` hands to diffusers as ``block_modules``."""+ import torch++ stacks = []+ dicts = set(_dict_block_modules(module)) if _diffusers_takes_block_modules() else set()+ for name, child in module.named_children():+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):+ stacks.append(child)+ elif name in dicts:+ stacks.extend(child.values())+ return stacks++def _stream_window_mib(pipe: Any) -> int:"""Device MiB the block prefetcher holds in flight: ``depth + 1`` of the largest denoiser block, or -1."""try:@@ -4110,9 +4137,7 @@ def _stream_window_mib(pipe: Any) -> int:continueif not isinstance(module, torch.nn.Module):continue- for child in module.children():- if not isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):- continue+ for child in _block_stacks(module):for block in child:seen: set[int] = set()nbytes = 0@@ -4168,9 +4193,8 @@ def _top_level_group_mib(pipe: Any) -> Optional[int]:if not isinstance(module, torch.nn.Module):continueblocks: set[int] = set()- for child in module.children():- if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):- blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))+ for child in _block_stacks(module):+ blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))seen: set[int] = set()nbytes = 0for t in (*module.parameters(), *module.buffers()):@@ -6854,6 +6878,51 @@ def _module_pin_mib(module: Any) -> int:return _module_host_mib(module)+# Block-level offload groups only direct ModuleList children; split FLUX.3's ModuleDict of block stacks too ("0": off).+GROUP_OFFLOAD_DICT_BLOCKS_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_DICT_BLOCKS"+_DICT_BLOCK_CLASSES = frozenset({"Flux3Transformer2DModel"})+++def _dict_block_modules(module: Any) -> list[str]:+ """Direct children of ``module`` that are ModuleDicts of block stacks."""+ if _env_off(GROUP_OFFLOAD_DICT_BLOCKS_ENV) or type(module).__name__ not in _DICT_BLOCK_CLASSES:+ return []+ try:+ import torch++ names = []+ for name, child in module.named_children():+ if (+ isinstance(child, torch.nn.ModuleDict)+ and len(child) > 0+ and all(+ isinstance(v, (torch.nn.ModuleList, torch.nn.Sequential))+ for v in child.values()+ )+ ):+ names.append(name)+ return names+ except Exception: # noqa: BLE001 - unreadable: the stock grouping+ return []+++def _with_dict_block_modules(+ apply: Callable[..., Any], module: Any, kwargs: dict[str, Any]+) -> dict[str, Any]:+ if kwargs.get("offload_type", "block_level") != "block_level" or "block_modules" in kwargs:+ return kwargs+ names = _dict_block_modules(module)+ if not names:+ return kwargs+ try:+ import inspect+ if "block_modules" not in inspect.signature(apply).parameters:+ return kwargs+ except (TypeError, ValueError):+ return kwargs+ return {**kwargs, "block_modules": names}++def _apply_group_offloading_packed(apply: Callable[..., Any],module: Any,@@ -6867,6 +6936,7 @@ def _apply_group_offloading_packed(pages, so overlap is unchanged. Lazy (``low_cpu_mem_usage``) and unstreamed applies run as before."""from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload+ kwargs = _with_dict_block_modules(apply, module, kwargs)pack = (bool(kwargs.get("use_stream"))and not kwargs.get("low_cpu_mem_usage", False)@@ -8322,7 +8392,12 @@ def _apply_group_offload(and not gkwargs.get("low_cpu_mem_usage", False)and not _torchao_weight_classes(module)):- apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})+ apply_group_offloading(+ module,+ **_with_dict_block_modules(+ apply_group_offloading, module, {**gkwargs, "low_cpu_mem_usage": True}+ ),+ )_defer_pinning(pipe, module, onload, logger)else:_apply_group_offloading_packed(@@ -8396,7 +8471,12 @@ def _apply_group_offload(if "low_cpu_mem_usage" in _params and use_stream:dkwargs["low_cpu_mem_usage"] = Trueapply_group_offloading(- dit, **_torchao_group_offload_kwargs(dit, dkwargs, pinned_mib)+ dit,+ **_with_dict_block_modules(+ apply_group_offloading,+ dit,+ _torchao_group_offload_kwargs(dit, dkwargs, pinned_mib),+ ),)installed += 1transformer_demoted = Truediff --git a/studio/backend/tests/test_diffusion_flux3_block_groups.py b/studio/backend/tests/test_diffusion_flux3_block_groups.pynew file mode 100644index 00000000000..8c300754e81--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_block_groups.py@@ -0,0 +1,150 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3's per-mode block stacks (ModuleDicts of ModuleLists) become their own block-level offload groups instead of+riding in the top-level group, which a streamed load onloads whole for every forward. CPU only."""++from __future__ import annotations++import pytest++import core.inference.diffusion_memory as dm+++def _flux3_like():+ import torch+ class Flux3Transformer2DModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.x_embedder = torch.nn.Linear(4, 8)+ self.mode_blocks = torch.nn.ModuleDict(+ {+ name: torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(3)])+ for name in ("image", "image_cond")+ }+ )+ self.modulations = torch.nn.ModuleDict({"image": torch.nn.Linear(8, 8)})+ self.txt_mode_blocks = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])++ def forward(self, x):+ x = self.x_embedder(x)+ for block in self.txt_mode_blocks:+ x = block(x)+ for block in self.mode_blocks["image"]:+ x = block(x)+ return x++ return Flux3Transformer2DModel()+++def test_flux3_mode_block_dicts_become_block_groups(monkeypatch):+ import torch++ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)+ model = _flux3_like()+ assert dm._dict_block_modules(model) == ["mode_blocks"]++ def apply(+ module,+ *,+ offload_type = "block_level",+ block_modules = None,+ **kw,+ ):+ return None++ def old_apply(+ module,+ *,+ offload_type = "block_level",+ **kw,+ ):+ return None++ assert dm._with_dict_block_modules(apply, model, {"offload_type": "block_level"})[+ "block_modules"+ ] == ["mode_blocks"]+ # leaf-level, an older diffusers without the parameter, and other classes keep the stock grouping+ assert "block_modules" not in dm._with_dict_block_modules(+ apply, model, {"offload_type": "leaf_level"}+ )+ assert "block_modules" not in dm._with_dict_block_modules(+ old_apply, model, {"offload_type": "block_level"}+ )+ assert (+ dm._dict_block_modules(+ torch.nn.ModuleDict({"a": torch.nn.ModuleList([torch.nn.Linear(2, 2)])})+ )+ == []+ )+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, "0")+ assert dm._dict_block_modules(model) == []+++def test_flux3_mode_blocks_get_their_own_diffusers_groups(monkeypatch):+ import inspect++ import torch++ from diffusers.hooks import apply_group_offloading++ # an older diffusers without block_modules keeps the stock grouping+ supported = "block_modules" in inspect.signature(apply_group_offloading).parameters+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)+ counts = {}+ for env in ("1", "0"):+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, env)+ model = _flux3_like()+ x = torch.randn(2, 4)+ want = model(x)+ kwargs = dict(+ onload_device = torch.device("cpu"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ )+ dm._apply_group_offloading_packed(apply_group_offloading, model, kwargs)+ groups = dm._offload_groups(model)+ counts[env] = len(groups)+ mode_groups = [+ g for g in groups if any(m is b for b in model.mode_blocks["image"] for m in g.modules)+ ]+ assert (len(mode_groups) == 3) is (env == "1" and supported)+ torch.testing.assert_close(model(x), want)+ # 6 mode blocks become groups of their own; the stock grouping folds them into the top-level group+ assert counts["1"] == counts["0"] + (6 if supported else 0)+++def _mib_model(monkeypatch):+ """A FLUX.3-like model whose parameter sizes are read from fake storage sizes (MiB per tensor)."""+ model = _flux3_like()+ sizes = {}+ for name, p in model.named_parameters():+ sizes[id(p)] = 64 if ".mode_blocks." in "." + name else 4+ monkeypatch.setattr(dm, "_storage_nbytes", lambda t: [sizes.get(id(t), 0) * 1024 * 1024])+ return model++[email protected]("supported", [True, False])+def test_top_level_group_leaves_the_mode_blocks_out(monkeypatch, supported):+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: supported)+ model = _mib_model(monkeypatch)+ pipe = type("P", (), {"components": {"transformer": model}})()+ top = dm._top_level_group_mib(pipe)+ # x_embedder + modulations (16 MiB) stay top level; the 6 mode blocks move out only if diffusers can split them+ assert top == (16 if supported else 16 + 6 * 128)+ monkeypatch.setenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, "0")+ assert dm._top_level_group_mib(pipe) == 16 + 6 * 128+++def test_stream_window_counts_the_mode_blocks(monkeypatch):+ monkeypatch.delenv(dm.GROUP_OFFLOAD_DICT_BLOCKS_ENV, raising = False)+ model = _mib_model(monkeypatch)+ pipe = type("P", (), {"components": {"transformer": model}})()+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: True)+ with_dict = dm._stream_window_mib(pipe)+ monkeypatch.setattr(dm, "_diffusers_takes_block_modules", lambda: False)+ without = dm._stream_window_mib(pipe)+ # the largest block is now a mode block (128 MiB) instead of a txt block (8 MiB)+ assert with_dict > without > 0