Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: prompt-length dims stay dynamic for positional block inputs under group offload hooks (#15653)

Daniel Han3773455def6c
diff --git a/studio/backend/core/inference/diffusion_dynamic_text.py b/studio/backend/core/inference/diffusion_dynamic_text.py
index 773bb3ef26e..7cab2b48de7 100644
--- a/studio/backend/core/inference/diffusion_dynamic_text.py
+++ b/studio/backend/core/inference/diffusion_dynamic_text.py
@@ -13,6 +13,12 @@ SLICE_IDENTITY_ENV = "UNSLOTH_DIFFUSION_SLICE_IDENTITY_FIX"
_SLICE_LOCK = threading.Lock()
_SLICE_STATE: dict[str, Any] = {}
+
+def _hooked_args(tail: str) -> tuple[str, ...]:
+ """Positional block input under group offload hooks (``L['args']``, or ``___stack0[0]`` after the graph break)."""
+ return tuple(root + tail + "$" for root in (r".*\['args'\]", r".*\['___stack\d+'\]\[0\]"))
+
+
# segments[0][0] (always 0) stays static: a symbol there trips torchao CantSplit (rows become ``s0 - s_start``).
_QWEN_IMAGE_21_SOURCES: tuple[str, ...] = (
"L['hidden_states']",
@@ -45,6 +51,9 @@ _MINIMAX_H3_SOURCES: tuple[str, ...] = (
"L['rotary_emb'][0]",
"L['rotary_emb'][1]",
"L['temb']",
+ *_hooked_args(r"\[[02]\]"),
+ *_hooked_args(r"\[3\]\[[01]\]"),
+ *_hooked_args(r"\[1\]"),
)
# FLUX.3 Klein (distilled and base): the pipeline pads the prompt to a multiple of 80 tokens, so every new bucket was
@@ -59,6 +68,10 @@ _FLUX3_SOURCES: tuple[str, ...] = (
"L['temb_mod_context']",
"L['image_rotary_emb'][0]",
"L['image_rotary_emb'][1]",
+ r".*\['(hidden_states|temb_mod|encoder_hidden_states|temb_mod_context)'\]$",
+ r".*\['image_rotary_emb'\]\[[01]\]$",
+ *_hooked_args(r"\[[01]\]"),
+ *_hooked_args(r"\[2\]\[[01]\]"),
)
# FLUX.1: Studio pads T5 to 256 tokens, or 512 for a longer prompt (diffusion_text_length.flux_t5_sequence_length),
@@ -82,7 +95,7 @@ _ZIMAGE_SOURCES: tuple[str, ...] = (
"L['x']",
"L['attn_mask']",
"L['freqs_cis']",
- r".*\['args'\]\[[0-2]\]$",
+ *_hooked_args(r"\[[0-2]\]"),
)
# Cosmos3 tokenizes the prompt unpadded (pipeline_cosmos3_omni.py ``_tokenize``): every prompt length is a new shape.
@@ -97,6 +110,8 @@ _MOSS_IMAGE_SOURCES: tuple[str, ...] = (
"L['x']",
"L['rope'][0]",
"L['rope'][1]",
+ *_hooked_args(r"\[0\]"),
+ *_hooked_args(r"\[1\]\[[01]\]"),
)
# Prompt-length policy for every denoiser class Studio loads (the ONE list; test_diffusion_prompt_length_policy checks
# it covers diffusion_families and video_families). A tuple: the denoiser's text length follows the prompt, and these
@@ -155,7 +170,7 @@ def _armed_when_dynamic(transformer: Any) -> bool:
# Unbacked: a backed symbol specialises size 1, and H3's temb has 1 row on step 1, 2 after (a second compile).
_FAMILY_UNBACKED: dict[str, tuple[str, ...]] = {
- "MiniMaxH3Transformer3DModel": ("L['temb']",),
+ "MiniMaxH3Transformer3DModel": ("L['temb']", *_hooked_args(r"\[1\]")),
}
diff --git a/studio/backend/tests/test_diffusion_dynamic_text.py b/studio/backend/tests/test_diffusion_dynamic_text.py
index 3b5d6a3bdeb..594667c2e49 100644
--- a/studio/backend/tests/test_diffusion_dynamic_text.py
+++ b/studio/backend/tests/test_diffusion_dynamic_text.py
@@ -346,7 +346,8 @@ def test_unbacked_list_is_restored_after_forward():
m = MiniMaxH3Transformer3DModel()
dt.install(m)
m(12, 1)
- assert m.seen_unbacked.split(",") == ["L['user_thing']", "L['temb']"]
+ assert m.seen_unbacked.split(",") == ["L['user_thing']", *dt.unbacked_sources_for(m)]
+ assert dt.unbacked_sources_for(m)[0] == "L['temb']"
assert cfg.unbacked_sources == "L['user_thing']"
dt.uninstall(m)
finally:
@@ -390,7 +391,11 @@ def test_static_compile_and_unarmed_families_are_not_hooked():
def test_fingerprint_for_dynamic_true_keys_only_on_unbacked():
h3 = MiniMaxH3Transformer3DModel()
- want = "unbacked:L['temb']" if dt.unbacked_supported() else None
+ want = (
+ "unbacked:" + ",".join(dt._FAMILY_UNBACKED[type(h3).__name__])
+ if dt.unbacked_supported()
+ else None
+ )
assert dt.fingerprint(h3, True) == want
assert dt.fingerprint(h3, False) is None
assert dt.fingerprint(QwenImage21Transformer2DModel(), True) is None
@@ -573,7 +578,10 @@ def test_minimax_h3_float8_weights_keep_temb_backed_dynamic():
assert "L['temb']" not in (m.seen_unbacked or "").split(",")
dt.uninstall(m)
# a plain (bf16 / int8) H3 keeps the unbacked temb
- assert dt.unbacked_sources_for(MiniMaxH3Transformer3DModel()) == ("L['temb']",)
+ assert dt.unbacked_sources_for(MiniMaxH3Transformer3DModel())[0] == "L['temb']"
+ assert not set(dt.unbacked_sources_for(MiniMaxH3Transformer3DModel())) & set(
+ dt.sources_for(MiniMaxH3Transformer3DModel())
+ )
def test_minimax_h3_float8_inside_a_plain_parameter_keeps_temb_backed_dynamic():
@@ -692,3 +700,229 @@ def test_flux1_new_prompt_length_and_resolution_reuse_the_first_graphs():
assert graphs_per_render(False)[2:] != [0, 0, 0]
assert graphs_per_render(True) == [2, 0, 0, 0, 0]
+
+
+class _Lin(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+
+class _Flux3ModeBlock(_Lin):
+ def forward(self, hidden_states, temb_mod, image_rotary_emb):
+ return self.proj(hidden_states) * image_rotary_emb[0] + image_rotary_emb[1] + temb_mod
+
+
+class _Flux3SingleBlock(_Lin):
+ def forward(
+ self, hidden_states, encoder_hidden_states, temb_mod, temb_mod_context, image_rotary_emb
+ ):
+ joint = (
+ torch.cat([encoder_hidden_states, hidden_states], 1) * image_rotary_emb[0]
+ + image_rotary_emb[1]
+ )
+ text = encoder_hidden_states.shape[1]
+ return self.proj(joint[:, :text]) + temb_mod_context, joint[:, text:] + temb_mod
+
+
+class _Flux3Hooked(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.transformer_blocks = torch.nn.ModuleList([_Flux3ModeBlock() for _ in range(2)])
+ self.single_transformer_blocks = torch.nn.ModuleList(
+ [_Flux3SingleBlock() for _ in range(2)]
+ )
+
+ def forward(
+ self,
+ text_len,
+ image_len = 64,
+ ):
+ txt, img, mod = (
+ torch.randn(1, text_len, 8),
+ torch.randn(1, image_len, 8),
+ torch.randn(1, 1, 8),
+ )
+ txt_rope = (torch.randn(text_len, 8), torch.randn(text_len, 8))
+ img_rope = (torch.randn(image_len, 8), torch.randn(image_len, 8))
+ for block in self.transformer_blocks:
+ txt = block(txt, mod, txt_rope)
+ for block in self.transformer_blocks:
+ img = block(img, mod, img_rope)
+ rope = (torch.cat([txt_rope[0], img_rope[0]]), torch.cat([txt_rope[1], img_rope[1]]))
+ for block in self.single_transformer_blocks:
+ txt, img = block(
+ hidden_states = img,
+ encoder_hidden_states = txt,
+ temb_mod = mod,
+ temb_mod_context = mod,
+ image_rotary_emb = rope,
+ )
+ return img
+
+
+class _ZImageBlock(_Lin):
+ def forward(self, x, attn_mask, freqs_cis):
+ return self.proj(x * freqs_cis).masked_fill(~attn_mask[..., None], 0.0) + x
+
+
+class _ZImageHooked(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.context_refiner = torch.nn.ModuleList([_ZImageBlock()])
+ self.noise_refiner = torch.nn.ModuleList([_ZImageBlock()])
+ self.layers = torch.nn.ModuleList([_ZImageBlock() for _ in range(2)])
+
+ def forward(
+ self,
+ text_len,
+ image_len = 64,
+ ):
+ def args(n):
+ return torch.randn(1, n, 8), torch.ones(1, n, dtype = torch.bool), torch.randn(1, n, 8)
+
+ cap, img = args(text_len), args(image_len)
+ for block in self.context_refiner:
+ cap = (block(*cap), *cap[1:])
+ for block in self.noise_refiner:
+ img = (block(*img), *img[1:])
+ x = tuple(torch.cat([a, b], 1) for a, b in zip(img, cap))
+ for block in self.layers:
+ x = (block(*x), *x[1:])
+ return x[0]
+
+
+class _MossBlock(_Lin):
+ def forward(self, x, rope, modulation):
+ return self.proj(x) * rope[0] + rope[1] + modulation
+
+
+class _MossHooked(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_MossBlock() for _ in range(2)])
+
+ def forward(
+ self,
+ text_len,
+ image_len = 64,
+ ):
+ n = text_len + image_len
+ x, rope, mod = (
+ torch.randn(1, n, 8),
+ (torch.randn(n, 8), torch.randn(n, 8)),
+ torch.randn(1, 1, 8),
+ )
+ for layer in self.layers:
+ x = layer(x, rope, mod)
+ return x
+
+
+class _H3Block(_Lin):
+ def forward(self, hidden_states, temb, adaln_indices, rotary_emb):
+ mod = temb.index_select(0, adaln_indices)
+ return (self.proj(hidden_states) * rotary_emb[0] + rotary_emb[1]) * (1 + mod)
+
+
+class _H3Hooked(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.transformer_blocks = torch.nn.ModuleList([_H3Block() for _ in range(2)])
+
+ def forward(
+ self,
+ text_len,
+ image_len = 64,
+ ):
+ n = text_len + image_len
+ x, rope = torch.randn(1, n, 8), (torch.randn(n, 8), torch.randn(n, 8))
+ for n_timesteps in (1, 2):
+ temb, idx = torch.randn(n_timesteps, 8), torch.arange(n) % n_timesteps
+ for block in self.transformer_blocks:
+ x = block(x, temb, idx, rope)
+ return x
+
+
+class _FluxHookedBlock(_Lin):
+ def forward(self, hidden_states, encoder_hidden_states, temb, image_rotary_emb):
+ joint = (
+ torch.cat([encoder_hidden_states, hidden_states], 1) * image_rotary_emb[0]
+ + image_rotary_emb[1]
+ )
+ text = encoder_hidden_states.shape[1]
+ return joint[:, :text], self.proj(joint[:, text:]) + temb[:, None]
+
+
+class _FluxHooked(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.transformer_blocks = torch.nn.ModuleList([_FluxHookedBlock() for _ in range(2)])
+
+ def forward(
+ self,
+ text_len,
+ image_len = 64,
+ ):
+ txt, img, temb = (
+ torch.randn(1, text_len, 8),
+ torch.randn(1, image_len, 8),
+ torch.randn(1, 8),
+ )
+ n = text_len + image_len
+ rope = (torch.randn(n, 8), torch.randn(n, 8))
+ for block in self.transformer_blocks:
+ txt, img = block(
+ hidden_states = img, encoder_hidden_states = txt, temb = temb, image_rotary_emb = rope
+ )
+ return img
+
+
+_HOOKED_FAMILIES = {
+ "FluxTransformer2DModel": _FluxHooked,
+ "Flux3Transformer2DModel": _Flux3Hooked,
+ "ZImageTransformer2DModel": _ZImageHooked,
+ "MOSSImageTransformer": _MossHooked,
+ "MiniMaxH3Transformer3DModel": _H3Hooked,
+}
+
+
+def _hooked_graphs_per_render(name):
+ hooks = pytest.importorskip("diffusers.hooks")
+ from torch._dynamo.utils import counters
+
+ from core.inference.diffusion_memory import install_group_offload_hooks_eager
+
+ torch._dynamo.reset()
+ counters.clear()
+ m = type(name, (_HOOKED_FAMILIES[name],), {})()
+ for block in m.modules():
+ if isinstance(block, _Lin):
+ block.compile(backend = "eager", dynamic = None)
+ assert dt.install(m) is True
+ install_group_offload_hooks_eager()
+ cpu = torch.device("cpu")
+ hooks.apply_group_offloading(
+ m, onload_device = cpu, offload_device = cpu, offload_type = "block_level", num_blocks_per_group = 1
+ )
+ seen = []
+ try:
+ with torch.no_grad():
+ for text_len in (16, 16, 24, 20, 40):
+ before = counters["stats"]["unique_graphs"]
+ m(text_len)
+ seen.append(counters["stats"]["unique_graphs"] - before)
+ finally:
+ dt.uninstall(m)
+ torch._dynamo.reset()
+ return seen
+
+
[email protected]("name", sorted(_HOOKED_FAMILIES))
+def test_new_prompt_length_under_group_offload_hooks_reuses_the_first_graphs(name):
+ """Group offload hooks wrap block inputs as ``L['args'][i]`` / ``L['kwargs'][...]`` / ``L['___stack0'][...]``."""
+ seen = _hooked_graphs_per_render(name)
+ assert seen[0] > 0 and seen[1:] == [0, 0, 0, 0], seen
+ if name == "ZImageTransformer2DModel" or (
+ name == "MiniMaxH3Transformer3DModel" and dt.unbacked_supported()
+ ):
+ assert seen[0] == 1, seen