Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: prompt-length dims stay dynamic for positional block inputs under group offload hooks (#15653)
Daniel Han3773455def6c
diff --git a/studio/backend/core/inference/diffusion_dynamic_text.py b/studio/backend/core/inference/diffusion_dynamic_text.pyindex 773bb3ef26e..7cab2b48de7 100644--- a/studio/backend/core/inference/diffusion_dynamic_text.py+++ b/studio/backend/core/inference/diffusion_dynamic_text.py@@ -13,6 +13,12 @@ SLICE_IDENTITY_ENV = "UNSLOTH_DIFFUSION_SLICE_IDENTITY_FIX"_SLICE_LOCK = threading.Lock()_SLICE_STATE: dict[str, Any] = {}++def _hooked_args(tail: str) -> tuple[str, ...]:+ """Positional block input under group offload hooks (``L['args']``, or ``___stack0[0]`` after the graph break)."""+ return tuple(root + tail + "$" for root in (r".*\['args'\]", r".*\['___stack\d+'\]\[0\]"))++# segments[0][0] (always 0) stays static: a symbol there trips torchao CantSplit (rows become ``s0 - s_start``)._QWEN_IMAGE_21_SOURCES: tuple[str, ...] = ("L['hidden_states']",@@ -45,6 +51,9 @@ _MINIMAX_H3_SOURCES: tuple[str, ...] = ("L['rotary_emb'][0]","L['rotary_emb'][1]","L['temb']",+ *_hooked_args(r"\[[02]\]"),+ *_hooked_args(r"\[3\]\[[01]\]"),+ *_hooked_args(r"\[1\]"),)# FLUX.3 Klein (distilled and base): the pipeline pads the prompt to a multiple of 80 tokens, so every new bucket was@@ -59,6 +68,10 @@ _FLUX3_SOURCES: tuple[str, ...] = ("L['temb_mod_context']","L['image_rotary_emb'][0]","L['image_rotary_emb'][1]",+ r".*\['(hidden_states|temb_mod|encoder_hidden_states|temb_mod_context)'\]$",+ r".*\['image_rotary_emb'\]\[[01]\]$",+ *_hooked_args(r"\[[01]\]"),+ *_hooked_args(r"\[2\]\[[01]\]"),)# FLUX.1: Studio pads T5 to 256 tokens, or 512 for a longer prompt (diffusion_text_length.flux_t5_sequence_length),@@ -82,7 +95,7 @@ _ZIMAGE_SOURCES: tuple[str, ...] = ("L['x']","L['attn_mask']","L['freqs_cis']",- r".*\['args'\]\[[0-2]\]$",+ *_hooked_args(r"\[[0-2]\]"),)# Cosmos3 tokenizes the prompt unpadded (pipeline_cosmos3_omni.py ``_tokenize``): every prompt length is a new shape.@@ -97,6 +110,8 @@ _MOSS_IMAGE_SOURCES: tuple[str, ...] = ("L['x']","L['rope'][0]","L['rope'][1]",+ *_hooked_args(r"\[0\]"),+ *_hooked_args(r"\[1\]\[[01]\]"),)# Prompt-length policy for every denoiser class Studio loads (the ONE list; test_diffusion_prompt_length_policy checks# it covers diffusion_families and video_families). A tuple: the denoiser's text length follows the prompt, and these@@ -155,7 +170,7 @@ def _armed_when_dynamic(transformer: Any) -> bool:# Unbacked: a backed symbol specialises size 1, and H3's temb has 1 row on step 1, 2 after (a second compile)._FAMILY_UNBACKED: dict[str, tuple[str, ...]] = {- "MiniMaxH3Transformer3DModel": ("L['temb']",),+ "MiniMaxH3Transformer3DModel": ("L['temb']", *_hooked_args(r"\[1\]")),}diff --git a/studio/backend/tests/test_diffusion_dynamic_text.py b/studio/backend/tests/test_diffusion_dynamic_text.pyindex 3b5d6a3bdeb..594667c2e49 100644--- a/studio/backend/tests/test_diffusion_dynamic_text.py+++ b/studio/backend/tests/test_diffusion_dynamic_text.py@@ -346,7 +346,8 @@ def test_unbacked_list_is_restored_after_forward():m = MiniMaxH3Transformer3DModel()dt.install(m)m(12, 1)- assert m.seen_unbacked.split(",") == ["L['user_thing']", "L['temb']"]+ assert m.seen_unbacked.split(",") == ["L['user_thing']", *dt.unbacked_sources_for(m)]+ assert dt.unbacked_sources_for(m)[0] == "L['temb']"assert cfg.unbacked_sources == "L['user_thing']"dt.uninstall(m)finally:@@ -390,7 +391,11 @@ def test_static_compile_and_unarmed_families_are_not_hooked():def test_fingerprint_for_dynamic_true_keys_only_on_unbacked():h3 = MiniMaxH3Transformer3DModel()- want = "unbacked:L['temb']" if dt.unbacked_supported() else None+ want = (+ "unbacked:" + ",".join(dt._FAMILY_UNBACKED[type(h3).__name__])+ if dt.unbacked_supported()+ else None+ )assert dt.fingerprint(h3, True) == wantassert dt.fingerprint(h3, False) is Noneassert dt.fingerprint(QwenImage21Transformer2DModel(), True) is None@@ -573,7 +578,10 @@ def test_minimax_h3_float8_weights_keep_temb_backed_dynamic():assert "L['temb']" not in (m.seen_unbacked or "").split(",")dt.uninstall(m)# a plain (bf16 / int8) H3 keeps the unbacked temb- assert dt.unbacked_sources_for(MiniMaxH3Transformer3DModel()) == ("L['temb']",)+ assert dt.unbacked_sources_for(MiniMaxH3Transformer3DModel())[0] == "L['temb']"+ assert not set(dt.unbacked_sources_for(MiniMaxH3Transformer3DModel())) & set(+ dt.sources_for(MiniMaxH3Transformer3DModel())+ )def test_minimax_h3_float8_inside_a_plain_parameter_keeps_temb_backed_dynamic():@@ -692,3 +700,229 @@ def test_flux1_new_prompt_length_and_resolution_reuse_the_first_graphs():assert graphs_per_render(False)[2:] != [0, 0, 0]assert graphs_per_render(True) == [2, 0, 0, 0, 0]+++class _Lin(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)+++class _Flux3ModeBlock(_Lin):+ def forward(self, hidden_states, temb_mod, image_rotary_emb):+ return self.proj(hidden_states) * image_rotary_emb[0] + image_rotary_emb[1] + temb_mod+++class _Flux3SingleBlock(_Lin):+ def forward(+ self, hidden_states, encoder_hidden_states, temb_mod, temb_mod_context, image_rotary_emb+ ):+ joint = (+ torch.cat([encoder_hidden_states, hidden_states], 1) * image_rotary_emb[0]+ + image_rotary_emb[1]+ )+ text = encoder_hidden_states.shape[1]+ return self.proj(joint[:, :text]) + temb_mod_context, joint[:, text:] + temb_mod+++class _Flux3Hooked(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.transformer_blocks = torch.nn.ModuleList([_Flux3ModeBlock() for _ in range(2)])+ self.single_transformer_blocks = torch.nn.ModuleList(+ [_Flux3SingleBlock() for _ in range(2)]+ )++ def forward(+ self,+ text_len,+ image_len = 64,+ ):+ txt, img, mod = (+ torch.randn(1, text_len, 8),+ torch.randn(1, image_len, 8),+ torch.randn(1, 1, 8),+ )+ txt_rope = (torch.randn(text_len, 8), torch.randn(text_len, 8))+ img_rope = (torch.randn(image_len, 8), torch.randn(image_len, 8))+ for block in self.transformer_blocks:+ txt = block(txt, mod, txt_rope)+ for block in self.transformer_blocks:+ img = block(img, mod, img_rope)+ rope = (torch.cat([txt_rope[0], img_rope[0]]), torch.cat([txt_rope[1], img_rope[1]]))+ for block in self.single_transformer_blocks:+ txt, img = block(+ hidden_states = img,+ encoder_hidden_states = txt,+ temb_mod = mod,+ temb_mod_context = mod,+ image_rotary_emb = rope,+ )+ return img+++class _ZImageBlock(_Lin):+ def forward(self, x, attn_mask, freqs_cis):+ return self.proj(x * freqs_cis).masked_fill(~attn_mask[..., None], 0.0) + x+++class _ZImageHooked(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.context_refiner = torch.nn.ModuleList([_ZImageBlock()])+ self.noise_refiner = torch.nn.ModuleList([_ZImageBlock()])+ self.layers = torch.nn.ModuleList([_ZImageBlock() for _ in range(2)])++ def forward(+ self,+ text_len,+ image_len = 64,+ ):+ def args(n):+ return torch.randn(1, n, 8), torch.ones(1, n, dtype = torch.bool), torch.randn(1, n, 8)++ cap, img = args(text_len), args(image_len)+ for block in self.context_refiner:+ cap = (block(*cap), *cap[1:])+ for block in self.noise_refiner:+ img = (block(*img), *img[1:])+ x = tuple(torch.cat([a, b], 1) for a, b in zip(img, cap))+ for block in self.layers:+ x = (block(*x), *x[1:])+ return x[0]+++class _MossBlock(_Lin):+ def forward(self, x, rope, modulation):+ return self.proj(x) * rope[0] + rope[1] + modulation+++class _MossHooked(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_MossBlock() for _ in range(2)])++ def forward(+ self,+ text_len,+ image_len = 64,+ ):+ n = text_len + image_len+ x, rope, mod = (+ torch.randn(1, n, 8),+ (torch.randn(n, 8), torch.randn(n, 8)),+ torch.randn(1, 1, 8),+ )+ for layer in self.layers:+ x = layer(x, rope, mod)+ return x+++class _H3Block(_Lin):+ def forward(self, hidden_states, temb, adaln_indices, rotary_emb):+ mod = temb.index_select(0, adaln_indices)+ return (self.proj(hidden_states) * rotary_emb[0] + rotary_emb[1]) * (1 + mod)+++class _H3Hooked(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.transformer_blocks = torch.nn.ModuleList([_H3Block() for _ in range(2)])++ def forward(+ self,+ text_len,+ image_len = 64,+ ):+ n = text_len + image_len+ x, rope = torch.randn(1, n, 8), (torch.randn(n, 8), torch.randn(n, 8))+ for n_timesteps in (1, 2):+ temb, idx = torch.randn(n_timesteps, 8), torch.arange(n) % n_timesteps+ for block in self.transformer_blocks:+ x = block(x, temb, idx, rope)+ return x+++class _FluxHookedBlock(_Lin):+ def forward(self, hidden_states, encoder_hidden_states, temb, image_rotary_emb):+ joint = (+ torch.cat([encoder_hidden_states, hidden_states], 1) * image_rotary_emb[0]+ + image_rotary_emb[1]+ )+ text = encoder_hidden_states.shape[1]+ return joint[:, :text], self.proj(joint[:, text:]) + temb[:, None]+++class _FluxHooked(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.transformer_blocks = torch.nn.ModuleList([_FluxHookedBlock() for _ in range(2)])++ def forward(+ self,+ text_len,+ image_len = 64,+ ):+ txt, img, temb = (+ torch.randn(1, text_len, 8),+ torch.randn(1, image_len, 8),+ torch.randn(1, 8),+ )+ n = text_len + image_len+ rope = (torch.randn(n, 8), torch.randn(n, 8))+ for block in self.transformer_blocks:+ txt, img = block(+ hidden_states = img, encoder_hidden_states = txt, temb = temb, image_rotary_emb = rope+ )+ return img+++_HOOKED_FAMILIES = {+ "FluxTransformer2DModel": _FluxHooked,+ "Flux3Transformer2DModel": _Flux3Hooked,+ "ZImageTransformer2DModel": _ZImageHooked,+ "MOSSImageTransformer": _MossHooked,+ "MiniMaxH3Transformer3DModel": _H3Hooked,+}+++def _hooked_graphs_per_render(name):+ hooks = pytest.importorskip("diffusers.hooks")+ from torch._dynamo.utils import counters++ from core.inference.diffusion_memory import install_group_offload_hooks_eager++ torch._dynamo.reset()+ counters.clear()+ m = type(name, (_HOOKED_FAMILIES[name],), {})()+ for block in m.modules():+ if isinstance(block, _Lin):+ block.compile(backend = "eager", dynamic = None)+ assert dt.install(m) is True+ install_group_offload_hooks_eager()+ cpu = torch.device("cpu")+ hooks.apply_group_offloading(+ m, onload_device = cpu, offload_device = cpu, offload_type = "block_level", num_blocks_per_group = 1+ )+ seen = []+ try:+ with torch.no_grad():+ for text_len in (16, 16, 24, 20, 40):+ before = counters["stats"]["unique_graphs"]+ m(text_len)+ seen.append(counters["stats"]["unique_graphs"] - before)+ finally:+ dt.uninstall(m)+ torch._dynamo.reset()+ return seen++[email protected]("name", sorted(_HOOKED_FAMILIES))+def test_new_prompt_length_under_group_offload_hooks_reuses_the_first_graphs(name):+ """Group offload hooks wrap block inputs as ``L['args'][i]`` / ``L['kwargs'][...]`` / ``L['___stack0'][...]``."""+ seen = _hooked_graphs_per_render(name)+ assert seen[0] > 0 and seen[1:] == [0, 0, 0, 0], seen+ if name == "ZImageTransformer2DModel" or (+ name == "MiniMaxH3Transformer3DModel" and dt.unbacked_supported()+ ):+ assert seen[0] == 1, seen