Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio FLUX.3: keep reference-image modulation at one row per span in the joint blocks (#17181)
Daniel Hanf2bb9c2195b6
diff --git a/studio/backend/core/inference/diffusion_cond_blocks.py b/studio/backend/core/inference/diffusion_cond_blocks.pyindex 00a5a8538b4..2cc7dfd2857 100644--- a/studio/backend/core/inference/diffusion_cond_blocks.py+++ b/studio/backend/core/inference/diffusion_cond_blocks.py@@ -5,7 +5,9 @@and end of each pipeline call; installed only on matching diffusers forward hash, else stock;UNSLOTH_DIFFUSION_COND_BLOCKS=0 disables. Offloaded (CUDA): computed inside the hooked forward,UNSLOTH_DIFFUSION_OFFLOAD_COND_CACHE=0 keeps stock. Modulation heads project one row per sample,-not per token (any FLUX.3), UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD=0 restores per-token rows"""+not per token (any FLUX.3), UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD=0 restores per-token rows. With reference+images the joint blocks take one row per span (image, references) instead of a per-token copy, bit-identical;+UNSLOTH_DIFFUSION_FLUX3_SPAN_MOD=0 restores the per-token rows"""from __future__ import annotations@@ -20,14 +22,21 @@ from .rocm_bf16 import is_rocm_torchCOND_BLOCKS_ENV = "UNSLOTH_DIFFUSION_COND_BLOCKS"OFFLOAD_COND_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_COND_CACHE"COMPACT_MOD_ENV = "UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD"+SPAN_MOD_ENV = "UNSLOTH_DIFFUSION_FLUX3_SPAN_MOD"+SPAN_KWARG = "unsloth_cond_mod"+SPAN_LEN_KWARG = "unsloth_cond_span"COND_KWARG = "unsloth_cond"_COMPACT_ATTR = "_unsloth_compact_mod"+_SPAN_ATTR = "_unsloth_span_mod"_OFFLOADED_ATTR = "_unsloth_cond_offloaded"# sha256[:16] of inspect.getsource(Flux3Transformer2DModel.forward) the replacement mirrors (d42ebf9c and 0.42.1)._FLUX3_FORWARD_SHA = frozenset({"f2f7c836416cfd58"})+# Same, for Flux3SingleTransformerBlock.forward (the span-modulated joint block mirrors it).+_FLUX3_SINGLE_SHA = frozenset({"92269bb072da8335"})_PATCHED: dict[type, Any] = {}+_PATCHED_SINGLE: dict[type, Any] = {}def _switch(name: str) -> bool:@@ -46,6 +55,10 @@ def compact_enabled() -> bool:return _switch(COMPACT_MOD_ENV)+def span_enabled() -> bool:+ return _switch(SPAN_MOD_ENV)++def _forward_sha(cls: type) -> Optional[str]:try:return hashlib.sha256(inspect.getsource(cls.forward).encode()).hexdigest()[:16]@@ -319,18 +332,30 @@ def _flux3_cached_forward(for block in self.mode_blocks["image"]:hidden_states = block(hidden_states, image_early_mod, image_rotary_emb)+ block_kwargs: dict = {}if has_cond:cond_states = unsloth_cond["cond_states"]cond_rotary_emb = (unsloth_cond["cond_cos"], unsloth_cond["cond_sin"])content_states = torch.cat([hidden_states, cond_states], dim = 1)- # image and reference spans carry different timesteps: per token here- content_mod = torch.cat(- [- _rows(image_single_mod, batch_size, image_seq_len),- _rows(unsloth_cond["cond_single_mod"], batch_size, cond_states.shape[1]),- ],- dim = 1,- )+ cond_single_mod = unsloth_cond["cond_single_mod"]+ if (+ self.__dict__.get(_SPAN_ATTR)+ and image_single_mod.shape[1] == 1+ and cond_single_mod.shape[1] == 1+ ):+ # one row per span; the span length rides on a storage-free tensor so block graphs can copy every input+ content_mod = image_single_mod+ block_kwargs[SPAN_KWARG] = cond_single_mod+ block_kwargs[SPAN_LEN_KWARG] = cond_states.new_empty((cond_states.shape[1], 0))+ else:+ # image and reference spans carry different timesteps: per token here+ content_mod = torch.cat(+ [+ _rows(image_single_mod, batch_size, image_seq_len),+ _rows(cond_single_mod, batch_size, cond_states.shape[1]),+ ],+ dim = 1,+ )joint_rotary_emb = (torch.cat([txt_rotary_emb[0], image_rotary_emb[0], cond_rotary_emb[0]], dim = 0),torch.cat([txt_rotary_emb[1], image_rotary_emb[1], cond_rotary_emb[1]], dim = 0),@@ -350,6 +375,7 @@ def _flux3_cached_forward(temb_mod = content_mod,temb_mod_context = txt_single_mod,image_rotary_emb = joint_rotary_emb,+ **block_kwargs,)hidden_states = content_states[:, :image_seq_len]@@ -361,6 +387,96 @@ def _flux3_cached_forward(return Transformer2DModelOutput(sample = output)+def _flux3_span_single_forward(+ self: Any,+ hidden_states: Any,+ encoder_hidden_states: Any,+ temb_mod: Any,+ temb_mod_context: Any,+ image_rotary_emb: Any,+ unsloth_cond_mod: Any = None,+ unsloth_cond_span: Any = None,+) -> Any:+ """Stock joint block, op for op, with one ``[B, 1, 3D]`` modulation row per span: ``temb_mod`` (image),+ ``unsloth_cond_mod`` (reference, length = ``unsloth_cond_span``'s ``[reference_tokens, 0]`` shape)."""+ if unsloth_cond_mod is None or unsloth_cond_span is None:+ return _stock_single_forward(self)(+ self,+ hidden_states = hidden_states,+ encoder_hidden_states = encoder_hidden_states,+ temb_mod = temb_mod,+ temb_mod_context = temb_mod_context,+ image_rotary_emb = image_rotary_emb,+ )+ import torch++ n_cond = unsloth_cond_span.shape[0]+ n_image = hidden_states.shape[1] - n_cond+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)+ r_shift, r_scale, r_gate = torch.chunk(unsloth_cond_mod, 3, dim = -1)+ c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim = -1)++ norm_hidden_states = self.norm(hidden_states)+ norm_image, norm_cond = torch.split(norm_hidden_states, [n_image, n_cond], dim = 1)+ norm_hidden_states = torch.cat(+ [(1 + mod_scale) * norm_image + mod_shift, (1 + r_scale) * norm_cond + r_shift], dim = 1+ )++ norm_encoder_hidden_states = self.norm(encoder_hidden_states)+ norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift++ combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)++ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)+ ff_output = self.ff(combined)+ output = attn_output + ff_output++ context_output, image_output, cond_output = torch.split(+ output, [encoder_hidden_states.shape[1], n_image, n_cond], dim = 1+ )+ encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output+ image_states, cond_states = torch.split(hidden_states, [n_image, n_cond], dim = 1)+ hidden_states = torch.cat(+ [image_states + mod_gate * image_output, cond_states + r_gate * cond_output], dim = 1+ )++ return encoder_hidden_states, hidden_states+++def _stock_single_forward(module: Any) -> Any:+ for cls in type(module).__mro__:+ stock = _PATCHED_SINGLE.get(cls)+ if stock is not None:+ return stock+ raise RuntimeError(f"{type(module).__name__} has no stock forward recorded")+++def patch_single_class(cls: type) -> Optional[str]:+ """Install the span-aware joint block forward on ``cls`` (at load, before compile or a hook binds it)."""+ if any(c in _PATCHED_SINGLE for c in cls.__mro__):+ return None+ sha = _forward_sha(cls)+ if sha not in _FLUX3_SINGLE_SHA:+ return f"diffusers joint block changed (source {sha})"+ _PATCHED_SINGLE[cls] = cls.forward+ cls.forward = _flux3_span_single_forward+ return None+++def _span_ready(transformer: Any) -> Optional[str]:+ """None when every joint block runs the span-aware forward; else why the per-token rows stay."""+ blocks = list(getattr(transformer, "single_transformer_blocks", None) or ())+ if not blocks:+ return "no joint blocks"+ for block in blocks:+ why = patch_single_class(type(block))+ if why is not None:+ return why+ if type(block).forward is not _flux3_span_single_forward:+ return "the joint block forward is not the patched one"+ return None++def _stock_forward(module: Any) -> Any:for cls in type(module).__mro__:stock = _PATCHED.get(cls)@@ -511,6 +627,13 @@ def prepare(transformer: Any, logger: Any = None) -> bool:transformer.__dict__[_COMPACT_ATTR] = compactif compact and logger is not None:logger.info("diffusion.cond_blocks: FLUX.3 modulation heads project one row per sample")+ span = False+ if compact and span_enabled():+ why = _span_ready(transformer)+ span = why is None+ if why is not None and logger is not None:+ logger.info("diffusion.cond_blocks: reference modulation stays per token (%s)", why)+ transformer.__dict__[_SPAN_ATTR] = spanif not cond_on:return Falseif applies is not None:diff --git a/studio/backend/tests/test_diffusion_cond_blocks.py b/studio/backend/tests/test_diffusion_cond_blocks.pyindex 1e7b45093e2..743cd05543c 100644--- a/studio/backend/tests/test_diffusion_cond_blocks.py+++ b/studio/backend/tests/test_diffusion_cond_blocks.py@@ -496,3 +496,146 @@ def test_switch_values(monkeypatch):for value, want in (("", True), ("1", True), ("0", False), ("off", False), ("False", False)):monkeypatch.setenv(cb.COND_BLOCKS_ENV, value)assert cb.enabled() is want+++def _span_inputs(+ batch: int,+ refs: int,+ dtype,+ seed: int = 3,+):+ g = torch.Generator().manual_seed(seed)+ img, txt = 16, 5+ kw = {+ "hidden_states": torch.randn(batch, img, 8, generator = g).to(dtype),+ "encoder_hidden_states": torch.randn(batch, txt, 24, generator = g).to(dtype),+ "timestep": torch.full((batch,), 0.7),+ "img_ids": torch.randint(0, 8, (img, 4), generator = g).float(),+ "txt_ids": torch.randint(0, 8, (txt, 4), generator = g).float(),+ "return_dict": False,+ }+ if refs:+ n = 7 * refs # odd span length, one span of 7 tokens per reference image+ kw["image_cond_hidden_states"] = torch.randn(batch, n, 8, generator = g).to(dtype)+ kw["image_cond_ids"] = torch.randint(0, 8, (n, 4), generator = g).float()+ return kw+++def _joint_mod_shapes(model):+ seen = []++ def spy(m, args, kwargs):+ span = kwargs.get(cb.SPAN_KWARG)+ if span is not None:+ assert tuple(span.shape[1:2]) == (1,) and kwargs[cb.SPAN_LEN_KWARG].numel() == 0+ # a block graph copies every input into a static buffer of the same strides: none may overlap itself+ from core.inference.diffusion_block_graph import _static_like+ for t in (kwargs["temb_mod"], span, kwargs[cb.SPAN_LEN_KWARG]):+ _static_like(t).copy_(t)+ seen.append((tuple(kwargs["temb_mod"].shape), span is not None))++ handles = [+ b.register_forward_pre_hook(spy, with_kwargs = True) for b in model.single_transformer_blocks+ ]+ return seen, handles++[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("batch", [1, 2])[email protected]("refs", [0, 1, 8])+def test_span_modulation_matches_per_token_rows_bitwise(dtype, batch, refs, monkeypatch):+ """One modulation row per span (image, references) is bitwise the per-token rows, and no per-token copy reaches+ the joint blocks."""+ cls = _flux3()+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)+ monkeypatch.delenv(cb.SPAN_MOD_ENV, raising = False)+ model = _tiny(cls).to(dtype)+ kw = _span_inputs(batch, refs, dtype)+ with torch.no_grad():+ assert cb.prepare(model)+ assert model.__dict__[cb._SPAN_ATTR] is True+ model.__dict__[cb._SPAN_ATTR] = False+ want = model(**kw)[0]+ model.__dict__[cb._SPAN_ATTR] = True+ seen, handles = _joint_mod_shapes(model)+ try:+ got = model(**kw)[0]+ finally:+ for h in handles:+ h.remove()+ assert torch.equal(+ got.view(torch.int16 if dtype == torch.bfloat16 else torch.int32),+ want.view(torch.int16 if dtype == torch.bfloat16 else torch.int32),+ )+ if refs:+ assert all(shape[1] == 1 and span for shape, span in seen), seen+ else:+ assert not any(span for _, span in seen)+++def test_span_modulation_kill_switch(monkeypatch):+ cls = _flux3()+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)+ monkeypatch.setenv(cb.SPAN_MOD_ENV, "0")+ model = _tiny(cls)+ cb.prepare(model)+ assert model.__dict__[cb._SPAN_ATTR] is False+ seen, handles = _joint_mod_shapes(model)+ with torch.no_grad():+ model(**_span_inputs(1, 1, torch.float32))+ for h in handles:+ h.remove()+ assert seen and all(shape[1] > 1 and not span for shape, span in seen)+++def test_span_block_without_the_kwarg_is_stock():+ cls = _flux3()+ model = _tiny(cls)+ block = model.single_transformer_blocks[0]+ assert cb.patch_single_class(type(block)) is None+ g = torch.Generator().manual_seed(5)+ h, e = torch.randn(1, 9, 32, generator = g), torch.randn(1, 4, 32, generator = g)+ mod, cmod = torch.randn(1, 9, 96, generator = g), torch.randn(1, 1, 96, generator = g)+ rope = (torch.ones(13, 16), torch.zeros(13, 16))+ stock = cb._PATCHED_SINGLE[type(block)]+ with torch.no_grad():+ want = stock(block, h, e, mod, cmod, rope)+ got = block(h, e, mod, cmod, rope)+ assert all(torch.equal(a, b) for a, b in zip(want, got))++[email protected]("refs", [1, 3])+def test_span_blocks_compile_without_breaks_and_match_compiled_per_token_rows(refs, monkeypatch):+ """Studio compiles each block: the span forward traces whole (no graph break), once across two reference counts+ under dynamic shapes, and matches the per-token rows compiled the same way."""+ from torch._dynamo.testing import CompileCounterWithBackend++ cls = _flux3()+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)+ monkeypatch.delenv(cb.SPAN_MOD_ENV, raising = False)+ model = _tiny(cls)+ assert cb.prepare(model)+ blocks = list(model.single_transformer_blocks)+ eager = [b.forward for b in blocks]+ outs = {}+ counters = {}+ for span in (False, True):+ torch._dynamo.reset()+ cc = CompileCounterWithBackend("inductor")+ for b in blocks:+ b.forward = torch.compile(+ type(b).forward.__get__(b), backend = cc, fullgraph = True, dynamic = True+ )+ model.__dict__[cb._SPAN_ATTR] = span+ try:+ with torch.no_grad():+ outs[span] = [+ model(**_span_inputs(1, n, torch.float32))[0] for n in (refs, refs + 1)+ ]+ finally:+ for b, f in zip(blocks, eager):+ b.forward = f+ counters[span] = cc.frame_count+ torch._dynamo.reset()+ assert counters[True] == len(blocks)+ assert all(torch.equal(a, b) for a, b in zip(outs[True], outs[False]))