Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio FLUX.3: keep reference-image modulation at one row per span in the joint blocks (#17181)

Daniel Hanf2bb9c2195b6
diff --git a/studio/backend/core/inference/diffusion_cond_blocks.py b/studio/backend/core/inference/diffusion_cond_blocks.py
index 00a5a8538b4..2cc7dfd2857 100644
--- a/studio/backend/core/inference/diffusion_cond_blocks.py
+++ b/studio/backend/core/inference/diffusion_cond_blocks.py
@@ -5,7 +5,9 @@
and end of each pipeline call; installed only on matching diffusers forward hash, else stock;
UNSLOTH_DIFFUSION_COND_BLOCKS=0 disables. Offloaded (CUDA): computed inside the hooked forward,
UNSLOTH_DIFFUSION_OFFLOAD_COND_CACHE=0 keeps stock. Modulation heads project one row per sample,
-not per token (any FLUX.3), UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD=0 restores per-token rows"""
+not per token (any FLUX.3), UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD=0 restores per-token rows. With reference
+images the joint blocks take one row per span (image, references) instead of a per-token copy, bit-identical;
+UNSLOTH_DIFFUSION_FLUX3_SPAN_MOD=0 restores the per-token rows"""
from __future__ import annotations
@@ -20,14 +22,21 @@ from .rocm_bf16 import is_rocm_torch
COND_BLOCKS_ENV = "UNSLOTH_DIFFUSION_COND_BLOCKS"
OFFLOAD_COND_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_COND_CACHE"
COMPACT_MOD_ENV = "UNSLOTH_DIFFUSION_FLUX3_COMPACT_MOD"
+SPAN_MOD_ENV = "UNSLOTH_DIFFUSION_FLUX3_SPAN_MOD"
+SPAN_KWARG = "unsloth_cond_mod"
+SPAN_LEN_KWARG = "unsloth_cond_span"
COND_KWARG = "unsloth_cond"
_COMPACT_ATTR = "_unsloth_compact_mod"
+_SPAN_ATTR = "_unsloth_span_mod"
_OFFLOADED_ATTR = "_unsloth_cond_offloaded"
# sha256[:16] of inspect.getsource(Flux3Transformer2DModel.forward) the replacement mirrors (d42ebf9c and 0.42.1).
_FLUX3_FORWARD_SHA = frozenset({"f2f7c836416cfd58"})
+# Same, for Flux3SingleTransformerBlock.forward (the span-modulated joint block mirrors it).
+_FLUX3_SINGLE_SHA = frozenset({"92269bb072da8335"})
_PATCHED: dict[type, Any] = {}
+_PATCHED_SINGLE: dict[type, Any] = {}
def _switch(name: str) -> bool:
@@ -46,6 +55,10 @@ def compact_enabled() -> bool:
return _switch(COMPACT_MOD_ENV)
+def span_enabled() -> bool:
+ return _switch(SPAN_MOD_ENV)
+
+
def _forward_sha(cls: type) -> Optional[str]:
try:
return hashlib.sha256(inspect.getsource(cls.forward).encode()).hexdigest()[:16]
@@ -319,18 +332,30 @@ def _flux3_cached_forward(
for block in self.mode_blocks["image"]:
hidden_states = block(hidden_states, image_early_mod, image_rotary_emb)
+ block_kwargs: dict = {}
if has_cond:
cond_states = unsloth_cond["cond_states"]
cond_rotary_emb = (unsloth_cond["cond_cos"], unsloth_cond["cond_sin"])
content_states = torch.cat([hidden_states, cond_states], dim = 1)
- # image and reference spans carry different timesteps: per token here
- content_mod = torch.cat(
- [
- _rows(image_single_mod, batch_size, image_seq_len),
- _rows(unsloth_cond["cond_single_mod"], batch_size, cond_states.shape[1]),
- ],
- dim = 1,
- )
+ cond_single_mod = unsloth_cond["cond_single_mod"]
+ if (
+ self.__dict__.get(_SPAN_ATTR)
+ and image_single_mod.shape[1] == 1
+ and cond_single_mod.shape[1] == 1
+ ):
+ # one row per span; the span length rides on a storage-free tensor so block graphs can copy every input
+ content_mod = image_single_mod
+ block_kwargs[SPAN_KWARG] = cond_single_mod
+ block_kwargs[SPAN_LEN_KWARG] = cond_states.new_empty((cond_states.shape[1], 0))
+ else:
+ # image and reference spans carry different timesteps: per token here
+ content_mod = torch.cat(
+ [
+ _rows(image_single_mod, batch_size, image_seq_len),
+ _rows(cond_single_mod, batch_size, cond_states.shape[1]),
+ ],
+ dim = 1,
+ )
joint_rotary_emb = (
torch.cat([txt_rotary_emb[0], image_rotary_emb[0], cond_rotary_emb[0]], dim = 0),
torch.cat([txt_rotary_emb[1], image_rotary_emb[1], cond_rotary_emb[1]], dim = 0),
@@ -350,6 +375,7 @@ def _flux3_cached_forward(
temb_mod = content_mod,
temb_mod_context = txt_single_mod,
image_rotary_emb = joint_rotary_emb,
+ **block_kwargs,
)
hidden_states = content_states[:, :image_seq_len]
@@ -361,6 +387,96 @@ def _flux3_cached_forward(
return Transformer2DModelOutput(sample = output)
+def _flux3_span_single_forward(
+ self: Any,
+ hidden_states: Any,
+ encoder_hidden_states: Any,
+ temb_mod: Any,
+ temb_mod_context: Any,
+ image_rotary_emb: Any,
+ unsloth_cond_mod: Any = None,
+ unsloth_cond_span: Any = None,
+) -> Any:
+ """Stock joint block, op for op, with one ``[B, 1, 3D]`` modulation row per span: ``temb_mod`` (image),
+ ``unsloth_cond_mod`` (reference, length = ``unsloth_cond_span``'s ``[reference_tokens, 0]`` shape)."""
+ if unsloth_cond_mod is None or unsloth_cond_span is None:
+ return _stock_single_forward(self)(
+ self,
+ hidden_states = hidden_states,
+ encoder_hidden_states = encoder_hidden_states,
+ temb_mod = temb_mod,
+ temb_mod_context = temb_mod_context,
+ image_rotary_emb = image_rotary_emb,
+ )
+ import torch
+
+ n_cond = unsloth_cond_span.shape[0]
+ n_image = hidden_states.shape[1] - n_cond
+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)
+ r_shift, r_scale, r_gate = torch.chunk(unsloth_cond_mod, 3, dim = -1)
+ c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim = -1)
+
+ norm_hidden_states = self.norm(hidden_states)
+ norm_image, norm_cond = torch.split(norm_hidden_states, [n_image, n_cond], dim = 1)
+ norm_hidden_states = torch.cat(
+ [(1 + mod_scale) * norm_image + mod_shift, (1 + r_scale) * norm_cond + r_shift], dim = 1
+ )
+
+ norm_encoder_hidden_states = self.norm(encoder_hidden_states)
+ norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift
+
+ combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)
+
+ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)
+ ff_output = self.ff(combined)
+ output = attn_output + ff_output
+
+ context_output, image_output, cond_output = torch.split(
+ output, [encoder_hidden_states.shape[1], n_image, n_cond], dim = 1
+ )
+ encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output
+ image_states, cond_states = torch.split(hidden_states, [n_image, n_cond], dim = 1)
+ hidden_states = torch.cat(
+ [image_states + mod_gate * image_output, cond_states + r_gate * cond_output], dim = 1
+ )
+
+ return encoder_hidden_states, hidden_states
+
+
+def _stock_single_forward(module: Any) -> Any:
+ for cls in type(module).__mro__:
+ stock = _PATCHED_SINGLE.get(cls)
+ if stock is not None:
+ return stock
+ raise RuntimeError(f"{type(module).__name__} has no stock forward recorded")
+
+
+def patch_single_class(cls: type) -> Optional[str]:
+ """Install the span-aware joint block forward on ``cls`` (at load, before compile or a hook binds it)."""
+ if any(c in _PATCHED_SINGLE for c in cls.__mro__):
+ return None
+ sha = _forward_sha(cls)
+ if sha not in _FLUX3_SINGLE_SHA:
+ return f"diffusers joint block changed (source {sha})"
+ _PATCHED_SINGLE[cls] = cls.forward
+ cls.forward = _flux3_span_single_forward
+ return None
+
+
+def _span_ready(transformer: Any) -> Optional[str]:
+ """None when every joint block runs the span-aware forward; else why the per-token rows stay."""
+ blocks = list(getattr(transformer, "single_transformer_blocks", None) or ())
+ if not blocks:
+ return "no joint blocks"
+ for block in blocks:
+ why = patch_single_class(type(block))
+ if why is not None:
+ return why
+ if type(block).forward is not _flux3_span_single_forward:
+ return "the joint block forward is not the patched one"
+ return None
+
+
def _stock_forward(module: Any) -> Any:
for cls in type(module).__mro__:
stock = _PATCHED.get(cls)
@@ -511,6 +627,13 @@ def prepare(transformer: Any, logger: Any = None) -> bool:
transformer.__dict__[_COMPACT_ATTR] = compact
if compact and logger is not None:
logger.info("diffusion.cond_blocks: FLUX.3 modulation heads project one row per sample")
+ span = False
+ if compact and span_enabled():
+ why = _span_ready(transformer)
+ span = why is None
+ if why is not None and logger is not None:
+ logger.info("diffusion.cond_blocks: reference modulation stays per token (%s)", why)
+ transformer.__dict__[_SPAN_ATTR] = span
if not cond_on:
return False
if applies is not None:
diff --git a/studio/backend/tests/test_diffusion_cond_blocks.py b/studio/backend/tests/test_diffusion_cond_blocks.py
index 1e7b45093e2..743cd05543c 100644
--- a/studio/backend/tests/test_diffusion_cond_blocks.py
+++ b/studio/backend/tests/test_diffusion_cond_blocks.py
@@ -496,3 +496,146 @@ def test_switch_values(monkeypatch):
for value, want in (("", True), ("1", True), ("0", False), ("off", False), ("False", False)):
monkeypatch.setenv(cb.COND_BLOCKS_ENV, value)
assert cb.enabled() is want
+
+
+def _span_inputs(
+ batch: int,
+ refs: int,
+ dtype,
+ seed: int = 3,
+):
+ g = torch.Generator().manual_seed(seed)
+ img, txt = 16, 5
+ kw = {
+ "hidden_states": torch.randn(batch, img, 8, generator = g).to(dtype),
+ "encoder_hidden_states": torch.randn(batch, txt, 24, generator = g).to(dtype),
+ "timestep": torch.full((batch,), 0.7),
+ "img_ids": torch.randint(0, 8, (img, 4), generator = g).float(),
+ "txt_ids": torch.randint(0, 8, (txt, 4), generator = g).float(),
+ "return_dict": False,
+ }
+ if refs:
+ n = 7 * refs # odd span length, one span of 7 tokens per reference image
+ kw["image_cond_hidden_states"] = torch.randn(batch, n, 8, generator = g).to(dtype)
+ kw["image_cond_ids"] = torch.randint(0, 8, (n, 4), generator = g).float()
+ return kw
+
+
+def _joint_mod_shapes(model):
+ seen = []
+
+ def spy(m, args, kwargs):
+ span = kwargs.get(cb.SPAN_KWARG)
+ if span is not None:
+ assert tuple(span.shape[1:2]) == (1,) and kwargs[cb.SPAN_LEN_KWARG].numel() == 0
+ # a block graph copies every input into a static buffer of the same strides: none may overlap itself
+ from core.inference.diffusion_block_graph import _static_like
+ for t in (kwargs["temb_mod"], span, kwargs[cb.SPAN_LEN_KWARG]):
+ _static_like(t).copy_(t)
+ seen.append((tuple(kwargs["temb_mod"].shape), span is not None))
+
+ handles = [
+ b.register_forward_pre_hook(spy, with_kwargs = True) for b in model.single_transformer_blocks
+ ]
+ return seen, handles
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("batch", [1, 2])
[email protected]("refs", [0, 1, 8])
+def test_span_modulation_matches_per_token_rows_bitwise(dtype, batch, refs, monkeypatch):
+ """One modulation row per span (image, references) is bitwise the per-token rows, and no per-token copy reaches
+ the joint blocks."""
+ cls = _flux3()
+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)
+ monkeypatch.delenv(cb.SPAN_MOD_ENV, raising = False)
+ model = _tiny(cls).to(dtype)
+ kw = _span_inputs(batch, refs, dtype)
+ with torch.no_grad():
+ assert cb.prepare(model)
+ assert model.__dict__[cb._SPAN_ATTR] is True
+ model.__dict__[cb._SPAN_ATTR] = False
+ want = model(**kw)[0]
+ model.__dict__[cb._SPAN_ATTR] = True
+ seen, handles = _joint_mod_shapes(model)
+ try:
+ got = model(**kw)[0]
+ finally:
+ for h in handles:
+ h.remove()
+ assert torch.equal(
+ got.view(torch.int16 if dtype == torch.bfloat16 else torch.int32),
+ want.view(torch.int16 if dtype == torch.bfloat16 else torch.int32),
+ )
+ if refs:
+ assert all(shape[1] == 1 and span for shape, span in seen), seen
+ else:
+ assert not any(span for _, span in seen)
+
+
+def test_span_modulation_kill_switch(monkeypatch):
+ cls = _flux3()
+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)
+ monkeypatch.setenv(cb.SPAN_MOD_ENV, "0")
+ model = _tiny(cls)
+ cb.prepare(model)
+ assert model.__dict__[cb._SPAN_ATTR] is False
+ seen, handles = _joint_mod_shapes(model)
+ with torch.no_grad():
+ model(**_span_inputs(1, 1, torch.float32))
+ for h in handles:
+ h.remove()
+ assert seen and all(shape[1] > 1 and not span for shape, span in seen)
+
+
+def test_span_block_without_the_kwarg_is_stock():
+ cls = _flux3()
+ model = _tiny(cls)
+ block = model.single_transformer_blocks[0]
+ assert cb.patch_single_class(type(block)) is None
+ g = torch.Generator().manual_seed(5)
+ h, e = torch.randn(1, 9, 32, generator = g), torch.randn(1, 4, 32, generator = g)
+ mod, cmod = torch.randn(1, 9, 96, generator = g), torch.randn(1, 1, 96, generator = g)
+ rope = (torch.ones(13, 16), torch.zeros(13, 16))
+ stock = cb._PATCHED_SINGLE[type(block)]
+ with torch.no_grad():
+ want = stock(block, h, e, mod, cmod, rope)
+ got = block(h, e, mod, cmod, rope)
+ assert all(torch.equal(a, b) for a, b in zip(want, got))
+
+
[email protected]("refs", [1, 3])
+def test_span_blocks_compile_without_breaks_and_match_compiled_per_token_rows(refs, monkeypatch):
+ """Studio compiles each block: the span forward traces whole (no graph break), once across two reference counts
+ under dynamic shapes, and matches the per-token rows compiled the same way."""
+ from torch._dynamo.testing import CompileCounterWithBackend
+
+ cls = _flux3()
+ monkeypatch.delenv(cb.COMPACT_MOD_ENV, raising = False)
+ monkeypatch.delenv(cb.SPAN_MOD_ENV, raising = False)
+ model = _tiny(cls)
+ assert cb.prepare(model)
+ blocks = list(model.single_transformer_blocks)
+ eager = [b.forward for b in blocks]
+ outs = {}
+ counters = {}
+ for span in (False, True):
+ torch._dynamo.reset()
+ cc = CompileCounterWithBackend("inductor")
+ for b in blocks:
+ b.forward = torch.compile(
+ type(b).forward.__get__(b), backend = cc, fullgraph = True, dynamic = True
+ )
+ model.__dict__[cb._SPAN_ATTR] = span
+ try:
+ with torch.no_grad():
+ outs[span] = [
+ model(**_span_inputs(1, n, torch.float32))[0] for n in (refs, refs + 1)
+ ]
+ finally:
+ for b, f in zip(blocks, eager):
+ b.forward = f
+ counters[span] = cc.frame_count
+ torch._dynamo.reset()
+ assert counters[True] == len(blocks)
+ assert all(torch.equal(a, b) for a, b in zip(outs[True], outs[False]))