Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: share the FLUX.3 joint block int8 ConvRot quant under the span forward (#17337)

Daniel Han762e84955612
diff --git a/studio/backend/core/inference/diffusion_cond_blocks.py b/studio/backend/core/inference/diffusion_cond_blocks.py
index 2cc7dfd2857..d70218ad63a 100644
--- a/studio/backend/core/inference/diffusion_cond_blocks.py
+++ b/studio/backend/core/inference/diffusion_cond_blocks.py
@@ -37,6 +37,9 @@ _FLUX3_SINGLE_SHA = frozenset({"92269bb072da8335"})
_PATCHED: dict[type, Any] = {}
_PATCHED_SINGLE: dict[type, Any] = {}
+# (attention output, MLP output) of a joint block on its combined input, bit-identical to ``self.attn`` / ``self.ff``,
+# or None for the modules: set by diffusion_flux_convrot_fused (one shared int8 ConvRot quant) while installed.
+JOINT_ATTN_FF: Any = None
def _switch(name: str) -> bool:
@@ -427,8 +430,12 @@ def _flux3_span_single_forward(
combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)
- attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)
- ff_output = self.ff(combined)
+ both = JOINT_ATTN_FF(self, combined, image_rotary_emb) if JOINT_ATTN_FF is not None else None
+ if both is None:
+ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)
+ ff_output = self.ff(combined)
+ else:
+ attn_output, ff_output = both
output = attn_output + ff_output
context_output, image_output, cond_output = torch.split(
diff --git a/studio/backend/core/inference/diffusion_flux_convrot_fused.py b/studio/backend/core/inference/diffusion_flux_convrot_fused.py
index f20a45b2d31..3206c83e5bc 100644
--- a/studio/backend/core/inference/diffusion_flux_convrot_fused.py
+++ b/studio/backend/core/inference/diffusion_flux_convrot_fused.py
@@ -25,9 +25,12 @@ import torch
from . import diffusion_int8_gemm as _gemm
FLUX_CONVROT_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"
+FLUX3_SPAN_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX3_SPAN_FUSED"
_MARK = "__unsloth_flux_convrot_fused__"
_LOCK = threading.Lock()
_STOCK: dict = {} # (owner, attr) -> original
+# Flux3SingleTransformerBlock class -> the diffusers forward diffusion_cond_blocks records as its stock (span patch)
+_SPAN_STOCK: dict = {}
_CALLS = {"shared": [0], "producer": [0], "block": [0]}
# Stock lines each replacement mirrors; a diffusers whose body lacks any keeps the stock function.
@@ -101,6 +104,17 @@ _NEEDLES = {
}
+def span_single_enabled() -> bool:
+ """FLUX.3 joint blocks under the span-aware forward share the quant too (``UNSLOTH_DIFFUSION_FLUX3_SPAN_FUSED=0``:
+ they keep a separate MLP-input quant, as before)."""
+ return (os.environ.get(FLUX3_SPAN_FUSED_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
def flux_convrot_fused_disabled() -> bool:
return (os.environ.get(FLUX_CONVROT_FUSED_ENV) or "").strip().lower() in (
"0",
@@ -433,6 +447,57 @@ def _patch(owner: Any, attr: str, key: str, make: Any) -> bool:
return True
+def _patch_span_single(cls: Any, mod: Any) -> bool:
+ """FLUX.3 joint block under ``diffusion_cond_blocks``' span-aware forward (installed first, at load): that forward
+ runs the stock diffusers one recorded in ``_PATCHED_SINGLE`` whenever a render has no reference span, so the
+ shared quant goes there; with a span its own body reads ``JOINT_ATTN_FF`` (``_f3_attn_ff``). Idempotent."""
+ try:
+ from . import diffusion_cond_blocks as cond
+ except Exception: # noqa: BLE001
+ return False
+ span_fwd = getattr(cond, "_flux3_span_single_forward", None)
+ registry = getattr(cond, "_PATCHED_SINGLE", None)
+ if span_fwd is None or registry is None or getattr(cls, "forward", None) is not span_fwd:
+ return False
+ stock = registry.get(cls)
+ if stock is None:
+ return False
+ if not getattr(stock, _MARK, False):
+ if not _has(stock, "f3_single"):
+ return False
+ _SPAN_STOCK[cls] = stock
+ registry[cls] = _make_f3_single_forward(stock, mod)
+ _SPAN_MOD[0] = mod
+ cond.JOINT_ATTN_FF = _span_attn_ff
+ return True
+
+
+# the diffusers FLUX.3 module whose apply_rotary_emb / dispatch_attention_fn the span path uses
+_SPAN_MOD: list = [None]
+
+
+def _span_attn_ff(block: Any, x: Any, image_rotary_emb: Any) -> Optional[tuple]:
+ """``diffusion_cond_blocks.JOINT_ATTN_FF``: one stable function, so a compiled span block never guards on a new
+ object per install."""
+ return _f3_attn_ff(block, x, image_rotary_emb, _SPAN_MOD[0])
+
+
+def _uninstall_span_single() -> None:
+ try:
+ from . import diffusion_cond_blocks as cond
+ except Exception: # noqa: BLE001
+ _SPAN_STOCK.clear()
+ return
+ registry = getattr(cond, "_PATCHED_SINGLE", None)
+ for cls, stock in list(_SPAN_STOCK.items()):
+ if registry is not None and getattr(registry.get(cls), _MARK, False):
+ registry[cls] = stock
+ _SPAN_STOCK.clear()
+ _SPAN_MOD[0] = None
+ if getattr(cond, "JOINT_ATTN_FF", None) is _span_attn_ff:
+ cond.JOINT_ATTN_FF = None
+
+
def _install_patches(logger: Any = None) -> dict:
done = {}
f2, f3 = _flux_modules()
@@ -488,6 +553,8 @@ def _install_patches(logger: Any = None) -> dict:
cls = getattr(f3, cls_name, None)
if cls is not None:
done[key] = _patch(cls, "forward", key, lambda s, m = make: m(s, f3))
+ if key == "f3_single" and not done[key] and span_single_enabled():
+ done[key] = _patch_span_single(cls, f3)
if logger is not None:
logger.info("diffusion.flux_convrot_fused: patches %s", done)
return done
@@ -556,6 +623,7 @@ def install(transformer: Any, logger: Any = None) -> dict:
def uninstall() -> None:
with _LOCK:
+ _uninstall_span_single()
for key, stock in list(_STOCK.items()):
owner, attr = key
if isinstance(owner, str):
@@ -567,7 +635,7 @@ def uninstall() -> None:
def is_installed() -> bool:
- return any(not isinstance(k[0], str) for k in _STOCK)
+ return bool(_SPAN_STOCK) or any(not isinstance(k[0], str) for k in _STOCK)
def call_counts() -> dict:
diff --git a/studio/backend/tests/test_diffusion_flux_convrot_fused.py b/studio/backend/tests/test_diffusion_flux_convrot_fused.py
index f415897695c..1ba019359e0 100644
--- a/studio/backend/tests/test_diffusion_flux_convrot_fused.py
+++ b/studio/backend/tests/test_diffusion_flux_convrot_fused.py
@@ -482,3 +482,94 @@ def test_probe_verdicts_round_trip_through_the_cache(monkeypatch, tmp_path):
assert cq._verdict_load(7) is None
path.write_text("{torn")
assert cq._verdict_load(7) is None
+
+
+# ------------------------------------------------------------------------- FLUX.3 joint block under the span forward
+def span_patched(env):
+ """diffusion_cond_blocks' span-aware joint block forward installed first, as the load does (cond_blocks.prepare
+ runs before the speed layer), restored after."""
+ torch, _f2, f3, fused, gemm = env
+ from core.inference import diffusion_cond_blocks as cond
+
+ cls = f3.Flux3SingleTransformerBlock
+ before_fwd, before_reg = cls.forward, dict(cond._PATCHED_SINGLE)
+ why = cond.patch_single_class(cls)
+ if why is not None:
+ pytest.skip(f"span forward not installable on this diffusers: {why}")
+ try:
+ yield torch, f3, fused, gemm, cond
+ finally:
+ fused.uninstall()
+ cls.forward = before_fwd
+ cond._PATCHED_SINGLE.clear()
+ cond._PATCHED_SINGLE.update(before_reg)
+ cond.JOINT_ATTN_FF = None
+
+
[email protected]("span", [False, True])
+def test_flux3_joint_block_under_span_forward_shares_the_quant(
+ span_patched, monkeypatch, span, mode
+):
+ """The span forward wins the class first, so the joint block patch goes to the stock it delegates to (no span)
+ and to its own attention + MLP call (span): one shared quant per block either way, outputs bit for bit stock."""
+ torch, f3, fused, gemm, cond = span_patched
+ calls = {"shared": 0, "producer": 0}
+ _force_on(monkeypatch, torch, fused, gemm, calls)
+ widths = []
+ plain = gemm.shared_rotq_linears
+
+ def shared(x, linears):
+ widths.append(len(linears))
+ return plain(x, linears)
+
+ monkeypatch.setattr(gemm, "shared_rotq_linears", shared)
+ dim, heads, hd = 64, 2, 32
+ hs = torch.randn(2, 12, dim)
+ enc = torch.randn(2, 5, dim)
+ mod = torch.randn(2, 1, 3 * dim) * 0.1
+ cmod = torch.randn(2, 1, 3 * dim) * 0.1
+ rope = _rope(torch, 17, hd)
+ kw = {}
+ if span:
+ kw = {
+ "unsloth_cond_mod": torch.randn(2, 1, 3 * dim) * 0.1,
+ "unsloth_cond_span": torch.empty(4, 0),
+ }
+ span_fwd = f3.Flux3SingleTransformerBlock.forward
+ ref, out, back = _run_stock_vs_patched(
+ torch,
+ fused,
+ lambda: f3.Flux3SingleTransformerBlock(dim, heads, hd),
+ lambda b: b(hs, enc, mod, cmod, rope, **kw),
+ mode,
+ )
+ assert _same(ref, out) and _same(ref, back)
+ assert widths == [
+ 4
+ ] # Q, K, V and the MLP input: one quant (main: Q / K / V, then the MLP input alone)
+ assert calls["producer"] == 1
+ assert f3.Flux3SingleTransformerBlock.forward is span_fwd # the span forward keeps the class
+ assert cond.JOINT_ATTN_FF is None and not fused.is_installed()
+
+
+def test_flux3_span_install_reports_and_restores(span_patched):
+ torch, f3, fused, _gemm, cond = span_patched
+ cls = f3.Flux3SingleTransformerBlock
+ stock = cond._PATCHED_SINGLE[cls]
+ done = fused._install_patches() # noqa: SLF001
+ assert done["f3_single"] is True
+ assert cond._PATCHED_SINGLE[cls] is not stock and cond.JOINT_ATTN_FF is not None
+ assert fused._install_patches() == done # noqa: SLF001 - idempotent
+ fused.uninstall()
+ assert cond._PATCHED_SINGLE[cls] is stock and cond.JOINT_ATTN_FF is None
+
+
+def test_flux3_span_switch_keeps_the_separate_quant(span_patched, monkeypatch):
+ torch, f3, fused, _gemm, cond = span_patched
+ monkeypatch.setenv(fused.FLUX3_SPAN_FUSED_ENV, "0")
+ stock = cond._PATCHED_SINGLE[f3.Flux3SingleTransformerBlock]
+ assert fused._install_patches()["f3_single"] is False # noqa: SLF001
+ assert (
+ cond._PATCHED_SINGLE[f3.Flux3SingleTransformerBlock] is stock and cond.JOINT_ATTN_FF is None
+ )