Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: share the FLUX.3 joint block int8 ConvRot quant under the span forward (#17337)
Daniel Han762e84955612
diff --git a/studio/backend/core/inference/diffusion_cond_blocks.py b/studio/backend/core/inference/diffusion_cond_blocks.pyindex 2cc7dfd2857..d70218ad63a 100644--- a/studio/backend/core/inference/diffusion_cond_blocks.py+++ b/studio/backend/core/inference/diffusion_cond_blocks.py@@ -37,6 +37,9 @@ _FLUX3_SINGLE_SHA = frozenset({"92269bb072da8335"})_PATCHED: dict[type, Any] = {}_PATCHED_SINGLE: dict[type, Any] = {}+# (attention output, MLP output) of a joint block on its combined input, bit-identical to ``self.attn`` / ``self.ff``,+# or None for the modules: set by diffusion_flux_convrot_fused (one shared int8 ConvRot quant) while installed.+JOINT_ATTN_FF: Any = Nonedef _switch(name: str) -> bool:@@ -427,8 +430,12 @@ def _flux3_span_single_forward(combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)- attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)- ff_output = self.ff(combined)+ both = JOINT_ATTN_FF(self, combined, image_rotary_emb) if JOINT_ATTN_FF is not None else None+ if both is None:+ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)+ ff_output = self.ff(combined)+ else:+ attn_output, ff_output = bothoutput = attn_output + ff_outputcontext_output, image_output, cond_output = torch.split(diff --git a/studio/backend/core/inference/diffusion_flux_convrot_fused.py b/studio/backend/core/inference/diffusion_flux_convrot_fused.pyindex f20a45b2d31..3206c83e5bc 100644--- a/studio/backend/core/inference/diffusion_flux_convrot_fused.py+++ b/studio/backend/core/inference/diffusion_flux_convrot_fused.py@@ -25,9 +25,12 @@ import torchfrom . import diffusion_int8_gemm as _gemmFLUX_CONVROT_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"+FLUX3_SPAN_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX3_SPAN_FUSED"_MARK = "__unsloth_flux_convrot_fused__"_LOCK = threading.Lock()_STOCK: dict = {} # (owner, attr) -> original+# Flux3SingleTransformerBlock class -> the diffusers forward diffusion_cond_blocks records as its stock (span patch)+_SPAN_STOCK: dict = {}_CALLS = {"shared": [0], "producer": [0], "block": [0]}# Stock lines each replacement mirrors; a diffusers whose body lacks any keeps the stock function.@@ -101,6 +104,17 @@ _NEEDLES = {}+def span_single_enabled() -> bool:+ """FLUX.3 joint blocks under the span-aware forward share the quant too (``UNSLOTH_DIFFUSION_FLUX3_SPAN_FUSED=0``:+ they keep a separate MLP-input quant, as before)."""+ return (os.environ.get(FLUX3_SPAN_FUSED_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )++def flux_convrot_fused_disabled() -> bool:return (os.environ.get(FLUX_CONVROT_FUSED_ENV) or "").strip().lower() in ("0",@@ -433,6 +447,57 @@ def _patch(owner: Any, attr: str, key: str, make: Any) -> bool:return True+def _patch_span_single(cls: Any, mod: Any) -> bool:+ """FLUX.3 joint block under ``diffusion_cond_blocks``' span-aware forward (installed first, at load): that forward+ runs the stock diffusers one recorded in ``_PATCHED_SINGLE`` whenever a render has no reference span, so the+ shared quant goes there; with a span its own body reads ``JOINT_ATTN_FF`` (``_f3_attn_ff``). Idempotent."""+ try:+ from . import diffusion_cond_blocks as cond+ except Exception: # noqa: BLE001+ return False+ span_fwd = getattr(cond, "_flux3_span_single_forward", None)+ registry = getattr(cond, "_PATCHED_SINGLE", None)+ if span_fwd is None or registry is None or getattr(cls, "forward", None) is not span_fwd:+ return False+ stock = registry.get(cls)+ if stock is None:+ return False+ if not getattr(stock, _MARK, False):+ if not _has(stock, "f3_single"):+ return False+ _SPAN_STOCK[cls] = stock+ registry[cls] = _make_f3_single_forward(stock, mod)+ _SPAN_MOD[0] = mod+ cond.JOINT_ATTN_FF = _span_attn_ff+ return True+++# the diffusers FLUX.3 module whose apply_rotary_emb / dispatch_attention_fn the span path uses+_SPAN_MOD: list = [None]+++def _span_attn_ff(block: Any, x: Any, image_rotary_emb: Any) -> Optional[tuple]:+ """``diffusion_cond_blocks.JOINT_ATTN_FF``: one stable function, so a compiled span block never guards on a new+ object per install."""+ return _f3_attn_ff(block, x, image_rotary_emb, _SPAN_MOD[0])+++def _uninstall_span_single() -> None:+ try:+ from . import diffusion_cond_blocks as cond+ except Exception: # noqa: BLE001+ _SPAN_STOCK.clear()+ return+ registry = getattr(cond, "_PATCHED_SINGLE", None)+ for cls, stock in list(_SPAN_STOCK.items()):+ if registry is not None and getattr(registry.get(cls), _MARK, False):+ registry[cls] = stock+ _SPAN_STOCK.clear()+ _SPAN_MOD[0] = None+ if getattr(cond, "JOINT_ATTN_FF", None) is _span_attn_ff:+ cond.JOINT_ATTN_FF = None++def _install_patches(logger: Any = None) -> dict:done = {}f2, f3 = _flux_modules()@@ -488,6 +553,8 @@ def _install_patches(logger: Any = None) -> dict:cls = getattr(f3, cls_name, None)if cls is not None:done[key] = _patch(cls, "forward", key, lambda s, m = make: m(s, f3))+ if key == "f3_single" and not done[key] and span_single_enabled():+ done[key] = _patch_span_single(cls, f3)if logger is not None:logger.info("diffusion.flux_convrot_fused: patches %s", done)return done@@ -556,6 +623,7 @@ def install(transformer: Any, logger: Any = None) -> dict:def uninstall() -> None:with _LOCK:+ _uninstall_span_single()for key, stock in list(_STOCK.items()):owner, attr = keyif isinstance(owner, str):@@ -567,7 +635,7 @@ def uninstall() -> None:def is_installed() -> bool:- return any(not isinstance(k[0], str) for k in _STOCK)+ return bool(_SPAN_STOCK) or any(not isinstance(k[0], str) for k in _STOCK)def call_counts() -> dict:diff --git a/studio/backend/tests/test_diffusion_flux_convrot_fused.py b/studio/backend/tests/test_diffusion_flux_convrot_fused.pyindex f415897695c..1ba019359e0 100644--- a/studio/backend/tests/test_diffusion_flux_convrot_fused.py+++ b/studio/backend/tests/test_diffusion_flux_convrot_fused.py@@ -482,3 +482,94 @@ def test_probe_verdicts_round_trip_through_the_cache(monkeypatch, tmp_path):assert cq._verdict_load(7) is Nonepath.write_text("{torn")assert cq._verdict_load(7) is None+++# ------------------------------------------------------------------------- FLUX.3 joint block under the span forward+def span_patched(env):+ """diffusion_cond_blocks' span-aware joint block forward installed first, as the load does (cond_blocks.prepare+ runs before the speed layer), restored after."""+ torch, _f2, f3, fused, gemm = env+ from core.inference import diffusion_cond_blocks as cond++ cls = f3.Flux3SingleTransformerBlock+ before_fwd, before_reg = cls.forward, dict(cond._PATCHED_SINGLE)+ why = cond.patch_single_class(cls)+ if why is not None:+ pytest.skip(f"span forward not installable on this diffusers: {why}")+ try:+ yield torch, f3, fused, gemm, cond+ finally:+ fused.uninstall()+ cls.forward = before_fwd+ cond._PATCHED_SINGLE.clear()+ cond._PATCHED_SINGLE.update(before_reg)+ cond.JOINT_ATTN_FF = None++[email protected]("span", [False, True])+def test_flux3_joint_block_under_span_forward_shares_the_quant(+ span_patched, monkeypatch, span, mode+):+ """The span forward wins the class first, so the joint block patch goes to the stock it delegates to (no span)+ and to its own attention + MLP call (span): one shared quant per block either way, outputs bit for bit stock."""+ torch, f3, fused, gemm, cond = span_patched+ calls = {"shared": 0, "producer": 0}+ _force_on(monkeypatch, torch, fused, gemm, calls)+ widths = []+ plain = gemm.shared_rotq_linears++ def shared(x, linears):+ widths.append(len(linears))+ return plain(x, linears)++ monkeypatch.setattr(gemm, "shared_rotq_linears", shared)+ dim, heads, hd = 64, 2, 32+ hs = torch.randn(2, 12, dim)+ enc = torch.randn(2, 5, dim)+ mod = torch.randn(2, 1, 3 * dim) * 0.1+ cmod = torch.randn(2, 1, 3 * dim) * 0.1+ rope = _rope(torch, 17, hd)+ kw = {}+ if span:+ kw = {+ "unsloth_cond_mod": torch.randn(2, 1, 3 * dim) * 0.1,+ "unsloth_cond_span": torch.empty(4, 0),+ }+ span_fwd = f3.Flux3SingleTransformerBlock.forward+ ref, out, back = _run_stock_vs_patched(+ torch,+ fused,+ lambda: f3.Flux3SingleTransformerBlock(dim, heads, hd),+ lambda b: b(hs, enc, mod, cmod, rope, **kw),+ mode,+ )+ assert _same(ref, out) and _same(ref, back)+ assert widths == [+ 4+ ] # Q, K, V and the MLP input: one quant (main: Q / K / V, then the MLP input alone)+ assert calls["producer"] == 1+ assert f3.Flux3SingleTransformerBlock.forward is span_fwd # the span forward keeps the class+ assert cond.JOINT_ATTN_FF is None and not fused.is_installed()+++def test_flux3_span_install_reports_and_restores(span_patched):+ torch, f3, fused, _gemm, cond = span_patched+ cls = f3.Flux3SingleTransformerBlock+ stock = cond._PATCHED_SINGLE[cls]+ done = fused._install_patches() # noqa: SLF001+ assert done["f3_single"] is True+ assert cond._PATCHED_SINGLE[cls] is not stock and cond.JOINT_ATTN_FF is not None+ assert fused._install_patches() == done # noqa: SLF001 - idempotent+ fused.uninstall()+ assert cond._PATCHED_SINGLE[cls] is stock and cond.JOINT_ATTN_FF is None+++def test_flux3_span_switch_keeps_the_separate_quant(span_patched, monkeypatch):+ torch, f3, fused, _gemm, cond = span_patched+ monkeypatch.setenv(fused.FLUX3_SPAN_FUSED_ENV, "0")+ stock = cond._PATCHED_SINGLE[f3.Flux3SingleTransformerBlock]+ assert fused._install_patches()["f3_single"] is False # noqa: SLF001+ assert (+ cond._PATCHED_SINGLE[f3.Flux3SingleTransformerBlock] is stock and cond.JOINT_ATTN_FF is None+ )