Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fuse the int8 ConvRot rotation into its producers on FLUX.2 / FLUX.3 (ConvRot cost +8.7% to +0.5%) (#17230)
Daniel Hanf4d17f4a59af
diff --git a/studio/backend/core/inference/diffusion_autoopt/patches.py b/studio/backend/core/inference/diffusion_autoopt/patches.pyindex fa0b2efd004..1a6c57f6bda 100644--- a/studio/backend/core/inference/diffusion_autoopt/patches.py+++ b/studio/backend/core/inference/diffusion_autoopt/patches.py@@ -504,6 +504,14 @@ def _builtin_specs() -> list[PatchSpec]:else bool(ctx.denoiser_offloaded),)+ def flux_convrot_fused(ctx: PatchContext) -> Any:+ from ..diffusion_flux_convrot_fused import install+ return install(ctx.transformer, ctx.logger)++ def flux_convrot_fused_uninstall() -> None:+ from ..diffusion_flux_convrot_fused import uninstall+ uninstall()+def moss_fused(ctx: PatchContext) -> Any:from ..diffusion_moss_fused import installreturn install(@@ -650,6 +658,16 @@ def _builtin_specs() -> list[PatchSpec]:warn = "qwen-image-2.1 dense gate/value GEMM",env = "UNSLOTH_DIFFUSION_Q21_DENSE_FUSED",),+ PatchSpec(+ "flux_convrot_fused",+ STAGE_PRE_COMPILE,+ flux_convrot_fused,+ flux_convrot_fused_uninstall,+ group = GROUP_FUSED_DIT,+ classes = ("Flux2Transformer2DModel", "Flux3Transformer2DModel"),+ warn = "flux convrot producer fusion",+ env = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED",+ ),PatchSpec("moss_fused",STAGE_PRE_COMPILE,diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.pyindex d59b031272d..cd775fbbcf2 100644--- a/studio/backend/core/inference/diffusion_convrot_quant.py+++ b/studio/backend/core/inference/diffusion_convrot_quant.py@@ -16,9 +16,11 @@ from .rocm_bf16 import is_rocm_torchINT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"INT8_ROTQUANT_H16_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16"INT8_ROTQUANT_SWIGLU_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_SWIGLU"+INT8_ROTQUANT_PRODUCER_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER"_OP_NAMESPACE = "unsloth_studio"_ROTQ_OP_NAME = "convrot_act_quant_int8"_SWIGLU_OP_NAME = "convrot_swiglu_act_quant_int8"+_PRODUCER_OP_NAME = "convrot_producer_act_quant_int8"# (BLOCK_M group rows, BLOCK_K, num_warps, num_stages) per (major, minor); measured end to end. Absent = stock._ROTQ_CONFIG = {@@ -52,6 +54,13 @@ _SWIGLU_DEVICE: dict = {}_H16_CALLS = [0]_SWIGLU_CALLS = [0]_SWIGLU_HANDLE: Any = None+# device index -> rotq_h16x (producer loads fused in front of the rotation) passed its probe+_PRODUCER_DEVICE: dict = {}+_PRODUCER_CALLS = [0]+_PRODUCER_HANDLE: Any = None+_PRODUCER_VERDICT_OFF: set = (+ set()+) # devices whose last load had the switch off (the probe verdict is kept)# 16 group rows per program: fastest (within 2%) on L4; 128 rows overflow 100 KB smem. One masked row per program at# K 12288 was 1.2x slower on A100 and spilled on B200._H16_TILE_ROWS = 16@@ -88,6 +97,11 @@ def rotquant_swiglu_enabled() -> bool:return _env_mode(INT8_ROTQUANT_SWIGLU_ENV) != "off"+def rotquant_producer_enabled() -> bool:+ """False only under ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER=0`` (the producer writes bf16, ``rotq_h16`` reads it)."""+ return _env_mode(INT8_ROTQUANT_PRODUCER_ENV) != "off"++def rotquant_mode() -> str:"""'off', 'force' or 'auto'. The int8 GEMM switch is the master: its ``0`` turns the whole swap off and its ``1``forces this lever too."""@@ -407,7 +421,407 @@ def _kernels() -> Optional[types.SimpleNamespace]:srow = pid.to(tl.int64) * ROWS + tl.arange(0, ROWS)tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = srow < M)- return types.SimpleNamespace(rotq_i8 = rotq_i8, rotq_h16 = rotq_h16, triton = triton)+ @triton.jit+ def _h16x_load(+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ grp,+ live,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ CAT: tl.constexpr, # groups below NA come from the attention prefix+ HD: tl.constexpr, # prefix head width+ ):+ # bf16 [TM, 256] of the activation the producer would have written: silu(gate) * value, after the prefix+ offs = tl.arange(0, 256)+ if CAT:+ pre = live & (grp < NA)+ col = grp[:, None] * 256 + offs[None, :]+ b = row // SEQ+ s = row - b * SEQ+ a_off = b[:, None] * sab + s[:, None] * sas + (col // HD) * sah + (col % HD)+ xa = tl.load(a_ptr + a_off, mask = pre[:, None], other = 0.0)+ mlp = live & (grp >= NA)+ mcol = tl.where(mlp, grp - NA, 0)[:, None] * 256 + offs[None, :]+ g = tl.load(g_ptr + row[:, None] * sg + mcol, mask = mlp[:, None], other = 0.0)+ v = tl.load(v_ptr + row[:, None] * sv + mcol, mask = mlp[:, None], other = 0.0)+ x = tl.where(pre[:, None], xa, _swiglu_bf16(g, v))+ else:+ col = grp[:, None] * 256 + offs[None, :]+ g = tl.load(g_ptr + row[:, None] * sg + col, mask = live[:, None], other = 0.0)+ v = tl.load(v_ptr + row[:, None] * sv + col, mask = live[:, None], other = 0.0)+ x = _swiglu_bf16(g, v)+ return x++ @triton.jit+ def _h16x_rot(+ x,+ h,+ h32,+ TM: tl.constexpr, # 256-wide groups per tile+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy+ ):+ # _h16_tile's rotation, op for op+ y = tl.dot(tl.reshape(x, (TM * 16, 16)), h, out_dtype = tl.float32)+ y = tl.reshape(tl.permute(tl.reshape(y, (TM, 16, 16)), (0, 2, 1)), (TM * 16, 16))+ if TF32X3:+ z = tl.dot(y, h32, input_precision = "tf32x3", out_dtype = tl.float32)+ else:+ z = tl.dot(y, h32, input_precision = "ieee", out_dtype = tl.float32)+ z = tl.reshape(tl.permute(tl.reshape(z, (TM, 16, 16)), (0, 2, 1)), (TM, 256))+ return _rbf16(z)++ @triton.jit+ def _h16x_chunk(+ c,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG: tl.constexpr, # groups per chunk+ ROWS: tl.constexpr, # activation rows per program+ CAT: tl.constexpr,+ HD: tl.constexpr,+ TF32X3: tl.constexpr,+ ):+ grp = c * CG + gl+ live = (row < M) & (grp < NG) # padded one-pass tiles mask the groups past the row+ r = tl.where(live, row, 0)+ x = _h16x_load(g_ptr, v_ptr, a_ptr, r, grp, live, sg, sv, NA, SEQ, sab, sas, sah, CAT, HD)+ return _h16x_rot(x, h, h32, ROWS * CG, TF32X3)++ @triton.jit+ def _h16x_amax(+ c,+ amax,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG: tl.constexpr,+ ROWS: tl.constexpr,+ CAT: tl.constexpr,+ HD: tl.constexpr,+ TF32X3: tl.constexpr,+ ):+ z = _h16x_chunk(+ c,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ )+ m = tl.where(row < M, tl.max(tl.abs(z), axis = 1), 0.0)+ return tl.maximum(amax, tl.max(tl.reshape(m, (ROWS, CG)), axis = 1))++ @triton.jit+ def _h16x_quant(+ c,+ inv_t,+ g_ptr,+ v_ptr,+ a_ptr,+ q_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG: tl.constexpr,+ ROWS: tl.constexpr,+ CAT: tl.constexpr,+ HD: tl.constexpr,+ TF32X3: tl.constexpr,+ QMIN: tl.constexpr,+ QMAX: tl.constexpr,+ FP32_SCALE: tl.constexpr,+ ):+ z = _h16x_chunk(+ c,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ )+ live = (row < M) & (c * CG + gl < NG)+ gidx = tl.where(live, row * NG + c * CG + gl, 0)+ _h16_quant_store(z, inv_t, q_ptr, gidx, live, QMIN, QMAX, FP32_SCALE)++ @triton.jit(do_not_specialize = ["M", "SEQ"])+ def rotq_h16x(+ g_ptr,+ v_ptr,+ a_ptr,+ h_ptr,+ h32_ptr,+ q_ptr,+ s_ptr,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ NCH: tl.constexpr, # chunks per row+ CG: tl.constexpr, # groups per chunk+ ROWS: tl.constexpr, # activation rows per program+ ONE_PASS: tl.constexpr, # whole row fits one chunk+ UNROLL: tl.constexpr, # unroll the chunk loop+ CAT: tl.constexpr, # attention prefix before the SwiGLU product (one row = [prefix | product])+ HD: tl.constexpr, # prefix head width+ TF32X3: tl.constexpr,+ QMIN: tl.constexpr,+ QMAX: tl.constexpr,+ DIV: tl.constexpr,+ EPS: tl.constexpr,+ FP32_SCALE: tl.constexpr,+ ):+ # rotq_h16 with the producer's loads in front: strided SwiGLU halves (and an attention prefix in any layout),+ # so the bf16 activation never reaches HBM. Same rotation, amax and rounding as rotq_h16.+ TM: tl.constexpr = ROWS * CG+ pid = tl.program_id(0)+ t = tl.arange(0, TM)+ row = pid.to(tl.int64) * ROWS + t // CG+ gl = t % CG+ o16 = tl.arange(0, 16)+ h = tl.load(h_ptr + o16[:, None] * 16 + o16[None, :])+ h32 = tl.load(h32_ptr + o16[:, None] * 16 + o16[None, :])+ if ONE_PASS:+ z = _h16x_chunk(+ 0,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ )+ m = tl.where(row < M, tl.max(tl.abs(z), axis = 1), 0.0)+ amax = tl.max(tl.reshape(m, (ROWS, CG)), axis = 1)+ else:+ amax = tl.zeros((ROWS,), dtype = tl.float32)+ if UNROLL:+ for c in tl.static_range(NCH):+ amax = _h16x_amax(+ c,+ amax,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ )+ else:+ for c in range(0, NCH):+ amax = _h16x_amax(+ c,+ amax,+ g_ptr,+ v_ptr,+ a_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ )+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))+ inv = libdevice.div_rn(tl.full((ROWS,), 1.0, tl.float32), s)+ if not FP32_SCALE:+ inv = _rbf16(inv)+ inv_t = tl.reshape(tl.broadcast_to(inv[:, None], (ROWS, CG)), (TM,))+ if ONE_PASS:+ live = (row < M) & (gl < NG)+ gidx = tl.where(live, row * NG + gl, 0)+ _h16_quant_store(z, inv_t, q_ptr, gidx, live, QMIN, QMAX, FP32_SCALE)+ elif UNROLL:+ for c in tl.static_range(NCH):+ _h16x_quant(+ c,+ inv_t,+ g_ptr,+ v_ptr,+ a_ptr,+ q_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ QMIN,+ QMAX,+ FP32_SCALE,+ )+ else:+ for c in range(0, NCH):+ _h16x_quant(+ c,+ inv_t,+ g_ptr,+ v_ptr,+ a_ptr,+ q_ptr,+ row,+ gl,+ M,+ NG,+ sg,+ sv,+ NA,+ SEQ,+ sab,+ sas,+ sah,+ h,+ h32,+ CG,+ ROWS,+ CAT,+ HD,+ TF32X3,+ QMIN,+ QMAX,+ FP32_SCALE,+ )+ srow = pid.to(tl.int64) * ROWS + tl.arange(0, ROWS)+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = srow < M)++ return types.SimpleNamespace(+ rotq_i8 = rotq_i8, rotq_h16 = rotq_h16, rotq_h16x = rotq_h16x, triton = triton+ )@lru_cache(maxsize = 1)@@ -603,6 +1017,169 @@ def _h16_launch(return q, s+# Per-(major, minor) warps and padded one-pass cap from a Colab sweep (A100: 557 vs 601 us two-pass at 4608 x 12288).+_H16X_WARPS = {(8, 0): (8, 4), (8, 9): (4, 4)}+_H16X_PAD_CAT = {(8, 0): 64}+++@lru_cache(maxsize = 16)+def _device_cap(index: Any) -> Optional[tuple]:+ try:+ import torch+ return tuple(torch.cuda.get_device_capability(index))+ except Exception: # noqa: BLE001+ return None+++def _h16x_pad(cap: Optional[tuple], ng: int, cat: bool) -> bool:+ limit = _H16X_PAD_CAT.get(cap, 0) if cat else 0+ return 1 < ng <= limit and ng & (ng - 1) != 0+++def _h16x_warps(cap: Optional[tuple], cat: bool, one: bool, tm: int) -> int:+ if one and tm > 16:+ return 16 if tm >= 64 else 8+ pair = _H16X_WARPS.get(cap)+ if pair is None:+ return _H16_WARPS if one else _H16_WARPS_TWO_PASS+ return pair[0] if cat else pair[1]+++def _prefix_layout(prefix: Any) -> Optional[tuple]:+ """(B, S, A, sab, sas, sah, HD) of an attention output ``[B, S, H, D]`` (any strides, unit last stride) or a 2D+ ``[M, A]`` prefix, else None."""+ if prefix.stride(-1) != 1:+ return None+ if prefix.dim() == 4:+ b, s, heads, hd = prefix.shape+ if hd not in (64, 128, 256):+ return None+ return b, s, heads * hd, prefix.stride(0), prefix.stride(1), prefix.stride(2), hd+ if prefix.dim() == 2:+ return 1, prefix.shape[0], prefix.shape[1], 0, prefix.stride(0), 256, 256+ return None+++def producer_layout_ok(prefix: Any, gate: Any, value: Any) -> bool:+ """``rotq_h16x`` reads this producer as is: bf16 CUDA, gate / value ``[M, F]`` with unit last stride, F and the+ prefix width multiples of 256, the prefix's rows matching."""+ import torch++ if (+ gate.dim() != 2+ or gate.shape != value.shape+ or gate.dtype != torch.bfloat16+ or value.dtype != torch.bfloat16+ or not gate.is_cuda+ or gate.device != value.device+ or gate.stride(-1) != 1+ or value.stride(-1) != 1+ or gate.shape[1] % 256+ or gate.shape[1] < 256+ ):+ return False+ if prefix is None:+ return True+ if prefix.dtype != torch.bfloat16 or prefix.device != gate.device:+ return False+ lay = _prefix_layout(prefix)+ return (+ lay is not None and lay[0] * lay[1] == gate.shape[0] and lay[2] % 256 == 0 and lay[2] >= 256+ )+++def producer_reference(prefix: Any, gate: Any, value: Any) -> Any:+ """The bf16 activation the stock producer writes: ``cat([prefix.flatten, silu(gate) * value], -1)``."""+ import torch++ prod = swiglu_reference(gate, value)+ if prefix is None:+ return prod+ return torch.cat([prefix.reshape(gate.shape[0], -1), prod], dim = -1)+++def _h16x_launch(+ prefix: Any,+ gate: Any,+ value: Any,+ kind: str,+ tile_rows: int = _H16_TILE_ROWS,+ tf32x3: Optional[bool] = None,+ warps: Optional[int] = None,+ pad: Optional[bool] = None,+) -> tuple:+ """``rotq_h16x`` on a producer accepted by ``producer_layout_ok``: (int8 [M, K], scale [M]). Strides stay+ specialized (multiple-of-16 hints keep the loads vectorized); only M and the sequence length are not."""+ import torch+ from .diffusion_convrot import build_convrot_hadamard++ kern = _kernels()+ m, f = gate.shape+ if prefix is None:+ a, seq, sab, sas, sah, hd, cat = 0, max(m, 1), 0, 0, 0, 256, False+ else:+ _b, seq, a, sab, sas, sah, hd = _prefix_layout(prefix)+ cat = True+ k = a + f+ ng = k // 256+ cap = _device_cap(gate.device.index)+ if pad is None:+ pad = _h16x_pad(cap, ng, cat)+ if pad:+ # one pass: the whole row in one masked power-of-two tile, so nothing is loaded or rotated twice+ cg, rows, nch, one = 1 << (ng - 1).bit_length(), 1, 1, True+ else:+ cg, rows, nch, one = _h16_plan(ng, tile_rows)+ if warps is None:+ warps = _h16x_warps(cap, cat, one, cg * rows)+ if tf32x3 is None:+ tf32x3 = _H16_DEVICE.get(gate.device.index) == "tf32x3"+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]+ h = build_convrot_hadamard(16, device = gate.device, dtype = torch.bfloat16)+ h32 = build_convrot_hadamard(16, device = gate.device, dtype = torch.float32)+ fp32_scale = _rotq_scale_fp32(kind)+ q = torch.empty((m, k), device = gate.device, dtype = torch.int8)+ s = torch.empty((m,), device = gate.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)+ if m == 0:+ return q, s+ with torch.cuda.device(gate.device):+ kern.rotq_h16x[(kern.triton.cdiv(m, rows),)](+ gate,+ value,+ gate if prefix is None else prefix,+ h,+ h32,+ q,+ s,+ m,+ ng,+ gate.stride(0),+ value.stride(0),+ a // 256,+ seq,+ sab,+ sas,+ sah,+ nch,+ CG = cg,+ ROWS = rows,+ ONE_PASS = one,+ UNROLL = nch <= _H16_UNROLL_MAX,+ CAT = cat,+ HD = hd,+ TF32X3 = bool(tf32x3),+ QMIN = qmin,+ QMAX = qmax,+ DIV = div,+ EPS = eps,+ FP32_SCALE = fp32_scale,+ num_warps = warps,+ num_stages = 1,+ enable_fp_fusion = False,+ )+ return q, s++def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:"""Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""import torch@@ -733,6 +1310,64 @@ def _swiglu_op() -> Any:return getattr(getattr(torch.ops, _OP_NAMESPACE), _SWIGLU_OP_NAME)+def _producer_run(prefix: Any, gate: Any, value: Any, group: int, v2: bool) -> tuple:+ """Op body: ``rotq_h16x`` where it passed its probe, else the stock product (and concat), rotation and quant."""+ _PRODUCER_CALLS[0] += 1+ kind = "v2" if v2 else "v1"+ index = gate.device.index+ if group == 256 and _PRODUCER_DEVICE.get(index) and producer_layout_ok(prefix, gate, value):+ try:+ return _h16x_launch(prefix, gate, value, kind)+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math+ _PRODUCER_DEVICE[index] = False+ return rotquant_reference(producer_reference(prefix, gate, value).contiguous(), group, kind)+++@lru_cache(maxsize = 1)+def _producer_op() -> Any:+ """``unsloth_studio::convrot_producer_act_quant_int8(prefix?, gate, value, group, v2)``: the rotated, quantized+ ``cat([prefix, silu(gate) * value])`` read straight from the producer's buffers (opaque to Inductor), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _PRODUCER_OP_NAME):+ return getattr(ns, _PRODUCER_OP_NAME)+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return None+ try:++ @custom_op(+ f"{_OP_NAMESPACE}::{_PRODUCER_OP_NAME}",+ mutates_args = (),+ schema = "(Tensor? prefix, Tensor gate, Tensor value, int group, bool v2) -> (Tensor, Tensor)",+ )+ def _convrot_producer_act_quant_int8(prefix, gate, value, group, v2):+ return _producer_run(prefix, gate, value, group, v2)++ @_convrot_producer_act_quant_int8.register_fake+ def _(prefix, gate, value, group, v2):+ k = gate.shape[1]+ if prefix is not None:+ k = k + (+ prefix.shape[-2] * prefix.shape[-1] if prefix.dim() == 4 else prefix.shape[-1]+ )+ return (+ gate.new_empty((gate.shape[0], k), dtype = torch.int8),+ gate.new_empty(+ (gate.shape[0],),+ dtype = torch.float32+ if _rotq_scale_fp32("v2" if v2 else "v1")+ else torch.bfloat16,+ ),+ )+ except Exception: # noqa: BLE001+ return None+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _PRODUCER_OP_NAME)++@lru_cache(maxsize = 1)def _rotq_op() -> Any:"""``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""@@ -782,6 +1417,9 @@ def rotquant_device_config(index: int) -> Optional[tuple]:and _kernels() is not Noneand _rotq_op() is not None):+ hit = _verdict_load(index)+ if hit is not None:+ return _adopt_verdict(index, hit)cap = torch.cuda.get_device_capability(index)want = rotquant_config(cap)for cand in (want, _ROTQ_FALLBACK) if want is not None else ():@@ -818,6 +1456,132 @@ def rotquant_device_config(index: int) -> Optional[tuple]:_ROTQ_HANDLE = _rotq_op()if sw:_SWIGLU_HANDLE = _swiglu_op()+ _verdict_store(+ index,+ {+ "cfg": list(cfg),+ "rules": [[lo, hi, list(t)] for lo, hi, t in rules],+ "h16": h16,+ "swiglu": bool(sw),+ },+ )+ return cfg+++# Probes cost ~50 s per process on an L4, so passing verdicts persist per card + toolchain fingerprint (+ this file and+# diffusion_convrot.py); failures are never stored (may be transient). UNSLOTH_DIFFUSION_PROBE_CACHE=0 disables.+_VERDICT_FILE = "diffusion_convrot_probe.json"+_VERDICT_SOURCES = ("diffusion_convrot_quant.py", "diffusion_convrot.py")+_VERDICT_MAX = 32+++@lru_cache(maxsize = 1)+def _verdict_sources() -> str:+ import hashlib+ from pathlib import Path++ h = hashlib.sha256()+ here = Path(__file__).resolve().parent+ for name in _VERDICT_SOURCES:+ try:+ h.update(name.encode())+ h.update((here / name).read_bytes())+ except OSError:+ h.update(b"<missing>")+ return h.hexdigest()+++def _verdict_slot(index: int) -> Optional[tuple]:+ """(cache file, key, fingerprint) for card ``index``, or None (cache off / unidentifiable card)."""+ try:+ import hashlib+ import json++ from . import diffusion_probe_cache as pc++ if not pc.enabled():+ return None+ path = pc._cache_file()+ fp = pc.fingerprint(f"cuda:{index}")+ if path is None or fp is None:+ return None+ fp = dict(fp, convrot_source = _verdict_sources())+ key = hashlib.sha256(json.dumps(fp, sort_keys = True).encode()).hexdigest()[:32]+ return path.with_name(_VERDICT_FILE), key, fp+ except Exception: # noqa: BLE001 - no cache: probe+ return None+++def _verdict_read(path: Any) -> dict:+ import json+ try:+ data = json.loads(path.read_text(encoding = "utf-8"))+ except Exception: # noqa: BLE001 - absent / torn reads as empty+ return {}+ return data if isinstance(data, dict) else {}+++def _verdict_load(index: int) -> Optional[dict]:+ slot = _verdict_slot(index)+ if slot is None:+ return None+ path, key, fp = slot+ entry = _verdict_read(path).get(key)+ if not isinstance(entry, dict) or entry.get("fingerprint") != fp:+ return None+ v = entry.get("verdicts")+ if not isinstance(v, dict) or not isinstance(v.get("cfg"), list) or len(v["cfg"]) != 4:+ return None+ return v+++def _verdict_store(index: int, update: dict) -> None:+ """Merge ``update`` into card ``index``'s entry (best effort, atomic replace)."""+ slot = _verdict_slot(index)+ if slot is None:+ return+ path, key, fp = slot+ import json+ import time++ try:+ data = _verdict_read(path)+ entry = data.get(key) if isinstance(data.get(key), dict) else {}+ verdicts = dict(entry.get("verdicts") or {}) if entry.get("fingerprint") == fp else {}+ verdicts.update(update)+ data[key] = {"fingerprint": fp, "verdicts": verdicts, "t": time.time()}+ if len(data) > _VERDICT_MAX:+ for k in sorted(data, key = lambda k: (data[k] or {}).get("t", 0))[+ : len(data) - _VERDICT_MAX+ ]:+ data.pop(k, None)+ path.parent.mkdir(parents = True, exist_ok = True)+ tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")+ tmp.write_text(json.dumps(data, sort_keys = True), encoding = "utf-8")+ os.replace(tmp, path)+ except Exception: # noqa: BLE001 - a cache write never fails a load+ pass+++def _adopt_verdict(index: int, v: dict) -> Optional[tuple]:+ """Install a persisted verdict as ``rotquant_device_config`` would after probing (the switches are part of the+ key, so the stored verdict already reflects them)."""+ global _ROTQ_HANDLE, _SWIGLU_HANDLE+ cfg = tuple(int(x) for x in v["cfg"])+ rules = tuple(+ (int(lo), int(hi), tuple(int(x) for x in t)) for lo, hi, t in v.get("rules") or ()+ )+ h16 = v.get("h16") if v.get("h16") in ("ieee", "tf32x3") else False+ sw = bool(v.get("swiglu")) and bool(h16) and _swiglu_op() is not None+ _ROTQ_DEVICE_K[index] = rules+ _ROTQ_DEVICE[index] = cfg+ _H16_DEVICE[index] = h16+ _SWIGLU_DEVICE[index] = sw+ if isinstance(v.get("producer"), bool) and h16:+ _PRODUCER_DEVICE[index] = v["producer"]+ _ROTQ_HANDLE = _rotq_op()+ if sw:+ _SWIGLU_HANDLE = _swiglu_op()return cfg@@ -951,6 +1715,109 @@ def _swiglu_probe(index: int) -> bool:return False+# (M, prefix (B, heads, head dim) or None, SwiGLU width)+_PRODUCER_PROBE_SHAPES = (+ (65, None, 256 * 4),+ (130, None, 256 * 36),+ (33, (1, 2, 128), 256 * 3),+ (2 * 41, (2, 24, 128), 256 * 36),+)+++def _producer_probe_inputs(m: int, pre: Optional[tuple], f: int, g: Any, dev: Any) -> tuple:+ import torch++ both = torch.cat(+ [(_probe_input(m, f, g).float() * 0.05).to(torch.bfloat16), _probe_input(m, f, g)], dim = 1+ ).to(dev)+ gate, value = both[:, :f], both[:, f:] # halves of one projection output: row stride 2F+ gate[0, :7] = torch.tensor([-90.0, -20.0, -1e-3, 0.0, 1e-3, 20.0, 90.0], device = dev)+ prefix = None+ if pre is not None:+ b, heads, hd = pre+ seq = m // b+ # [B, H, S, D] storage viewed as [B, S, H, D]: what SDPA hands back+ prefix = (+ _probe_input(b * heads * seq, hd, g).to(dev).view(b, heads, seq, hd).transpose(1, 2)+ )+ return prefix, gate, value+++def _producer_probe(index: int) -> bool:+ """``rotq_h16x`` vs the stock producer (``producer_reference``: ATen SiLU product, concat) then the stock rotation ++ quant: ``codes_close`` on every probe shape, v1 and v2."""+ import torch++ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(5)+ try:+ for m, pre, f in _PRODUCER_PROBE_SHAPES:+ prefix, gate, value = _producer_probe_inputs(m, pre, f, g, dev)+ if not producer_layout_ok(prefix, gate, value):+ return False+ ref = producer_reference(prefix, gate, value).contiguous()+ for kind in ("v1", "v2"):+ rq, rs = rotquant_reference(ref, 256, kind)+ q, s = _h16x_launch(prefix, gate, value, kind)+ if not codes_close(q, s, rq, rs):+ return False+ from .diffusion_cuda_graph import synchronize_outside_capture++ synchronize_outside_capture(dev)+ return True+ except Exception: # noqa: BLE001+ return False+++def _arm_producer(index: int) -> bool:+ """Per load, outside any trace, after ``rotq_h16`` was armed on the device: the producer variant's probe (once per+ device); publishes its op handle. The switch is read here, so it acts without a restart."""+ global _PRODUCER_HANDLE+ if not rotquant_producer_enabled() or not _H16_DEVICE.get(index):+ _PRODUCER_VERDICT_OFF.add(index)+ return False+ _PRODUCER_VERDICT_OFF.discard(index)+ if index not in _PRODUCER_DEVICE:+ ok = False+ try:+ ok = _producer_op() is not None and _producer_probe(index)+ except Exception: # noqa: BLE001 - the stock producer stays+ ok = False+ _PRODUCER_DEVICE[index] = ok+ if ok and _ROTQ_DEVICE.get(index) is not None:+ _verdict_store(index, {"producer": True})+ if _PRODUCER_DEVICE[index]:+ _PRODUCER_HANDLE = _producer_op()+ return bool(_PRODUCER_DEVICE[index])+++def producer_armed(index: Any) -> bool:+ """``rotq_h16x`` serves card ``index`` (its probe passed and the switch is on)."""+ return (+ _PRODUCER_HANDLE is not None+ and index not in _PRODUCER_VERDICT_OFF+ and bool(_PRODUCER_DEVICE.get(index))+ )+++def convrot_producer_act_quant(+ prefix: Any, gate: Any, value: Any, group: int, v2: bool+) -> Optional[tuple]:+ """``convrot_act_quant`` of ``cat([prefix, silu(gate) * value], -1)`` read from the producer's own buffers (the+ projection output halves and the attention output in its native layout), so neither the product nor the concat+ reaches HBM. None when this device / layout / group is not covered: the caller keeps the stock producer and its+ own rotation + quant. Safe inside a traced forward: no host sync, the probe ran at install time."""+ if group != 256 or _PRODUCER_HANDLE is None or not producer_armed(gate.device.index):+ return None+ if not producer_layout_ok(prefix, gate, value):+ return None+ return _PRODUCER_HANDLE(prefix, gate, value, group, v2)+++def producer_call_count() -> int:+ return _PRODUCER_CALLS[0]++def rotquant_call_count() -> int:return _ROTQ_CALLS[0]diff --git a/studio/backend/core/inference/diffusion_flux_convrot_fused.py b/studio/backend/core/inference/diffusion_flux_convrot_fused.pynew file mode 100644index 00000000000..f20a45b2d31--- /dev/null+++ b/studio/backend/core/inference/diffusion_flux_convrot_fused.py@@ -0,0 +1,574 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.2 / FLUX.3 int8 ConvRot without the separate rotate + quantize pass over each bf16 activation.++On the int8 GEMM swap (``diffusion_int8_gemm``) every rotated Linear quantizes its own input with ``rotq_h16``: the+producer writes bf16, the kernel reads it back. Here:+ * Linears reading one activation (Q / K / V, and on FLUX.3 also the MLP input) share one rotation + act quant+ (``shared_rotq_linears``: the same op on the same tensor, so bit-identical);+ * the MLP down projection and FLUX.2's single-block output projection read the SwiGLU halves (and the attention+ output) straight from the producers' buffers (``rotq_h16x``), so neither the product nor the concat is written.+Every patch checks its Linears per call and runs the stock code when they are not swapped rotated int8 Linears.+Kill switch ``UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED=0``; ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER=0`` keeps only the+shared quants."""++from __future__ import annotations++import functools+import os+import threading+from typing import Any, Optional++import torch++from . import diffusion_int8_gemm as _gemm++FLUX_CONVROT_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"+_MARK = "__unsloth_flux_convrot_fused__"+_LOCK = threading.Lock()+_STOCK: dict = {} # (owner, attr) -> original+_CALLS = {"shared": [0], "producer": [0], "block": [0]}++# Stock lines each replacement mirrors; a diffusers whose body lacks any keeps the stock function.+_NEEDLES = {+ "f2_projections": (+ "query = attn.to_q(hidden_states)",+ "key = attn.to_k(hidden_states)",+ "value = attn.to_v(hidden_states)",+ "encoder_query = attn.add_q_proj(encoder_hidden_states)",+ "encoder_key = attn.add_k_proj(encoder_hidden_states)",+ "encoder_value = attn.add_v_proj(encoder_hidden_states)",+ ),+ "f3_projections": (+ "return attn.to_q(hidden_states), attn.to_k(hidden_states), attn.to_v(hidden_states)",+ ),+ "ff": ("x = self.linear_in(x)", "x = self.act_fn(x)", "x = self.linear_out(x)"),+ "swiglu": ("half = x.shape[-1] // 2", "x = self.gate_fn(x[..., :half]) * x[..., half:]"),+ "f2_parallel": (+ "hidden_states = attn.to_qkv_mlp_proj(hidden_states)",+ "query, key, value = qkv.chunk(3, dim=-1)",+ "query = attn.norm_q(query)",+ "key = attn.norm_k(key)",+ "query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1)",+ "key = apply_rotary_emb(key, image_rotary_emb, sequence_dim=1)",+ "attn_mask=attention_mask,",+ "backend=self._attention_backend,",+ "parallel_config=self._parallel_config,",+ "hidden_states = hidden_states.flatten(2, 3)",+ "hidden_states = hidden_states.to(query.dtype)",+ "mlp_hidden_states = attn.mlp_act_fn(mlp_hidden_states)",+ "hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim=-1)",+ "hidden_states = attn.to_out(hidden_states)",+ ),+ "f3_attn": (+ "query, key, value = _get_qkv_projections(attn, hidden_states)",+ "query = query.unflatten(-1, (attn.heads, -1))",+ "query = attn.norm_q(query.float())",+ "key = attn.norm_k(key.float())",+ "query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1)",+ "query, key = query.to(value.dtype), key.to(value.dtype)",+ "attn_mask=attention_mask,",+ "hidden_states = hidden_states.flatten(2, 3)",+ "hidden_states = hidden_states.to(query.dtype)",+ "hidden_states = attn.to_out[0](hidden_states)",+ "hidden_states = attn.to_out[1](hidden_states)",+ ),+ "f3_attn_forward": (+ "return self.processor(self, hidden_states, attention_mask, image_rotary_emb)",+ ),+ "f3_mode": (+ "mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim=-1)",+ "norm_hidden_states = self.norm(hidden_states)",+ "norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift",+ "attn_output = self.attn(hidden_states=norm_hidden_states, image_rotary_emb=image_rotary_emb)",+ "ff_output = self.ff(norm_hidden_states)",+ "return hidden_states + mod_gate * (attn_output + ff_output)",+ ),+ "f3_single": (+ "mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim=-1)",+ "c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim=-1)",+ "norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift",+ "norm_encoder_hidden_states = self.norm(encoder_hidden_states)",+ "norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift",+ "combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim=1)",+ "attn_output = self.attn(hidden_states=combined, image_rotary_emb=image_rotary_emb)",+ "ff_output = self.ff(combined)",+ "output = attn_output + ff_output",+ "encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output",+ "hidden_states = hidden_states + mod_gate * content_output",+ ),+}+++def flux_convrot_fused_disabled() -> bool:+ return (os.environ.get(FLUX_CONVROT_FUSED_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ )+++def _flux_modules() -> tuple:+ try:+ from diffusers.models.transformers import transformer_flux2 as f2+ except Exception: # noqa: BLE001 - diffusers without this family+ f2 = None+ try:+ from diffusers.models.transformers import transformer_flux3 as f3+ except Exception: # noqa: BLE001+ f3 = None+ return f2, f3+++def _count(kind: str) -> None:+ """Engagement census, eager calls only (a traced forward counts nothing: no side effect in the graph)."""+ if not torch.compiler.is_compiling():+ _CALLS[kind][0] += 1+++def _stock(owner: Any, attr: str) -> Any:+ """The original ``owner.attr`` (ours stashed it), else the live one."""+ return _STOCK.get((owner, attr), getattr(owner, attr, None))+++def _has(fn: Any, key: str) -> bool:+ from .diffusion_autoopt.patches import body_has+ return fn is not None and body_has(fn, *_NEEDLES[key])+++def _rotated(lin: Any) -> bool:+ """``lin`` is a swapped rotated int8 Linear whose act quant the fused rotation covers."""+ rec = _gemm._swapped_rec(lin)+ return rec is not None and bool(rec[2]) and rec[1] == 256+++def _plain_swiglu(act: Any, cls_name: str) -> bool:+ """``act`` is the stock FLUX SwiGLU (``silu(x[:half]) * x[half:]``) with an unhooked ``nn.SiLU``."""+ return (+ act is not None+ and type(act) is _CLS.get(cls_name)+ and not _gemm._hooked(act)+ and _gemm._plain_silu(getattr(act, "gate_fn", None))+ )+++# Traced gates compare classes, never functions: dynamo resolves ``f.__func__ is g`` to False.+_CLS: dict = {}+++def _ff_ready(ff: Any, act_cls: str) -> bool:+ return (+ not _gemm._hooked(ff)+ and _plain_swiglu(getattr(ff, "act_fn", None), act_cls)+ and _rotated(ff.linear_out)+ )+++def _ff_tail(ff: Any, h: Any) -> Any:+ """``ff.linear_out(ff.act_fn(h))``: the down projection reads the two halves of ``h`` directly when it can."""+ half = h.shape[-1] // 2+ y = _gemm.producer_linear(ff.linear_out, None, h[..., :half], h[..., half:])+ if y is None:+ return ff.linear_out(ff.act_fn(h))+ _count("producer")+ return y+++def _make_ff_forward(stock: Any, act_cls: str) -> Any:+ def forward(self, x):+ if not _ff_ready(self, act_cls):+ return stock(self, x)+ return _ff_tail(self, self.linear_in(x))++ functools.update_wrapper(+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()+ )+ setattr(forward, _MARK, True)+ return forward+++def _make_f2_projections(stock: Any) -> Any:+ def _get_projections(+ attn,+ hidden_states,+ encoder_hidden_states = None,+ ):+ qkv = None+ if _rotated(attn.to_q):+ qkv = _gemm.shared_rotq_linears(hidden_states, (attn.to_q, attn.to_k, attn.to_v))+ if qkv is None:+ return stock(attn, hidden_states, encoder_hidden_states)+ _count("shared")+ query, key, value = qkv+ encoder_query = encoder_key = encoder_value = None+ if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None:+ added = None+ if _rotated(attn.add_q_proj):+ added = _gemm.shared_rotq_linears(+ encoder_hidden_states, (attn.add_q_proj, attn.add_k_proj, attn.add_v_proj)+ )+ if added is None:+ added = (+ attn.add_q_proj(encoder_hidden_states),+ attn.add_k_proj(encoder_hidden_states),+ attn.add_v_proj(encoder_hidden_states),+ )+ encoder_query, encoder_key, encoder_value = added+ return query, key, value, encoder_query, encoder_key, encoder_value++ functools.update_wrapper(_get_projections, stock, assigned = ("__name__", "__doc__"), updated = ())+ setattr(_get_projections, _MARK, True)+ return _get_projections+++def _make_f3_projections(stock: Any) -> Any:+ def _get_projections(attn, hidden_states):+ qkv = None+ if _rotated(attn.to_q):+ qkv = _gemm.shared_rotq_linears(hidden_states, (attn.to_q, attn.to_k, attn.to_v))+ if qkv is None:+ return stock(attn, hidden_states)+ _count("shared")+ return tuple(qkv)++ functools.update_wrapper(_get_projections, stock, assigned = ("__name__", "__doc__"), updated = ())+ setattr(_get_projections, _MARK, True)+ return _get_projections+++def _parallel_ready(attn: Any) -> bool:+ """FLUX.2 single-block attention whose output projection can read [attention | SwiGLU] from the producers.+ Traced: plain attribute reads only (no try / except)."""+ proj = getattr(attn, "to_qkv_mlp_proj", None)+ out = getattr(attn, "to_out", None)+ act = getattr(attn, "mlp_act_fn", None)+ if proj is None or out is None or act is None or getattr(attn, "mlp_mult_factor", None) != 2:+ return False+ return (+ getattr(proj, "out_features", None)+ == 3 * attn.inner_dim + 2 * attn.mlp_hidden_dim # unsharded+ and getattr(out, "in_features", None) == attn.inner_dim + attn.mlp_hidden_dim+ and _plain_swiglu(act, "Flux2SwiGLU")+ and _rotated(out)+ )+++def _make_f2_parallel_call(stock: Any, mod: Any) -> Any:+ def __call__(+ self,+ attn,+ hidden_states,+ attention_mask = None,+ image_rotary_emb = None,+ ):+ if not _parallel_ready(attn):+ return stock(self, attn, hidden_states, attention_mask, image_rotary_emb)+ hidden_states = attn.to_qkv_mlp_proj(hidden_states)+ qkv_dim = 3 * attn.inner_dim+ qkv, mlp_hidden_states = torch.split(+ hidden_states, [qkv_dim, hidden_states.shape[-1] - qkv_dim], dim = -1+ )+ query, key, value = qkv.chunk(3, dim = -1)+ query = query.unflatten(-1, (-1, attn.head_dim))+ key = key.unflatten(-1, (-1, attn.head_dim))+ value = value.unflatten(-1, (-1, attn.head_dim))+ query = attn.norm_q(query)+ key = attn.norm_k(key)+ if image_rotary_emb is not None:+ # looked up per call: diffusion_flux2_rope may have replaced it+ query = mod.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)+ key = mod.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)+ hidden_states = mod.dispatch_attention_fn(+ query,+ key,+ value,+ attn_mask = attention_mask,+ backend = self._attention_backend,+ parallel_config = self._parallel_config,+ )+ hidden_states = hidden_states.to(query.dtype)+ half = mlp_hidden_states.shape[-1] // 2+ out = _gemm.producer_linear(+ attn.to_out, hidden_states, mlp_hidden_states[..., :half], mlp_hidden_states[..., half:]+ )+ if out is not None:+ _count("producer")+ return out+ hidden_states = hidden_states.flatten(2, 3)+ mlp_hidden_states = attn.mlp_act_fn(mlp_hidden_states)+ hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim = -1)+ return attn.to_out(hidden_states)++ functools.update_wrapper(+ __call__, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()+ )+ setattr(__call__, _MARK, True)+ return __call__+++def _f3_block_ready(block: Any) -> bool:+ """A FLUX.3 block whose attention and MLP can share one rotation + act quant of their common input. Traced: plain+ attribute reads only."""+ attn, ff = getattr(block, "attn", None), getattr(block, "ff", None)+ if attn is None or ff is None or getattr(attn, "fused_projections", False):+ return False+ proc = getattr(attn, "processor", None)+ return (+ type(proc) is _CLS.get("Flux3AttnProcessor")+ and type(attn) is _CLS.get("Flux3Attention")+ and not _gemm._hooked(attn)+ and type(ff) is _CLS.get("Flux3FeedForward")+ and _ff_ready(ff, "Flux3SwiGLU")+ and _rotated(attn.to_q)+ and _rotated(attn.to_k)+ and _rotated(attn.to_v)+ and _rotated(ff.linear_in)+ )+++def _f3_attn_ff(block: Any, x: Any, image_rotary_emb: Any, mod: Any) -> Optional[tuple]:+ """(attention output, MLP output) of ``x`` with Q / K / V and the MLP input on one rotation + act quant; None:+ the caller runs the stock modules. Mirrors ``Flux3AttnProcessor.__call__`` below the projections."""+ if not _f3_block_ready(block):+ return None+ attn, ff = block.attn, block.ff+ parts = _gemm.shared_rotq_linears(x, (attn.to_q, attn.to_k, attn.to_v, ff.linear_in))+ if parts is None:+ return None+ _count("block")+ query, key, value, h = parts+ proc = attn.processor+ query = query.unflatten(-1, (attn.heads, -1))+ key = key.unflatten(-1, (attn.heads, -1))+ value = value.unflatten(-1, (attn.heads, -1))+ query = attn.norm_q(query.float())+ key = attn.norm_k(key.float())+ if image_rotary_emb is not None:+ # looked up per call: diffusion_flux3_rope may have replaced it+ query = mod.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)+ key = mod.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)+ query, key = query.to(value.dtype), key.to(value.dtype)+ hidden_states = mod.dispatch_attention_fn(+ query,+ key,+ value,+ attn_mask = None,+ backend = proc._attention_backend,+ parallel_config = proc._parallel_config,+ )+ hidden_states = hidden_states.flatten(2, 3)+ hidden_states = hidden_states.to(query.dtype)+ hidden_states = attn.to_out[0](hidden_states)+ hidden_states = attn.to_out[1](hidden_states)+ return hidden_states, _ff_tail(ff, h)+++def _make_f3_mode_forward(stock: Any, mod: Any) -> Any:+ def forward(self, hidden_states, temb_mod, image_rotary_emb):+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)+ norm_hidden_states = self.norm(hidden_states)+ norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift+ both = _f3_attn_ff(self, norm_hidden_states, image_rotary_emb, mod)+ if both is None:+ attn_output = self.attn(+ hidden_states = norm_hidden_states, image_rotary_emb = image_rotary_emb+ )+ ff_output = self.ff(norm_hidden_states)+ else:+ attn_output, ff_output = both+ return hidden_states + mod_gate * (attn_output + ff_output)++ functools.update_wrapper(+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()+ )+ setattr(forward, _MARK, True)+ return forward+++def _make_f3_single_forward(stock: Any, mod: Any) -> Any:+ def forward(+ self, hidden_states, encoder_hidden_states, temb_mod, temb_mod_context, image_rotary_emb+ ):+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)+ c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim = -1)+ norm_hidden_states = self.norm(hidden_states)+ norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift+ norm_encoder_hidden_states = self.norm(encoder_hidden_states)+ norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift+ combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)+ both = _f3_attn_ff(self, combined, image_rotary_emb, mod)+ if both is None:+ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)+ ff_output = self.ff(combined)+ else:+ attn_output, ff_output = both+ output = attn_output + ff_output+ context_output, content_output = torch.split(+ output, [encoder_hidden_states.shape[1], hidden_states.shape[1]], dim = 1+ )+ encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output+ hidden_states = hidden_states + mod_gate * content_output+ return encoder_hidden_states, hidden_states++ functools.update_wrapper(+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()+ )+ setattr(forward, _MARK, True)+ return forward+++def _patch(owner: Any, attr: str, key: str, make: Any) -> bool:+ """Swap ``owner.attr`` for ``make(stock)`` when the stock body carries ``_NEEDLES[key]``; idempotent."""+ current = getattr(owner, attr, None)+ if current is None:+ return False+ if getattr(current, _MARK, False):+ return True+ if not _has(current, key):+ return False+ _STOCK[(owner, attr)] = current+ setattr(owner, attr, make(current))+ return True+++def _install_patches(logger: Any = None) -> dict:+ done = {}+ f2, f3 = _flux_modules()+ with _LOCK:+ for mod, prefix, swiglu, ff_cls in (+ (f2, "f2", "Flux2SwiGLU", "Flux2FeedForward"),+ (f3, "f3", "Flux3SwiGLU", "Flux3FeedForward"),+ ):+ if mod is None:+ continue+ act = getattr(mod, swiglu, None)+ if act is not None and _has(act.forward, "swiglu"):+ _CLS[swiglu] = act+ proj_key = f"{prefix}_projections"+ done[proj_key] = _patch(+ mod,+ "_get_projections",+ proj_key,+ _make_f2_projections if prefix == "f2" else _make_f3_projections,+ )+ ff = getattr(mod, ff_cls, None)+ if ff is not None and swiglu in _CLS:+ done[f"{prefix}_ff"] = _patch(+ ff, "forward", "ff", lambda s, c = swiglu: _make_ff_forward(s, c)+ )+ if done[f"{prefix}_ff"]:+ _CLS[ff_cls] = ff+ if f2 is not None and "Flux2SwiGLU" in _CLS:+ proc = getattr(f2, "Flux2ParallelSelfAttnProcessor", None)+ if proc is not None:+ done["f2_parallel"] = _patch(+ proc, "__call__", "f2_parallel", lambda s: _make_f2_parallel_call(s, f2)+ )+ if f3 is not None and "Flux3FeedForward" in _CLS:+ proc, attn_cls = (+ getattr(f3, "Flux3AttnProcessor", None),+ getattr(f3, "Flux3Attention", None),+ )+ ff = getattr(f3, "Flux3FeedForward", None)+ if (+ proc is not None+ and attn_cls is not None+ and ff is not None+ and _has(proc.__call__, "f3_attn")+ and _has(attn_cls.forward, "f3_attn_forward")+ ):+ _CLS["Flux3AttnProcessor"] = proc+ _CLS["Flux3Attention"] = attn_cls+ for cls_name, key, make in (+ ("Flux3ModeTransformerBlock", "f3_mode", _make_f3_mode_forward),+ ("Flux3SingleTransformerBlock", "f3_single", _make_f3_single_forward),+ ):+ cls = getattr(f3, cls_name, None)+ if cls is not None:+ done[key] = _patch(cls, "forward", key, lambda s, m = make: m(s, f3))+ if logger is not None:+ logger.info("diffusion.flux_convrot_fused: patches %s", done)+ return done+++def _rotated_count(transformer: Any) -> int:+ n = 0+ for module in transformer.modules():+ rec = _gemm._swapped_rec(module) if hasattr(module, "weight") else None+ if rec is not None and rec[2] and rec[1] == 256:+ n += 1+ return n+++def _rotated_linears(transformer: Any) -> int:+ """ConvRot (group 256) Linears in the denoiser, swapped yet or not."""+ from .diffusion_convrot import is_rotated_linear+ return sum(+ 1+ for m in transformer.modules()+ if is_rotated_linear(m) and int(getattr(m, "convrot_groupsize", 0) or 0) == 256+ )+++def _arm_cards(transformer: Any, logger: Any = None) -> list:+ """``rotq_h16x`` on every card the denoiser sits on (outside any trace / capture)."""+ from .diffusion_convrot_quant import _arm_producer+ from .diffusion_int8_fused import resident_cuda_cards++ cards = [index for index in resident_cuda_cards(transformer) or () if _arm_producer(index)]+ if logger is not None:+ logger.info(+ "diffusion.flux_convrot_fused: %d swapped rotated int8 Linears; producer-fused rotation on cards %s",+ _rotated_count(transformer),+ cards,+ )+ return cards+++def install(transformer: Any, logger: Any = None) -> dict:+ """After the int8 GEMM swap, before the regional compile: patch the FLUX.2 / FLUX.3 helpers (each checks its+ Linears per call) and arm ``rotq_h16x`` on the denoiser's cards, at the first forward when the weights (and so+ the swap) are not on the GPU yet. Nothing without ConvRot Linears."""+ result = {"rotated": 0, "producer_cards": None, "patches": {}}+ if flux_convrot_fused_disabled():+ uninstall()+ return result+ if type(transformer).__name__ not in ("Flux2Transformer2DModel", "Flux3Transformer2DModel"):+ return result+ try:+ result["rotated"] = _rotated_linears(transformer)+ if not result["rotated"]:+ return result+ result["patches"] = _install_patches(logger)+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call++ if resident_cuda_cards(transformer) and _rotated_count(transformer):+ result["producer_cards"] = _arm_cards(transformer, logger)+ else:+ run_on_first_call(transformer, "flux_convrot_fused", lambda t: _arm_cards(t, logger))+ except Exception as exc: # noqa: BLE001 - optimisation only: stock helpers stay+ if logger is not None:+ logger.info("diffusion.flux_convrot_fused: not installed (%s)", exc)+ return result+++def uninstall() -> None:+ with _LOCK:+ for key, stock in list(_STOCK.items()):+ owner, attr = key+ if isinstance(owner, str):+ continue+ if getattr(getattr(owner, attr, None), _MARK, False):+ setattr(owner, attr, stock)+ _STOCK.clear()+ _CLS.clear()+++def is_installed() -> bool:+ return any(not isinstance(k[0], str) for k in _STOCK)+++def call_counts() -> dict:+ return {k: v[0] for k, v in _CALLS.items()}diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 8f0522784b3..a1a30803139 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -33,6 +33,9 @@ _OP_NAMESPACE = "unsloth_studio"_OP_NAME = "int8_mm_dequant"_REC = "_unsloth_i8_gemm"_MARK = "_unsloth_i8_gemm_prev"+# The bound ``_linear_forward`` the swap installed: a traced forward checks the instance ``forward`` by identity with+# it (dynamo resolves ``forward.__func__ is _linear_forward`` to False, so the shared paths never engaged compiled).+_FWD = "_unsloth_i8_gemm_fwd"# on a SwiGLU MLP whose down projection takes its codes from the gate / value halves_FF_MARK = "_unsloth_i8_gemm_swiglu_prev"_SWIGLU_FF_CLASSES = ("QwenImage21SwiGLUFeedForward",)@@ -1111,7 +1114,11 @@ def _swapped_rec(module: Any) -> Optional[tuple]:rec = d.get(_REC)if rec is None or _MARK not in d or "_diffusers_hook" in d or "_hf_hook" in d:return None- if getattr(d.get("forward"), "__func__", None) is not _linear_forward:+ fwd = d.get("forward")+ if torch.compiler.is_compiling():+ if fwd is None or fwd is not d.get(_FWD):+ return None+ elif getattr(fwd, "__func__", None) is not _linear_forward:return Noneif d.get("_forward_hooks") or d.get("_forward_pre_hooks"):return None@@ -1190,6 +1197,50 @@ def _swiglu_down(down: Any, gate: Any, value: Any) -> Optional[Any]:return y.reshape(*gate.shape[:-1], y.shape[-1])+def producer_linear(down: Any, prefix: Any, gate: Any, value: Any) -> Optional[Any]:+ """``down(cat([prefix.flatten(-2), silu(gate) * value], -1))`` (no prefix: ``down(silu(gate) * value)``) with the+ product, concat, rotation and act quant in one kernel reading the producers' buffers, else None (caller: stock).+ ``gate`` / ``value``: ``[..., F]`` halves of one projection output; ``prefix``: the attention output+ ``[B, S, H, D]`` as the attention backend returned it."""+ import torch++ rec = _swapped_rec(down)+ if rec is None:+ return None+ kind, group, rotq, weight, fused = rec+ lead = gate.shape[:-1]+ f = gate.shape[-1]+ width = f if prefix is None else f + prefix.shape[-2] * prefix.shape[-1]+ if (+ not rotq+ or group != 256+ or gate.dtype != torch.bfloat16+ or not gate.is_cuda+ or gate.shape != value.shape+ or width != down.in_features+ or gate.numel() < _MIN_ROWS * f+ ):+ return None+ if prefix is not None and (prefix.dim() != 4 or prefix.shape[:2] != lead):+ return None+ wq, ws = _payload(kind, weight)+ if wq.device != gate.device:+ return None+ from .diffusion_convrot_quant import convrot_producer_act_quant++ q = convrot_producer_act_quant(+ prefix, gate.reshape(-1, f), value.reshape(-1, f), group, kind == "v2"+ )+ if q is None:+ return None+ xq, xs = q+ if fused:+ y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, down.bias)+ else:+ y = stock_gemm(xq, wq, xs, ws, down.bias)+ return y.reshape(*lead, y.shape[-1])++def _plain_silu(act: Any) -> bool:"""``act`` is an exact, unhooked ``nn.SiLU`` (the fused prologue hard-codes SiLU); ``type() is`` avoids a graph break."""from torch import nn@@ -1544,6 +1595,7 @@ def _swap_on()module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)+ module.__dict__[_FWD] = module.forwardcount += 1ff = (_install_swiglu_ff(@@ -1657,6 +1709,7 @@ def uninstall(transformer: Any = None) -> None:if prev is None:continuemodule.__dict__.pop(_REC, None)+ module.__dict__.pop(_FWD, None)if prev is _NO_PREV:module.__dict__.pop("forward", None)else:diff --git a/studio/backend/tests/test_diffusion_autoopt_core.py b/studio/backend/tests/test_diffusion_autoopt_core.pyindex c14e7bb9f9b..badb7a92497 100644--- a/studio/backend/tests/test_diffusion_autoopt_core.py+++ b/studio/backend/tests/test_diffusion_autoopt_core.py@@ -206,6 +206,7 @@ def test_builtin_specs_cover_every_former_install_site():"zimage_fused","qwenimage21_fused","qwenimage21_dense_fused",+ "flux_convrot_fused","moss_fused",}# the former hand-kept teardown list is now a registry group@@ -218,6 +219,7 @@ def test_builtin_specs_cover_every_former_install_site():"zimage_fused","qwenimage21_fused","qwenimage21_dense_fused",+ "flux_convrot_fused","moss_fused",}# every spec answers its module's existing kill switch name@@ -225,6 +227,7 @@ def test_builtin_specs_cover_every_former_install_site():assert envs["arch_fusions"] == "UNSLOTH_DIFFUSION_ARCH_PATCHES"assert envs["wan_fused"] == "UNSLOTH_DIFFUSION_WAN_FUSED_ADALN"assert envs["zimage_fused"] == "UNSLOTH_DIFFUSION_ZIMAGE_FUSED"+ assert envs["flux_convrot_fused"] == "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"# order inside the regional compile: Q2.1 real RoPE before the int8 swap, the rest after itassert [s.name for s in ap.specs(ap.STAGE_PRE_INT8)] == ["qwenimage21_real_rope"]assert [s.name for s in ap.specs(ap.STAGE_PRE_COMPILE)] == [@@ -232,6 +235,7 @@ def test_builtin_specs_cover_every_former_install_site():"zimage_fused","qwenimage21_fused","qwenimage21_dense_fused",+ "flux_convrot_fused","moss_fused",]diff --git a/studio/backend/tests/test_diffusion_autoopt_parity.py b/studio/backend/tests/test_diffusion_autoopt_parity.pyindex e4ef1e6644d..2fa1ffa4fb6 100644--- a/studio/backend/tests/test_diffusion_autoopt_parity.py+++ b/studio/backend/tests/test_diffusion_autoopt_parity.py@@ -180,6 +180,7 @@ def _teardown() -> None:"diffusion_qwenimage_rope","diffusion_zimage_fused","diffusion_qwenimage21_fused",+ "diffusion_flux_convrot_fused","diffusion_rocm_fused","diffusion_qwenimage21","diffusion_qwenimage21_rope",diff --git a/studio/backend/tests/test_diffusion_flux_convrot_fused.py b/studio/backend/tests/test_diffusion_flux_convrot_fused.pynew file mode 100644index 00000000000..f415897695c--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux_convrot_fused.py@@ -0,0 +1,484 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.2 / FLUX.3 int8 ConvRot producer fusion (diffusion_flux_convrot_fused). The patched helpers and forwards must+compute what the stock diffusers ones do: here the int8 pieces (the shared quant and the producer-fused down+projection) are replaced by the plain modules, so a forced-on patch has to match stock bit for bit on CPU. The GPU+kernel itself is checked against the stock path by its own probe and the GPU parity run."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))+++def env(monkeypatch):+ torch = pytest.importorskip("torch")+ pytest.importorskip("diffusers")+ f2 = pytest.importorskip("diffusers.models.transformers.transformer_flux2")+ f3 = pytest.importorskip("diffusers.models.transformers.transformer_flux3")+ from core.inference import diffusion_flux_convrot_fused as fused+ from core.inference import diffusion_int8_gemm as gemm++ fused.uninstall()+ monkeypatch.delenv(fused.FLUX_CONVROT_FUSED_ENV, raising = False)+ yield torch, f2, f3, fused, gemm+ fused.uninstall()+++def _force_on(monkeypatch, torch, fused, gemm, calls):+ """Every Linear counts as a swapped rotated int8 Linear; the int8 pieces run the plain modules."""+ import torch.nn.functional as F++ monkeypatch.setattr(fused, "_rotated", lambda lin: True)++ def shared(x, linears):+ calls["shared"] += 1+ return [m(x) for m in linears]++ def producer(down, prefix, gate, value):+ calls["producer"] += 1+ h = F.silu(gate) * value+ if prefix is not None:+ h = torch.cat([prefix.flatten(2, 3), h], dim = -1)+ return down(h)++ monkeypatch.setattr(gemm, "shared_rotq_linears", shared)+ monkeypatch.setattr(gemm, "producer_linear", producer)+++def _rope(torch, seq, dim):+ g = torch.Generator().manual_seed(1)+ ang = torch.rand(seq, dim, generator = g) * 3+ return ang.cos(), ang.sin()++[email protected](params = ["eager", "compiled"])+def mode(request):+ return request.param+++def _run_stock_vs_patched(+ torch,+ fused,+ build,+ call,+ mode = "eager",+):+ """Stock, patched (eager, or traced by dynamo: the per-call gates must hold inside the regional compile) and+ restored outputs of one block."""+ torch.manual_seed(0)+ block = build().eval()+ with torch.no_grad():+ ref = call(block)+ assert fused._install_patches() # noqa: SLF001 - the install minus the int8 gates+ if mode == "compiled":+ torch._dynamo.reset()+ compiled = torch.compile(block, backend = "eager", fullgraph = True)+ out = call(compiled)+ torch._dynamo.reset()+ else:+ out = call(block)+ fused.uninstall()+ with torch.no_grad():+ back = call(block)+ return ref, out, back+++def _same(a, b):+ if isinstance(a, (tuple, list)):+ return all(_same(x, y) for x, y in zip(a, b)) and len(a) == len(b)+ return a.shape == b.shape and bool((a == b).all())+++def test_flux2_double_block_matches_stock(env, monkeypatch, mode):+ torch, f2, _f3, fused, gemm = env+ calls = {"shared": 0, "producer": 0}+ _force_on(monkeypatch, torch, fused, gemm, calls)+ dim, heads, hd = 64, 2, 32+ hs = torch.randn(2, 12, dim)+ enc = torch.randn(2, 5, dim)+ mod_i, mod_t = torch.randn(2, 6 * dim) * 0.1, torch.randn(2, 6 * dim) * 0.1+ rope = _rope(torch, 17, hd)+ ref, out, back = _run_stock_vs_patched(+ torch,+ fused,+ lambda: f2.Flux2TransformerBlock(dim, heads, hd),+ lambda b: b(hs, enc, mod_i, mod_t, image_rotary_emb = rope),+ mode,+ )+ assert _same(ref, out) and _same(ref, back)+ assert calls["shared"] == 2 # image Q/K/V and text Q/K/V, one quant each+ assert calls["producer"] == 2 # ff and ff_context down projections+++def test_flux2_single_block_matches_stock(env, monkeypatch, mode):+ torch, f2, _f3, fused, gemm = env+ calls = {"shared": 0, "producer": 0}+ _force_on(monkeypatch, torch, fused, gemm, calls)+ dim, heads, hd = 64, 2, 32+ hs = torch.randn(2, 12, dim)+ enc = torch.randn(2, 5, dim)+ mod = torch.randn(2, 3 * dim) * 0.1+ rope = _rope(torch, 17, hd)+ ref, out, back = _run_stock_vs_patched(+ torch,+ fused,+ lambda: f2.Flux2SingleTransformerBlock(dim, heads, hd),+ lambda b: b(hs, enc, mod, image_rotary_emb = rope, split_hidden_states = True),+ mode,+ )+ assert _same(ref, out) and _same(ref, back)+ assert calls["producer"] == 1 # [attention | SwiGLU] into to_out, read from the producers++[email protected]("kind", ["mode", "single"])+def test_flux3_blocks_match_stock(env, monkeypatch, kind, mode):+ torch, _f2, f3, fused, gemm = env+ calls = {"shared": 0, "producer": 0}+ _force_on(monkeypatch, torch, fused, gemm, calls)+ dim, heads, hd = 64, 2, 32+ hs = torch.randn(2, 12, dim)+ enc = torch.randn(2, 5, dim)+ mod = torch.randn(2, 1, 3 * dim) * 0.1+ cmod = torch.randn(2, 1, 3 * dim) * 0.1+ if kind == "mode":+ rope = _rope(torch, 12, hd)+ build = lambda: f3.Flux3ModeTransformerBlock(dim, heads, hd) # noqa: E731+ call = lambda b: b(hs, mod, rope) # noqa: E731+ else:+ rope = _rope(torch, 17, hd)+ build = lambda: f3.Flux3SingleTransformerBlock(dim, heads, hd) # noqa: E731+ call = lambda b: b(hs, enc, mod, cmod, rope) # noqa: E731+ ref, out, back = _run_stock_vs_patched(torch, fused, build, call, mode)+ assert _same(ref, out) and _same(ref, back)+ assert calls["shared"] == 1 # Q, K, V and the MLP input: one quant+ assert calls["producer"] == 1+++def test_gates_off_runs_stock(env, monkeypatch):+ """Unswapped Linears (CPU, plain bf16, LoRA...): every patch takes the stock path, so nothing int8 is called."""+ torch, f2, f3, fused, gemm = env++ def boom(*a, **k):+ raise AssertionError("int8 path reached without swapped Linears")++ monkeypatch.setattr(gemm, "shared_rotq_linears", boom)+ monkeypatch.setattr(gemm, "producer_linear", boom)+ dim, heads, hd = 64, 2, 32+ torch.manual_seed(0)+ single = f2.Flux2SingleTransformerBlock(dim, heads, hd).eval()+ mode = f3.Flux3ModeTransformerBlock(dim, heads, hd).eval()+ hs = torch.randn(1, 8, dim)+ smod, mmod, rope = (+ torch.randn(1, 3 * dim) * 0.1,+ torch.randn(1, 1, 3 * dim) * 0.1,+ _rope(torch, 8, hd),+ )+ with torch.no_grad():+ a, b = single(hs, None, smod), mode(hs, mmod, rope)+ assert all(fused._install_patches().values()) # noqa: SLF001+ assert torch.equal(single(hs, None, smod), a)+ assert torch.equal(mode(hs, mmod, rope), b)+++def test_drifted_body_keeps_stock(env, monkeypatch):+ torch, f2, _f3, fused, _gemm = env+ stock = f2.Flux2ParallelSelfAttnProcessor.__call__+ monkeypatch.setitem(+ fused._NEEDLES,+ "f2_parallel",+ fused._NEEDLES["f2_parallel"] + ("a line diffusers never had",),+ )+ done = fused._install_patches() # noqa: SLF001+ assert done.get("f2_parallel") is False+ assert f2.Flux2ParallelSelfAttnProcessor.__call__ is stock+++def test_uninstall_restores_everything(env):+ torch, f2, f3, fused, _gemm = env+ before = {+ "p2": f2._get_projections,+ "p3": f3._get_projections,+ "ff2": f2.Flux2FeedForward.forward,+ "ff3": f3.Flux3FeedForward.forward,+ "par": f2.Flux2ParallelSelfAttnProcessor.__call__,+ "mode": f3.Flux3ModeTransformerBlock.forward,+ "single": f3.Flux3SingleTransformerBlock.forward,+ }+ done = fused._install_patches() # noqa: SLF001+ assert all(done.values()), done+ assert fused.is_installed()+ assert fused._install_patches() == done # noqa: SLF001 - idempotent+ fused.uninstall()+ after = {+ "p2": f2._get_projections,+ "p3": f3._get_projections,+ "ff2": f2.Flux2FeedForward.forward,+ "ff3": f3.Flux3FeedForward.forward,+ "par": f2.Flux2ParallelSelfAttnProcessor.__call__,+ "mode": f3.Flux3ModeTransformerBlock.forward,+ "single": f3.Flux3SingleTransformerBlock.forward,+ }+ assert after == before+ assert not fused.is_installed()+++def test_kill_switch(env, monkeypatch):+ torch, f2, _f3, fused, _gemm = env+ monkeypatch.setenv(fused.FLUX_CONVROT_FUSED_ENV, "0")++ class Flux2Transformer2DModel:+ pass++ assert fused.install(Flux2Transformer2DModel())["patches"] == {}+ assert not fused.is_installed()+++def test_install_needs_rotated_linears(env):+ torch, _f2, _f3, fused, _gemm = env++ class Flux2Transformer2DModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ out = fused.install(Flux2Transformer2DModel())+ assert out["rotated"] == 0 and out["patches"] == {} and not fused.is_installed()+++# ------------------------------------------------------------------------------------------------ rotq_h16x kernel+def test_prefix_layout_reads_any_attention_output_stride():+ torch = pytest.importorskip("torch")+ from core.inference import diffusion_convrot_quant as cq++ heads_major = torch.zeros(2, 24, 41, 128).transpose(+ 1, 2+ ) # SDPA's [B, H, S, D] storage seen as [B, S, H, D]+ assert cq._prefix_layout(heads_major) == (2, 41, 3072, 24 * 41 * 128, 128, 41 * 128, 128)+ seq_major = torch.zeros(1, 41, 24, 128)+ assert cq._prefix_layout(seq_major) == (1, 41, 3072, 41 * 3072, 3072, 128, 128)+ flat = torch.zeros(41, 3072)+ assert cq._prefix_layout(flat) == (1, 41, 3072, 0, 3072, 256, 256)+ assert cq._prefix_layout(torch.zeros(41, 3072).t()) is None # last stride must be 1+ assert (+ cq._prefix_layout(torch.zeros(1, 41, 24, 96)) is None+ ) # head width the kernel does not index+ assert not cq.producer_layout_ok(+ None, torch.zeros(8, 256, dtype = torch.bfloat16), torch.zeros(8, 256, dtype = torch.bfloat16)+ )+++def test_launch_shape_tables():+ pytest.importorskip("torch")+ from core.inference import diffusion_convrot_quant as cq++ assert cq._h16x_pad((8, 0), 48, True) and not cq._h16x_pad((8, 0), 36, False)+ assert not cq._h16x_pad((8, 0), 16, True) # a power of two is already one exact pass+ assert not cq._h16x_pad((8, 9), 48, True) and not cq._h16x_pad(None, 48, True)+ assert (+ cq._h16x_warps((8, 0), True, False, 16) == 8+ and cq._h16x_warps((8, 0), False, False, 16) == 4+ )+ assert cq._h16x_warps((8, 0), True, True, 64) == 16+ assert cq._h16x_warps((12, 0), True, False, 16) == cq._H16_WARPS_TWO_PASS+++def _cuda_ready():+ try:+ import torch++ if not torch.cuda.is_available() or torch.cuda.get_device_capability(0) < (8, 0):+ return False+ if getattr(torch.version, "hip", None):+ return False+ import torchao # noqa: F401+ import triton # noqa: F401+ except Exception: # noqa: BLE001+ return False+ return True+++needs_cuda = pytest.mark.skipif(+ not _cuda_ready(), reason = "needs NVIDIA sm80+ CUDA, Triton and torchao"+)+++def producer_armed(monkeypatch):+ import torch++ from core.inference import diffusion_convrot_quant as cq++ monkeypatch.delenv(cq.INT8_ROTQUANT_PRODUCER_ENV, raising = False)+ index = torch.cuda.current_device()+ cq.rotquant_device_config(index)+ if not cq._H16_DEVICE.get(index):+ cq.unlisted_device_ok(index)+ if not cq._arm_producer(index):+ pytest.skip("rotq_h16 / rotq_h16x probes refused this device")+ return cq+++def _producer(torch, m, pre, f, seed):+ from core.inference import diffusion_convrot_quant as cq+ g = torch.Generator(device = "cpu").manual_seed(seed)+ return cq._producer_probe_inputs(+ m, pre, f, g, torch.device("cuda", torch.cuda.current_device())+ )+++@needs_cuda+ "m, pre, f",+ [+ (300, None, 256 * 36), # FLUX MLP down projection (two-pass, looped)+ (77, None, 256 * 16), # one pass+ (2 * 300, (2, 24, 128), 256 * 36), # FLUX.2 single-block output projection, batch 2+ (129, (1, 4, 128), 256 * 12),+ ],+)[email protected]("kind", ["v1", "v2"])+def test_producer_kernel_is_bit_identical_to_the_current_path(producer_armed, m, pre, f, kind):+ """rotq_h16x == the bf16 producer output (ATen SiLU product, concat) fed to the rotq op the current path runs."""+ import torch++ cq = producer_armed+ prefix, gate, value = _producer(torch, m, pre, f, m + f)+ q, s = cq.convrot_producer_act_quant(prefix, gate, value, 256, kind == "v2")+ rq, rs = cq._rotq_run(+ cq.producer_reference(prefix, gate, value).contiguous(), 256, kind == "v2"+ )+ assert torch.equal(q, rq) and torch.equal(s, rs)+ # every launch shape gives the same codes: the rotation and the rounding are per group / per row+ for kw in ({"pad": True}, {"pad": False, "warps": 8}, {"pad": False, "tile_rows": 32}):+ q2, s2 = cq._h16x_launch(prefix, gate, value, kind, **kw)+ assert torch.equal(q, q2) and torch.equal(s, s2), kw+++@needs_cuda+def test_producer_declines_what_it_cannot_read(producer_armed):+ import torch++ cq = producer_armed+ prefix, gate, value = _producer(torch, 64, (1, 2, 128), 512, 1)+ assert cq.convrot_producer_act_quant(prefix, gate, value, 128, True) is None # other group+ assert (+ cq.convrot_producer_act_quant(prefix[:, :32], gate, value, 256, True) is None+ ) # rows disagree+ assert (+ cq.convrot_producer_act_quant(None, gate[:, :200], value[:, :200], 256, True) is None+ ) # off-grid width+ assert cq.convrot_producer_act_quant(None, gate.float(), value.float(), 256, True) is None+++@needs_cuda+def test_producer_kill_switch(producer_armed, monkeypatch):+ import torch++ cq = producer_armed+ index = torch.cuda.current_device()+ monkeypatch.setenv(cq.INT8_ROTQUANT_PRODUCER_ENV, "0")+ assert cq._arm_producer(index) is False+ prefix, gate, value = _producer(torch, 64, None, 512, 2)+ assert cq.convrot_producer_act_quant(prefix, gate, value, 256, True) is None+ monkeypatch.delenv(cq.INT8_ROTQUANT_PRODUCER_ENV)+ assert cq._arm_producer(index) is True # the probe verdict was kept: no second probe+ assert cq.convrot_producer_act_quant(prefix, gate, value, 256, True) is not None+++def test_swapped_record_is_seen_inside_a_compiled_block():+ """The shared / producer paths read ``_swapped_rec`` inside the regional compile. Dynamo resolves+ ``forward.__func__ is _linear_forward`` to False, so the traced check compares the instance forward with the bound+ method the swap stored: a swapped Linear is seen, a forward replaced after the swap is not."""+ import types++ torch = pytest.importorskip("torch")+ from core.inference import diffusion_int8_gemm as g8++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, x):+ return x + 1 if g8._swapped_rec(self.proj) is not None else x - 1++ def swapped():+ block = Block()+ lin = block.proj+ lin.__dict__[g8._REC] = ("v2", 256, True, lin.weight, True)+ lin.__dict__[g8._MARK] = g8._NO_PREV+ lin.forward = types.MethodType(g8._linear_forward, lin)+ lin.__dict__[g8._FWD] = lin.forward+ return block++ x = torch.zeros(1)+ torch._dynamo.reset()+ assert torch.compile(swapped(), backend = "eager", fullgraph = True)(x).item() == 1+ replaced = swapped()+ replaced.proj.forward = types.MethodType(lambda self, t: t, replaced.proj)+ torch._dynamo.reset()+ assert torch.compile(replaced, backend = "eager", fullgraph = True)(x).item() == -1+ unmarked = swapped()+ del unmarked.proj.__dict__[g8._FWD]+ torch._dynamo.reset()+ assert torch.compile(unmarked, backend = "eager", fullgraph = True)(x).item() == -1+++def test_probe_verdicts_round_trip_through_the_cache(monkeypatch, tmp_path):+ """A passing device's probe verdicts persist per card + toolchain fingerprint; a later process adopts them without+ probing, and a changed fingerprint (driver, torch, switches, these sources) is a miss."""+ pytest.importorskip("torch")+ from core.inference import diffusion_convrot_quant as cq++ path = tmp_path / cq._VERDICT_FILE+ fp = {"torch": "x", "gpu_uuid": "GPU-1"}+ monkeypatch.setattr(cq, "_verdict_slot", lambda index: (path, f"k{index}", dict(fp)))+ for name in (+ "_ROTQ_DEVICE",+ "_ROTQ_DEVICE_K",+ "_H16_DEVICE",+ "_SWIGLU_DEVICE",+ "_PRODUCER_DEVICE",+ ):+ monkeypatch.setattr(cq, name, {})+ monkeypatch.setattr(cq, "_rotq_op", lambda: "rotq-op")+ monkeypatch.setattr(cq, "_swiglu_op", lambda: "swiglu-op")+ assert cq._verdict_load(7) is None+ cq._verdict_store(+ 7,+ {+ "cfg": [128, 32, 8, 3],+ "rules": [[256, 8192, [64, 32, 4, 3]]],+ "h16": "ieee",+ "swiglu": True,+ },+ )+ cq._verdict_store(7, {"producer": True})+ v = cq._verdict_load(7)+ assert v == {+ "cfg": [128, 32, 8, 3],+ "rules": [[256, 8192, [64, 32, 4, 3]]],+ "h16": "ieee",+ "swiglu": True,+ "producer": True,+ }+ assert cq._adopt_verdict(7, v) == (128, 32, 8, 3)+ assert cq._ROTQ_DEVICE_K[7] == ((256, 8192, (64, 32, 4, 3)),)+ assert (+ cq._H16_DEVICE[7] == "ieee"+ and cq._SWIGLU_DEVICE[7] is True+ and cq._PRODUCER_DEVICE[7] is True+ )+ fp["torch"] = "y" # another toolchain: a miss, never a stale verdict+ assert cq._verdict_load(7) is None+ path.write_text("{torn")+ assert cq._verdict_load(7) is Nonediff --git a/studio/backend/tests/test_diffusion_triton_resolution_args.py b/studio/backend/tests/test_diffusion_triton_resolution_args.pyindex 25418e5e860..da1755f655f 100644--- a/studio/backend/tests/test_diffusion_triton_resolution_args.py+++ b/studio/backend/tests/test_diffusion_triton_resolution_args.py@@ -13,7 +13,11 @@ INF = pathlib.Path(__file__).resolve().parents[1] / "core" / "inference"CASES = {"diffusion_int8_gemm.py": {"i8mm_dq": ("M",), "i8mm_dq_tma": ("M",)},- "diffusion_convrot_quant.py": {"rotq_i8": ("M",), "rotq_h16": ("M",)},+ "diffusion_convrot_quant.py": {+ "rotq_i8": ("M",),+ "rotq_h16": ("M",),+ "rotq_h16x": ("M", "SEQ"),+ },"diffusion_flux2_rope.py": {"_rope_fwd": ("S",)},"diffusion_rocm_fused.py": {"_adaln_fwd": ("S",)},"diffusion_moss_fused.py": {"kernel": ("S",)},