Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fuse the int8 ConvRot rotation into its producers on FLUX.2 / FLUX.3 (ConvRot cost +8.7% to +0.5%) (#17230)

Daniel Hanf4d17f4a59af
diff --git a/studio/backend/core/inference/diffusion_autoopt/patches.py b/studio/backend/core/inference/diffusion_autoopt/patches.py
index fa0b2efd004..1a6c57f6bda 100644
--- a/studio/backend/core/inference/diffusion_autoopt/patches.py
+++ b/studio/backend/core/inference/diffusion_autoopt/patches.py
@@ -504,6 +504,14 @@ def _builtin_specs() -> list[PatchSpec]:
else bool(ctx.denoiser_offloaded),
)
+ def flux_convrot_fused(ctx: PatchContext) -> Any:
+ from ..diffusion_flux_convrot_fused import install
+ return install(ctx.transformer, ctx.logger)
+
+ def flux_convrot_fused_uninstall() -> None:
+ from ..diffusion_flux_convrot_fused import uninstall
+ uninstall()
+
def moss_fused(ctx: PatchContext) -> Any:
from ..diffusion_moss_fused import install
return install(
@@ -650,6 +658,16 @@ def _builtin_specs() -> list[PatchSpec]:
warn = "qwen-image-2.1 dense gate/value GEMM",
env = "UNSLOTH_DIFFUSION_Q21_DENSE_FUSED",
),
+ PatchSpec(
+ "flux_convrot_fused",
+ STAGE_PRE_COMPILE,
+ flux_convrot_fused,
+ flux_convrot_fused_uninstall,
+ group = GROUP_FUSED_DIT,
+ classes = ("Flux2Transformer2DModel", "Flux3Transformer2DModel"),
+ warn = "flux convrot producer fusion",
+ env = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED",
+ ),
PatchSpec(
"moss_fused",
STAGE_PRE_COMPILE,
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
index d59b031272d..cd775fbbcf2 100644
--- a/studio/backend/core/inference/diffusion_convrot_quant.py
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -16,9 +16,11 @@ from .rocm_bf16 import is_rocm_torch
INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"
INT8_ROTQUANT_H16_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16"
INT8_ROTQUANT_SWIGLU_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_SWIGLU"
+INT8_ROTQUANT_PRODUCER_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER"
_OP_NAMESPACE = "unsloth_studio"
_ROTQ_OP_NAME = "convrot_act_quant_int8"
_SWIGLU_OP_NAME = "convrot_swiglu_act_quant_int8"
+_PRODUCER_OP_NAME = "convrot_producer_act_quant_int8"
# (BLOCK_M group rows, BLOCK_K, num_warps, num_stages) per (major, minor); measured end to end. Absent = stock.
_ROTQ_CONFIG = {
@@ -52,6 +54,13 @@ _SWIGLU_DEVICE: dict = {}
_H16_CALLS = [0]
_SWIGLU_CALLS = [0]
_SWIGLU_HANDLE: Any = None
+# device index -> rotq_h16x (producer loads fused in front of the rotation) passed its probe
+_PRODUCER_DEVICE: dict = {}
+_PRODUCER_CALLS = [0]
+_PRODUCER_HANDLE: Any = None
+_PRODUCER_VERDICT_OFF: set = (
+ set()
+) # devices whose last load had the switch off (the probe verdict is kept)
# 16 group rows per program: fastest (within 2%) on L4; 128 rows overflow 100 KB smem. One masked row per program at
# K 12288 was 1.2x slower on A100 and spilled on B200.
_H16_TILE_ROWS = 16
@@ -88,6 +97,11 @@ def rotquant_swiglu_enabled() -> bool:
return _env_mode(INT8_ROTQUANT_SWIGLU_ENV) != "off"
+def rotquant_producer_enabled() -> bool:
+ """False only under ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER=0`` (the producer writes bf16, ``rotq_h16`` reads it)."""
+ return _env_mode(INT8_ROTQUANT_PRODUCER_ENV) != "off"
+
+
def rotquant_mode() -> str:
"""'off', 'force' or 'auto'. The int8 GEMM switch is the master: its ``0`` turns the whole swap off and its ``1``
forces this lever too."""
@@ -407,7 +421,407 @@ def _kernels() -> Optional[types.SimpleNamespace]:
srow = pid.to(tl.int64) * ROWS + tl.arange(0, ROWS)
tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = srow < M)
- return types.SimpleNamespace(rotq_i8 = rotq_i8, rotq_h16 = rotq_h16, triton = triton)
+ @triton.jit
+ def _h16x_load(
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ grp,
+ live,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ CAT: tl.constexpr, # groups below NA come from the attention prefix
+ HD: tl.constexpr, # prefix head width
+ ):
+ # bf16 [TM, 256] of the activation the producer would have written: silu(gate) * value, after the prefix
+ offs = tl.arange(0, 256)
+ if CAT:
+ pre = live & (grp < NA)
+ col = grp[:, None] * 256 + offs[None, :]
+ b = row // SEQ
+ s = row - b * SEQ
+ a_off = b[:, None] * sab + s[:, None] * sas + (col // HD) * sah + (col % HD)
+ xa = tl.load(a_ptr + a_off, mask = pre[:, None], other = 0.0)
+ mlp = live & (grp >= NA)
+ mcol = tl.where(mlp, grp - NA, 0)[:, None] * 256 + offs[None, :]
+ g = tl.load(g_ptr + row[:, None] * sg + mcol, mask = mlp[:, None], other = 0.0)
+ v = tl.load(v_ptr + row[:, None] * sv + mcol, mask = mlp[:, None], other = 0.0)
+ x = tl.where(pre[:, None], xa, _swiglu_bf16(g, v))
+ else:
+ col = grp[:, None] * 256 + offs[None, :]
+ g = tl.load(g_ptr + row[:, None] * sg + col, mask = live[:, None], other = 0.0)
+ v = tl.load(v_ptr + row[:, None] * sv + col, mask = live[:, None], other = 0.0)
+ x = _swiglu_bf16(g, v)
+ return x
+
+ @triton.jit
+ def _h16x_rot(
+ x,
+ h,
+ h32,
+ TM: tl.constexpr, # 256-wide groups per tile
+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy
+ ):
+ # _h16_tile's rotation, op for op
+ y = tl.dot(tl.reshape(x, (TM * 16, 16)), h, out_dtype = tl.float32)
+ y = tl.reshape(tl.permute(tl.reshape(y, (TM, 16, 16)), (0, 2, 1)), (TM * 16, 16))
+ if TF32X3:
+ z = tl.dot(y, h32, input_precision = "tf32x3", out_dtype = tl.float32)
+ else:
+ z = tl.dot(y, h32, input_precision = "ieee", out_dtype = tl.float32)
+ z = tl.reshape(tl.permute(tl.reshape(z, (TM, 16, 16)), (0, 2, 1)), (TM, 256))
+ return _rbf16(z)
+
+ @triton.jit
+ def _h16x_chunk(
+ c,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG: tl.constexpr, # groups per chunk
+ ROWS: tl.constexpr, # activation rows per program
+ CAT: tl.constexpr,
+ HD: tl.constexpr,
+ TF32X3: tl.constexpr,
+ ):
+ grp = c * CG + gl
+ live = (row < M) & (grp < NG) # padded one-pass tiles mask the groups past the row
+ r = tl.where(live, row, 0)
+ x = _h16x_load(g_ptr, v_ptr, a_ptr, r, grp, live, sg, sv, NA, SEQ, sab, sas, sah, CAT, HD)
+ return _h16x_rot(x, h, h32, ROWS * CG, TF32X3)
+
+ @triton.jit
+ def _h16x_amax(
+ c,
+ amax,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG: tl.constexpr,
+ ROWS: tl.constexpr,
+ CAT: tl.constexpr,
+ HD: tl.constexpr,
+ TF32X3: tl.constexpr,
+ ):
+ z = _h16x_chunk(
+ c,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ )
+ m = tl.where(row < M, tl.max(tl.abs(z), axis = 1), 0.0)
+ return tl.maximum(amax, tl.max(tl.reshape(m, (ROWS, CG)), axis = 1))
+
+ @triton.jit
+ def _h16x_quant(
+ c,
+ inv_t,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ q_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG: tl.constexpr,
+ ROWS: tl.constexpr,
+ CAT: tl.constexpr,
+ HD: tl.constexpr,
+ TF32X3: tl.constexpr,
+ QMIN: tl.constexpr,
+ QMAX: tl.constexpr,
+ FP32_SCALE: tl.constexpr,
+ ):
+ z = _h16x_chunk(
+ c,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ )
+ live = (row < M) & (c * CG + gl < NG)
+ gidx = tl.where(live, row * NG + c * CG + gl, 0)
+ _h16_quant_store(z, inv_t, q_ptr, gidx, live, QMIN, QMAX, FP32_SCALE)
+
+ @triton.jit(do_not_specialize = ["M", "SEQ"])
+ def rotq_h16x(
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ h_ptr,
+ h32_ptr,
+ q_ptr,
+ s_ptr,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ NCH: tl.constexpr, # chunks per row
+ CG: tl.constexpr, # groups per chunk
+ ROWS: tl.constexpr, # activation rows per program
+ ONE_PASS: tl.constexpr, # whole row fits one chunk
+ UNROLL: tl.constexpr, # unroll the chunk loop
+ CAT: tl.constexpr, # attention prefix before the SwiGLU product (one row = [prefix | product])
+ HD: tl.constexpr, # prefix head width
+ TF32X3: tl.constexpr,
+ QMIN: tl.constexpr,
+ QMAX: tl.constexpr,
+ DIV: tl.constexpr,
+ EPS: tl.constexpr,
+ FP32_SCALE: tl.constexpr,
+ ):
+ # rotq_h16 with the producer's loads in front: strided SwiGLU halves (and an attention prefix in any layout),
+ # so the bf16 activation never reaches HBM. Same rotation, amax and rounding as rotq_h16.
+ TM: tl.constexpr = ROWS * CG
+ pid = tl.program_id(0)
+ t = tl.arange(0, TM)
+ row = pid.to(tl.int64) * ROWS + t // CG
+ gl = t % CG
+ o16 = tl.arange(0, 16)
+ h = tl.load(h_ptr + o16[:, None] * 16 + o16[None, :])
+ h32 = tl.load(h32_ptr + o16[:, None] * 16 + o16[None, :])
+ if ONE_PASS:
+ z = _h16x_chunk(
+ 0,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ )
+ m = tl.where(row < M, tl.max(tl.abs(z), axis = 1), 0.0)
+ amax = tl.max(tl.reshape(m, (ROWS, CG)), axis = 1)
+ else:
+ amax = tl.zeros((ROWS,), dtype = tl.float32)
+ if UNROLL:
+ for c in tl.static_range(NCH):
+ amax = _h16x_amax(
+ c,
+ amax,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ )
+ else:
+ for c in range(0, NCH):
+ amax = _h16x_amax(
+ c,
+ amax,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ )
+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))
+ inv = libdevice.div_rn(tl.full((ROWS,), 1.0, tl.float32), s)
+ if not FP32_SCALE:
+ inv = _rbf16(inv)
+ inv_t = tl.reshape(tl.broadcast_to(inv[:, None], (ROWS, CG)), (TM,))
+ if ONE_PASS:
+ live = (row < M) & (gl < NG)
+ gidx = tl.where(live, row * NG + gl, 0)
+ _h16_quant_store(z, inv_t, q_ptr, gidx, live, QMIN, QMAX, FP32_SCALE)
+ elif UNROLL:
+ for c in tl.static_range(NCH):
+ _h16x_quant(
+ c,
+ inv_t,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ q_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ QMIN,
+ QMAX,
+ FP32_SCALE,
+ )
+ else:
+ for c in range(0, NCH):
+ _h16x_quant(
+ c,
+ inv_t,
+ g_ptr,
+ v_ptr,
+ a_ptr,
+ q_ptr,
+ row,
+ gl,
+ M,
+ NG,
+ sg,
+ sv,
+ NA,
+ SEQ,
+ sab,
+ sas,
+ sah,
+ h,
+ h32,
+ CG,
+ ROWS,
+ CAT,
+ HD,
+ TF32X3,
+ QMIN,
+ QMAX,
+ FP32_SCALE,
+ )
+ srow = pid.to(tl.int64) * ROWS + tl.arange(0, ROWS)
+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = srow < M)
+
+ return types.SimpleNamespace(
+ rotq_i8 = rotq_i8, rotq_h16 = rotq_h16, rotq_h16x = rotq_h16x, triton = triton
+ )
@lru_cache(maxsize = 1)
@@ -603,6 +1017,169 @@ def _h16_launch(
return q, s
+# Per-(major, minor) warps and padded one-pass cap from a Colab sweep (A100: 557 vs 601 us two-pass at 4608 x 12288).
+_H16X_WARPS = {(8, 0): (8, 4), (8, 9): (4, 4)}
+_H16X_PAD_CAT = {(8, 0): 64}
+
+
+@lru_cache(maxsize = 16)
+def _device_cap(index: Any) -> Optional[tuple]:
+ try:
+ import torch
+ return tuple(torch.cuda.get_device_capability(index))
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _h16x_pad(cap: Optional[tuple], ng: int, cat: bool) -> bool:
+ limit = _H16X_PAD_CAT.get(cap, 0) if cat else 0
+ return 1 < ng <= limit and ng & (ng - 1) != 0
+
+
+def _h16x_warps(cap: Optional[tuple], cat: bool, one: bool, tm: int) -> int:
+ if one and tm > 16:
+ return 16 if tm >= 64 else 8
+ pair = _H16X_WARPS.get(cap)
+ if pair is None:
+ return _H16_WARPS if one else _H16_WARPS_TWO_PASS
+ return pair[0] if cat else pair[1]
+
+
+def _prefix_layout(prefix: Any) -> Optional[tuple]:
+ """(B, S, A, sab, sas, sah, HD) of an attention output ``[B, S, H, D]`` (any strides, unit last stride) or a 2D
+ ``[M, A]`` prefix, else None."""
+ if prefix.stride(-1) != 1:
+ return None
+ if prefix.dim() == 4:
+ b, s, heads, hd = prefix.shape
+ if hd not in (64, 128, 256):
+ return None
+ return b, s, heads * hd, prefix.stride(0), prefix.stride(1), prefix.stride(2), hd
+ if prefix.dim() == 2:
+ return 1, prefix.shape[0], prefix.shape[1], 0, prefix.stride(0), 256, 256
+ return None
+
+
+def producer_layout_ok(prefix: Any, gate: Any, value: Any) -> bool:
+ """``rotq_h16x`` reads this producer as is: bf16 CUDA, gate / value ``[M, F]`` with unit last stride, F and the
+ prefix width multiples of 256, the prefix's rows matching."""
+ import torch
+
+ if (
+ gate.dim() != 2
+ or gate.shape != value.shape
+ or gate.dtype != torch.bfloat16
+ or value.dtype != torch.bfloat16
+ or not gate.is_cuda
+ or gate.device != value.device
+ or gate.stride(-1) != 1
+ or value.stride(-1) != 1
+ or gate.shape[1] % 256
+ or gate.shape[1] < 256
+ ):
+ return False
+ if prefix is None:
+ return True
+ if prefix.dtype != torch.bfloat16 or prefix.device != gate.device:
+ return False
+ lay = _prefix_layout(prefix)
+ return (
+ lay is not None and lay[0] * lay[1] == gate.shape[0] and lay[2] % 256 == 0 and lay[2] >= 256
+ )
+
+
+def producer_reference(prefix: Any, gate: Any, value: Any) -> Any:
+ """The bf16 activation the stock producer writes: ``cat([prefix.flatten, silu(gate) * value], -1)``."""
+ import torch
+
+ prod = swiglu_reference(gate, value)
+ if prefix is None:
+ return prod
+ return torch.cat([prefix.reshape(gate.shape[0], -1), prod], dim = -1)
+
+
+def _h16x_launch(
+ prefix: Any,
+ gate: Any,
+ value: Any,
+ kind: str,
+ tile_rows: int = _H16_TILE_ROWS,
+ tf32x3: Optional[bool] = None,
+ warps: Optional[int] = None,
+ pad: Optional[bool] = None,
+) -> tuple:
+ """``rotq_h16x`` on a producer accepted by ``producer_layout_ok``: (int8 [M, K], scale [M]). Strides stay
+ specialized (multiple-of-16 hints keep the loads vectorized); only M and the sequence length are not."""
+ import torch
+ from .diffusion_convrot import build_convrot_hadamard
+
+ kern = _kernels()
+ m, f = gate.shape
+ if prefix is None:
+ a, seq, sab, sas, sah, hd, cat = 0, max(m, 1), 0, 0, 0, 256, False
+ else:
+ _b, seq, a, sab, sas, sah, hd = _prefix_layout(prefix)
+ cat = True
+ k = a + f
+ ng = k // 256
+ cap = _device_cap(gate.device.index)
+ if pad is None:
+ pad = _h16x_pad(cap, ng, cat)
+ if pad:
+ # one pass: the whole row in one masked power-of-two tile, so nothing is loaded or rotated twice
+ cg, rows, nch, one = 1 << (ng - 1).bit_length(), 1, 1, True
+ else:
+ cg, rows, nch, one = _h16_plan(ng, tile_rows)
+ if warps is None:
+ warps = _h16x_warps(cap, cat, one, cg * rows)
+ if tf32x3 is None:
+ tf32x3 = _H16_DEVICE.get(gate.device.index) == "tf32x3"
+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]
+ h = build_convrot_hadamard(16, device = gate.device, dtype = torch.bfloat16)
+ h32 = build_convrot_hadamard(16, device = gate.device, dtype = torch.float32)
+ fp32_scale = _rotq_scale_fp32(kind)
+ q = torch.empty((m, k), device = gate.device, dtype = torch.int8)
+ s = torch.empty((m,), device = gate.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)
+ if m == 0:
+ return q, s
+ with torch.cuda.device(gate.device):
+ kern.rotq_h16x[(kern.triton.cdiv(m, rows),)](
+ gate,
+ value,
+ gate if prefix is None else prefix,
+ h,
+ h32,
+ q,
+ s,
+ m,
+ ng,
+ gate.stride(0),
+ value.stride(0),
+ a // 256,
+ seq,
+ sab,
+ sas,
+ sah,
+ nch,
+ CG = cg,
+ ROWS = rows,
+ ONE_PASS = one,
+ UNROLL = nch <= _H16_UNROLL_MAX,
+ CAT = cat,
+ HD = hd,
+ TF32X3 = bool(tf32x3),
+ QMIN = qmin,
+ QMAX = qmax,
+ DIV = div,
+ EPS = eps,
+ FP32_SCALE = fp32_scale,
+ num_warps = warps,
+ num_stages = 1,
+ enable_fp_fusion = False,
+ )
+ return q, s
+
+
def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:
"""Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""
import torch
@@ -733,6 +1310,64 @@ def _swiglu_op() -> Any:
return getattr(getattr(torch.ops, _OP_NAMESPACE), _SWIGLU_OP_NAME)
+def _producer_run(prefix: Any, gate: Any, value: Any, group: int, v2: bool) -> tuple:
+ """Op body: ``rotq_h16x`` where it passed its probe, else the stock product (and concat), rotation and quant."""
+ _PRODUCER_CALLS[0] += 1
+ kind = "v2" if v2 else "v1"
+ index = gate.device.index
+ if group == 256 and _PRODUCER_DEVICE.get(index) and producer_layout_ok(prefix, gate, value):
+ try:
+ return _h16x_launch(prefix, gate, value, kind)
+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math
+ _PRODUCER_DEVICE[index] = False
+ return rotquant_reference(producer_reference(prefix, gate, value).contiguous(), group, kind)
+
+
+@lru_cache(maxsize = 1)
+def _producer_op() -> Any:
+ """``unsloth_studio::convrot_producer_act_quant_int8(prefix?, gate, value, group, v2)``: the rotated, quantized
+ ``cat([prefix, silu(gate) * value])`` read straight from the producer's buffers (opaque to Inductor), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _PRODUCER_OP_NAME):
+ return getattr(ns, _PRODUCER_OP_NAME)
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return None
+ try:
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_PRODUCER_OP_NAME}",
+ mutates_args = (),
+ schema = "(Tensor? prefix, Tensor gate, Tensor value, int group, bool v2) -> (Tensor, Tensor)",
+ )
+ def _convrot_producer_act_quant_int8(prefix, gate, value, group, v2):
+ return _producer_run(prefix, gate, value, group, v2)
+
+ @_convrot_producer_act_quant_int8.register_fake
+ def _(prefix, gate, value, group, v2):
+ k = gate.shape[1]
+ if prefix is not None:
+ k = k + (
+ prefix.shape[-2] * prefix.shape[-1] if prefix.dim() == 4 else prefix.shape[-1]
+ )
+ return (
+ gate.new_empty((gate.shape[0], k), dtype = torch.int8),
+ gate.new_empty(
+ (gate.shape[0],),
+ dtype = torch.float32
+ if _rotq_scale_fp32("v2" if v2 else "v1")
+ else torch.bfloat16,
+ ),
+ )
+ except Exception: # noqa: BLE001
+ return None
+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _PRODUCER_OP_NAME)
+
+
@lru_cache(maxsize = 1)
def _rotq_op() -> Any:
"""``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""
@@ -782,6 +1417,9 @@ def rotquant_device_config(index: int) -> Optional[tuple]:
and _kernels() is not None
and _rotq_op() is not None
):
+ hit = _verdict_load(index)
+ if hit is not None:
+ return _adopt_verdict(index, hit)
cap = torch.cuda.get_device_capability(index)
want = rotquant_config(cap)
for cand in (want, _ROTQ_FALLBACK) if want is not None else ():
@@ -818,6 +1456,132 @@ def rotquant_device_config(index: int) -> Optional[tuple]:
_ROTQ_HANDLE = _rotq_op()
if sw:
_SWIGLU_HANDLE = _swiglu_op()
+ _verdict_store(
+ index,
+ {
+ "cfg": list(cfg),
+ "rules": [[lo, hi, list(t)] for lo, hi, t in rules],
+ "h16": h16,
+ "swiglu": bool(sw),
+ },
+ )
+ return cfg
+
+
+# Probes cost ~50 s per process on an L4, so passing verdicts persist per card + toolchain fingerprint (+ this file and
+# diffusion_convrot.py); failures are never stored (may be transient). UNSLOTH_DIFFUSION_PROBE_CACHE=0 disables.
+_VERDICT_FILE = "diffusion_convrot_probe.json"
+_VERDICT_SOURCES = ("diffusion_convrot_quant.py", "diffusion_convrot.py")
+_VERDICT_MAX = 32
+
+
+@lru_cache(maxsize = 1)
+def _verdict_sources() -> str:
+ import hashlib
+ from pathlib import Path
+
+ h = hashlib.sha256()
+ here = Path(__file__).resolve().parent
+ for name in _VERDICT_SOURCES:
+ try:
+ h.update(name.encode())
+ h.update((here / name).read_bytes())
+ except OSError:
+ h.update(b"<missing>")
+ return h.hexdigest()
+
+
+def _verdict_slot(index: int) -> Optional[tuple]:
+ """(cache file, key, fingerprint) for card ``index``, or None (cache off / unidentifiable card)."""
+ try:
+ import hashlib
+ import json
+
+ from . import diffusion_probe_cache as pc
+
+ if not pc.enabled():
+ return None
+ path = pc._cache_file()
+ fp = pc.fingerprint(f"cuda:{index}")
+ if path is None or fp is None:
+ return None
+ fp = dict(fp, convrot_source = _verdict_sources())
+ key = hashlib.sha256(json.dumps(fp, sort_keys = True).encode()).hexdigest()[:32]
+ return path.with_name(_VERDICT_FILE), key, fp
+ except Exception: # noqa: BLE001 - no cache: probe
+ return None
+
+
+def _verdict_read(path: Any) -> dict:
+ import json
+ try:
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ except Exception: # noqa: BLE001 - absent / torn reads as empty
+ return {}
+ return data if isinstance(data, dict) else {}
+
+
+def _verdict_load(index: int) -> Optional[dict]:
+ slot = _verdict_slot(index)
+ if slot is None:
+ return None
+ path, key, fp = slot
+ entry = _verdict_read(path).get(key)
+ if not isinstance(entry, dict) or entry.get("fingerprint") != fp:
+ return None
+ v = entry.get("verdicts")
+ if not isinstance(v, dict) or not isinstance(v.get("cfg"), list) or len(v["cfg"]) != 4:
+ return None
+ return v
+
+
+def _verdict_store(index: int, update: dict) -> None:
+ """Merge ``update`` into card ``index``'s entry (best effort, atomic replace)."""
+ slot = _verdict_slot(index)
+ if slot is None:
+ return
+ path, key, fp = slot
+ import json
+ import time
+
+ try:
+ data = _verdict_read(path)
+ entry = data.get(key) if isinstance(data.get(key), dict) else {}
+ verdicts = dict(entry.get("verdicts") or {}) if entry.get("fingerprint") == fp else {}
+ verdicts.update(update)
+ data[key] = {"fingerprint": fp, "verdicts": verdicts, "t": time.time()}
+ if len(data) > _VERDICT_MAX:
+ for k in sorted(data, key = lambda k: (data[k] or {}).get("t", 0))[
+ : len(data) - _VERDICT_MAX
+ ]:
+ data.pop(k, None)
+ path.parent.mkdir(parents = True, exist_ok = True)
+ tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+ tmp.write_text(json.dumps(data, sort_keys = True), encoding = "utf-8")
+ os.replace(tmp, path)
+ except Exception: # noqa: BLE001 - a cache write never fails a load
+ pass
+
+
+def _adopt_verdict(index: int, v: dict) -> Optional[tuple]:
+ """Install a persisted verdict as ``rotquant_device_config`` would after probing (the switches are part of the
+ key, so the stored verdict already reflects them)."""
+ global _ROTQ_HANDLE, _SWIGLU_HANDLE
+ cfg = tuple(int(x) for x in v["cfg"])
+ rules = tuple(
+ (int(lo), int(hi), tuple(int(x) for x in t)) for lo, hi, t in v.get("rules") or ()
+ )
+ h16 = v.get("h16") if v.get("h16") in ("ieee", "tf32x3") else False
+ sw = bool(v.get("swiglu")) and bool(h16) and _swiglu_op() is not None
+ _ROTQ_DEVICE_K[index] = rules
+ _ROTQ_DEVICE[index] = cfg
+ _H16_DEVICE[index] = h16
+ _SWIGLU_DEVICE[index] = sw
+ if isinstance(v.get("producer"), bool) and h16:
+ _PRODUCER_DEVICE[index] = v["producer"]
+ _ROTQ_HANDLE = _rotq_op()
+ if sw:
+ _SWIGLU_HANDLE = _swiglu_op()
return cfg
@@ -951,6 +1715,109 @@ def _swiglu_probe(index: int) -> bool:
return False
+# (M, prefix (B, heads, head dim) or None, SwiGLU width)
+_PRODUCER_PROBE_SHAPES = (
+ (65, None, 256 * 4),
+ (130, None, 256 * 36),
+ (33, (1, 2, 128), 256 * 3),
+ (2 * 41, (2, 24, 128), 256 * 36),
+)
+
+
+def _producer_probe_inputs(m: int, pre: Optional[tuple], f: int, g: Any, dev: Any) -> tuple:
+ import torch
+
+ both = torch.cat(
+ [(_probe_input(m, f, g).float() * 0.05).to(torch.bfloat16), _probe_input(m, f, g)], dim = 1
+ ).to(dev)
+ gate, value = both[:, :f], both[:, f:] # halves of one projection output: row stride 2F
+ gate[0, :7] = torch.tensor([-90.0, -20.0, -1e-3, 0.0, 1e-3, 20.0, 90.0], device = dev)
+ prefix = None
+ if pre is not None:
+ b, heads, hd = pre
+ seq = m // b
+ # [B, H, S, D] storage viewed as [B, S, H, D]: what SDPA hands back
+ prefix = (
+ _probe_input(b * heads * seq, hd, g).to(dev).view(b, heads, seq, hd).transpose(1, 2)
+ )
+ return prefix, gate, value
+
+
+def _producer_probe(index: int) -> bool:
+ """``rotq_h16x`` vs the stock producer (``producer_reference``: ATen SiLU product, concat) then the stock rotation +
+ quant: ``codes_close`` on every probe shape, v1 and v2."""
+ import torch
+
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(5)
+ try:
+ for m, pre, f in _PRODUCER_PROBE_SHAPES:
+ prefix, gate, value = _producer_probe_inputs(m, pre, f, g, dev)
+ if not producer_layout_ok(prefix, gate, value):
+ return False
+ ref = producer_reference(prefix, gate, value).contiguous()
+ for kind in ("v1", "v2"):
+ rq, rs = rotquant_reference(ref, 256, kind)
+ q, s = _h16x_launch(prefix, gate, value, kind)
+ if not codes_close(q, s, rq, rs):
+ return False
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _arm_producer(index: int) -> bool:
+ """Per load, outside any trace, after ``rotq_h16`` was armed on the device: the producer variant's probe (once per
+ device); publishes its op handle. The switch is read here, so it acts without a restart."""
+ global _PRODUCER_HANDLE
+ if not rotquant_producer_enabled() or not _H16_DEVICE.get(index):
+ _PRODUCER_VERDICT_OFF.add(index)
+ return False
+ _PRODUCER_VERDICT_OFF.discard(index)
+ if index not in _PRODUCER_DEVICE:
+ ok = False
+ try:
+ ok = _producer_op() is not None and _producer_probe(index)
+ except Exception: # noqa: BLE001 - the stock producer stays
+ ok = False
+ _PRODUCER_DEVICE[index] = ok
+ if ok and _ROTQ_DEVICE.get(index) is not None:
+ _verdict_store(index, {"producer": True})
+ if _PRODUCER_DEVICE[index]:
+ _PRODUCER_HANDLE = _producer_op()
+ return bool(_PRODUCER_DEVICE[index])
+
+
+def producer_armed(index: Any) -> bool:
+ """``rotq_h16x`` serves card ``index`` (its probe passed and the switch is on)."""
+ return (
+ _PRODUCER_HANDLE is not None
+ and index not in _PRODUCER_VERDICT_OFF
+ and bool(_PRODUCER_DEVICE.get(index))
+ )
+
+
+def convrot_producer_act_quant(
+ prefix: Any, gate: Any, value: Any, group: int, v2: bool
+) -> Optional[tuple]:
+ """``convrot_act_quant`` of ``cat([prefix, silu(gate) * value], -1)`` read from the producer's own buffers (the
+ projection output halves and the attention output in its native layout), so neither the product nor the concat
+ reaches HBM. None when this device / layout / group is not covered: the caller keeps the stock producer and its
+ own rotation + quant. Safe inside a traced forward: no host sync, the probe ran at install time."""
+ if group != 256 or _PRODUCER_HANDLE is None or not producer_armed(gate.device.index):
+ return None
+ if not producer_layout_ok(prefix, gate, value):
+ return None
+ return _PRODUCER_HANDLE(prefix, gate, value, group, v2)
+
+
+def producer_call_count() -> int:
+ return _PRODUCER_CALLS[0]
+
+
def rotquant_call_count() -> int:
return _ROTQ_CALLS[0]
diff --git a/studio/backend/core/inference/diffusion_flux_convrot_fused.py b/studio/backend/core/inference/diffusion_flux_convrot_fused.py
new file mode 100644
index 00000000000..f20a45b2d31
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_flux_convrot_fused.py
@@ -0,0 +1,574 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.2 / FLUX.3 int8 ConvRot without the separate rotate + quantize pass over each bf16 activation.
+
+On the int8 GEMM swap (``diffusion_int8_gemm``) every rotated Linear quantizes its own input with ``rotq_h16``: the
+producer writes bf16, the kernel reads it back. Here:
+ * Linears reading one activation (Q / K / V, and on FLUX.3 also the MLP input) share one rotation + act quant
+ (``shared_rotq_linears``: the same op on the same tensor, so bit-identical);
+ * the MLP down projection and FLUX.2's single-block output projection read the SwiGLU halves (and the attention
+ output) straight from the producers' buffers (``rotq_h16x``), so neither the product nor the concat is written.
+Every patch checks its Linears per call and runs the stock code when they are not swapped rotated int8 Linears.
+Kill switch ``UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED=0``; ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_PRODUCER=0`` keeps only the
+shared quants."""
+
+from __future__ import annotations
+
+import functools
+import os
+import threading
+from typing import Any, Optional
+
+import torch
+
+from . import diffusion_int8_gemm as _gemm
+
+FLUX_CONVROT_FUSED_ENV = "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"
+_MARK = "__unsloth_flux_convrot_fused__"
+_LOCK = threading.Lock()
+_STOCK: dict = {} # (owner, attr) -> original
+_CALLS = {"shared": [0], "producer": [0], "block": [0]}
+
+# Stock lines each replacement mirrors; a diffusers whose body lacks any keeps the stock function.
+_NEEDLES = {
+ "f2_projections": (
+ "query = attn.to_q(hidden_states)",
+ "key = attn.to_k(hidden_states)",
+ "value = attn.to_v(hidden_states)",
+ "encoder_query = attn.add_q_proj(encoder_hidden_states)",
+ "encoder_key = attn.add_k_proj(encoder_hidden_states)",
+ "encoder_value = attn.add_v_proj(encoder_hidden_states)",
+ ),
+ "f3_projections": (
+ "return attn.to_q(hidden_states), attn.to_k(hidden_states), attn.to_v(hidden_states)",
+ ),
+ "ff": ("x = self.linear_in(x)", "x = self.act_fn(x)", "x = self.linear_out(x)"),
+ "swiglu": ("half = x.shape[-1] // 2", "x = self.gate_fn(x[..., :half]) * x[..., half:]"),
+ "f2_parallel": (
+ "hidden_states = attn.to_qkv_mlp_proj(hidden_states)",
+ "query, key, value = qkv.chunk(3, dim=-1)",
+ "query = attn.norm_q(query)",
+ "key = attn.norm_k(key)",
+ "query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1)",
+ "key = apply_rotary_emb(key, image_rotary_emb, sequence_dim=1)",
+ "attn_mask=attention_mask,",
+ "backend=self._attention_backend,",
+ "parallel_config=self._parallel_config,",
+ "hidden_states = hidden_states.flatten(2, 3)",
+ "hidden_states = hidden_states.to(query.dtype)",
+ "mlp_hidden_states = attn.mlp_act_fn(mlp_hidden_states)",
+ "hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim=-1)",
+ "hidden_states = attn.to_out(hidden_states)",
+ ),
+ "f3_attn": (
+ "query, key, value = _get_qkv_projections(attn, hidden_states)",
+ "query = query.unflatten(-1, (attn.heads, -1))",
+ "query = attn.norm_q(query.float())",
+ "key = attn.norm_k(key.float())",
+ "query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1)",
+ "query, key = query.to(value.dtype), key.to(value.dtype)",
+ "attn_mask=attention_mask,",
+ "hidden_states = hidden_states.flatten(2, 3)",
+ "hidden_states = hidden_states.to(query.dtype)",
+ "hidden_states = attn.to_out[0](hidden_states)",
+ "hidden_states = attn.to_out[1](hidden_states)",
+ ),
+ "f3_attn_forward": (
+ "return self.processor(self, hidden_states, attention_mask, image_rotary_emb)",
+ ),
+ "f3_mode": (
+ "mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim=-1)",
+ "norm_hidden_states = self.norm(hidden_states)",
+ "norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift",
+ "attn_output = self.attn(hidden_states=norm_hidden_states, image_rotary_emb=image_rotary_emb)",
+ "ff_output = self.ff(norm_hidden_states)",
+ "return hidden_states + mod_gate * (attn_output + ff_output)",
+ ),
+ "f3_single": (
+ "mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim=-1)",
+ "c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim=-1)",
+ "norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift",
+ "norm_encoder_hidden_states = self.norm(encoder_hidden_states)",
+ "norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift",
+ "combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim=1)",
+ "attn_output = self.attn(hidden_states=combined, image_rotary_emb=image_rotary_emb)",
+ "ff_output = self.ff(combined)",
+ "output = attn_output + ff_output",
+ "encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output",
+ "hidden_states = hidden_states + mod_gate * content_output",
+ ),
+}
+
+
+def flux_convrot_fused_disabled() -> bool:
+ return (os.environ.get(FLUX_CONVROT_FUSED_ENV) or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _flux_modules() -> tuple:
+ try:
+ from diffusers.models.transformers import transformer_flux2 as f2
+ except Exception: # noqa: BLE001 - diffusers without this family
+ f2 = None
+ try:
+ from diffusers.models.transformers import transformer_flux3 as f3
+ except Exception: # noqa: BLE001
+ f3 = None
+ return f2, f3
+
+
+def _count(kind: str) -> None:
+ """Engagement census, eager calls only (a traced forward counts nothing: no side effect in the graph)."""
+ if not torch.compiler.is_compiling():
+ _CALLS[kind][0] += 1
+
+
+def _stock(owner: Any, attr: str) -> Any:
+ """The original ``owner.attr`` (ours stashed it), else the live one."""
+ return _STOCK.get((owner, attr), getattr(owner, attr, None))
+
+
+def _has(fn: Any, key: str) -> bool:
+ from .diffusion_autoopt.patches import body_has
+ return fn is not None and body_has(fn, *_NEEDLES[key])
+
+
+def _rotated(lin: Any) -> bool:
+ """``lin`` is a swapped rotated int8 Linear whose act quant the fused rotation covers."""
+ rec = _gemm._swapped_rec(lin)
+ return rec is not None and bool(rec[2]) and rec[1] == 256
+
+
+def _plain_swiglu(act: Any, cls_name: str) -> bool:
+ """``act`` is the stock FLUX SwiGLU (``silu(x[:half]) * x[half:]``) with an unhooked ``nn.SiLU``."""
+ return (
+ act is not None
+ and type(act) is _CLS.get(cls_name)
+ and not _gemm._hooked(act)
+ and _gemm._plain_silu(getattr(act, "gate_fn", None))
+ )
+
+
+# Traced gates compare classes, never functions: dynamo resolves ``f.__func__ is g`` to False.
+_CLS: dict = {}
+
+
+def _ff_ready(ff: Any, act_cls: str) -> bool:
+ return (
+ not _gemm._hooked(ff)
+ and _plain_swiglu(getattr(ff, "act_fn", None), act_cls)
+ and _rotated(ff.linear_out)
+ )
+
+
+def _ff_tail(ff: Any, h: Any) -> Any:
+ """``ff.linear_out(ff.act_fn(h))``: the down projection reads the two halves of ``h`` directly when it can."""
+ half = h.shape[-1] // 2
+ y = _gemm.producer_linear(ff.linear_out, None, h[..., :half], h[..., half:])
+ if y is None:
+ return ff.linear_out(ff.act_fn(h))
+ _count("producer")
+ return y
+
+
+def _make_ff_forward(stock: Any, act_cls: str) -> Any:
+ def forward(self, x):
+ if not _ff_ready(self, act_cls):
+ return stock(self, x)
+ return _ff_tail(self, self.linear_in(x))
+
+ functools.update_wrapper(
+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()
+ )
+ setattr(forward, _MARK, True)
+ return forward
+
+
+def _make_f2_projections(stock: Any) -> Any:
+ def _get_projections(
+ attn,
+ hidden_states,
+ encoder_hidden_states = None,
+ ):
+ qkv = None
+ if _rotated(attn.to_q):
+ qkv = _gemm.shared_rotq_linears(hidden_states, (attn.to_q, attn.to_k, attn.to_v))
+ if qkv is None:
+ return stock(attn, hidden_states, encoder_hidden_states)
+ _count("shared")
+ query, key, value = qkv
+ encoder_query = encoder_key = encoder_value = None
+ if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None:
+ added = None
+ if _rotated(attn.add_q_proj):
+ added = _gemm.shared_rotq_linears(
+ encoder_hidden_states, (attn.add_q_proj, attn.add_k_proj, attn.add_v_proj)
+ )
+ if added is None:
+ added = (
+ attn.add_q_proj(encoder_hidden_states),
+ attn.add_k_proj(encoder_hidden_states),
+ attn.add_v_proj(encoder_hidden_states),
+ )
+ encoder_query, encoder_key, encoder_value = added
+ return query, key, value, encoder_query, encoder_key, encoder_value
+
+ functools.update_wrapper(_get_projections, stock, assigned = ("__name__", "__doc__"), updated = ())
+ setattr(_get_projections, _MARK, True)
+ return _get_projections
+
+
+def _make_f3_projections(stock: Any) -> Any:
+ def _get_projections(attn, hidden_states):
+ qkv = None
+ if _rotated(attn.to_q):
+ qkv = _gemm.shared_rotq_linears(hidden_states, (attn.to_q, attn.to_k, attn.to_v))
+ if qkv is None:
+ return stock(attn, hidden_states)
+ _count("shared")
+ return tuple(qkv)
+
+ functools.update_wrapper(_get_projections, stock, assigned = ("__name__", "__doc__"), updated = ())
+ setattr(_get_projections, _MARK, True)
+ return _get_projections
+
+
+def _parallel_ready(attn: Any) -> bool:
+ """FLUX.2 single-block attention whose output projection can read [attention | SwiGLU] from the producers.
+ Traced: plain attribute reads only (no try / except)."""
+ proj = getattr(attn, "to_qkv_mlp_proj", None)
+ out = getattr(attn, "to_out", None)
+ act = getattr(attn, "mlp_act_fn", None)
+ if proj is None or out is None or act is None or getattr(attn, "mlp_mult_factor", None) != 2:
+ return False
+ return (
+ getattr(proj, "out_features", None)
+ == 3 * attn.inner_dim + 2 * attn.mlp_hidden_dim # unsharded
+ and getattr(out, "in_features", None) == attn.inner_dim + attn.mlp_hidden_dim
+ and _plain_swiglu(act, "Flux2SwiGLU")
+ and _rotated(out)
+ )
+
+
+def _make_f2_parallel_call(stock: Any, mod: Any) -> Any:
+ def __call__(
+ self,
+ attn,
+ hidden_states,
+ attention_mask = None,
+ image_rotary_emb = None,
+ ):
+ if not _parallel_ready(attn):
+ return stock(self, attn, hidden_states, attention_mask, image_rotary_emb)
+ hidden_states = attn.to_qkv_mlp_proj(hidden_states)
+ qkv_dim = 3 * attn.inner_dim
+ qkv, mlp_hidden_states = torch.split(
+ hidden_states, [qkv_dim, hidden_states.shape[-1] - qkv_dim], dim = -1
+ )
+ query, key, value = qkv.chunk(3, dim = -1)
+ query = query.unflatten(-1, (-1, attn.head_dim))
+ key = key.unflatten(-1, (-1, attn.head_dim))
+ value = value.unflatten(-1, (-1, attn.head_dim))
+ query = attn.norm_q(query)
+ key = attn.norm_k(key)
+ if image_rotary_emb is not None:
+ # looked up per call: diffusion_flux2_rope may have replaced it
+ query = mod.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)
+ key = mod.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)
+ hidden_states = mod.dispatch_attention_fn(
+ query,
+ key,
+ value,
+ attn_mask = attention_mask,
+ backend = self._attention_backend,
+ parallel_config = self._parallel_config,
+ )
+ hidden_states = hidden_states.to(query.dtype)
+ half = mlp_hidden_states.shape[-1] // 2
+ out = _gemm.producer_linear(
+ attn.to_out, hidden_states, mlp_hidden_states[..., :half], mlp_hidden_states[..., half:]
+ )
+ if out is not None:
+ _count("producer")
+ return out
+ hidden_states = hidden_states.flatten(2, 3)
+ mlp_hidden_states = attn.mlp_act_fn(mlp_hidden_states)
+ hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim = -1)
+ return attn.to_out(hidden_states)
+
+ functools.update_wrapper(
+ __call__, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()
+ )
+ setattr(__call__, _MARK, True)
+ return __call__
+
+
+def _f3_block_ready(block: Any) -> bool:
+ """A FLUX.3 block whose attention and MLP can share one rotation + act quant of their common input. Traced: plain
+ attribute reads only."""
+ attn, ff = getattr(block, "attn", None), getattr(block, "ff", None)
+ if attn is None or ff is None or getattr(attn, "fused_projections", False):
+ return False
+ proc = getattr(attn, "processor", None)
+ return (
+ type(proc) is _CLS.get("Flux3AttnProcessor")
+ and type(attn) is _CLS.get("Flux3Attention")
+ and not _gemm._hooked(attn)
+ and type(ff) is _CLS.get("Flux3FeedForward")
+ and _ff_ready(ff, "Flux3SwiGLU")
+ and _rotated(attn.to_q)
+ and _rotated(attn.to_k)
+ and _rotated(attn.to_v)
+ and _rotated(ff.linear_in)
+ )
+
+
+def _f3_attn_ff(block: Any, x: Any, image_rotary_emb: Any, mod: Any) -> Optional[tuple]:
+ """(attention output, MLP output) of ``x`` with Q / K / V and the MLP input on one rotation + act quant; None:
+ the caller runs the stock modules. Mirrors ``Flux3AttnProcessor.__call__`` below the projections."""
+ if not _f3_block_ready(block):
+ return None
+ attn, ff = block.attn, block.ff
+ parts = _gemm.shared_rotq_linears(x, (attn.to_q, attn.to_k, attn.to_v, ff.linear_in))
+ if parts is None:
+ return None
+ _count("block")
+ query, key, value, h = parts
+ proc = attn.processor
+ query = query.unflatten(-1, (attn.heads, -1))
+ key = key.unflatten(-1, (attn.heads, -1))
+ value = value.unflatten(-1, (attn.heads, -1))
+ query = attn.norm_q(query.float())
+ key = attn.norm_k(key.float())
+ if image_rotary_emb is not None:
+ # looked up per call: diffusion_flux3_rope may have replaced it
+ query = mod.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)
+ key = mod.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)
+ query, key = query.to(value.dtype), key.to(value.dtype)
+ hidden_states = mod.dispatch_attention_fn(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ backend = proc._attention_backend,
+ parallel_config = proc._parallel_config,
+ )
+ hidden_states = hidden_states.flatten(2, 3)
+ hidden_states = hidden_states.to(query.dtype)
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+ return hidden_states, _ff_tail(ff, h)
+
+
+def _make_f3_mode_forward(stock: Any, mod: Any) -> Any:
+ def forward(self, hidden_states, temb_mod, image_rotary_emb):
+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)
+ norm_hidden_states = self.norm(hidden_states)
+ norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift
+ both = _f3_attn_ff(self, norm_hidden_states, image_rotary_emb, mod)
+ if both is None:
+ attn_output = self.attn(
+ hidden_states = norm_hidden_states, image_rotary_emb = image_rotary_emb
+ )
+ ff_output = self.ff(norm_hidden_states)
+ else:
+ attn_output, ff_output = both
+ return hidden_states + mod_gate * (attn_output + ff_output)
+
+ functools.update_wrapper(
+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()
+ )
+ setattr(forward, _MARK, True)
+ return forward
+
+
+def _make_f3_single_forward(stock: Any, mod: Any) -> Any:
+ def forward(
+ self, hidden_states, encoder_hidden_states, temb_mod, temb_mod_context, image_rotary_emb
+ ):
+ mod_shift, mod_scale, mod_gate = torch.chunk(temb_mod, 3, dim = -1)
+ c_mod_shift, c_mod_scale, c_mod_gate = torch.chunk(temb_mod_context, 3, dim = -1)
+ norm_hidden_states = self.norm(hidden_states)
+ norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift
+ norm_encoder_hidden_states = self.norm(encoder_hidden_states)
+ norm_encoder_hidden_states = (1 + c_mod_scale) * norm_encoder_hidden_states + c_mod_shift
+ combined = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim = 1)
+ both = _f3_attn_ff(self, combined, image_rotary_emb, mod)
+ if both is None:
+ attn_output = self.attn(hidden_states = combined, image_rotary_emb = image_rotary_emb)
+ ff_output = self.ff(combined)
+ else:
+ attn_output, ff_output = both
+ output = attn_output + ff_output
+ context_output, content_output = torch.split(
+ output, [encoder_hidden_states.shape[1], hidden_states.shape[1]], dim = 1
+ )
+ encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output
+ hidden_states = hidden_states + mod_gate * content_output
+ return encoder_hidden_states, hidden_states
+
+ functools.update_wrapper(
+ forward, stock, assigned = ("__name__", "__qualname__", "__doc__"), updated = ()
+ )
+ setattr(forward, _MARK, True)
+ return forward
+
+
+def _patch(owner: Any, attr: str, key: str, make: Any) -> bool:
+ """Swap ``owner.attr`` for ``make(stock)`` when the stock body carries ``_NEEDLES[key]``; idempotent."""
+ current = getattr(owner, attr, None)
+ if current is None:
+ return False
+ if getattr(current, _MARK, False):
+ return True
+ if not _has(current, key):
+ return False
+ _STOCK[(owner, attr)] = current
+ setattr(owner, attr, make(current))
+ return True
+
+
+def _install_patches(logger: Any = None) -> dict:
+ done = {}
+ f2, f3 = _flux_modules()
+ with _LOCK:
+ for mod, prefix, swiglu, ff_cls in (
+ (f2, "f2", "Flux2SwiGLU", "Flux2FeedForward"),
+ (f3, "f3", "Flux3SwiGLU", "Flux3FeedForward"),
+ ):
+ if mod is None:
+ continue
+ act = getattr(mod, swiglu, None)
+ if act is not None and _has(act.forward, "swiglu"):
+ _CLS[swiglu] = act
+ proj_key = f"{prefix}_projections"
+ done[proj_key] = _patch(
+ mod,
+ "_get_projections",
+ proj_key,
+ _make_f2_projections if prefix == "f2" else _make_f3_projections,
+ )
+ ff = getattr(mod, ff_cls, None)
+ if ff is not None and swiglu in _CLS:
+ done[f"{prefix}_ff"] = _patch(
+ ff, "forward", "ff", lambda s, c = swiglu: _make_ff_forward(s, c)
+ )
+ if done[f"{prefix}_ff"]:
+ _CLS[ff_cls] = ff
+ if f2 is not None and "Flux2SwiGLU" in _CLS:
+ proc = getattr(f2, "Flux2ParallelSelfAttnProcessor", None)
+ if proc is not None:
+ done["f2_parallel"] = _patch(
+ proc, "__call__", "f2_parallel", lambda s: _make_f2_parallel_call(s, f2)
+ )
+ if f3 is not None and "Flux3FeedForward" in _CLS:
+ proc, attn_cls = (
+ getattr(f3, "Flux3AttnProcessor", None),
+ getattr(f3, "Flux3Attention", None),
+ )
+ ff = getattr(f3, "Flux3FeedForward", None)
+ if (
+ proc is not None
+ and attn_cls is not None
+ and ff is not None
+ and _has(proc.__call__, "f3_attn")
+ and _has(attn_cls.forward, "f3_attn_forward")
+ ):
+ _CLS["Flux3AttnProcessor"] = proc
+ _CLS["Flux3Attention"] = attn_cls
+ for cls_name, key, make in (
+ ("Flux3ModeTransformerBlock", "f3_mode", _make_f3_mode_forward),
+ ("Flux3SingleTransformerBlock", "f3_single", _make_f3_single_forward),
+ ):
+ cls = getattr(f3, cls_name, None)
+ if cls is not None:
+ done[key] = _patch(cls, "forward", key, lambda s, m = make: m(s, f3))
+ if logger is not None:
+ logger.info("diffusion.flux_convrot_fused: patches %s", done)
+ return done
+
+
+def _rotated_count(transformer: Any) -> int:
+ n = 0
+ for module in transformer.modules():
+ rec = _gemm._swapped_rec(module) if hasattr(module, "weight") else None
+ if rec is not None and rec[2] and rec[1] == 256:
+ n += 1
+ return n
+
+
+def _rotated_linears(transformer: Any) -> int:
+ """ConvRot (group 256) Linears in the denoiser, swapped yet or not."""
+ from .diffusion_convrot import is_rotated_linear
+ return sum(
+ 1
+ for m in transformer.modules()
+ if is_rotated_linear(m) and int(getattr(m, "convrot_groupsize", 0) or 0) == 256
+ )
+
+
+def _arm_cards(transformer: Any, logger: Any = None) -> list:
+ """``rotq_h16x`` on every card the denoiser sits on (outside any trace / capture)."""
+ from .diffusion_convrot_quant import _arm_producer
+ from .diffusion_int8_fused import resident_cuda_cards
+
+ cards = [index for index in resident_cuda_cards(transformer) or () if _arm_producer(index)]
+ if logger is not None:
+ logger.info(
+ "diffusion.flux_convrot_fused: %d swapped rotated int8 Linears; producer-fused rotation on cards %s",
+ _rotated_count(transformer),
+ cards,
+ )
+ return cards
+
+
+def install(transformer: Any, logger: Any = None) -> dict:
+ """After the int8 GEMM swap, before the regional compile: patch the FLUX.2 / FLUX.3 helpers (each checks its
+ Linears per call) and arm ``rotq_h16x`` on the denoiser's cards, at the first forward when the weights (and so
+ the swap) are not on the GPU yet. Nothing without ConvRot Linears."""
+ result = {"rotated": 0, "producer_cards": None, "patches": {}}
+ if flux_convrot_fused_disabled():
+ uninstall()
+ return result
+ if type(transformer).__name__ not in ("Flux2Transformer2DModel", "Flux3Transformer2DModel"):
+ return result
+ try:
+ result["rotated"] = _rotated_linears(transformer)
+ if not result["rotated"]:
+ return result
+ result["patches"] = _install_patches(logger)
+ from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call
+
+ if resident_cuda_cards(transformer) and _rotated_count(transformer):
+ result["producer_cards"] = _arm_cards(transformer, logger)
+ else:
+ run_on_first_call(transformer, "flux_convrot_fused", lambda t: _arm_cards(t, logger))
+ except Exception as exc: # noqa: BLE001 - optimisation only: stock helpers stay
+ if logger is not None:
+ logger.info("diffusion.flux_convrot_fused: not installed (%s)", exc)
+ return result
+
+
+def uninstall() -> None:
+ with _LOCK:
+ for key, stock in list(_STOCK.items()):
+ owner, attr = key
+ if isinstance(owner, str):
+ continue
+ if getattr(getattr(owner, attr, None), _MARK, False):
+ setattr(owner, attr, stock)
+ _STOCK.clear()
+ _CLS.clear()
+
+
+def is_installed() -> bool:
+ return any(not isinstance(k[0], str) for k in _STOCK)
+
+
+def call_counts() -> dict:
+ return {k: v[0] for k, v in _CALLS.items()}
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 8f0522784b3..a1a30803139 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -33,6 +33,9 @@ _OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_mm_dequant"
_REC = "_unsloth_i8_gemm"
_MARK = "_unsloth_i8_gemm_prev"
+# The bound ``_linear_forward`` the swap installed: a traced forward checks the instance ``forward`` by identity with
+# it (dynamo resolves ``forward.__func__ is _linear_forward`` to False, so the shared paths never engaged compiled).
+_FWD = "_unsloth_i8_gemm_fwd"
# on a SwiGLU MLP whose down projection takes its codes from the gate / value halves
_FF_MARK = "_unsloth_i8_gemm_swiglu_prev"
_SWIGLU_FF_CLASSES = ("QwenImage21SwiGLUFeedForward",)
@@ -1111,7 +1114,11 @@ def _swapped_rec(module: Any) -> Optional[tuple]:
rec = d.get(_REC)
if rec is None or _MARK not in d or "_diffusers_hook" in d or "_hf_hook" in d:
return None
- if getattr(d.get("forward"), "__func__", None) is not _linear_forward:
+ fwd = d.get("forward")
+ if torch.compiler.is_compiling():
+ if fwd is None or fwd is not d.get(_FWD):
+ return None
+ elif getattr(fwd, "__func__", None) is not _linear_forward:
return None
if d.get("_forward_hooks") or d.get("_forward_pre_hooks"):
return None
@@ -1190,6 +1197,50 @@ def _swiglu_down(down: Any, gate: Any, value: Any) -> Optional[Any]:
return y.reshape(*gate.shape[:-1], y.shape[-1])
+def producer_linear(down: Any, prefix: Any, gate: Any, value: Any) -> Optional[Any]:
+ """``down(cat([prefix.flatten(-2), silu(gate) * value], -1))`` (no prefix: ``down(silu(gate) * value)``) with the
+ product, concat, rotation and act quant in one kernel reading the producers' buffers, else None (caller: stock).
+ ``gate`` / ``value``: ``[..., F]`` halves of one projection output; ``prefix``: the attention output
+ ``[B, S, H, D]`` as the attention backend returned it."""
+ import torch
+
+ rec = _swapped_rec(down)
+ if rec is None:
+ return None
+ kind, group, rotq, weight, fused = rec
+ lead = gate.shape[:-1]
+ f = gate.shape[-1]
+ width = f if prefix is None else f + prefix.shape[-2] * prefix.shape[-1]
+ if (
+ not rotq
+ or group != 256
+ or gate.dtype != torch.bfloat16
+ or not gate.is_cuda
+ or gate.shape != value.shape
+ or width != down.in_features
+ or gate.numel() < _MIN_ROWS * f
+ ):
+ return None
+ if prefix is not None and (prefix.dim() != 4 or prefix.shape[:2] != lead):
+ return None
+ wq, ws = _payload(kind, weight)
+ if wq.device != gate.device:
+ return None
+ from .diffusion_convrot_quant import convrot_producer_act_quant
+
+ q = convrot_producer_act_quant(
+ prefix, gate.reshape(-1, f), value.reshape(-1, f), group, kind == "v2"
+ )
+ if q is None:
+ return None
+ xq, xs = q
+ if fused:
+ y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, down.bias)
+ else:
+ y = stock_gemm(xq, wq, xs, ws, down.bias)
+ return y.reshape(*lead, y.shape[-1])
+
+
def _plain_silu(act: Any) -> bool:
"""``act`` is an exact, unhooked ``nn.SiLU`` (the fused prologue hard-codes SiLU); ``type() is`` avoids a graph break."""
from torch import nn
@@ -1544,6 +1595,7 @@ def _swap_on(
)
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
+ module.__dict__[_FWD] = module.forward
count += 1
ff = (
_install_swiglu_ff(
@@ -1657,6 +1709,7 @@ def uninstall(transformer: Any = None) -> None:
if prev is None:
continue
module.__dict__.pop(_REC, None)
+ module.__dict__.pop(_FWD, None)
if prev is _NO_PREV:
module.__dict__.pop("forward", None)
else:
diff --git a/studio/backend/tests/test_diffusion_autoopt_core.py b/studio/backend/tests/test_diffusion_autoopt_core.py
index c14e7bb9f9b..badb7a92497 100644
--- a/studio/backend/tests/test_diffusion_autoopt_core.py
+++ b/studio/backend/tests/test_diffusion_autoopt_core.py
@@ -206,6 +206,7 @@ def test_builtin_specs_cover_every_former_install_site():
"zimage_fused",
"qwenimage21_fused",
"qwenimage21_dense_fused",
+ "flux_convrot_fused",
"moss_fused",
}
# the former hand-kept teardown list is now a registry group
@@ -218,6 +219,7 @@ def test_builtin_specs_cover_every_former_install_site():
"zimage_fused",
"qwenimage21_fused",
"qwenimage21_dense_fused",
+ "flux_convrot_fused",
"moss_fused",
}
# every spec answers its module's existing kill switch name
@@ -225,6 +227,7 @@ def test_builtin_specs_cover_every_former_install_site():
assert envs["arch_fusions"] == "UNSLOTH_DIFFUSION_ARCH_PATCHES"
assert envs["wan_fused"] == "UNSLOTH_DIFFUSION_WAN_FUSED_ADALN"
assert envs["zimage_fused"] == "UNSLOTH_DIFFUSION_ZIMAGE_FUSED"
+ assert envs["flux_convrot_fused"] == "UNSLOTH_DIFFUSION_FLUX_CONVROT_FUSED"
# order inside the regional compile: Q2.1 real RoPE before the int8 swap, the rest after it
assert [s.name for s in ap.specs(ap.STAGE_PRE_INT8)] == ["qwenimage21_real_rope"]
assert [s.name for s in ap.specs(ap.STAGE_PRE_COMPILE)] == [
@@ -232,6 +235,7 @@ def test_builtin_specs_cover_every_former_install_site():
"zimage_fused",
"qwenimage21_fused",
"qwenimage21_dense_fused",
+ "flux_convrot_fused",
"moss_fused",
]
diff --git a/studio/backend/tests/test_diffusion_autoopt_parity.py b/studio/backend/tests/test_diffusion_autoopt_parity.py
index e4ef1e6644d..2fa1ffa4fb6 100644
--- a/studio/backend/tests/test_diffusion_autoopt_parity.py
+++ b/studio/backend/tests/test_diffusion_autoopt_parity.py
@@ -180,6 +180,7 @@ def _teardown() -> None:
"diffusion_qwenimage_rope",
"diffusion_zimage_fused",
"diffusion_qwenimage21_fused",
+ "diffusion_flux_convrot_fused",
"diffusion_rocm_fused",
"diffusion_qwenimage21",
"diffusion_qwenimage21_rope",
diff --git a/studio/backend/tests/test_diffusion_flux_convrot_fused.py b/studio/backend/tests/test_diffusion_flux_convrot_fused.py
new file mode 100644
index 00000000000..f415897695c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux_convrot_fused.py
@@ -0,0 +1,484 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.2 / FLUX.3 int8 ConvRot producer fusion (diffusion_flux_convrot_fused). The patched helpers and forwards must
+compute what the stock diffusers ones do: here the int8 pieces (the shared quant and the producer-fused down
+projection) are replaced by the plain modules, so a forced-on patch has to match stock bit for bit on CPU. The GPU
+kernel itself is checked against the stock path by its own probe and the GPU parity run."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+
+def env(monkeypatch):
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("diffusers")
+ f2 = pytest.importorskip("diffusers.models.transformers.transformer_flux2")
+ f3 = pytest.importorskip("diffusers.models.transformers.transformer_flux3")
+ from core.inference import diffusion_flux_convrot_fused as fused
+ from core.inference import diffusion_int8_gemm as gemm
+
+ fused.uninstall()
+ monkeypatch.delenv(fused.FLUX_CONVROT_FUSED_ENV, raising = False)
+ yield torch, f2, f3, fused, gemm
+ fused.uninstall()
+
+
+def _force_on(monkeypatch, torch, fused, gemm, calls):
+ """Every Linear counts as a swapped rotated int8 Linear; the int8 pieces run the plain modules."""
+ import torch.nn.functional as F
+
+ monkeypatch.setattr(fused, "_rotated", lambda lin: True)
+
+ def shared(x, linears):
+ calls["shared"] += 1
+ return [m(x) for m in linears]
+
+ def producer(down, prefix, gate, value):
+ calls["producer"] += 1
+ h = F.silu(gate) * value
+ if prefix is not None:
+ h = torch.cat([prefix.flatten(2, 3), h], dim = -1)
+ return down(h)
+
+ monkeypatch.setattr(gemm, "shared_rotq_linears", shared)
+ monkeypatch.setattr(gemm, "producer_linear", producer)
+
+
+def _rope(torch, seq, dim):
+ g = torch.Generator().manual_seed(1)
+ ang = torch.rand(seq, dim, generator = g) * 3
+ return ang.cos(), ang.sin()
+
+
[email protected](params = ["eager", "compiled"])
+def mode(request):
+ return request.param
+
+
+def _run_stock_vs_patched(
+ torch,
+ fused,
+ build,
+ call,
+ mode = "eager",
+):
+ """Stock, patched (eager, or traced by dynamo: the per-call gates must hold inside the regional compile) and
+ restored outputs of one block."""
+ torch.manual_seed(0)
+ block = build().eval()
+ with torch.no_grad():
+ ref = call(block)
+ assert fused._install_patches() # noqa: SLF001 - the install minus the int8 gates
+ if mode == "compiled":
+ torch._dynamo.reset()
+ compiled = torch.compile(block, backend = "eager", fullgraph = True)
+ out = call(compiled)
+ torch._dynamo.reset()
+ else:
+ out = call(block)
+ fused.uninstall()
+ with torch.no_grad():
+ back = call(block)
+ return ref, out, back
+
+
+def _same(a, b):
+ if isinstance(a, (tuple, list)):
+ return all(_same(x, y) for x, y in zip(a, b)) and len(a) == len(b)
+ return a.shape == b.shape and bool((a == b).all())
+
+
+def test_flux2_double_block_matches_stock(env, monkeypatch, mode):
+ torch, f2, _f3, fused, gemm = env
+ calls = {"shared": 0, "producer": 0}
+ _force_on(monkeypatch, torch, fused, gemm, calls)
+ dim, heads, hd = 64, 2, 32
+ hs = torch.randn(2, 12, dim)
+ enc = torch.randn(2, 5, dim)
+ mod_i, mod_t = torch.randn(2, 6 * dim) * 0.1, torch.randn(2, 6 * dim) * 0.1
+ rope = _rope(torch, 17, hd)
+ ref, out, back = _run_stock_vs_patched(
+ torch,
+ fused,
+ lambda: f2.Flux2TransformerBlock(dim, heads, hd),
+ lambda b: b(hs, enc, mod_i, mod_t, image_rotary_emb = rope),
+ mode,
+ )
+ assert _same(ref, out) and _same(ref, back)
+ assert calls["shared"] == 2 # image Q/K/V and text Q/K/V, one quant each
+ assert calls["producer"] == 2 # ff and ff_context down projections
+
+
+def test_flux2_single_block_matches_stock(env, monkeypatch, mode):
+ torch, f2, _f3, fused, gemm = env
+ calls = {"shared": 0, "producer": 0}
+ _force_on(monkeypatch, torch, fused, gemm, calls)
+ dim, heads, hd = 64, 2, 32
+ hs = torch.randn(2, 12, dim)
+ enc = torch.randn(2, 5, dim)
+ mod = torch.randn(2, 3 * dim) * 0.1
+ rope = _rope(torch, 17, hd)
+ ref, out, back = _run_stock_vs_patched(
+ torch,
+ fused,
+ lambda: f2.Flux2SingleTransformerBlock(dim, heads, hd),
+ lambda b: b(hs, enc, mod, image_rotary_emb = rope, split_hidden_states = True),
+ mode,
+ )
+ assert _same(ref, out) and _same(ref, back)
+ assert calls["producer"] == 1 # [attention | SwiGLU] into to_out, read from the producers
+
+
[email protected]("kind", ["mode", "single"])
+def test_flux3_blocks_match_stock(env, monkeypatch, kind, mode):
+ torch, _f2, f3, fused, gemm = env
+ calls = {"shared": 0, "producer": 0}
+ _force_on(monkeypatch, torch, fused, gemm, calls)
+ dim, heads, hd = 64, 2, 32
+ hs = torch.randn(2, 12, dim)
+ enc = torch.randn(2, 5, dim)
+ mod = torch.randn(2, 1, 3 * dim) * 0.1
+ cmod = torch.randn(2, 1, 3 * dim) * 0.1
+ if kind == "mode":
+ rope = _rope(torch, 12, hd)
+ build = lambda: f3.Flux3ModeTransformerBlock(dim, heads, hd) # noqa: E731
+ call = lambda b: b(hs, mod, rope) # noqa: E731
+ else:
+ rope = _rope(torch, 17, hd)
+ build = lambda: f3.Flux3SingleTransformerBlock(dim, heads, hd) # noqa: E731
+ call = lambda b: b(hs, enc, mod, cmod, rope) # noqa: E731
+ ref, out, back = _run_stock_vs_patched(torch, fused, build, call, mode)
+ assert _same(ref, out) and _same(ref, back)
+ assert calls["shared"] == 1 # Q, K, V and the MLP input: one quant
+ assert calls["producer"] == 1
+
+
+def test_gates_off_runs_stock(env, monkeypatch):
+ """Unswapped Linears (CPU, plain bf16, LoRA...): every patch takes the stock path, so nothing int8 is called."""
+ torch, f2, f3, fused, gemm = env
+
+ def boom(*a, **k):
+ raise AssertionError("int8 path reached without swapped Linears")
+
+ monkeypatch.setattr(gemm, "shared_rotq_linears", boom)
+ monkeypatch.setattr(gemm, "producer_linear", boom)
+ dim, heads, hd = 64, 2, 32
+ torch.manual_seed(0)
+ single = f2.Flux2SingleTransformerBlock(dim, heads, hd).eval()
+ mode = f3.Flux3ModeTransformerBlock(dim, heads, hd).eval()
+ hs = torch.randn(1, 8, dim)
+ smod, mmod, rope = (
+ torch.randn(1, 3 * dim) * 0.1,
+ torch.randn(1, 1, 3 * dim) * 0.1,
+ _rope(torch, 8, hd),
+ )
+ with torch.no_grad():
+ a, b = single(hs, None, smod), mode(hs, mmod, rope)
+ assert all(fused._install_patches().values()) # noqa: SLF001
+ assert torch.equal(single(hs, None, smod), a)
+ assert torch.equal(mode(hs, mmod, rope), b)
+
+
+def test_drifted_body_keeps_stock(env, monkeypatch):
+ torch, f2, _f3, fused, _gemm = env
+ stock = f2.Flux2ParallelSelfAttnProcessor.__call__
+ monkeypatch.setitem(
+ fused._NEEDLES,
+ "f2_parallel",
+ fused._NEEDLES["f2_parallel"] + ("a line diffusers never had",),
+ )
+ done = fused._install_patches() # noqa: SLF001
+ assert done.get("f2_parallel") is False
+ assert f2.Flux2ParallelSelfAttnProcessor.__call__ is stock
+
+
+def test_uninstall_restores_everything(env):
+ torch, f2, f3, fused, _gemm = env
+ before = {
+ "p2": f2._get_projections,
+ "p3": f3._get_projections,
+ "ff2": f2.Flux2FeedForward.forward,
+ "ff3": f3.Flux3FeedForward.forward,
+ "par": f2.Flux2ParallelSelfAttnProcessor.__call__,
+ "mode": f3.Flux3ModeTransformerBlock.forward,
+ "single": f3.Flux3SingleTransformerBlock.forward,
+ }
+ done = fused._install_patches() # noqa: SLF001
+ assert all(done.values()), done
+ assert fused.is_installed()
+ assert fused._install_patches() == done # noqa: SLF001 - idempotent
+ fused.uninstall()
+ after = {
+ "p2": f2._get_projections,
+ "p3": f3._get_projections,
+ "ff2": f2.Flux2FeedForward.forward,
+ "ff3": f3.Flux3FeedForward.forward,
+ "par": f2.Flux2ParallelSelfAttnProcessor.__call__,
+ "mode": f3.Flux3ModeTransformerBlock.forward,
+ "single": f3.Flux3SingleTransformerBlock.forward,
+ }
+ assert after == before
+ assert not fused.is_installed()
+
+
+def test_kill_switch(env, monkeypatch):
+ torch, f2, _f3, fused, _gemm = env
+ monkeypatch.setenv(fused.FLUX_CONVROT_FUSED_ENV, "0")
+
+ class Flux2Transformer2DModel:
+ pass
+
+ assert fused.install(Flux2Transformer2DModel())["patches"] == {}
+ assert not fused.is_installed()
+
+
+def test_install_needs_rotated_linears(env):
+ torch, _f2, _f3, fused, _gemm = env
+
+ class Flux2Transformer2DModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ out = fused.install(Flux2Transformer2DModel())
+ assert out["rotated"] == 0 and out["patches"] == {} and not fused.is_installed()
+
+
+# ------------------------------------------------------------------------------------------------ rotq_h16x kernel
+def test_prefix_layout_reads_any_attention_output_stride():
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_convrot_quant as cq
+
+ heads_major = torch.zeros(2, 24, 41, 128).transpose(
+ 1, 2
+ ) # SDPA's [B, H, S, D] storage seen as [B, S, H, D]
+ assert cq._prefix_layout(heads_major) == (2, 41, 3072, 24 * 41 * 128, 128, 41 * 128, 128)
+ seq_major = torch.zeros(1, 41, 24, 128)
+ assert cq._prefix_layout(seq_major) == (1, 41, 3072, 41 * 3072, 3072, 128, 128)
+ flat = torch.zeros(41, 3072)
+ assert cq._prefix_layout(flat) == (1, 41, 3072, 0, 3072, 256, 256)
+ assert cq._prefix_layout(torch.zeros(41, 3072).t()) is None # last stride must be 1
+ assert (
+ cq._prefix_layout(torch.zeros(1, 41, 24, 96)) is None
+ ) # head width the kernel does not index
+ assert not cq.producer_layout_ok(
+ None, torch.zeros(8, 256, dtype = torch.bfloat16), torch.zeros(8, 256, dtype = torch.bfloat16)
+ )
+
+
+def test_launch_shape_tables():
+ pytest.importorskip("torch")
+ from core.inference import diffusion_convrot_quant as cq
+
+ assert cq._h16x_pad((8, 0), 48, True) and not cq._h16x_pad((8, 0), 36, False)
+ assert not cq._h16x_pad((8, 0), 16, True) # a power of two is already one exact pass
+ assert not cq._h16x_pad((8, 9), 48, True) and not cq._h16x_pad(None, 48, True)
+ assert (
+ cq._h16x_warps((8, 0), True, False, 16) == 8
+ and cq._h16x_warps((8, 0), False, False, 16) == 4
+ )
+ assert cq._h16x_warps((8, 0), True, True, 64) == 16
+ assert cq._h16x_warps((12, 0), True, False, 16) == cq._H16_WARPS_TWO_PASS
+
+
+def _cuda_ready():
+ try:
+ import torch
+
+ if not torch.cuda.is_available() or torch.cuda.get_device_capability(0) < (8, 0):
+ return False
+ if getattr(torch.version, "hip", None):
+ return False
+ import torchao # noqa: F401
+ import triton # noqa: F401
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+needs_cuda = pytest.mark.skipif(
+ not _cuda_ready(), reason = "needs NVIDIA sm80+ CUDA, Triton and torchao"
+)
+
+
+def producer_armed(monkeypatch):
+ import torch
+
+ from core.inference import diffusion_convrot_quant as cq
+
+ monkeypatch.delenv(cq.INT8_ROTQUANT_PRODUCER_ENV, raising = False)
+ index = torch.cuda.current_device()
+ cq.rotquant_device_config(index)
+ if not cq._H16_DEVICE.get(index):
+ cq.unlisted_device_ok(index)
+ if not cq._arm_producer(index):
+ pytest.skip("rotq_h16 / rotq_h16x probes refused this device")
+ return cq
+
+
+def _producer(torch, m, pre, f, seed):
+ from core.inference import diffusion_convrot_quant as cq
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ return cq._producer_probe_inputs(
+ m, pre, f, g, torch.device("cuda", torch.cuda.current_device())
+ )
+
+
+@needs_cuda
+ "m, pre, f",
+ [
+ (300, None, 256 * 36), # FLUX MLP down projection (two-pass, looped)
+ (77, None, 256 * 16), # one pass
+ (2 * 300, (2, 24, 128), 256 * 36), # FLUX.2 single-block output projection, batch 2
+ (129, (1, 4, 128), 256 * 12),
+ ],
+)
[email protected]("kind", ["v1", "v2"])
+def test_producer_kernel_is_bit_identical_to_the_current_path(producer_armed, m, pre, f, kind):
+ """rotq_h16x == the bf16 producer output (ATen SiLU product, concat) fed to the rotq op the current path runs."""
+ import torch
+
+ cq = producer_armed
+ prefix, gate, value = _producer(torch, m, pre, f, m + f)
+ q, s = cq.convrot_producer_act_quant(prefix, gate, value, 256, kind == "v2")
+ rq, rs = cq._rotq_run(
+ cq.producer_reference(prefix, gate, value).contiguous(), 256, kind == "v2"
+ )
+ assert torch.equal(q, rq) and torch.equal(s, rs)
+ # every launch shape gives the same codes: the rotation and the rounding are per group / per row
+ for kw in ({"pad": True}, {"pad": False, "warps": 8}, {"pad": False, "tile_rows": 32}):
+ q2, s2 = cq._h16x_launch(prefix, gate, value, kind, **kw)
+ assert torch.equal(q, q2) and torch.equal(s, s2), kw
+
+
+@needs_cuda
+def test_producer_declines_what_it_cannot_read(producer_armed):
+ import torch
+
+ cq = producer_armed
+ prefix, gate, value = _producer(torch, 64, (1, 2, 128), 512, 1)
+ assert cq.convrot_producer_act_quant(prefix, gate, value, 128, True) is None # other group
+ assert (
+ cq.convrot_producer_act_quant(prefix[:, :32], gate, value, 256, True) is None
+ ) # rows disagree
+ assert (
+ cq.convrot_producer_act_quant(None, gate[:, :200], value[:, :200], 256, True) is None
+ ) # off-grid width
+ assert cq.convrot_producer_act_quant(None, gate.float(), value.float(), 256, True) is None
+
+
+@needs_cuda
+def test_producer_kill_switch(producer_armed, monkeypatch):
+ import torch
+
+ cq = producer_armed
+ index = torch.cuda.current_device()
+ monkeypatch.setenv(cq.INT8_ROTQUANT_PRODUCER_ENV, "0")
+ assert cq._arm_producer(index) is False
+ prefix, gate, value = _producer(torch, 64, None, 512, 2)
+ assert cq.convrot_producer_act_quant(prefix, gate, value, 256, True) is None
+ monkeypatch.delenv(cq.INT8_ROTQUANT_PRODUCER_ENV)
+ assert cq._arm_producer(index) is True # the probe verdict was kept: no second probe
+ assert cq.convrot_producer_act_quant(prefix, gate, value, 256, True) is not None
+
+
+def test_swapped_record_is_seen_inside_a_compiled_block():
+ """The shared / producer paths read ``_swapped_rec`` inside the regional compile. Dynamo resolves
+ ``forward.__func__ is _linear_forward`` to False, so the traced check compares the instance forward with the bound
+ method the swap stored: a swapped Linear is seen, a forward replaced after the swap is not."""
+ import types
+
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_int8_gemm as g8
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, x):
+ return x + 1 if g8._swapped_rec(self.proj) is not None else x - 1
+
+ def swapped():
+ block = Block()
+ lin = block.proj
+ lin.__dict__[g8._REC] = ("v2", 256, True, lin.weight, True)
+ lin.__dict__[g8._MARK] = g8._NO_PREV
+ lin.forward = types.MethodType(g8._linear_forward, lin)
+ lin.__dict__[g8._FWD] = lin.forward
+ return block
+
+ x = torch.zeros(1)
+ torch._dynamo.reset()
+ assert torch.compile(swapped(), backend = "eager", fullgraph = True)(x).item() == 1
+ replaced = swapped()
+ replaced.proj.forward = types.MethodType(lambda self, t: t, replaced.proj)
+ torch._dynamo.reset()
+ assert torch.compile(replaced, backend = "eager", fullgraph = True)(x).item() == -1
+ unmarked = swapped()
+ del unmarked.proj.__dict__[g8._FWD]
+ torch._dynamo.reset()
+ assert torch.compile(unmarked, backend = "eager", fullgraph = True)(x).item() == -1
+
+
+def test_probe_verdicts_round_trip_through_the_cache(monkeypatch, tmp_path):
+ """A passing device's probe verdicts persist per card + toolchain fingerprint; a later process adopts them without
+ probing, and a changed fingerprint (driver, torch, switches, these sources) is a miss."""
+ pytest.importorskip("torch")
+ from core.inference import diffusion_convrot_quant as cq
+
+ path = tmp_path / cq._VERDICT_FILE
+ fp = {"torch": "x", "gpu_uuid": "GPU-1"}
+ monkeypatch.setattr(cq, "_verdict_slot", lambda index: (path, f"k{index}", dict(fp)))
+ for name in (
+ "_ROTQ_DEVICE",
+ "_ROTQ_DEVICE_K",
+ "_H16_DEVICE",
+ "_SWIGLU_DEVICE",
+ "_PRODUCER_DEVICE",
+ ):
+ monkeypatch.setattr(cq, name, {})
+ monkeypatch.setattr(cq, "_rotq_op", lambda: "rotq-op")
+ monkeypatch.setattr(cq, "_swiglu_op", lambda: "swiglu-op")
+ assert cq._verdict_load(7) is None
+ cq._verdict_store(
+ 7,
+ {
+ "cfg": [128, 32, 8, 3],
+ "rules": [[256, 8192, [64, 32, 4, 3]]],
+ "h16": "ieee",
+ "swiglu": True,
+ },
+ )
+ cq._verdict_store(7, {"producer": True})
+ v = cq._verdict_load(7)
+ assert v == {
+ "cfg": [128, 32, 8, 3],
+ "rules": [[256, 8192, [64, 32, 4, 3]]],
+ "h16": "ieee",
+ "swiglu": True,
+ "producer": True,
+ }
+ assert cq._adopt_verdict(7, v) == (128, 32, 8, 3)
+ assert cq._ROTQ_DEVICE_K[7] == ((256, 8192, (64, 32, 4, 3)),)
+ assert (
+ cq._H16_DEVICE[7] == "ieee"
+ and cq._SWIGLU_DEVICE[7] is True
+ and cq._PRODUCER_DEVICE[7] is True
+ )
+ fp["torch"] = "y" # another toolchain: a miss, never a stale verdict
+ assert cq._verdict_load(7) is None
+ path.write_text("{torn")
+ assert cq._verdict_load(7) is None
diff --git a/studio/backend/tests/test_diffusion_triton_resolution_args.py b/studio/backend/tests/test_diffusion_triton_resolution_args.py
index 25418e5e860..da1755f655f 100644
--- a/studio/backend/tests/test_diffusion_triton_resolution_args.py
+++ b/studio/backend/tests/test_diffusion_triton_resolution_args.py
@@ -13,7 +13,11 @@ INF = pathlib.Path(__file__).resolve().parents[1] / "core" / "inference"
CASES = {
"diffusion_int8_gemm.py": {"i8mm_dq": ("M",), "i8mm_dq_tma": ("M",)},
- "diffusion_convrot_quant.py": {"rotq_i8": ("M",), "rotq_h16": ("M",)},
+ "diffusion_convrot_quant.py": {
+ "rotq_i8": ("M",),
+ "rotq_h16": ("M",),
+ "rotq_h16x": ("M", "SEQ"),
+ },
"diffusion_flux2_rope.py": {"_rope_fwd": ("S",)},
"diffusion_rocm_fused.py": {"_adaln_fwd": ("S",)},
"diffusion_moss_fused.py": {"kernel": ("S",)},