Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fused int8 MLP kernels and real-arithmetic RoPE for DiT denoisers (FLUX.1 11% less GPU time per step) (#12083)

Daniel Han8f615aceae20
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 2057c25b7db..01ec92befa8 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -1738,6 +1738,18 @@ def _auto_quant_eager_reason(
return reason
+def _uninstall_fused_dit_patches() -> None:
+ """Restore the process-global fused DiT patches so the next load honours its own kill switches."""
+ try:
+ from .diffusion_qwenimage_rope import uninstall as uninstall_qwen_real_rope
+ from .diffusion_zimage_fused import uninstall as uninstall_zimage_fused
+
+ uninstall_qwen_real_rope()
+ uninstall_zimage_fused()
+ except Exception: # noqa: BLE001 - teardown is best effort
+ pass
+
+
def _clear_exception_frames(exc: BaseException) -> None:
"""Release failed-call locals while preserving traceback locations."""
errors, seen = [exc], set()
@@ -6931,6 +6943,7 @@ class DiffusionBackend:
reset_nvfp4_state()
except Exception: # noqa: BLE001 - teardown is best effort
pass
+ _uninstall_fused_dit_patches()
if eager_patched:
uninstall_patches()
uninstall_arch_patches()
@@ -9249,6 +9262,7 @@ class DiffusionBackend:
reset_nvfp4_state()
except Exception: # noqa: BLE001 - teardown is best effort
pass
+ _uninstall_fused_dit_patches()
if state.eager_patched:
# Lazy import to keep diffusion.py torch-free to import.
from .diffusion_eager_patches import uninstall_patches
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
new file mode 100644
index 00000000000..faeb8d67151
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -0,0 +1,1080 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Fused int8 activation path between two torchao int8 GEMMs of a DiT feed-forward: one Triton kernel
+reads the int32 ``_int_mm`` output, applies dequant + GELU(tanh) / SwiGLU and writes int8 + the fp32 row scale.
+
+NUMERICS: bit-identical to EAGER torchao + ATen (every bf16 rounding kept, fp contraction off so
+``y * w_scale + bias`` rounds twice, row scale ``bf16(amax / 127.5)`` clamped at fp32 eps, quantizer multiplies by
+the correctly rounded reciprocal). The compiled stock path is NOT eager-exact (Inductor keeps fp32 chains).
+
+Anything but a plain dynamic symmetric per-row ``Int8Tensor`` on CUDA + Triton >= 3.2 keeps the stock path.
+Kill switch: ``UNSLOTH_DIFFUSION_INT8_FUSED=0``.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import types
+from functools import lru_cache
+from typing import Any, Optional
+
+INT8_FUSED_ENV = "UNSLOTH_DIFFUSION_INT8_FUSED"
+_MIN_TRITON = (3, 2)
+# torch._int_mm needs more than 16 rows; torchao's own safe_int_mm handles the rest, so small M keeps the stock path.
+_MIN_ROWS = 17
+_OP_NAMESPACE = "unsloth_studio"
+_OP_NAME = "int8_dq_gelu_quant"
+_OP_NAME_SWIGLU = "int8_dq_swiglu_quant"
+_SWIGLU_ATTR = "_unsloth_i8_swiglu"
+
+_LOCK = threading.Lock()
+# Read by the traced forwards: dynamo must not trace into the lru_cache'd registration.
+_OP_HANDLE: Any = None
+# Marker on each patched module (no global registry: it would pin an unloaded transformer).
+_MARK = "_unsloth_i8_fused_prev"
+_NO_PREV = object()
+
+
+def int8_fused_disabled() -> bool:
+ return (os.environ.get(INT8_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _triton_version_ok(version: Optional[str] = None) -> bool:
+ if version is None:
+ try:
+ import triton
+ version = str(triton.__version__)
+ except Exception: # noqa: BLE001
+ return False
+ import re
+
+ match = re.match(r"(\d+)\.(\d+)", version)
+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON
+
+
+@lru_cache(maxsize = 1)
+def _triton_jit_toolchain_ok() -> bool:
+ """Windows: Triton needs the MSVC CRT headers. Elsewhere, or if the probe fails, True."""
+ if sys.platform != "win32":
+ return True
+ try:
+ from .._msvc_env import crt_headers_reachable
+ return bool(crt_headers_reachable())
+ except Exception: # noqa: BLE001
+ return True
+
+
+@lru_cache(maxsize = 1)
+def _kernels() -> Optional[types.SimpleNamespace]:
+ """Compile-on-first-use Triton kernel, or None when Triton is unavailable."""
+ try:
+ import triton
+ import triton.language as tl
+ except Exception: # noqa: BLE001 - no Triton means the stock path
+ return None
+ # Triton <= 3.2 resolves string annotations against module globals, not this function's locals
+ globals().update(triton = triton, tl = tl)
+
+ @triton.jit
+ def _rbf16(x):
+ # Round-to-nearest-even to bf16, kept in fp32 (one cvt; faster here than the integer-op form).
+ return x.to(tl.bfloat16).to(tl.float32)
+
+ @triton.jit
+ def _gelu_tanh(y):
+ # 0.5 * y * (1 + tanh(u)) == y * sigmoid(2u) == y / (1 + exp(-2u)), u = sqrt(2/pi) * (y + 0.044715 y^3)
+ u = 0.7978845608028654 * (y + 0.044715 * (y * y * y))
+ return y / (1.0 + tl.exp2(u * -2.8853900817779268))
+
+ @triton.jit
+ def _pre(
+ c_ptr,
+ xs,
+ ws_ptr,
+ b_ptr,
+ offs,
+ mask,
+ HAS_BIAS: tl.constexpr,
+ WS_FP32: tl.constexpr,
+ EVICT: tl.constexpr,
+ FINAL_ROUND: tl.constexpr,
+ ):
+ # torchao Int8Tensor linear epilogue: (int32 * x_scale).to(bf16) * w_scale (+ bias), then .to(bf16).
+ # FINAL_ROUND=False leaves the last rounding to the caller (it commutes with a row max).
+ c = tl.load(c_ptr + offs, mask = mask, other = 0, eviction_policy = EVICT).to(tl.float32)
+ y = _rbf16(c * xs)
+ y = y * tl.load(ws_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(
+ tl.float32
+ )
+ if not WS_FP32:
+ y = _rbf16(y)
+ if HAS_BIAS:
+ y = y + tl.load(b_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(
+ tl.float32
+ )
+ if FINAL_ROUND:
+ y = _rbf16(y)
+ return y
+
+ @triton.jit
+ def dq_gelu_quant(
+ c_ptr,
+ xs_ptr,
+ ws_ptr,
+ b_ptr,
+ p_ptr,
+ q_ptr,
+ s_ptr,
+ N,
+ NA,
+ S,
+ stride_c,
+ stride_q,
+ p_sb,
+ p_ss,
+ p_sh,
+ HAS_BIAS: tl.constexpr,
+ WS_FP32: tl.constexpr,
+ HAS_PREFIX: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ CHUNK: tl.constexpr,
+ CHUNK_A: tl.constexpr,
+ ):
+ row = tl.program_id(0).to(tl.int64)
+ xs = tl.load(xs_ptr + row).to(tl.float32)
+ base = tl.arange(0, CHUNK)
+ crow = c_ptr + row * stride_c
+ qrow = q_ptr + row * stride_q
+ # pass 1a: max of the pre-activation. GELU is increasing for y > -0.75 and |gelu(y)| < 0.16997 for y < 0,
+ # so max|bf16(gelu)| over the row is bf16(gelu(max y)) whenever that is >= 0.171875; otherwise recompute.
+ vmax = tl.full((CHUNK,), float("-inf"), tl.float32)
+ for k in range(0, N, CHUNK):
+ offs = k + base
+ m = offs < N
+ y = _pre(crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_last", False)
+ vmax = tl.maximum(vmax, tl.where(m, y, float("-inf")))
+ # bf16 rounding is monotone, so max(round(v)) == round(max(v)): round once, after the reduction.
+ amax = _rbf16(_gelu_tanh(_rbf16(tl.max(vmax, axis = 0))))
+ if amax < 0.171875:
+ acc = tl.zeros((CHUNK,), dtype = tl.float32)
+ for k in range(0, N, CHUNK):
+ offs = k + base
+ m = offs < N
+ g = _rbf16(
+ _gelu_tanh(
+ _pre(
+ crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_last", True
+ )
+ )
+ )
+ acc = tl.maximum(acc, tl.where(m, tl.abs(g), 0.0))
+ amax = tl.max(acc, axis = 0)
+ if HAS_PREFIX:
+ # FluxSingleTransformerBlock: the bf16 attention output sits in front of the GELU branch in the row.
+ b_idx = row // S
+ s_idx = row % S
+ pbase = p_ptr + b_idx * p_sb + s_idx * p_ss
+ abase = tl.arange(0, CHUNK_A)
+ pacc = tl.zeros((CHUNK_A,), dtype = tl.float32)
+ for k in range(0, NA, CHUNK_A):
+ j = k + abase
+ m = j < NA
+ v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(
+ tl.float32
+ )
+ pacc = tl.maximum(pacc, tl.abs(v))
+ amax = tl.maximum(amax, tl.max(pacc, axis = 0))
+ scale = _rbf16(tl.math.div_rn(amax, 127.5))
+ scale = tl.maximum(scale, 1.1920928955078125e-07)
+ inv = tl.math.div_rn(1.0, scale)
+ tl.store(s_ptr + row, scale)
+ if HAS_PREFIX:
+ for k in range(0, NA, CHUNK_A):
+ j = k + abase
+ m = j < NA
+ v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(
+ tl.float32
+ )
+ qi = tl.extra.cuda.libdevice.nearbyint(v * inv)
+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
+ tl.store(qrow + j, qi.to(tl.int8), mask = m)
+ # pass 2: rebuild (the int32 row is mostly still in L2) and quantize.
+ for k in range(0, N, CHUNK):
+ offs = k + base
+ m = offs < N
+ g = _rbf16(
+ _gelu_tanh(
+ _pre(crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_first", True)
+ )
+ )
+ qi = tl.extra.cuda.libdevice.nearbyint(g * inv)
+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
+ tl.store(qrow + NA + offs, qi.to(tl.int8), mask = m)
+
+ @triton.jit
+ def _silu(y):
+ # ATen: x / (1 + exp(-x)) in fp32
+ return y / (1.0 + tl.exp(-y))
+
+ @triton.jit
+ def _swiglu_val(
+ crow,
+ xs,
+ ws_ptr,
+ b_ptr,
+ g0,
+ v0,
+ offs,
+ m,
+ HAS_BIAS: tl.constexpr,
+ WS_FP32: tl.constexpr,
+ EVICT: tl.constexpr,
+ ):
+ g = _pre(crow + g0, xs, ws_ptr + g0, b_ptr + g0, offs, m, HAS_BIAS, WS_FP32, EVICT, True)
+ v = _pre(crow + v0, xs, ws_ptr + v0, b_ptr + v0, offs, m, HAS_BIAS, WS_FP32, EVICT, True)
+ return _rbf16(_rbf16(_silu(g)) * v)
+
+ @triton.jit
+ def dq_swiglu_quant(
+ c_ptr,
+ xs_ptr,
+ ws_ptr,
+ b_ptr,
+ h_ptr,
+ q_ptr,
+ s_ptr,
+ N,
+ G0,
+ V0,
+ stride_c,
+ stride_q,
+ HAS_BIAS: tl.constexpr,
+ WS_FP32: tl.constexpr,
+ CHUNK: tl.constexpr,
+ ):
+ # c row = the fused GEMM output; the gate half starts at column G0, the value half at V0 (both N wide).
+ # Two int32 rows per output row make recomputing in pass 2 ALU-bound, so pass 1 parks the bf16 product in a
+ # scratch row (still in L2 when pass 2 reads it back).
+ row = tl.program_id(0).to(tl.int64)
+ xs = tl.load(xs_ptr + row).to(tl.float32)
+ base = tl.arange(0, CHUNK)
+ crow = c_ptr + row * stride_c
+ hrow = h_ptr + row * N
+ acc = tl.zeros((CHUNK,), dtype = tl.float32)
+ for k in range(0, N, CHUNK):
+ offs = k + base
+ m = offs < N
+ h = _swiglu_val(
+ crow, xs, ws_ptr, b_ptr, G0, V0, offs, m, HAS_BIAS, WS_FP32, "evict_first"
+ )
+ tl.store(hrow + offs, h.to(tl.bfloat16), mask = m, eviction_policy = "evict_last")
+ acc = tl.maximum(acc, tl.where(m, tl.abs(h), 0.0))
+ amax = tl.max(acc, axis = 0)
+ scale = _rbf16(tl.math.div_rn(amax, 127.5))
+ scale = tl.maximum(scale, 1.1920928955078125e-07)
+ inv = tl.math.div_rn(1.0, scale)
+ tl.store(s_ptr + row, scale)
+ qrow = q_ptr + row * stride_q
+ for k in range(0, N, CHUNK):
+ offs = k + base
+ m = offs < N
+ h = tl.load(hrow + offs, mask = m, other = 0.0, eviction_policy = "evict_first").to(
+ tl.float32
+ )
+ qi = tl.extra.cuda.libdevice.nearbyint(h * inv)
+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
+ tl.store(qrow + offs, qi.to(tl.int8), mask = m)
+
+ return types.SimpleNamespace(
+ dq_gelu_quant = dq_gelu_quant, dq_swiglu_quant = dq_swiglu_quant, triton = triton
+ )
+
+
+def _launch(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:
+ """Run the kernel. ``prefix`` is None or a [B, S, H, D] view (any strides, last dim contiguous)."""
+ import torch
+
+ k = _kernels()
+ m_rows, n = c.shape
+ if prefix is not None:
+ b, s, h, d = prefix.shape
+ na = h * d
+ p_sb, p_ss, p_sh = prefix.stride(0), prefix.stride(1), prefix.stride(2)
+ else:
+ s, d, na = 1, 1, 0
+ p_sb = p_ss = p_sh = 0
+ q = torch.empty((m_rows, na + n), device = c.device, dtype = torch.int8)
+ scale = torch.empty((m_rows,), device = c.device, dtype = torch.float32)
+ with torch.cuda.device(c.device):
+ k.dq_gelu_quant[(m_rows,)](
+ c,
+ xs,
+ ws,
+ bias if bias is not None else ws,
+ prefix if prefix is not None else ws,
+ q,
+ scale,
+ n,
+ na,
+ s,
+ c.stride(0),
+ q.stride(0),
+ p_sb,
+ p_ss,
+ p_sh,
+ HAS_BIAS = bias is not None,
+ WS_FP32 = ws.dtype == torch.float32,
+ HAS_PREFIX = prefix is not None,
+ HEAD_DIM = d,
+ CHUNK = 2048,
+ CHUNK_A = 1024,
+ num_warps = 8,
+ enable_fp_fusion = False,
+ )
+ return q, scale
+
+
+def _launch_swiglu(
+ c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int
+) -> tuple:
+ """SwiGLU on the int32 output ``c`` of one fused GEMM: gate columns [gate_col, +n), value columns [value_col, +n)."""
+ import torch
+
+ k = _kernels()
+ m_rows = c.shape[0]
+ q = torch.empty((m_rows, n), device = c.device, dtype = torch.int8)
+ scale = torch.empty((m_rows,), device = c.device, dtype = torch.float32)
+ scratch = torch.empty((m_rows, n), device = c.device, dtype = torch.bfloat16)
+ with torch.cuda.device(c.device):
+ k.dq_swiglu_quant[(m_rows,)](
+ c,
+ xs,
+ ws,
+ bias if bias is not None else ws,
+ scratch,
+ q,
+ scale,
+ n,
+ gate_col,
+ value_col,
+ c.stride(0),
+ q.stride(0),
+ HAS_BIAS = bias is not None,
+ WS_FP32 = ws.dtype == torch.float32,
+ CHUNK = 2048,
+ num_warps = 8,
+ enable_fp_fusion = False,
+ )
+ return q, scale
+
+
+def reference_dq_swiglu_quant(
+ c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int
+) -> tuple:
+ """Eager mirror: two torchao epilogues (gate, value), ATen SiLU, bf16 product, torchao per-row act quant."""
+ import torch
+ import torch.nn.functional as F
+
+ def part(col):
+ y = (c[:, col : col + n] * xs.reshape(-1, 1)).to(torch.bfloat16) * ws[col : col + n]
+ if bias is not None:
+ y = y + bias[col : col + n]
+ return y.to(torch.bfloat16)
+
+ h = F.silu(part(gate_col)) * part(value_col)
+ amax = torch.maximum(-h.amin(dim = 1).clamp(max = 0), h.amax(dim = 1).clamp(min = 0))
+ scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(torch.float32)
+ q = torch.clamp(torch.round(h * (1.0 / scale).reshape(-1, 1)), -128, 127).to(torch.int8)
+ return q, scale
+
+
+@lru_cache(maxsize = 1)
+def _op() -> Any:
+ """The torch.library op (opaque to dynamo, CUDA-graph safe: no host sync, allocations only), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ qualname = f"{_OP_NAMESPACE}::{_OP_NAME}"
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _OP_NAME) and hasattr(ns, _OP_NAME_SWIGLU):
+ return types.SimpleNamespace(
+ gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU)
+ )
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None: # torch < 2.4
+ return None
+ try:
+ # Explicit schema: this module's annotations are strings (``from __future__ import annotations``) and torch is
+ # imported lazily, so schema inference from them would not resolve.
+ @custom_op(
+ qualname,
+ mutates_args = (),
+ schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, Tensor? prefix) -> (Tensor, Tensor)",
+ )
+ def _dq_gelu_quant(c, xs, ws, bias, prefix):
+ return _launch(c, xs, ws, bias, prefix)
+
+ @_dq_gelu_quant.register_fake
+ def _(c, xs, ws, bias, prefix):
+ na = 0 if prefix is None else prefix.shape[-2] * prefix.shape[-1]
+ return (
+ c.new_empty((c.shape[0], na + c.shape[1]), dtype = torch.int8),
+ c.new_empty((c.shape[0],), dtype = torch.float32),
+ )
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_OP_NAME_SWIGLU}",
+ mutates_args = (),
+ schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, int gate_col, int value_col, int n) -> (Tensor, Tensor)",
+ )
+ def _dq_swiglu_quant(c, xs, ws, bias, gate_col, value_col, n):
+ return _launch_swiglu(c, xs, ws, bias, gate_col, value_col, n)
+
+ @_dq_swiglu_quant.register_fake
+ def _(c, xs, ws, bias, gate_col, value_col, n):
+ return (
+ c.new_empty((c.shape[0], n), dtype = torch.int8),
+ c.new_empty((c.shape[0],), dtype = torch.float32),
+ )
+ except Exception: # noqa: BLE001 - a registration failure keeps the stock path
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE)
+ return types.SimpleNamespace(gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU))
+
+
+@lru_cache(maxsize = 8)
+def _device_ok(index: int) -> bool:
+ """CUDA NVIDIA, Triton new enough, toolchain present, kernel compiles and matches the reference once."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ if (
+ not (_triton_version_ok() and _triton_jit_toolchain_ok())
+ or _kernels() is None
+ or _op() is None
+ ):
+ return False
+ try:
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ c = torch.randint(-(2**20), 2**20, (33, 200), generator = g, dtype = torch.int32).to(dev)
+ xs = (torch.rand(33, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16).float().to(dev)
+ ws = (torch.rand(200, generator = g) * 1e-4 + 1e-6).to(dev)
+ bias = (torch.randn(200, generator = g) * 0.1).to(torch.bfloat16).to(dev)
+ q, s = _launch(c, xs, ws, bias, None)
+ q_ref, s_ref = reference_dq_gelu_quant(c, xs, ws, bias, None)
+ ok = bool(torch.equal(q, q_ref) and torch.equal(s, s_ref))
+ q, s = _launch_swiglu(c, xs, ws, bias, 104, 0, 96)
+ q_ref, s_ref = reference_dq_swiglu_quant(c, xs, ws, bias, 104, 0, 96)
+ return ok and bool(torch.equal(q, q_ref) and torch.equal(s, s_ref))
+ except Exception: # noqa: BLE001 - any build / launch failure keeps the stock path
+ return False
+
+
+def reference_dq_gelu_quant(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:
+ """Eager torch mirror of the stock chain (torchao epilogue, ATen GELU, torchao per-row act quant)."""
+ import torch
+ import torch.nn.functional as F
+
+ y = (c * xs.reshape(-1, 1)).to(torch.bfloat16)
+ y = y * ws.flatten()
+ if bias is not None:
+ y = y + bias
+ g = F.gelu(y.to(torch.bfloat16), approximate = "tanh")
+ if prefix is not None:
+ g = torch.cat(
+ [prefix.reshape(prefix.shape[0] * prefix.shape[1], -1).to(g.dtype), g], dim = -1
+ )
+ amax = torch.maximum(-g.amin(dim = 1).clamp(max = 0), g.amax(dim = 1).clamp(min = 0))
+ scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(torch.float32)
+ q = torch.clamp(torch.round(g * (1.0 / scale).reshape(-1, 1)), -128, 127).to(torch.int8)
+ return q, scale
+
+
+def _plain_int8_weight(w: Any) -> bool:
+ """A torchao Int8Tensor with dynamic, symmetric, per-row activation quant and nothing else attached."""
+ try:
+ if type(w).__name__ != "Int8Tensor":
+ return False
+ kw = getattr(w, "act_quant_kwargs", None)
+ if kw is None or getattr(w, "act_pre_scale", None) is not None:
+ return False
+ if (
+ getattr(w, "act_quant_scale", None) is not None
+ or getattr(w, "act_quant_zero_point", None) is not None
+ ):
+ return False
+ if getattr(kw, "reduce_range", False) or getattr(w, "reduce_range", False):
+ return False
+ if "SYMMETRIC" not in str(getattr(kw, "mapping_type", "")) or "ASYMMETRIC" in str(
+ getattr(kw, "mapping_type", "")
+ ):
+ return False
+ if type(getattr(kw, "granularity", None)).__name__ != "PerRow":
+ return False
+ qd, sc = w.qdata, w.scale
+ if qd.dim() != 2 or list(getattr(w, "block_size", [])) != [1, qd.shape[1]]:
+ return False
+ if sc.dim() != 2 or tuple(sc.shape) != (qd.shape[0], 1):
+ return False
+ return qd.shape[0] % 8 == 0 and qd.shape[1] % 8 == 0
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _act_quant(x2d: Any, weight: Any) -> tuple:
+ """torchao's own dynamic activation quant for ``weight`` (the stock path's call), as (int8 rows, fp32 scale)."""
+ from torchao.quantization.quantize_.common.quantize_tensor_kwargs import (
+ _choose_quant_func_and_quantize_tensor,
+ )
+
+ act = _choose_quant_func_and_quantize_tensor(
+ x2d,
+ weight.act_quant_kwargs,
+ scale = weight.act_quant_scale,
+ zero_point = weight.act_quant_zero_point,
+ )
+ return act.qdata, act.scale
+
+
+def _int_mm(a: Any, weight: Any) -> Any:
+ from torchao.kernel.intmm import safe_int_mm
+ return safe_int_mm(a, weight.qdata.contiguous().t())
+
+
+def _linear_from_q(q: Any, xs: Any, weight: Any, bias: Any, out_dtype: Any) -> Any:
+ """torchao Int8Tensor linear on an already-quantized activation: same ops, same order, as the stock epilogue."""
+ import torch
+
+ y = (_int_mm(q, weight) * xs.reshape(-1, 1).to(torch.float32)).to(out_dtype)
+ y = y * weight.scale.flatten()
+ if bias is not None:
+ y = y + bias
+ return y.to(out_dtype)
+
+
+def _ff_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
+ """``FeedForward.forward`` for GELU(tanh) -> Linear with plain int8 weights."""
+ import torch
+
+ proj = self.net[0].proj
+ down = self.net[2]
+ lead = hidden_states.shape[:-1]
+ x2d = hidden_states.reshape(-1, hidden_states.shape[-1])
+ if (
+ args
+ or kwargs
+ or x2d.shape[0] < _MIN_ROWS
+ or not hidden_states.is_cuda
+ or hidden_states.dtype != torch.bfloat16
+ ):
+ return type(self).forward(self, hidden_states, *args, **kwargs)
+ xq, xs = _act_quant(x2d, proj.weight)
+ c = _int_mm(xq.reshape(-1, xq.shape[-1]), proj.weight)
+ q, s = _OP_HANDLE.gelu(c, xs.reshape(-1), proj.weight.scale.flatten(), proj.bias, None)
+ y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)
+ for extra in self.net[3:]:
+ y = extra(y)
+ return y.reshape(*lead, y.shape[-1])
+
+
+def _flux_single_forward(
+ self: Any,
+ hidden_states: Any,
+ encoder_hidden_states: Any,
+ temb: Any,
+ image_rotary_emb: Any = None,
+ joint_attention_kwargs: Any = None,
+) -> Any:
+ """``FluxSingleTransformerBlock.forward`` with the GELU branch, the concat and ``proj_out``'s activation
+ quant fused into one kernel. Mirrors the installed class forward (stock or Studio's addcmul arch patch)."""
+ import torch
+
+ text_seq_len = encoder_hidden_states.shape[1]
+ rows = hidden_states.shape[0] * (hidden_states.shape[1] + text_seq_len)
+ if rows < _MIN_ROWS or hidden_states.dtype != torch.bfloat16 or not hidden_states.is_cuda:
+ return type(self).forward(
+ self,
+ hidden_states,
+ encoder_hidden_states,
+ temb,
+ image_rotary_emb,
+ joint_attention_kwargs,
+ )
+ hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim = 1)
+ residual = hidden_states
+ norm_hidden_states, gate = self.norm(hidden_states, emb = temb)
+ x2d = norm_hidden_states.reshape(rows, -1)
+ xq, xs = _act_quant(x2d, self.proj_mlp.weight)
+ c = _int_mm(xq.reshape(rows, -1), self.proj_mlp.weight)
+ joint_attention_kwargs = joint_attention_kwargs or {}
+ attn_output = self.attn(
+ hidden_states = norm_hidden_states,
+ image_rotary_emb = image_rotary_emb,
+ **joint_attention_kwargs,
+ )
+ heads = self.attn.heads
+ prefix = attn_output.unflatten(-1, (heads, attn_output.shape[-1] // heads))
+ if prefix.stride(-1) != 1:
+ prefix = prefix.contiguous()
+ q, s = _OP_HANDLE.gelu(
+ c, xs.reshape(-1), self.proj_mlp.weight.scale.flatten(), self.proj_mlp.bias, prefix
+ )
+ out = _linear_from_q(q, s, self.proj_out.weight, self.proj_out.bias, hidden_states.dtype)
+ out = out.reshape(*hidden_states.shape[:-1], out.shape[-1])
+ gate = gate.unsqueeze(1)
+ if getattr(self, "_unsloth_i8_addcmul", False):
+ hidden_states = torch.addcmul(residual, gate, out)
+ else:
+ hidden_states = residual + gate * out
+ if hidden_states.dtype == torch.float16:
+ hidden_states = hidden_states.clip(-65504, 65504)
+ return hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:]
+
+
+def _swiglu_spec(module: Any) -> Optional[tuple]:
+ """(fused in-proj name, None, down name, width, gate column, value column) for the SwiGLU MLPs whose two halves
+ already come from ONE Linear, else None."""
+ name = type(module).__name__
+ try:
+ if name == "FeedForward" and type(module).__module__ == "diffusers.models.attention":
+ from diffusers.models.activations import SwiGLU
+
+ net = getattr(module, "net", None)
+ if net is None or len(net) < 3 or type(net[0]) is not SwiGLU:
+ return None
+ if any(type(extra).__name__ != "Dropout" for extra in (net[1], *net[3:])):
+ return None
+ n = net[0].proj.out_features // 2
+ # SwiGLU.forward: hidden, gate = proj(x).chunk(2) -> value first, gate second
+ return ("net.0.proj", None, "net.2", n, n, 0)
+ if (
+ name == "Flux2FeedForward"
+ and type(getattr(module, "act_fn", None)).__name__ == "Flux2SwiGLU"
+ ):
+ n = module.linear_in.out_features // 2
+ return ("linear_in", None, "linear_out", n, 0, n) # gate first, value second (FLUX.2)
+ except Exception: # noqa: BLE001
+ return None
+ return None
+
+
+def _split_spec(module: Any) -> Optional[tuple]:
+ """Two separate projections (gate, value) + down: Z-Image FeedForward (w1, w3, w2), Qwen-Image-2.1 SwiGLU."""
+ name = type(module).__name__
+ mod = type(module).__module__
+ if (
+ name == "FeedForward"
+ and mod.endswith("transformer_z_image")
+ and all(hasattr(module, n) for n in ("w1", "w2", "w3"))
+ ):
+ return ("w1", "w3", "w2")
+ if name == "QwenImage21SwiGLUFeedForward":
+ return ("gate_layer", "proj", "out")
+ return None
+
+
+# SwiGLU kernel is bit-exact vs eager but moved FLUX.2-klein / Qwen-Image-2.1 compiled renders further from eager bf16
+# (LPIPS) than the stock path: Z-Image only until understood.
+_SWIGLU_ALL_LAYOUTS = False
+
+
+def _is_zimage_ff(module: Any) -> bool:
+ return _split_spec(module) == ("w1", "w3", "w2") and type(module).__module__.endswith(
+ "transformer_z_image"
+ )
+
+
+def _swiglu_layout_allowed(module: Any) -> bool:
+ return _SWIGLU_ALL_LAYOUTS or _is_zimage_ff(module)
+
+
+def _get(module: Any, dotted: str) -> Any:
+ for part in dotted.split("."):
+ module = module[int(part)] if part.isdigit() else getattr(module, part)
+ return module
+
+
+def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
+ """SwiGLU MLP with plain int8 weights: one act quant, one GEMM over [gate | value], fused SiLU-product quant."""
+ import torch
+
+ rec = self.__dict__.get(_SWIGLU_ATTR)
+ lead = hidden_states.shape[:-1]
+ x2d = hidden_states.reshape(-1, hidden_states.shape[-1])
+ ok = (
+ rec is not None
+ and not args
+ and not kwargs
+ and x2d.shape[0] >= _MIN_ROWS
+ and hidden_states.is_cuda
+ )
+ ok = ok and hidden_states.dtype == torch.bfloat16
+ if ok:
+ fused_in, parts, down_name, n, gate_col, value_col = rec
+ ok = all(_get(self, nm) is mod for nm, mod in parts)
+ if not ok:
+ return type(self).forward(self, hidden_states, *args, **kwargs)
+ down = _get(self, down_name)
+ xq, xs = _act_quant(x2d, fused_in.weight)
+ c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)
+ q, s = _OP_HANDLE.swiglu(
+ c, xs.reshape(-1), fused_in.weight.scale.flatten(), fused_in.bias, gate_col, value_col, n
+ )
+ y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)
+ return y.reshape(*lead, y.shape[-1])
+
+
+def _prepare_swiglu(module: Any) -> bool:
+ """Attach the (off-tree) record the SwiGLU forward reads; False keeps the stock forward."""
+ from torch import nn
+
+ if not _swiglu_layout_allowed(module):
+ return False
+ spec = _swiglu_spec(module)
+ if spec is not None:
+ in_name, _unused, down_name, n, gate_col, value_col = spec
+ lin, down = _get(module, in_name), _get(module, down_name)
+ if type(lin) is not nn.Linear or type(down) is not nn.Linear:
+ return False
+ if not (_plain_int8_weight(lin.weight) and _plain_int8_weight(down.weight)):
+ return False
+ module.__dict__[_SWIGLU_ATTR] = (
+ lin,
+ ((in_name, lin), (down_name, down)),
+ down_name,
+ n,
+ gate_col,
+ value_col,
+ )
+ return True
+ split = _split_spec(module)
+ if split is None:
+ return False
+ gate_name, value_name, down_name = split
+ gate, value, down = _get(module, gate_name), _get(module, value_name), _get(module, down_name)
+ if any(type(m) is not nn.Linear for m in (gate, value, down)):
+ return False
+ if not all(_plain_int8_weight(m.weight) for m in (gate, value, down)):
+ return False
+ from .diffusion_zimage_fused import _fuse_linears, _share_storage
+
+ fused = _fuse_linears([gate, value])
+ if fused is None or not _share_storage(fused, [gate, value]):
+ return False
+ n = gate.weight.shape[0]
+ parts = ((gate_name, gate), (value_name, value), (down_name, down))
+ module.__dict__[_SWIGLU_ATTR] = (fused, parts, down_name, n, 0, n)
+ return True
+
+
+def _exact_linear(*modules: Any) -> bool:
+ """Plain nn.Linear only: a subclass (ConvRotLinear) may transform the input the fused _int_mm would skip."""
+ from torch import nn
+ return all(type(m) is nn.Linear for m in modules)
+
+
+def _ff_eligible(module: Any) -> bool:
+ try:
+ from diffusers.models.activations import GELU
+ from diffusers.models.attention import FeedForward
+ except Exception: # noqa: BLE001
+ return False
+ if type(module) is not FeedForward:
+ return False
+ net = getattr(module, "net", None)
+ if net is None or len(net) < 3:
+ return False
+ act, drop, down = net[0], net[1], net[2]
+ if type(act) is not GELU or getattr(act, "approximate", None) != "tanh":
+ return False
+ if type(drop).__name__ != "Dropout" or not _exact_linear(act.proj, down):
+ return False
+ if any(type(extra).__name__ != "Dropout" for extra in net[3:]):
+ return False
+ return _plain_int8_weight(act.proj.weight) and _plain_int8_weight(down.weight)
+
+
+def _flux_single_eligible(module: Any) -> bool:
+ if type(module).__name__ != "FluxSingleTransformerBlock":
+ return False
+ try:
+ act = module.act_mlp
+ if type(act).__name__ != "GELU" or getattr(act, "approximate", None) != "tanh":
+ return False
+ if getattr(module.attn, "heads", None) is None or not getattr(
+ module.attn, "pre_only", False
+ ):
+ return False
+ if not _exact_linear(module.proj_mlp, module.proj_out):
+ return False
+ return _plain_int8_weight(module.proj_mlp.weight) and _plain_int8_weight(
+ module.proj_out.weight
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _flux_single_class_is_arch_patched(module: Any) -> bool:
+ try:
+ from . import diffusion_arch_patches as ap
+ return getattr(type(module), "forward", None) is getattr(
+ ap, "_flux_single_forward", object()
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def resident_cuda_device(module: Any) -> Any:
+ """The one CUDA device every parameter of ``module`` lives on (NVIDIA, not ROCm), else None."""
+ try:
+ import torch
+ if getattr(torch.version, "hip", None):
+ return None
+ devices = {p.device for p in module.parameters()}
+ except Exception: # noqa: BLE001
+ return None
+ if len(devices) != 1:
+ return None
+ dev = next(iter(devices))
+ return dev if dev.type == "cuda" else None
+
+
+def run_on_first_call(module: Any, key: str, fn: Any) -> None:
+ """Run ``fn(module)`` once, eagerly, right before ``module``'s first forward (after load-time placement moved
+ the weights, before the regional compile traces the blocks and before any CUDA-graph capture). Idempotent per key."""
+ hooks = module.__dict__.setdefault("_unsloth_first_call_hooks", {})
+ if key in hooks:
+ return
+
+ def _pre_hook(mod, args, kwargs):
+ handle = hooks.pop(key, None)
+ if handle is not None:
+ handle.remove()
+ try:
+ import torch
+
+ # Studio renders under inference_mode, where detaching / viewing a torchao weight subclass raises
+ # "Cannot set version_counter for inference tensor" and the fused SwiGLU / QKV silently stayed off.
+ with torch.inference_mode(False), torch.no_grad():
+ fn(mod)
+ except Exception: # noqa: BLE001 - an optimisation: the stock path stays
+ pass
+ return None
+
+ hooks[key] = module.register_forward_pre_hook(_pre_hook, with_kwargs = True)
+
+
+def cancel_first_call(module: Any, key: str) -> None:
+ """Drop a ``run_on_first_call`` hook that has not fired yet (no-op if it has, or was never registered)."""
+ hooks = getattr(module, "__dict__", {}).get("_unsloth_first_call_hooks")
+ handle = hooks.pop(key, None) if hooks else None
+ if handle is not None:
+ handle.remove()
+
+
+def install(
+ transformer: Any,
+ logger: Any = None,
+ offload_active: bool = False,
+) -> int:
+ """Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""
+ if int8_fused_disabled() or transformer is None or offload_active:
+ return 0
+ if resident_cuda_device(transformer) is not None:
+ return _finalize(transformer, logger)
+ candidates = sum(
+ 1
+ for m in transformer.modules()
+ if _ff_eligible(m) or _flux_single_eligible(m) or _swiglu_candidate(m)
+ )
+ if candidates:
+ run_on_first_call(transformer, "int8_fused", lambda t: _finalize(t, logger))
+ return candidates
+
+
+def _swiglu_candidate(module: Any) -> bool:
+ return (
+ _swiglu_spec(module) is not None or _split_spec(module) is not None
+ ) and _swiglu_layout_allowed(module)
+
+
+def _swiglu_eligible(module: Any) -> bool:
+ """The checks ``_prepare_swiglu`` makes, without fusing anything."""
+ from torch import nn
+
+ if not _swiglu_layout_allowed(module):
+ return False
+ try:
+ spec = _swiglu_spec(module)
+ if spec is not None:
+ parts = (_get(module, spec[0]), _get(module, spec[2]))
+ else:
+ split = _split_spec(module)
+ if split is None:
+ return False
+ parts = tuple(_get(module, name) for name in split)
+ return all(type(m) is nn.Linear and _plain_int8_weight(m.weight) for m in parts)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _has_eligible(transformer: Any) -> bool:
+ return any(
+ _MARK in m.__dict__ or _ff_eligible(m) or _swiglu_eligible(m) or _flux_single_eligible(m)
+ for m in transformer.modules()
+ )
+
+
+def _finalize(transformer: Any, logger: Any = None) -> int:
+ global _OP_HANDLE
+ dev = resident_cuda_device(transformer)
+ if dev is None:
+ return 0
+ # Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.
+ if not _has_eligible(transformer):
+ return 0
+ import torch
+
+ if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):
+ return 0
+ count = 0
+ rearm = False
+ with _LOCK:
+ _OP_HANDLE = _op()
+ for _name, module in transformer.named_modules():
+ if _MARK in module.__dict__:
+ count += 1
+ continue
+ if _ff_eligible(module):
+ fn = _ff_forward
+ elif _prepare_swiglu(module):
+ fn = _swiglu_forward
+ elif _flux_single_eligible(module):
+ fn = _flux_single_forward
+ module._unsloth_i8_addcmul = _flux_single_class_is_arch_patched(module)
+ else:
+ continue
+ bound = types.MethodType(fn, module)
+ slot = _hooked_forward_slot(module)
+ if slot is not None:
+ # A step cache (FBCache / MagCache) engaged before the speed layer: its hook owns the instance forward,
+ # so the swap goes under it. Overwriting the wrapper would drop the skip logic and the tail residuals.
+ ref, attr = slot
+ stock_inner = _drop_compiled_inner(module)
+ module.__dict__[_MARK] = (
+ stock_inner if stock_inner is not None else getattr(ref, attr)
+ )
+ setattr(ref, attr, bound)
+ rearm = rearm or stock_inner is not None
+ else:
+ module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
+ module.forward = bound
+ count += 1
+ if rearm:
+ try:
+ from .diffusion_cache import _compile_hooked_block_inners
+ _compile_hooked_block_inners(transformer, logger)
+ except Exception: # noqa: BLE001 - the fused inner still runs, eager
+ pass
+ if logger is not None and count:
+ logger.info(
+ "diffusion.int8_fused: %d int8 MLP region(s) run the fused dequant/activation/quant kernel",
+ count,
+ )
+ return count
+
+
+def _hooked_forward_slot(module: Any) -> Optional[tuple]:
+ """(fn_ref, attribute) holding the module's own forward in the innermost ``HookFunctionReference``, else None."""
+ registry = getattr(module, "_diffusers_hook", None)
+ fn_refs = getattr(registry, "_fn_refs", None)
+ if not fn_refs or "forward" not in getattr(module, "__dict__", {}):
+ return None
+ ref = fn_refs[0]
+ if getattr(ref, "original_forward", None) is not None:
+ return ref, "original_forward"
+ if getattr(ref, "forward", None) is not None:
+ return ref, "forward"
+ return None
+
+
+def _is_fused_forward(fn: Any, module: Any) -> bool:
+ return getattr(fn, "__self__", None) is module and getattr(fn, "__func__", None) in (
+ _ff_forward,
+ _swiglu_forward,
+ _flux_single_forward,
+ )
+
+
+def _class_forward(module: Any) -> Any:
+ return types.MethodType(type(module).forward, module)
+
+
+def _cache_hooks(module: Any) -> list:
+ try:
+ from .diffusion_cache import _CACHE_HOOK_NAMES
+ except Exception: # noqa: BLE001
+ return []
+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
+ return [hooks[name] for name in _CACHE_HOOK_NAMES if name in hooks]
+
+
+def _drop_compiled_inner(module: Any) -> Any:
+ """Drop a cache hook's compiled wrapper of the STOCK inner forward (deferred install); return that inner, else None."""
+ stock = None
+ for hook in _cache_hooks(module):
+ inner = getattr(hook, "_unsloth_orig_inner", None)
+ if inner is not None:
+ hook._unsloth_orig_inner = None
+ stock = inner
+ return stock
+
+
+def _disarm_fused_inner(module: Any, prev: Any) -> None:
+ """Uninstall under an armed cache hook: its compiled inner wraps the fused forward, point it back at stock."""
+ for hook in _cache_hooks(module):
+ inner = getattr(hook, "_unsloth_orig_inner", None)
+ if inner is not None and _is_fused_forward(inner, module):
+ stock = prev if prev is not _NO_PREV else _class_forward(module)
+ hook._unsloth_orig_inner = None
+ try:
+ hook.fn_ref.original_forward = stock
+ except Exception: # noqa: BLE001
+ pass
+
+
+def uninstall(transformer: Any = None) -> None:
+ """Restore the stock forwards under ``transformer`` (a dropped transformer needs nothing: the patch is per instance)."""
+ if transformer is None:
+ return
+ # A deferred install still pending (weights not yet on the GPU) must not fire at the next forward.
+ cancel_first_call(transformer, "int8_fused")
+ with _LOCK:
+ for module in transformer.modules():
+ prev = module.__dict__.pop(_MARK, None)
+ if prev is None:
+ continue
+ slot = _hooked_forward_slot(module)
+ if slot is not None and _is_fused_forward(getattr(*slot), module):
+ # Swapped under a hook, or a cache engaged after the swap: restore the hook's inner, keep its wrapper.
+ setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))
+ elif _is_fused_forward(module.__dict__.get("forward"), module):
+ if prev is _NO_PREV:
+ module.__dict__.pop("forward", None)
+ else:
+ module.forward = prev
+ _disarm_fused_inner(module, prev)
+ module.__dict__.pop("_unsloth_i8_addcmul", None)
+ module.__dict__.pop(_SWIGLU_ATTR, None)
+
+
+def is_installed(module: Any) -> bool:
+ return _MARK in getattr(module, "__dict__", {})
diff --git a/studio/backend/core/inference/diffusion_qwenimage_rope.py b/studio/backend/core/inference/diffusion_qwenimage_rope.py
new file mode 100644
index 00000000000..41a53a309d7
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_qwenimage_rope.py
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image RoPE in real arithmetic (Inductor cannot lower the complex multiply): points
+``ROPE_PER_DEVICE["cuda"]`` at ``diffusion_qwenimage21_rope``'s probed fma wrapper, bit-identical to the complex product.
+Kill switch: ``UNSLOTH_DIFFUSION_QWEN_REAL_ROPE=0`` (``UNSLOTH_DIFFUSION_Q21_REAL_ROPE=0`` also disables it).
+"""
+
+from __future__ import annotations
+
+import functools
+import os
+import threading
+from typing import Any
+
+QWEN_REAL_ROPE_ENV = "UNSLOTH_DIFFUSION_QWEN_REAL_ROPE"
+_MODULE = "diffusers.models.transformers.transformer_qwenimage"
+_FINGERPRINT = frozenset({"ba81dd3fc907c23a"}) # apply_rotary_emb_qwen, shared with Qwen-Image-2.1
+_LOCK = threading.Lock()
+_STOCK: dict = {}
+
+
+def disabled() -> bool:
+ from .diffusion_qwenimage21_rope import real_rope_disabled
+ return (os.environ.get(QWEN_REAL_ROPE_ENV) or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ) or real_rope_disabled()
+
+
+def _patch_table(index: int, logger: Any = None) -> bool:
+ import importlib
+
+ import torch
+
+ from . import diffusion_qwenimage21_rope as q21
+
+ if not q21.inductor_addcmul_is_fma():
+ return False
+ mod = importlib.import_module(_MODULE)
+ table = getattr(mod, "ROPE_PER_DEVICE", None)
+ entry = table.get("cuda") if isinstance(table, dict) else None
+ if entry is None:
+ return False
+ with _LOCK:
+ stock = getattr(entry, "func", None)
+ patched = getattr(stock, "__unsloth_real_rope__", False)
+ if not patched and (
+ stock is None
+ or dict(getattr(entry, "keywords", {}) or {}) != {"use_real": False}
+ or q21._digest(stock) not in _FINGERPRINT
+ ):
+ if logger is not None:
+ logger.info("diffusion.qwenimage_rope: complex RoPE kept: RoPE table entry differs")
+ return False
+ # Per device, even once the table is patched: the wrapper keeps the complex path on an unprobed index.
+ if index not in q21._FUSION:
+ q21._FUSION[index] = q21.probe_fusion(torch.device("cuda", index))
+ if q21._FUSION[index] is None:
+ return False
+ if patched:
+ return True
+ q21._NEEDS_EMULATE[0] = q21._addcmul_lowering()[1]
+ wrapper = q21._WRAPPERS.get(stock)
+ if wrapper is None:
+ wrapper = q21._WRAPPERS[stock] = q21._make_rope(stock)
+ _STOCK[id(table)] = (table, entry)
+ table["cuda"] = functools.partial(wrapper, use_real = False)
+ if logger is not None:
+ logger.info(
+ "diffusion.qwenimage_rope: RoPE runs in real arithmetic inside the compiled blocks"
+ )
+ return True
+
+
+def install(transformer: Any, logger: Any = None) -> bool:
+ """Before the first compile; patches at the first forward if the DiT is not yet on the GPU."""
+ if disabled():
+ uninstall() # a previous load may have patched the process-global table
+ return False
+ if type(transformer).__name__ != "QwenImageTransformer2DModel":
+ return False
+ from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
+
+ def _finish(t: Any) -> bool:
+ dev = resident_cuda_device(t)
+ if dev is None:
+ return False
+ import torch
+
+ return _patch_table(
+ dev.index if dev.index is not None else torch.cuda.current_device(), logger
+ )
+
+ if resident_cuda_device(transformer) is not None:
+ return _finish(transformer)
+ run_on_first_call(transformer, "qwenimage_rope", _finish)
+ return True
+
+
+def uninstall(transformer: Any = None) -> None:
+ """Restore the stock table entry; drop ``transformer``'s pending first-forward install."""
+ if transformer is not None:
+ from .diffusion_int8_fused import cancel_first_call
+ cancel_first_call(transformer, "qwenimage_rope")
+ with _LOCK:
+ for table, entry in list(_STOCK.values()):
+ table["cuda"] = entry
+ _STOCK.clear()
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 07f913d1e57..ff3b5c8f6c2 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -841,6 +841,24 @@ def _compile_repeated_blocks(
install_real_rope(logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "qwen-image-2.1 real rope", exc)
+ # Before the compile: the regional compile traces whatever forward the blocks carry at the first call.
+ try:
+ from .diffusion_int8_fused import install as install_int8_fused
+ install_int8_fused(transformer, logger, offload_active = offload_active)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "int8 fused mlp", exc)
+ if type(transformer).__name__ == "QwenImageTransformer2DModel":
+ try:
+ from .diffusion_qwenimage_rope import install as install_qwen_real_rope
+ install_qwen_real_rope(transformer, logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "qwen-image real rope", exc)
+ if type(transformer).__name__ == "ZImageTransformer2DModel":
+ try:
+ from .diffusion_zimage_fused import install as install_zimage_fused
+ install_zimage_fused(transformer, logger, offload_active = offload_active)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "z-image fused attention", exc)
try:
transformer.compile_repeated_blocks(**dit_kwargs)
engaged = True
diff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.py
new file mode 100644
index 00000000000..857f14876fa
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_zimage_fused.py
@@ -0,0 +1,340 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Z-Image-Turbo attention inside the regional compile: RoPE as ``addcmul`` in the card's probed fma form
+(bit-identical to the complex multiply Inductor cannot lower) and one fused QKV GEMM (exact for int8:
+per-row scales). ``to_q/to_k/to_v`` weights become views of the fused one; the fused Linear is used only
+while they are still the modules it was built from, so LoRA / later wrappers fall back to stock.
+Kill switch: ``UNSLOTH_DIFFUSION_ZIMAGE_FUSED=0``.
+"""
+
+from __future__ import annotations
+
+import functools
+import os
+import threading
+from typing import Any, Optional
+
+ZIMAGE_FUSED_ENV = "UNSLOTH_DIFFUSION_ZIMAGE_FUSED"
+_MODULE = "diffusers.models.transformers.transformer_z_image"
+_FINGERPRINTS = {"ZSingleStreamAttnProcessor.__call__": frozenset({"c87f9c27d7df695f"})}
+_QKV_ATTR = "_unsloth_fused_qkv"
+
+_LOCK = threading.Lock()
+_STATE: dict = {}
+# device index -> (even lanes, odd lanes) fma form of this card's complex multiply, or None
+_FUSION: dict = {}
+
+
+def zimage_fused_disabled() -> bool:
+ return (os.environ.get(ZIMAGE_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _stock_digest(cls: Any) -> Optional[str]:
+ from .diffusion_qwenimage21_rope import _digest
+ return _digest(getattr(cls, "__call__", None))
+
+
+def _real_rope(x: Any, freqs_cis: Any, fusion: tuple) -> Any:
+ """``view_as_real(view_as_complex(x.float()) * freqs_cis.unsqueeze(2)).flatten(3).type_as(x)`` in real ops.
+ ``x``: [B, S, H, D]; ``freqs_cis``: [B, S, D/2] complex64."""
+ import torch
+
+ xf = x.float()
+ f = torch.view_as_real(freqs_cis).unsqueeze(2)
+ cos = f[..., :1].expand(*f.shape[:-1], 2).flatten(-2)
+ sin = torch.stack([-f[..., 1], f[..., 1]], dim = -1).flatten(-2)
+ swapped = xf.unflatten(-1, (-1, 2)).flip(-1).flatten(-2)
+ on_x = torch.addcmul(swapped * sin, xf, cos) if "x" in fusion else None
+ on_s = torch.addcmul(xf * cos, swapped, sin) if "s" in fusion else None
+ if on_s is None:
+ out = on_x
+ elif on_x is None:
+ out = on_s
+ else:
+ even = torch.arange(x.shape[-1], device = x.device) % 2 == 0
+ out = torch.where(even, on_x, on_s) if fusion[0] == "x" else torch.where(even, on_s, on_x)
+ return out.type_as(x)
+
+
+def _qkv_intact(attn: Any) -> Any:
+ """The fused Linear, if ``to_q/to_k/to_v`` are still the exact modules it was built from."""
+ rec = attn.__dict__.get(_QKV_ATTR)
+ if rec is None:
+ return None
+ lin, q, k, v = rec
+ if attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:
+ return None
+ return lin
+
+
+def _make_call(stock: Any) -> Any:
+ import torch
+
+ def __call__(
+ self,
+ attn,
+ hidden_states,
+ encoder_hidden_states = None,
+ attention_mask = None,
+ freqs_cis = None,
+ ):
+ if not torch.compiler.is_compiling():
+ return stock(
+ self, attn, hidden_states, encoder_hidden_states, attention_mask, freqs_cis
+ )
+ from diffusers.models.attention_dispatch import dispatch_attention_fn
+
+ fused = _qkv_intact(attn)
+ if fused is not None:
+ query, key, value = fused(hidden_states).chunk(3, dim = -1)
+ else:
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(hidden_states)
+ value = attn.to_v(hidden_states)
+ query = query.unflatten(-1, (attn.heads, -1))
+ key = key.unflatten(-1, (attn.heads, -1))
+ value = value.unflatten(-1, (attn.heads, -1))
+ if attn.norm_q is not None:
+ query = attn.norm_q(query)
+ if attn.norm_k is not None:
+ key = attn.norm_k(key)
+ if freqs_cis is not None:
+ fusion = _FUSION.get(query.device.index) if query.is_cuda else None
+ if (
+ fusion is not None
+ and freqs_cis.dtype == torch.complex64
+ and query.shape[-1] % 2 == 0
+ ):
+ query = _real_rope(query, freqs_cis, fusion)
+ key = _real_rope(key, freqs_cis, fusion)
+ else:
+ query = _stock_rope(query, freqs_cis)
+ key = _stock_rope(key, freqs_cis)
+ dtype = query.dtype
+ query, key = query.to(dtype), key.to(dtype)
+ if attention_mask is not None and attention_mask.ndim == 2:
+ attention_mask = attention_mask[:, None, None, :]
+ hidden_states = dispatch_attention_fn(
+ query,
+ key,
+ value,
+ attn_mask = attention_mask,
+ dropout_p = 0.0,
+ is_causal = False,
+ backend = self._attention_backend,
+ parallel_config = self._parallel_config,
+ )
+ hidden_states = hidden_states.flatten(2, 3).to(dtype)
+ output = attn.to_out[0](hidden_states)
+ if len(attn.to_out) > 1:
+ output = attn.to_out[1](output)
+ return output
+
+ functools.update_wrapper(__call__, stock, assigned = ("__name__", "__doc__"), updated = ())
+ __call__.__unsloth_zimage_fused__ = True
+ return __call__
+
+
+def _stock_rope(x_in: Any, freqs_cis: Any) -> Any:
+ import torch
+ with torch.amp.autocast("cuda", enabled = False):
+ x = torch.view_as_complex(x_in.float().reshape(*x_in.shape[:-1], -1, 2))
+ x_out = torch.view_as_real(x * freqs_cis.unsqueeze(2)).flatten(3)
+ return x_out.type_as(x_in)
+
+
+def _fuse_linears(linears: list) -> Any:
+ """One ``nn.Linear`` over the concatenated outputs, or None; torchao tensors are rebuilt from their data/attribute lists."""
+ import torch
+ from torch import nn
+
+ ws = [lin.weight for lin in linears]
+ if any(type(lin) is not nn.Linear for lin in linears):
+ return None
+ has_bias = [lin.bias is not None for lin in linears]
+ if any(has_bias) and not all(has_bias):
+ return None
+ cls = type(ws[0])
+ if any(type(w) is not cls for w in ws) or any(
+ w.dtype != ws[0].dtype or w.device != ws[0].device for w in ws
+ ):
+ return None
+ if any(w.shape[1] != ws[0].shape[1] for w in ws):
+ return None
+ if cls in (torch.Tensor, nn.Parameter):
+ fused_w = torch.cat([w.detach() for w in ws], dim = 0)
+ else:
+ names = list(getattr(cls, "tensor_data_names", ()))
+ attrs = list(getattr(cls, "tensor_attribute_names", ()))
+ opt_t = list(getattr(cls, "optional_tensor_data_names", ()))
+ opt_a = list(getattr(cls, "optional_tensor_attribute_names", ()))
+ if not names:
+ return None
+ kwargs: dict = {}
+ for name in names + opt_t:
+ parts = [getattr(w, name, None) for w in ws]
+ if all(p is None for p in parts):
+ continue
+ if any(
+ p is None or p.dim() == 0 or p.shape[0] != w.shape[0] for p, w in zip(parts, ws)
+ ):
+ return None
+ kwargs[name] = torch.cat(parts, dim = 0)
+ for name in attrs + opt_a:
+ first = getattr(ws[0], name, None)
+ if any(getattr(w, name, None) != first for w in ws[1:]):
+ return None
+ kwargs[name] = first
+ try:
+ fused_w = cls(**kwargs)
+ except Exception: # noqa: BLE001 - an unknown constructor keeps the stock projections
+ return None
+ out = nn.Linear(ws[0].shape[1], sum(w.shape[0] for w in ws), bias = False, device = "meta")
+ out.weight = nn.Parameter(fused_w, requires_grad = False)
+ if all(has_bias):
+ out.bias = nn.Parameter(
+ torch.cat([lin.bias.detach() for lin in linears]), requires_grad = False
+ )
+ return out
+
+
+def _share_storage(fused: Any, linears: list) -> bool:
+ from torch import nn
+
+ start = 0
+ views = []
+ try:
+ whole = fused.weight.detach()
+ for lin in linears:
+ n = lin.weight.shape[0]
+ view = whole[start : start + n]
+ if tuple(view.shape) != tuple(lin.weight.shape) or type(view) is not type(
+ lin.weight.detach()
+ ):
+ return False
+ views.append(view)
+ start += n
+ for lin, view in zip(linears, views):
+ lin.weight = nn.Parameter(view, requires_grad = False)
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+def install(
+ transformer: Any,
+ logger: Any = None,
+ offload_active: bool = False,
+) -> dict:
+ """Before the first compile; probe + QKV fusion wait for the first forward if the weights are not on the GPU."""
+ if zimage_fused_disabled():
+ uninstall() # a previous load may have patched the process-global class
+ return {"real_rope": False, "fused_qkv": 0}
+ if type(transformer).__name__ != "ZImageTransformer2DModel":
+ return {"real_rope": False, "fused_qkv": 0}
+ if not _patch_class(logger):
+ return {"real_rope": False, "fused_qkv": 0}
+ from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
+
+ if resident_cuda_device(transformer) is not None:
+ return install_modules(transformer, logger, fuse_qkv = not offload_active)
+ run_on_first_call(
+ transformer,
+ "zimage_fused",
+ lambda t: install_modules(t, logger, fuse_qkv = not offload_active),
+ )
+ return {"real_rope": None, "fused_qkv": None}
+
+
+def _patch_class(logger: Any = None) -> bool:
+ try:
+ import importlib
+ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
+ except Exception: # noqa: BLE001
+ return False
+ with _LOCK:
+ current = cls.__dict__.get("__call__")
+ if getattr(current, "__unsloth_zimage_fused__", False):
+ return True
+ digest = _stock_digest(cls)
+ if digest not in _FINGERPRINTS["ZSingleStreamAttnProcessor.__call__"]:
+ if logger is not None:
+ logger.info(
+ "diffusion.zimage_fused: stock attention kept: processor differs (%s)", digest
+ )
+ return False
+ _STATE["stock"] = current
+ cls.__call__ = _make_call(current)
+ return True
+
+
+def install_modules(
+ root: Any,
+ logger: Any = None,
+ fuse_qkv: bool = True,
+) -> dict:
+ result = {"real_rope": False, "fused_qkv": 0}
+ if zimage_fused_disabled() or not _patch_class(logger):
+ return result
+ import importlib
+
+ import torch
+
+ from .diffusion_int8_fused import resident_cuda_device
+
+ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
+ dev = resident_cuda_device(root)
+ if dev is None:
+ return result
+ with _LOCK:
+ index = dev.index if dev.index is not None else torch.cuda.current_device()
+ if index not in _FUSION:
+ try:
+ from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion
+ _FUSION[index] = (
+ probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None
+ )
+ except Exception: # noqa: BLE001
+ _FUSION[index] = None
+ result["real_rope"] = _FUSION[index] is not None
+ for _name, module in root.named_modules():
+ if not fuse_qkv or type(getattr(module, "processor", None)) is not cls:
+ continue
+ if _QKV_ATTR in module.__dict__:
+ result["fused_qkv"] += 1
+ continue
+ parts = [getattr(module, n, None) for n in ("to_q", "to_k", "to_v")]
+ if any(p is None for p in parts):
+ continue
+ fused = _fuse_linears(parts)
+ if fused is None or not _share_storage(fused, parts):
+ continue
+ module.__dict__[_QKV_ATTR] = (fused, *parts)
+ result["fused_qkv"] += 1
+ if logger is not None:
+ logger.info(
+ "diffusion.zimage_fused: real-arithmetic RoPE %s, fused QKV on %d attention module(s)",
+ "on" if result["real_rope"] else "off (card's complex multiply not a known fma form)",
+ result["fused_qkv"],
+ )
+ return result
+
+
+def uninstall(transformer: Any = None) -> None:
+ """Restore the stock processor; drop the fused Linears (the per-row views stay valid) and pending install."""
+ if transformer is not None:
+ from .diffusion_int8_fused import cancel_first_call
+ cancel_first_call(transformer, "zimage_fused")
+ with _LOCK:
+ if "stock" in _STATE:
+ try:
+ import importlib
+ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
+ if getattr(cls.__dict__.get("__call__"), "__unsloth_zimage_fused__", False):
+ cls.__call__ = _STATE.pop("stock")
+ except Exception: # noqa: BLE001
+ pass
+ if transformer is not None:
+ for module in transformer.modules():
+ module.__dict__.pop(_QKV_ATTR, None)
diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.py
new file mode 100644
index 00000000000..2f666a23328
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_int8_fused.py
@@ -0,0 +1,665 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for ``diffusion_int8_fused.py``. The kernel / model tests need CUDA, Triton and torchao's Int8Tensor."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_int8_fused as fused
+
+torch = pytest.importorskip("torch")
+
+
+def _cuda_int8_ready() -> bool:
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ try:
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig # noqa: F401
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor # noqa: F401
+ except Exception: # noqa: BLE001
+ return False
+ return fused._device_ok(torch.cuda.current_device())
+
+
+needs_cuda = pytest.mark.skipif(
+ not _cuda_int8_ready(), reason = "needs CUDA (not ROCm), Triton and torchao Int8Tensor"
+)
+
+
+def _require_int8tensor(module):
+ """Skip when the torchao (<= 0.17) int8 config builds the legacy tensor: the fused path then keeps stock."""
+ for m in module.modules():
+ if isinstance(m, torch.nn.Linear) and type(m.weight).__name__ not in (
+ "Parameter",
+ "Tensor",
+ ):
+ if type(m.weight).__name__ != "Int8Tensor":
+ pytest.skip(f"torchao int8 config builds {type(m.weight).__name__}, not Int8Tensor")
+ return module
+ return module
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)
+ yield
+
+
+ "version, ok",
+ [
+ ("3.7.1", True),
+ ("3.2.0", True),
+ ("3.1.0", False),
+ ("2.3.1", False),
+ ("garbage", False),
+ ("4.0", True),
+ ],
+)
+def test_triton_version_gate(version, ok):
+ assert fused._triton_version_ok(version) is ok
+
+
+ "value, off", [("0", True), ("off", True), ("false", True), ("1", False), ("", False)]
+)
+def test_kill_switch(monkeypatch, value, off):
+ monkeypatch.setenv(fused.INT8_FUSED_ENV, value)
+ assert fused.int8_fused_disabled() is off
+
+
+def test_install_noop_when_disabled(monkeypatch):
+ monkeypatch.setenv(fused.INT8_FUSED_ENV, "0")
+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8))
+ assert fused.install(mod) == 0
+ assert "forward" not in mod[0].__dict__
+
+
+def test_plain_tensor_is_not_eligible():
+ assert fused._plain_int8_weight(torch.zeros(8, 8)) is False
+ assert fused._plain_int8_weight(None) is False
+
+
+def test_install_noop_on_cpu_module():
+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
+
+ ff = FeedForward(64, 64, activation_fn = "gelu-approximate")
+ assert fused.install(ff) == 0
+ assert "forward" not in ff.__dict__
+
+
+def _rand_inputs(
+ m,
+ n,
+ *,
+ ws_dtype = torch.float32,
+ bias = True,
+ seed = 0,
+):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ c = torch.randint(-(2**20), 2**20, (m, n), generator = g, dtype = torch.int32).cuda()
+ xs = (torch.rand(m, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16).float().cuda()
+ ws = (torch.rand(n, generator = g) * 1e-4 + 1e-6).to(ws_dtype).cuda()
+ b = (torch.randn(n, generator = g) * 0.1).to(torch.bfloat16).cuda() if bias else None
+ return c, xs, ws, b
+
+
+@needs_cuda
+ "m, n, ws_dtype, bias",
+ [
+ (4096, 12288, torch.float32, True), # Qwen-Image / FLUX.1 MLP
+ (4101, 3000, torch.float32, True), # odd rows, N not a multiple of the chunk
+ (512, 12288, torch.bfloat16, False), # bf16 weight scale, no bias
+ (17, 8, torch.float32, True), # smallest eligible
+ ],
+)
+def test_kernel_bit_exact_vs_eager_reference(m, n, ws_dtype, bias):
+ c, xs, ws, b = _rand_inputs(m, n, ws_dtype = ws_dtype, bias = bias)
+ q, s = fused._launch(c, xs, ws, b, None)
+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)
+ assert torch.equal(s, s_ref)
+ assert torch.equal(q, q_ref)
+
+
+@needs_cuda
+def test_kernel_small_activations_take_the_exact_amax_path():
+ # Every pre-activation negative: max|gelu| comes from the negative lobe, not gelu(max y).
+ c, xs, ws, b = _rand_inputs(256, 1024, bias = False)
+ c = -c.abs() - 1
+ q, s = fused._launch(c, xs, ws, b, None)
+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)
+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)
+
+
+@needs_cuda
[email protected]("transposed", [False, True])
+def test_kernel_prefix_segment(transposed):
+ bsz, seq, heads, hd, n = 2, 300, 4, 64, 1024
+ c, xs, ws, b = _rand_inputs(bsz * seq, n)
+ if transposed: # SDPA output layout [B, H, S, D] seen as [B, S, H, D]
+ prefix = (
+ (torch.randn(bsz, heads, seq, hd, device = "cuda") * 3).to(torch.bfloat16).transpose(1, 2)
+ )
+ else:
+ prefix = (torch.randn(bsz, seq, heads, hd, device = "cuda") * 3).to(torch.bfloat16)
+ q, s = fused._launch(c, xs, ws, b, prefix)
+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, prefix)
+ assert q.shape == (bsz * seq, heads * hd + n)
+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)
+
+
+def _quantized_ff(
+ dim = 256,
+ inner = 1024,
+ seed = 0,
+):
+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ torch.manual_seed(seed)
+ ff = (
+ FeedForward(dim, dim, mult = inner // dim, activation_fn = "gelu-approximate")
+ .cuda()
+ .to(torch.bfloat16)
+ .eval()
+ )
+ for p in ff.parameters():
+ p.data.normal_(0, 0.05)
+ quantize_(ff, Int8DynamicActivationInt8WeightConfig())
+ return _require_int8tensor(ff)
+
+
+@needs_cuda
+def test_feedforward_bit_identical_to_stock_eager():
+ ff = _quantized_ff()
+ x = torch.randn(2, 300, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = ff(x)
+ assert fused.install(ff) == 1
+ out = ff(x)
+ assert torch.equal(out, ref)
+
+
+@needs_cuda
+def test_feedforward_small_m_keeps_stock_path():
+ ff = _quantized_ff()
+ x = torch.randn(1, 8, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = ff(x)
+ fused.install(ff)
+ out = ff(x)
+ assert torch.equal(out, ref)
+
+
+@needs_cuda
+def test_feedforward_compiles_without_graph_break():
+ ff = _quantized_ff()
+ x = torch.randn(1, 512, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ eager = ff(x)
+ torch._dynamo.reset()
+ stock_compiled = torch.compile(ff, fullgraph = True)(x)
+ fused.install(ff)
+ torch._dynamo.reset()
+ out = torch.compile(ff, fullgraph = True)(x)
+ _assert_within_compile_floor(out, stock_compiled, eager)
+
+
+@needs_cuda
+def test_install_is_idempotent_and_holds_no_global_reference():
+ import gc
+ import weakref
+
+ ff = _quantized_ff()
+ assert fused.install(ff) == 1
+ assert fused.install(ff) == 1
+ ref = weakref.ref(ff)
+ del ff
+ gc.collect()
+ assert ref() is None
+
+
+@needs_cuda
+def test_uninstall_restores_stock_forward():
+ ff = _quantized_ff()
+ fused.install(ff)
+ assert "forward" in ff.__dict__
+ fused.uninstall(ff)
+ assert "forward" not in ff.__dict__
+ assert not fused.is_installed(ff)
+
+
+@needs_cuda
+def test_flux_single_block_bit_identical_to_stock_eager():
+ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ torch.manual_seed(0)
+ blk = (
+ tf.FluxSingleTransformerBlock(dim = 256, num_attention_heads = 4, attention_head_dim = 64)
+ .cuda()
+ .to(torch.bfloat16)
+ .eval()
+ )
+ for p in blk.parameters():
+ p.data.normal_(0, 0.05)
+ quantize_(
+ blk,
+ Int8DynamicActivationInt8WeightConfig(),
+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and "norm" not in fqn,
+ )
+ _require_int8tensor(blk)
+ hid = torch.randn(1, 200, 256, device = "cuda", dtype = torch.bfloat16)
+ enc = torch.randn(1, 40, 256, device = "cuda", dtype = torch.bfloat16)
+ temb = torch.randn(1, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = blk(hid, enc, temb)
+ assert fused.install(blk) == 1
+ out = blk(hid, enc, temb)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
+
+
+@needs_cuda
+ "m, n, gate_col, value_col", [(4352, 10240, 0, 10240), (300, 1024, 1024, 0), (17, 64, 0, 64)]
+)
+def test_swiglu_kernel_bit_exact_vs_eager_reference(m, n, gate_col, value_col):
+ c, xs, ws, b = _rand_inputs(m, 2 * n, bias = False)
+ q, s = fused._launch_swiglu(c, xs, ws, None, gate_col, value_col, n)
+ q_ref, s_ref = fused.reference_dq_swiglu_quant(c, xs, ws, None, gate_col, value_col, n)
+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)
+
+
+def _quantize(module):
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ for p in module.parameters():
+ p.data.normal_(0, 0.05)
+ quantize_(module, Int8DynamicActivationInt8WeightConfig())
+ return _require_int8tensor(module)
+
+
+@needs_cuda
[email protected]("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])
+def test_swiglu_mlps_bit_identical_to_stock_eager(kind, monkeypatch):
+ monkeypatch.setattr(fused, "_SWIGLU_ALL_LAYOUTS", True)
+ torch.manual_seed(0)
+ if kind == "diffusers_swiglu":
+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
+ ff = FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
+ elif kind == "zimage":
+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
+ ff = zmod.FeedForward(256, 512)
+ elif kind == "flux2":
+ f2 = pytest.importorskip("diffusers.models.transformers.transformer_flux2")
+ ff = f2.Flux2FeedForward(256, inner_dim = 512)
+ else:
+ q21 = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
+ ff = q21.QwenImage21SwiGLUFeedForward(256, 512)
+ ff = _quantize(ff.cuda().to(torch.bfloat16).eval())
+ x = torch.randn(2, 150, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = ff(x)
+ torch._dynamo.reset()
+ stock_compiled = torch.compile(ff, fullgraph = True)(x)
+ assert fused.install(ff) == 1
+ out = ff(x)
+ torch._dynamo.reset()
+ compiled = torch.compile(ff, fullgraph = True)(x)
+ assert torch.equal(out, ref)
+ _assert_within_compile_floor(compiled, stock_compiled, ref)
+
+
+def _assert_within_compile_floor(compiled, stock_compiled, eager):
+ # Inductor's own act-quant codegen is not eager-exact, so the bar is the stock compile's distance from eager.
+ floor = (stock_compiled.float() - eager.float()).abs()
+ ours = (compiled.float() - eager.float()).abs()
+ assert ours.max().item() <= 1.5 * floor.max().item() + 1e-6
+ assert ours.mean().item() <= 1.1 * floor.mean().item() + 1e-6
+
+
+@needs_cuda
+def test_cpu_placed_model_is_swapped_at_the_first_forward():
+ ff = _quantized_ff().cpu()
+ assert fused.install(ff) == 1
+ assert not fused.is_installed(ff)
+ ff = ff.cuda()
+ x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ff(x)
+ assert fused.is_installed(ff)
+ assert (
+ "_unsloth_first_call_hooks" in ff.__dict__ and not ff.__dict__["_unsloth_first_call_hooks"]
+ )
+
+
+@needs_cuda
+def test_zimage_swiglu_deferred_swap_engages_under_inference_mode():
+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
+ torch.manual_seed(0)
+ ff = _quantize(zmod.FeedForward(256, 512).to(torch.bfloat16).eval())
+ assert fused.install(ff) == 1
+ ff = ff.cuda()
+ x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ ref = type(ff).forward(ff, x)
+ out = ff(x)
+ assert fused.is_installed(ff)
+ assert torch.equal(out, ref)
+
+
+def test_offload_skips_install():
+ assert fused.install(torch.nn.Linear(8, 8), offload_active = True) == 0
+
+
+@needs_cuda
[email protected]("kind", ["gelu", "swiglu"])
+def test_convrot_linears_keep_the_stock_forward(kind, monkeypatch):
+ # MiniMax-H3's ConvRotLinear rotates the input before the GEMM; the fused _int_mm would skip it.
+ from core.inference.diffusion_convrot import _install_rotation
+
+ if kind == "gelu":
+ ff = _quantized_ff()
+ lins = (ff.net[0].proj, ff.net[2])
+ else:
+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
+
+ torch.manual_seed(0)
+ ff = _quantize(
+ FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
+ .cuda()
+ .to(torch.bfloat16)
+ .eval()
+ )
+ lins = (ff.net[0].proj, ff.net[2])
+ monkeypatch.setattr(
+ fused, "_SWIGLU_ALL_LAYOUTS", True
+ ) # the layout gate alone would already refuse diffusers SwiGLU
+ for lin in lins:
+ _install_rotation(lin, 16)
+ assert fused.install(ff) == 0
+ assert not fused.is_installed(ff)
+
+
+def _swiglu_module(kind):
+ if kind == "diffusers_swiglu":
+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
+ return FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
+ if kind == "zimage":
+ return pytest.importorskip("diffusers.models.transformers.transformer_z_image").FeedForward(
+ 256, 512
+ )
+ if kind == "flux2":
+ return pytest.importorskip(
+ "diffusers.models.transformers.transformer_flux2"
+ ).Flux2FeedForward(256, inner_dim = 512)
+ return pytest.importorskip(
+ "diffusers.models.transformers.transformer_qwenimage21"
+ ).QwenImage21SwiGLUFeedForward(256, 512)
+
+
[email protected]("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])
+def test_swiglu_quality_gate_allows_zimage_only(kind):
+ # See _SWIGLU_ALL_LAYOUTS.
+ ff = _swiglu_module(kind)
+ assert fused._swiglu_candidate(ff) is (kind == "zimage")
+ assert fused._swiglu_layout_allowed(ff) is (kind == "zimage")
+
+
+@needs_cuda
[email protected]("kind", ["zimage", "flux2", "qwenimage21"])
+def test_swiglu_install_count_follows_the_quality_gate(kind):
+ torch.manual_seed(0)
+ ff = _quantize(_swiglu_module(kind).cuda().to(torch.bfloat16).eval())
+ assert fused.install(ff) == (1 if kind == "zimage" else 0)
+ assert fused.is_installed(ff) is (kind == "zimage")
+
+
+def test_ineligible_resident_model_never_probes_the_kernels(monkeypatch):
+ # A bf16 / fp16 DiT already on the GPU has nothing to fuse: the Triton validation launch must not run.
+ probed = []
+ monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))
+ monkeypatch.setattr(fused, "_device_ok", lambda index: probed.append(index) or True)
+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.GELU(), torch.nn.Linear(8, 8))
+ assert fused.install(mod) == 0
+ assert probed == []
+ assert "forward" not in mod[0].__dict__
+
+
+def test_uninstall_cancels_the_deferred_install(monkeypatch):
+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8))
+ target = mod[0]
+ monkeypatch.setattr(fused, "_ff_eligible", lambda m: m is target)
+ finalized = []
+ monkeypatch.setattr(fused, "_finalize", lambda t, logger = None: finalized.append(t) or 1)
+ assert fused.install(mod) == 1
+ assert "int8_fused" in mod.__dict__["_unsloth_first_call_hooks"]
+ fused.uninstall(mod)
+ mod(torch.randn(2, 8))
+ assert finalized == []
+ assert "int8_fused" not in mod.__dict__.get("_unsloth_first_call_hooks", {})
+ assert not mod._forward_pre_hooks
+
+
+def _tiny_flux():
+ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
+ torch.manual_seed(0)
+ return tf.FluxTransformer2DModel(
+ patch_size = 1,
+ in_channels = 4,
+ num_layers = 1,
+ num_single_layers = 3,
+ attention_head_dim = 16,
+ num_attention_heads = 2,
+ joint_attention_dim = 32,
+ pooled_projection_dim = 32,
+ axes_dims_rope = (4, 6, 6),
+ ).eval()
+
+
+def _flux_inputs():
+ g = torch.Generator().manual_seed(1)
+ return dict(
+ hidden_states = torch.randn(1, 16, 4, generator = g),
+ encoder_hidden_states = torch.randn(1, 8, 32, generator = g),
+ pooled_projections = torch.randn(1, 32, generator = g),
+ timestep = torch.tensor([1.0]),
+ img_ids = torch.zeros(16, 3),
+ txt_ids = torch.zeros(8, 3),
+ return_dict = False,
+ )
+
+
+def _fake_cuda_install(monkeypatch, model):
+ """Real _finalize swap on CPU (fused forwards fall back off CUDA); returns the list the swapped forward appends to."""
+ calls = []
+ real = fused._flux_single_forward
+
+ def spy(self, *args, **kwargs):
+ calls.append(self)
+ return real(self, *args, **kwargs)
+
+ monkeypatch.setattr(fused, "_flux_single_forward", spy)
+ monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))
+ monkeypatch.setattr(fused, "_device_ok", lambda index: True)
+ monkeypatch.setattr(fused, "_op", lambda: None)
+ monkeypatch.setattr(fused, "_ff_eligible", lambda m: False)
+ monkeypatch.setattr(fused, "_prepare_swiglu", lambda m: False)
+ monkeypatch.setattr(fused, "_has_eligible", lambda t: True)
+ monkeypatch.setattr(
+ fused, "_flux_single_eligible", lambda m: type(m).__name__ == "FluxSingleTransformerBlock"
+ )
+ assert fused.install(model) == len(model.single_transformer_blocks)
+ return calls
+
+
+def _fbcache(model):
+ hooks = pytest.importorskip("diffusers.hooks")
+ model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))
+
+
+def _two_steps(model):
+ outs = []
+ with torch.no_grad(), model.cache_context("cond"):
+ for _ in range(2):
+ outs.append(model(**_flux_inputs())[0])
+ return outs
+
+
+def test_fused_flux_single_keeps_fbcache_hooks_installed_before(monkeypatch):
+ # Swap must go under the FBCache hooks, else the tail hook never records residuals and reuse reads None.
+ import copy
+
+ model = _tiny_flux()
+ ref_model = copy.deepcopy(model)
+ _fbcache(ref_model)
+ ref = _two_steps(ref_model)
+
+ _fbcache(model)
+ wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]
+ calls = _fake_cuda_install(monkeypatch, model)
+ out = _two_steps(model)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
+ assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers
+ assert len(calls) == len(model.single_transformer_blocks)
+
+ # Turning the cache off splices the hook's inner forward back: the fused forward must survive it.
+ model.disable_cache()
+ for b in model.single_transformer_blocks:
+ assert b.forward.__func__ is fused._flux_single_forward
+ fused.uninstall(model)
+ for b in model.single_transformer_blocks:
+ assert not fused.is_installed(b)
+ assert b.forward.__func__ is type(b).forward
+
+
+def test_fused_flux_single_then_fbcache_and_uninstall_keeps_hooks(monkeypatch):
+ import copy
+
+ model = _tiny_flux()
+ ref_model = copy.deepcopy(model)
+ _fbcache(ref_model)
+ ref = _two_steps(ref_model)
+
+ calls = _fake_cuda_install(monkeypatch, model)
+ _fbcache(model)
+ out = _two_steps(model)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
+ assert len(calls) == len(model.single_transformer_blocks)
+ wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]
+ fused.uninstall(model)
+ assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers
+ calls.clear()
+ out = _two_steps(model)
+ # Both caches now hold the first run's residuals; a fresh reference would round step 0 differently on some CPUs.
+ ref = _two_steps(ref_model)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]) and not calls
+
+
+def test_fused_flux_single_rearms_a_compiled_cache_inner(monkeypatch):
+ # Deferred swap must re-arm the hooks' compiled inner on the fused forward, not bypass it.
+ import copy
+
+ from core.inference import diffusion_cache
+
+ model = _tiny_flux()
+ ref_model = copy.deepcopy(model)
+ _fbcache(ref_model)
+ ref = _two_steps(ref_model)
+
+ compiled = []
+
+ def fake_compile(fn, **kwargs):
+ def wrapper(*args, **kw):
+ return fn(*args, **kw)
+
+ wrapper.inner = fn
+ compiled.append(fn)
+ return wrapper
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ _fbcache(model)
+ for b in model.single_transformer_blocks:
+ b._compiled_call_impl = b._call_impl # stands in for compile_repeated_blocks
+ armed = diffusion_cache._compile_hooked_block_inners(model)
+ assert armed == len(model.single_transformer_blocks)
+ calls = _fake_cuda_install(monkeypatch, model)
+ for b in model.single_transformer_blocks:
+ inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward
+ assert inner.inner.__func__ is fused._flux_single_forward
+ out = _two_steps(model)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
+ assert len(calls) == len(model.single_transformer_blocks)
+
+ fused.uninstall(model)
+ for b in model.single_transformer_blocks:
+ inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward
+ assert inner.__func__ is type(b).forward
+
+
+@needs_cuda
+def test_int8_flux_under_fbcache_renders_through_the_fused_kernel():
+ import copy
+
+ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
+ hooks = pytest.importorskip("diffusers.hooks")
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ torch.manual_seed(0)
+ model = (
+ tf.FluxTransformer2DModel(
+ patch_size = 1,
+ in_channels = 16,
+ num_layers = 1,
+ num_single_layers = 2,
+ attention_head_dim = 64,
+ num_attention_heads = 4,
+ joint_attention_dim = 64,
+ pooled_projection_dim = 64,
+ axes_dims_rope = (16, 24, 24),
+ )
+ .cuda()
+ .to(torch.bfloat16)
+ .eval()
+ )
+ for p in model.parameters():
+ p.data.normal_(0, 0.05)
+ quantize_(
+ model,
+ Int8DynamicActivationInt8WeightConfig(),
+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear)
+ and "single_transformer_blocks" in fqn
+ and "norm" not in fqn,
+ )
+ _require_int8tensor(model)
+ ref_model = copy.deepcopy(model)
+ g = torch.Generator(device = "cuda").manual_seed(1)
+ kwargs = dict(
+ hidden_states = torch.randn(1, 256, 16, device = "cuda", dtype = torch.bfloat16, generator = g),
+ encoder_hidden_states = torch.randn(
+ 1, 64, 64, device = "cuda", dtype = torch.bfloat16, generator = g
+ ),
+ pooled_projections = torch.randn(1, 64, device = "cuda", dtype = torch.bfloat16, generator = g),
+ timestep = torch.tensor([1.0], device = "cuda"),
+ img_ids = torch.zeros(256, 3, device = "cuda"),
+ txt_ids = torch.zeros(64, 3, device = "cuda"),
+ return_dict = False,
+ )
+
+ def run(m):
+ outs = []
+ with torch.no_grad(), m.cache_context("cond"):
+ for _ in range(2):
+ outs.append(m(**kwargs)[0])
+ return outs
+
+ ref_model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))
+ ref = run(ref_model)
+ model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))
+ assert fused.install(model) == 2
+ out = run(model)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
+ assert all(fused.is_installed(b) for b in model.single_transformer_blocks)
diff --git a/studio/backend/tests/test_diffusion_qwenimage_rope.py b/studio/backend/tests/test_diffusion_qwenimage_rope.py
new file mode 100644
index 00000000000..ff766fb4d8a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_qwenimage_rope.py
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for ``diffusion_qwenimage_rope.py``: the Qwen-Image RoPE table entry swap."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_qwenimage21_rope as q21
+from core.inference import diffusion_qwenimage_rope as qr
+
+torch = pytest.importorskip("torch")
+qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage")
+
+needs_cuda = pytest.mark.skipif(
+ not (
+ torch.cuda.is_available()
+ and not getattr(torch.version, "hip", None)
+ and q21.inductor_addcmul_is_fma()
+ ),
+ reason = "needs CUDA and inductor's fma addcmul lowering",
+)
+
+
[email protected](autouse = True)
+def _restore(monkeypatch):
+ monkeypatch.delenv(qr.QWEN_REAL_ROPE_ENV, raising = False)
+ yield
+ qr.uninstall()
+
+
+def test_stock_entry_fingerprint_matches():
+ entry = qmod.ROPE_PER_DEVICE["cuda"]
+ assert q21._digest(entry.func) in qr._FINGERPRINT
+ assert entry.keywords == {"use_real": False}
+
+
+def test_disabled_and_other_models_are_noops(monkeypatch):
+ assert qr.install(torch.nn.Linear(4, 4)) is False
+ monkeypatch.setenv(qr.QWEN_REAL_ROPE_ENV, "0")
+ assert qr.disabled()
+
+
+@needs_cuda
+def test_compiled_rope_bit_identical_to_complex_product():
+ index = torch.cuda.current_device()
+ stock = qmod.ROPE_PER_DEVICE["cuda"]
+ assert qr._patch_table(index)
+ patched = qmod.ROPE_PER_DEVICE["cuda"]
+ assert patched is not stock
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ x = torch.randn(2, 300, 24, 128, generator = g).cuda().to(torch.bfloat16)
+ ang = torch.rand(300, 64, generator = g) * 6.28
+ freqs = torch.polar(torch.ones_like(ang), ang).to(torch.complex64).cuda()
+ ref = stock(x, freqs)
+ torch._dynamo.reset()
+ out = torch.compile(lambda a, f: qmod.ROPE_PER_DEVICE["cuda"](a, f))(x, freqs)
+ assert torch.equal(out, ref)
+ # Eager calls keep the stock complex path.
+ assert torch.equal(patched(x, freqs), ref)
+ qr.uninstall()
+ assert qmod.ROPE_PER_DEVICE["cuda"] is stock
+
+
+def _fake_probe(monkeypatch):
+ probed = []
+ monkeypatch.setattr(q21, "inductor_addcmul_is_fma", lambda: True)
+ monkeypatch.setattr(q21, "_addcmul_lowering", lambda: (True, False))
+ monkeypatch.setattr(q21, "_FUSION", {})
+ monkeypatch.setattr(q21, "probe_fusion", lambda dev: probed.append(dev.index) or ("x", "x"))
+ return probed
+
+
+def test_every_device_is_probed_once_the_table_is_patched(monkeypatch):
+ probed = _fake_probe(monkeypatch)
+ assert qr._patch_table(0)
+ assert qr._patch_table(1)
+ assert probed == [0, 1]
+ assert q21._FUSION == {0: ("x", "x"), 1: ("x", "x")}
+
+
+class QwenImageTransformer2DModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+
[email protected]("env", [qr.QWEN_REAL_ROPE_ENV, q21.REAL_ROPE_ENV])
+def test_kill_switch_on_a_later_load_restores_the_table(monkeypatch, env):
+ _fake_probe(monkeypatch)
+ stock = qmod.ROPE_PER_DEVICE["cuda"]
+ assert qr._patch_table(0)
+ assert qmod.ROPE_PER_DEVICE["cuda"] is not stock
+ monkeypatch.setenv(env, "0")
+ assert qr.install(QwenImageTransformer2DModel()) is False
+ assert qmod.ROPE_PER_DEVICE["cuda"] is stock
diff --git a/studio/backend/tests/test_diffusion_zimage_fused.py b/studio/backend/tests/test_diffusion_zimage_fused.py
new file mode 100644
index 00000000000..3bf49476f0b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_zimage_fused.py
@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for ``diffusion_zimage_fused.py``; the bit-identity tests need CUDA and inductor's fma addcmul lowering."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_zimage_fused as zf
+
+torch = pytest.importorskip("torch")
+zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
+
+
+def _ready() -> bool:
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ from core.inference.diffusion_qwenimage21_rope import inductor_addcmul_is_fma
+ return inductor_addcmul_is_fma()
+
+
+needs_cuda = pytest.mark.skipif(
+ not _ready(), reason = "needs CUDA (not ROCm) and inductor's fma addcmul lowering"
+)
+
+
[email protected](autouse = True)
+def _restore(monkeypatch):
+ monkeypatch.delenv(zf.ZIMAGE_FUSED_ENV, raising = False)
+ yield
+ zf.uninstall()
+
+
+def test_stock_processor_fingerprint_matches_installed_diffusers():
+ assert (
+ zf._stock_digest(zmod.ZSingleStreamAttnProcessor)
+ in zf._FINGERPRINTS["ZSingleStreamAttnProcessor.__call__"]
+ )
+
+
+def test_disabled_and_wrong_model_are_noops(monkeypatch):
+ assert zf.install(torch.nn.Linear(4, 4)) == {"real_rope": False, "fused_qkv": 0}
+ monkeypatch.setenv(zf.ZIMAGE_FUSED_ENV, "0")
+ assert zf.install_modules(torch.nn.Linear(4, 4)) == {"real_rope": False, "fused_qkv": 0}
+
+
+def test_fuse_linears_rejects_mismatched_parts():
+ a, b = torch.nn.Linear(8, 8, bias = False), torch.nn.Linear(8, 8, bias = True)
+ assert zf._fuse_linears([a, b]) is None
+ assert zf._fuse_linears([a, torch.nn.Linear(4, 8, bias = False)]) is None
+
+
+def test_fuse_linears_plain_bf16_matches_and_shares_storage():
+ parts = [torch.nn.Linear(16, 8, bias = False) for _ in range(3)]
+ x = torch.randn(5, 16)
+ # One GEMM over the stacked weights: split GEMMs can round differently (AVX512 BLAS, AMD EPYC 9B45).
+ ref = torch.nn.functional.linear(x, torch.cat([p.weight for p in parts]))
+ fused = zf._fuse_linears(parts)
+ assert zf._share_storage(fused, parts)
+ assert torch.equal(fused(x), ref)
+ assert (
+ parts[1].weight.data_ptr() == fused.weight.data_ptr() + 8 * 16 * fused.weight.element_size()
+ )
+
+
+def _block(quant: str):
+ torch.manual_seed(0)
+ blk = (
+ zmod.ZImageTransformerBlock(0, 384, 4, 4, 1e-5, True, modulation = True)
+ .cuda()
+ .to(torch.bfloat16)
+ .eval()
+ )
+ for p in blk.parameters():
+ p.data.normal_(0, 0.05)
+ if quant == "int8":
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+ quantize_(
+ blk,
+ Int8DynamicActivationInt8WeightConfig(),
+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and "adaLN" not in fqn,
+ )
+ return blk
+
+
+def _inputs(seq = 96):
+ g = torch.Generator(device = "cpu").manual_seed(1)
+ x = torch.randn(1, seq, 384, generator = g).cuda().to(torch.bfloat16)
+ mask = torch.ones(1, seq, dtype = torch.bool).cuda()
+ ang = torch.rand(1, seq, 48, generator = g) * 6.28
+ freqs = torch.polar(torch.ones_like(ang), ang).to(torch.complex64).cuda()
+ adaln = torch.randn(1, 256, generator = g).cuda().to(torch.bfloat16)
+ return x, mask, freqs, adaln
+
+
+@needs_cuda
[email protected]("quant", ["bf16", "int8"])
+def test_compiled_block_stays_within_the_compile_floor(quant):
+ # Not bit-identical to stock compiled (fusion boundaries move bf16 roundings): bar = stock compile's distance from eager.
+ if quant == "int8":
+ pytest.importorskip("torchao.quantization")
+ blk = _block(quant)
+ if quant == "int8" and type(blk.attention.to_q.weight).__name__ != "Int8Tensor":
+ pytest.skip(
+ "this torchao's int8 config builds the legacy tensor, which keeps the stock projections"
+ )
+ x, mask, freqs, adaln = _inputs()
+ with torch.no_grad():
+ eager = blk(x, mask, freqs, adaln)
+ torch._dynamo.reset()
+ ref = torch.compile(blk)(x, mask, freqs, adaln)
+ res = zf.install_modules(blk)
+ assert res["fused_qkv"] == 1
+ torch._dynamo.reset()
+ out = torch.compile(blk)(x, mask, freqs, adaln)
+ floor = (ref != eager).sum().item()
+ assert (out != eager).sum().item() <= 1.1 * floor + 16
+ assert (out.float() - eager.float()).abs().max().item() <= 2 * (
+ ref.float() - eager.float()
+ ).abs().max().item() + 1e-6
+
+
+@needs_cuda
+def test_eager_calls_keep_the_stock_processor():
+ blk = _block("bf16")
+ x, mask, freqs, adaln = _inputs()
+ with torch.no_grad():
+ ref = blk(x, mask, freqs, adaln)
+ zf.install_modules(blk)
+ out = blk(x, mask, freqs, adaln)
+ assert torch.equal(out, ref)
+
+
+@needs_cuda
+def test_wrapped_projection_falls_back_to_stock_projections():
+ blk = _block("bf16")
+ zf.install_modules(blk)
+ attn = blk.attention
+ assert zf._qkv_intact(attn) is not None
+ attn.to_k = torch.nn.Sequential(attn.to_k) # stand-in for a LoRA wrapper
+ assert zf._qkv_intact(attn) is None
+
+
+class ZImageTransformer2DModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+
+def test_kill_switch_on_a_later_load_restores_the_processor(monkeypatch):
+ cls = zmod.ZSingleStreamAttnProcessor
+ stock = cls.__dict__["__call__"]
+ zf.install(ZImageTransformer2DModel())
+ assert getattr(cls.__dict__["__call__"], "__unsloth_zimage_fused__", False)
+ monkeypatch.setenv(zf.ZIMAGE_FUSED_ENV, "0")
+ assert zf.install(ZImageTransformer2DModel()) == {"real_rope": False, "fused_qkv": 0}
+ assert cls.__dict__["__call__"] is stock
+
+
+def test_model_unload_restores_the_process_global_patches(monkeypatch):
+ from core.inference import diffusion
+ from core.inference import diffusion_qwenimage21_rope as q21
+ from core.inference import diffusion_qwenimage_rope as qr
+
+ qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage")
+ monkeypatch.setattr(q21, "inductor_addcmul_is_fma", lambda: True)
+ monkeypatch.setattr(q21, "_addcmul_lowering", lambda: (True, False))
+ monkeypatch.setattr(q21, "_FUSION", {})
+ monkeypatch.setattr(q21, "probe_fusion", lambda dev: ("x", "x"))
+ rope_stock = qmod.ROPE_PER_DEVICE["cuda"]
+ call_stock = zmod.ZSingleStreamAttnProcessor.__dict__["__call__"]
+ try:
+ assert qr._patch_table(0) and zf._patch_class()
+ assert qmod.ROPE_PER_DEVICE["cuda"] is not rope_stock
+ assert zmod.ZSingleStreamAttnProcessor.__dict__["__call__"] is not call_stock
+ for name in ("clear_gpu_cache", "release_pinned_host_memory", "reclaim_host_memory"):
+ monkeypatch.setattr(diffusion, name, lambda *a, **k: None)
+ backend = diffusion.DiffusionBackend()
+ backend._state = diffusion._LoadState(object(), None, "r", "b", "cpu", "float32", False)
+ backend._unload_locked()
+ assert backend._state is None
+ assert qmod.ROPE_PER_DEVICE["cuda"] is rope_stock
+ assert zmod.ZSingleStreamAttnProcessor.__dict__["__call__"] is call_stock
+ finally:
+ qr.uninstall()
+
+
+def test_uninstall_cancels_the_deferred_module_install(monkeypatch):
+ fired = []
+ monkeypatch.setattr(zf, "install_modules", lambda t, *a, **k: fired.append(t))
+ model = ZImageTransformer2DModel()
+ zf.install(model)
+ assert "zimage_fused" in model.__dict__["_unsloth_first_call_hooks"]
+ zf.uninstall(model)
+ model(torch.randn(2, 4))
+ assert fired == [] and not model._forward_pre_hooks