Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fused int8 MLP kernels and real-arithmetic RoPE for DiT denoisers (FLUX.1 11% less GPU time per step) (#12083)
Daniel Han8f615aceae20
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 2057c25b7db..01ec92befa8 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -1738,6 +1738,18 @@ def _auto_quant_eager_reason(return reason+def _uninstall_fused_dit_patches() -> None:+ """Restore the process-global fused DiT patches so the next load honours its own kill switches."""+ try:+ from .diffusion_qwenimage_rope import uninstall as uninstall_qwen_real_rope+ from .diffusion_zimage_fused import uninstall as uninstall_zimage_fused++ uninstall_qwen_real_rope()+ uninstall_zimage_fused()+ except Exception: # noqa: BLE001 - teardown is best effort+ pass++def _clear_exception_frames(exc: BaseException) -> None:"""Release failed-call locals while preserving traceback locations."""errors, seen = [exc], set()@@ -6931,6 +6943,7 @@ class DiffusionBackend:reset_nvfp4_state()except Exception: # noqa: BLE001 - teardown is best effortpass+ _uninstall_fused_dit_patches()if eager_patched:uninstall_patches()uninstall_arch_patches()@@ -9249,6 +9262,7 @@ class DiffusionBackend:reset_nvfp4_state()except Exception: # noqa: BLE001 - teardown is best effortpass+ _uninstall_fused_dit_patches()if state.eager_patched:# Lazy import to keep diffusion.py torch-free to import.from .diffusion_eager_patches import uninstall_patchesdiff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pynew file mode 100644index 00000000000..faeb8d67151--- /dev/null+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -0,0 +1,1080 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Fused int8 activation path between two torchao int8 GEMMs of a DiT feed-forward: one Triton kernel+reads the int32 ``_int_mm`` output, applies dequant + GELU(tanh) / SwiGLU and writes int8 + the fp32 row scale.++NUMERICS: bit-identical to EAGER torchao + ATen (every bf16 rounding kept, fp contraction off so+``y * w_scale + bias`` rounds twice, row scale ``bf16(amax / 127.5)`` clamped at fp32 eps, quantizer multiplies by+the correctly rounded reciprocal). The compiled stock path is NOT eager-exact (Inductor keeps fp32 chains).++Anything but a plain dynamic symmetric per-row ``Int8Tensor`` on CUDA + Triton >= 3.2 keeps the stock path.+Kill switch: ``UNSLOTH_DIFFUSION_INT8_FUSED=0``.+"""++from __future__ import annotations++import os+import sys+import threading+import types+from functools import lru_cache+from typing import Any, Optional++INT8_FUSED_ENV = "UNSLOTH_DIFFUSION_INT8_FUSED"+_MIN_TRITON = (3, 2)+# torch._int_mm needs more than 16 rows; torchao's own safe_int_mm handles the rest, so small M keeps the stock path.+_MIN_ROWS = 17+_OP_NAMESPACE = "unsloth_studio"+_OP_NAME = "int8_dq_gelu_quant"+_OP_NAME_SWIGLU = "int8_dq_swiglu_quant"+_SWIGLU_ATTR = "_unsloth_i8_swiglu"++_LOCK = threading.Lock()+# Read by the traced forwards: dynamo must not trace into the lru_cache'd registration.+_OP_HANDLE: Any = None+# Marker on each patched module (no global registry: it would pin an unloaded transformer).+_MARK = "_unsloth_i8_fused_prev"+_NO_PREV = object()+++def int8_fused_disabled() -> bool:+ return (os.environ.get(INT8_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def _triton_version_ok(version: Optional[str] = None) -> bool:+ if version is None:+ try:+ import triton+ version = str(triton.__version__)+ except Exception: # noqa: BLE001+ return False+ import re++ match = re.match(r"(\d+)\.(\d+)", version)+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON+++@lru_cache(maxsize = 1)+def _triton_jit_toolchain_ok() -> bool:+ """Windows: Triton needs the MSVC CRT headers. Elsewhere, or if the probe fails, True."""+ if sys.platform != "win32":+ return True+ try:+ from .._msvc_env import crt_headers_reachable+ return bool(crt_headers_reachable())+ except Exception: # noqa: BLE001+ return True+++@lru_cache(maxsize = 1)+def _kernels() -> Optional[types.SimpleNamespace]:+ """Compile-on-first-use Triton kernel, or None when Triton is unavailable."""+ try:+ import triton+ import triton.language as tl+ except Exception: # noqa: BLE001 - no Triton means the stock path+ return None+ # Triton <= 3.2 resolves string annotations against module globals, not this function's locals+ globals().update(triton = triton, tl = tl)++ @triton.jit+ def _rbf16(x):+ # Round-to-nearest-even to bf16, kept in fp32 (one cvt; faster here than the integer-op form).+ return x.to(tl.bfloat16).to(tl.float32)++ @triton.jit+ def _gelu_tanh(y):+ # 0.5 * y * (1 + tanh(u)) == y * sigmoid(2u) == y / (1 + exp(-2u)), u = sqrt(2/pi) * (y + 0.044715 y^3)+ u = 0.7978845608028654 * (y + 0.044715 * (y * y * y))+ return y / (1.0 + tl.exp2(u * -2.8853900817779268))++ @triton.jit+ def _pre(+ c_ptr,+ xs,+ ws_ptr,+ b_ptr,+ offs,+ mask,+ HAS_BIAS: tl.constexpr,+ WS_FP32: tl.constexpr,+ EVICT: tl.constexpr,+ FINAL_ROUND: tl.constexpr,+ ):+ # torchao Int8Tensor linear epilogue: (int32 * x_scale).to(bf16) * w_scale (+ bias), then .to(bf16).+ # FINAL_ROUND=False leaves the last rounding to the caller (it commutes with a row max).+ c = tl.load(c_ptr + offs, mask = mask, other = 0, eviction_policy = EVICT).to(tl.float32)+ y = _rbf16(c * xs)+ y = y * tl.load(ws_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(+ tl.float32+ )+ if not WS_FP32:+ y = _rbf16(y)+ if HAS_BIAS:+ y = y + tl.load(b_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(+ tl.float32+ )+ if FINAL_ROUND:+ y = _rbf16(y)+ return y++ @triton.jit+ def dq_gelu_quant(+ c_ptr,+ xs_ptr,+ ws_ptr,+ b_ptr,+ p_ptr,+ q_ptr,+ s_ptr,+ N,+ NA,+ S,+ stride_c,+ stride_q,+ p_sb,+ p_ss,+ p_sh,+ HAS_BIAS: tl.constexpr,+ WS_FP32: tl.constexpr,+ HAS_PREFIX: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ CHUNK: tl.constexpr,+ CHUNK_A: tl.constexpr,+ ):+ row = tl.program_id(0).to(tl.int64)+ xs = tl.load(xs_ptr + row).to(tl.float32)+ base = tl.arange(0, CHUNK)+ crow = c_ptr + row * stride_c+ qrow = q_ptr + row * stride_q+ # pass 1a: max of the pre-activation. GELU is increasing for y > -0.75 and |gelu(y)| < 0.16997 for y < 0,+ # so max|bf16(gelu)| over the row is bf16(gelu(max y)) whenever that is >= 0.171875; otherwise recompute.+ vmax = tl.full((CHUNK,), float("-inf"), tl.float32)+ for k in range(0, N, CHUNK):+ offs = k + base+ m = offs < N+ y = _pre(crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_last", False)+ vmax = tl.maximum(vmax, tl.where(m, y, float("-inf")))+ # bf16 rounding is monotone, so max(round(v)) == round(max(v)): round once, after the reduction.+ amax = _rbf16(_gelu_tanh(_rbf16(tl.max(vmax, axis = 0))))+ if amax < 0.171875:+ acc = tl.zeros((CHUNK,), dtype = tl.float32)+ for k in range(0, N, CHUNK):+ offs = k + base+ m = offs < N+ g = _rbf16(+ _gelu_tanh(+ _pre(+ crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_last", True+ )+ )+ )+ acc = tl.maximum(acc, tl.where(m, tl.abs(g), 0.0))+ amax = tl.max(acc, axis = 0)+ if HAS_PREFIX:+ # FluxSingleTransformerBlock: the bf16 attention output sits in front of the GELU branch in the row.+ b_idx = row // S+ s_idx = row % S+ pbase = p_ptr + b_idx * p_sb + s_idx * p_ss+ abase = tl.arange(0, CHUNK_A)+ pacc = tl.zeros((CHUNK_A,), dtype = tl.float32)+ for k in range(0, NA, CHUNK_A):+ j = k + abase+ m = j < NA+ v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(+ tl.float32+ )+ pacc = tl.maximum(pacc, tl.abs(v))+ amax = tl.maximum(amax, tl.max(pacc, axis = 0))+ scale = _rbf16(tl.math.div_rn(amax, 127.5))+ scale = tl.maximum(scale, 1.1920928955078125e-07)+ inv = tl.math.div_rn(1.0, scale)+ tl.store(s_ptr + row, scale)+ if HAS_PREFIX:+ for k in range(0, NA, CHUNK_A):+ j = k + abase+ m = j < NA+ v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(+ tl.float32+ )+ qi = tl.extra.cuda.libdevice.nearbyint(v * inv)+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)+ tl.store(qrow + j, qi.to(tl.int8), mask = m)+ # pass 2: rebuild (the int32 row is mostly still in L2) and quantize.+ for k in range(0, N, CHUNK):+ offs = k + base+ m = offs < N+ g = _rbf16(+ _gelu_tanh(+ _pre(crow, xs, ws_ptr, b_ptr, offs, m, HAS_BIAS, WS_FP32, "evict_first", True)+ )+ )+ qi = tl.extra.cuda.libdevice.nearbyint(g * inv)+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)+ tl.store(qrow + NA + offs, qi.to(tl.int8), mask = m)++ @triton.jit+ def _silu(y):+ # ATen: x / (1 + exp(-x)) in fp32+ return y / (1.0 + tl.exp(-y))++ @triton.jit+ def _swiglu_val(+ crow,+ xs,+ ws_ptr,+ b_ptr,+ g0,+ v0,+ offs,+ m,+ HAS_BIAS: tl.constexpr,+ WS_FP32: tl.constexpr,+ EVICT: tl.constexpr,+ ):+ g = _pre(crow + g0, xs, ws_ptr + g0, b_ptr + g0, offs, m, HAS_BIAS, WS_FP32, EVICT, True)+ v = _pre(crow + v0, xs, ws_ptr + v0, b_ptr + v0, offs, m, HAS_BIAS, WS_FP32, EVICT, True)+ return _rbf16(_rbf16(_silu(g)) * v)++ @triton.jit+ def dq_swiglu_quant(+ c_ptr,+ xs_ptr,+ ws_ptr,+ b_ptr,+ h_ptr,+ q_ptr,+ s_ptr,+ N,+ G0,+ V0,+ stride_c,+ stride_q,+ HAS_BIAS: tl.constexpr,+ WS_FP32: tl.constexpr,+ CHUNK: tl.constexpr,+ ):+ # c row = the fused GEMM output; the gate half starts at column G0, the value half at V0 (both N wide).+ # Two int32 rows per output row make recomputing in pass 2 ALU-bound, so pass 1 parks the bf16 product in a+ # scratch row (still in L2 when pass 2 reads it back).+ row = tl.program_id(0).to(tl.int64)+ xs = tl.load(xs_ptr + row).to(tl.float32)+ base = tl.arange(0, CHUNK)+ crow = c_ptr + row * stride_c+ hrow = h_ptr + row * N+ acc = tl.zeros((CHUNK,), dtype = tl.float32)+ for k in range(0, N, CHUNK):+ offs = k + base+ m = offs < N+ h = _swiglu_val(+ crow, xs, ws_ptr, b_ptr, G0, V0, offs, m, HAS_BIAS, WS_FP32, "evict_first"+ )+ tl.store(hrow + offs, h.to(tl.bfloat16), mask = m, eviction_policy = "evict_last")+ acc = tl.maximum(acc, tl.where(m, tl.abs(h), 0.0))+ amax = tl.max(acc, axis = 0)+ scale = _rbf16(tl.math.div_rn(amax, 127.5))+ scale = tl.maximum(scale, 1.1920928955078125e-07)+ inv = tl.math.div_rn(1.0, scale)+ tl.store(s_ptr + row, scale)+ qrow = q_ptr + row * stride_q+ for k in range(0, N, CHUNK):+ offs = k + base+ m = offs < N+ h = tl.load(hrow + offs, mask = m, other = 0.0, eviction_policy = "evict_first").to(+ tl.float32+ )+ qi = tl.extra.cuda.libdevice.nearbyint(h * inv)+ qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)+ tl.store(qrow + offs, qi.to(tl.int8), mask = m)++ return types.SimpleNamespace(+ dq_gelu_quant = dq_gelu_quant, dq_swiglu_quant = dq_swiglu_quant, triton = triton+ )+++def _launch(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:+ """Run the kernel. ``prefix`` is None or a [B, S, H, D] view (any strides, last dim contiguous)."""+ import torch++ k = _kernels()+ m_rows, n = c.shape+ if prefix is not None:+ b, s, h, d = prefix.shape+ na = h * d+ p_sb, p_ss, p_sh = prefix.stride(0), prefix.stride(1), prefix.stride(2)+ else:+ s, d, na = 1, 1, 0+ p_sb = p_ss = p_sh = 0+ q = torch.empty((m_rows, na + n), device = c.device, dtype = torch.int8)+ scale = torch.empty((m_rows,), device = c.device, dtype = torch.float32)+ with torch.cuda.device(c.device):+ k.dq_gelu_quant[(m_rows,)](+ c,+ xs,+ ws,+ bias if bias is not None else ws,+ prefix if prefix is not None else ws,+ q,+ scale,+ n,+ na,+ s,+ c.stride(0),+ q.stride(0),+ p_sb,+ p_ss,+ p_sh,+ HAS_BIAS = bias is not None,+ WS_FP32 = ws.dtype == torch.float32,+ HAS_PREFIX = prefix is not None,+ HEAD_DIM = d,+ CHUNK = 2048,+ CHUNK_A = 1024,+ num_warps = 8,+ enable_fp_fusion = False,+ )+ return q, scale+++def _launch_swiglu(+ c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int+) -> tuple:+ """SwiGLU on the int32 output ``c`` of one fused GEMM: gate columns [gate_col, +n), value columns [value_col, +n)."""+ import torch++ k = _kernels()+ m_rows = c.shape[0]+ q = torch.empty((m_rows, n), device = c.device, dtype = torch.int8)+ scale = torch.empty((m_rows,), device = c.device, dtype = torch.float32)+ scratch = torch.empty((m_rows, n), device = c.device, dtype = torch.bfloat16)+ with torch.cuda.device(c.device):+ k.dq_swiglu_quant[(m_rows,)](+ c,+ xs,+ ws,+ bias if bias is not None else ws,+ scratch,+ q,+ scale,+ n,+ gate_col,+ value_col,+ c.stride(0),+ q.stride(0),+ HAS_BIAS = bias is not None,+ WS_FP32 = ws.dtype == torch.float32,+ CHUNK = 2048,+ num_warps = 8,+ enable_fp_fusion = False,+ )+ return q, scale+++def reference_dq_swiglu_quant(+ c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int+) -> tuple:+ """Eager mirror: two torchao epilogues (gate, value), ATen SiLU, bf16 product, torchao per-row act quant."""+ import torch+ import torch.nn.functional as F++ def part(col):+ y = (c[:, col : col + n] * xs.reshape(-1, 1)).to(torch.bfloat16) * ws[col : col + n]+ if bias is not None:+ y = y + bias[col : col + n]+ return y.to(torch.bfloat16)++ h = F.silu(part(gate_col)) * part(value_col)+ amax = torch.maximum(-h.amin(dim = 1).clamp(max = 0), h.amax(dim = 1).clamp(min = 0))+ scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(torch.float32)+ q = torch.clamp(torch.round(h * (1.0 / scale).reshape(-1, 1)), -128, 127).to(torch.int8)+ return q, scale+++@lru_cache(maxsize = 1)+def _op() -> Any:+ """The torch.library op (opaque to dynamo, CUDA-graph safe: no host sync, allocations only), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ qualname = f"{_OP_NAMESPACE}::{_OP_NAME}"+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _OP_NAME) and hasattr(ns, _OP_NAME_SWIGLU):+ return types.SimpleNamespace(+ gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU)+ )+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None: # torch < 2.4+ return None+ try:+ # Explicit schema: this module's annotations are strings (``from __future__ import annotations``) and torch is+ # imported lazily, so schema inference from them would not resolve.+ @custom_op(+ qualname,+ mutates_args = (),+ schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, Tensor? prefix) -> (Tensor, Tensor)",+ )+ def _dq_gelu_quant(c, xs, ws, bias, prefix):+ return _launch(c, xs, ws, bias, prefix)++ @_dq_gelu_quant.register_fake+ def _(c, xs, ws, bias, prefix):+ na = 0 if prefix is None else prefix.shape[-2] * prefix.shape[-1]+ return (+ c.new_empty((c.shape[0], na + c.shape[1]), dtype = torch.int8),+ c.new_empty((c.shape[0],), dtype = torch.float32),+ )++ @custom_op(+ f"{_OP_NAMESPACE}::{_OP_NAME_SWIGLU}",+ mutates_args = (),+ schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, int gate_col, int value_col, int n) -> (Tensor, Tensor)",+ )+ def _dq_swiglu_quant(c, xs, ws, bias, gate_col, value_col, n):+ return _launch_swiglu(c, xs, ws, bias, gate_col, value_col, n)++ @_dq_swiglu_quant.register_fake+ def _(c, xs, ws, bias, gate_col, value_col, n):+ return (+ c.new_empty((c.shape[0], n), dtype = torch.int8),+ c.new_empty((c.shape[0],), dtype = torch.float32),+ )+ except Exception: # noqa: BLE001 - a registration failure keeps the stock path+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE)+ return types.SimpleNamespace(gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU))+++@lru_cache(maxsize = 8)+def _device_ok(index: int) -> bool:+ """CUDA NVIDIA, Triton new enough, toolchain present, kernel compiles and matches the reference once."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return False+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ if (+ not (_triton_version_ok() and _triton_jit_toolchain_ok())+ or _kernels() is None+ or _op() is None+ ):+ return False+ try:+ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(0)+ c = torch.randint(-(2**20), 2**20, (33, 200), generator = g, dtype = torch.int32).to(dev)+ xs = (torch.rand(33, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16).float().to(dev)+ ws = (torch.rand(200, generator = g) * 1e-4 + 1e-6).to(dev)+ bias = (torch.randn(200, generator = g) * 0.1).to(torch.bfloat16).to(dev)+ q, s = _launch(c, xs, ws, bias, None)+ q_ref, s_ref = reference_dq_gelu_quant(c, xs, ws, bias, None)+ ok = bool(torch.equal(q, q_ref) and torch.equal(s, s_ref))+ q, s = _launch_swiglu(c, xs, ws, bias, 104, 0, 96)+ q_ref, s_ref = reference_dq_swiglu_quant(c, xs, ws, bias, 104, 0, 96)+ return ok and bool(torch.equal(q, q_ref) and torch.equal(s, s_ref))+ except Exception: # noqa: BLE001 - any build / launch failure keeps the stock path+ return False+++def reference_dq_gelu_quant(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:+ """Eager torch mirror of the stock chain (torchao epilogue, ATen GELU, torchao per-row act quant)."""+ import torch+ import torch.nn.functional as F++ y = (c * xs.reshape(-1, 1)).to(torch.bfloat16)+ y = y * ws.flatten()+ if bias is not None:+ y = y + bias+ g = F.gelu(y.to(torch.bfloat16), approximate = "tanh")+ if prefix is not None:+ g = torch.cat(+ [prefix.reshape(prefix.shape[0] * prefix.shape[1], -1).to(g.dtype), g], dim = -1+ )+ amax = torch.maximum(-g.amin(dim = 1).clamp(max = 0), g.amax(dim = 1).clamp(min = 0))+ scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(torch.float32)+ q = torch.clamp(torch.round(g * (1.0 / scale).reshape(-1, 1)), -128, 127).to(torch.int8)+ return q, scale+++def _plain_int8_weight(w: Any) -> bool:+ """A torchao Int8Tensor with dynamic, symmetric, per-row activation quant and nothing else attached."""+ try:+ if type(w).__name__ != "Int8Tensor":+ return False+ kw = getattr(w, "act_quant_kwargs", None)+ if kw is None or getattr(w, "act_pre_scale", None) is not None:+ return False+ if (+ getattr(w, "act_quant_scale", None) is not None+ or getattr(w, "act_quant_zero_point", None) is not None+ ):+ return False+ if getattr(kw, "reduce_range", False) or getattr(w, "reduce_range", False):+ return False+ if "SYMMETRIC" not in str(getattr(kw, "mapping_type", "")) or "ASYMMETRIC" in str(+ getattr(kw, "mapping_type", "")+ ):+ return False+ if type(getattr(kw, "granularity", None)).__name__ != "PerRow":+ return False+ qd, sc = w.qdata, w.scale+ if qd.dim() != 2 or list(getattr(w, "block_size", [])) != [1, qd.shape[1]]:+ return False+ if sc.dim() != 2 or tuple(sc.shape) != (qd.shape[0], 1):+ return False+ return qd.shape[0] % 8 == 0 and qd.shape[1] % 8 == 0+ except Exception: # noqa: BLE001+ return False+++def _act_quant(x2d: Any, weight: Any) -> tuple:+ """torchao's own dynamic activation quant for ``weight`` (the stock path's call), as (int8 rows, fp32 scale)."""+ from torchao.quantization.quantize_.common.quantize_tensor_kwargs import (+ _choose_quant_func_and_quantize_tensor,+ )++ act = _choose_quant_func_and_quantize_tensor(+ x2d,+ weight.act_quant_kwargs,+ scale = weight.act_quant_scale,+ zero_point = weight.act_quant_zero_point,+ )+ return act.qdata, act.scale+++def _int_mm(a: Any, weight: Any) -> Any:+ from torchao.kernel.intmm import safe_int_mm+ return safe_int_mm(a, weight.qdata.contiguous().t())+++def _linear_from_q(q: Any, xs: Any, weight: Any, bias: Any, out_dtype: Any) -> Any:+ """torchao Int8Tensor linear on an already-quantized activation: same ops, same order, as the stock epilogue."""+ import torch++ y = (_int_mm(q, weight) * xs.reshape(-1, 1).to(torch.float32)).to(out_dtype)+ y = y * weight.scale.flatten()+ if bias is not None:+ y = y + bias+ return y.to(out_dtype)+++def _ff_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:+ """``FeedForward.forward`` for GELU(tanh) -> Linear with plain int8 weights."""+ import torch++ proj = self.net[0].proj+ down = self.net[2]+ lead = hidden_states.shape[:-1]+ x2d = hidden_states.reshape(-1, hidden_states.shape[-1])+ if (+ args+ or kwargs+ or x2d.shape[0] < _MIN_ROWS+ or not hidden_states.is_cuda+ or hidden_states.dtype != torch.bfloat16+ ):+ return type(self).forward(self, hidden_states, *args, **kwargs)+ xq, xs = _act_quant(x2d, proj.weight)+ c = _int_mm(xq.reshape(-1, xq.shape[-1]), proj.weight)+ q, s = _OP_HANDLE.gelu(c, xs.reshape(-1), proj.weight.scale.flatten(), proj.bias, None)+ y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)+ for extra in self.net[3:]:+ y = extra(y)+ return y.reshape(*lead, y.shape[-1])+++def _flux_single_forward(+ self: Any,+ hidden_states: Any,+ encoder_hidden_states: Any,+ temb: Any,+ image_rotary_emb: Any = None,+ joint_attention_kwargs: Any = None,+) -> Any:+ """``FluxSingleTransformerBlock.forward`` with the GELU branch, the concat and ``proj_out``'s activation+ quant fused into one kernel. Mirrors the installed class forward (stock or Studio's addcmul arch patch)."""+ import torch++ text_seq_len = encoder_hidden_states.shape[1]+ rows = hidden_states.shape[0] * (hidden_states.shape[1] + text_seq_len)+ if rows < _MIN_ROWS or hidden_states.dtype != torch.bfloat16 or not hidden_states.is_cuda:+ return type(self).forward(+ self,+ hidden_states,+ encoder_hidden_states,+ temb,+ image_rotary_emb,+ joint_attention_kwargs,+ )+ hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim = 1)+ residual = hidden_states+ norm_hidden_states, gate = self.norm(hidden_states, emb = temb)+ x2d = norm_hidden_states.reshape(rows, -1)+ xq, xs = _act_quant(x2d, self.proj_mlp.weight)+ c = _int_mm(xq.reshape(rows, -1), self.proj_mlp.weight)+ joint_attention_kwargs = joint_attention_kwargs or {}+ attn_output = self.attn(+ hidden_states = norm_hidden_states,+ image_rotary_emb = image_rotary_emb,+ **joint_attention_kwargs,+ )+ heads = self.attn.heads+ prefix = attn_output.unflatten(-1, (heads, attn_output.shape[-1] // heads))+ if prefix.stride(-1) != 1:+ prefix = prefix.contiguous()+ q, s = _OP_HANDLE.gelu(+ c, xs.reshape(-1), self.proj_mlp.weight.scale.flatten(), self.proj_mlp.bias, prefix+ )+ out = _linear_from_q(q, s, self.proj_out.weight, self.proj_out.bias, hidden_states.dtype)+ out = out.reshape(*hidden_states.shape[:-1], out.shape[-1])+ gate = gate.unsqueeze(1)+ if getattr(self, "_unsloth_i8_addcmul", False):+ hidden_states = torch.addcmul(residual, gate, out)+ else:+ hidden_states = residual + gate * out+ if hidden_states.dtype == torch.float16:+ hidden_states = hidden_states.clip(-65504, 65504)+ return hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:]+++def _swiglu_spec(module: Any) -> Optional[tuple]:+ """(fused in-proj name, None, down name, width, gate column, value column) for the SwiGLU MLPs whose two halves+ already come from ONE Linear, else None."""+ name = type(module).__name__+ try:+ if name == "FeedForward" and type(module).__module__ == "diffusers.models.attention":+ from diffusers.models.activations import SwiGLU++ net = getattr(module, "net", None)+ if net is None or len(net) < 3 or type(net[0]) is not SwiGLU:+ return None+ if any(type(extra).__name__ != "Dropout" for extra in (net[1], *net[3:])):+ return None+ n = net[0].proj.out_features // 2+ # SwiGLU.forward: hidden, gate = proj(x).chunk(2) -> value first, gate second+ return ("net.0.proj", None, "net.2", n, n, 0)+ if (+ name == "Flux2FeedForward"+ and type(getattr(module, "act_fn", None)).__name__ == "Flux2SwiGLU"+ ):+ n = module.linear_in.out_features // 2+ return ("linear_in", None, "linear_out", n, 0, n) # gate first, value second (FLUX.2)+ except Exception: # noqa: BLE001+ return None+ return None+++def _split_spec(module: Any) -> Optional[tuple]:+ """Two separate projections (gate, value) + down: Z-Image FeedForward (w1, w3, w2), Qwen-Image-2.1 SwiGLU."""+ name = type(module).__name__+ mod = type(module).__module__+ if (+ name == "FeedForward"+ and mod.endswith("transformer_z_image")+ and all(hasattr(module, n) for n in ("w1", "w2", "w3"))+ ):+ return ("w1", "w3", "w2")+ if name == "QwenImage21SwiGLUFeedForward":+ return ("gate_layer", "proj", "out")+ return None+++# SwiGLU kernel is bit-exact vs eager but moved FLUX.2-klein / Qwen-Image-2.1 compiled renders further from eager bf16+# (LPIPS) than the stock path: Z-Image only until understood.+_SWIGLU_ALL_LAYOUTS = False+++def _is_zimage_ff(module: Any) -> bool:+ return _split_spec(module) == ("w1", "w3", "w2") and type(module).__module__.endswith(+ "transformer_z_image"+ )+++def _swiglu_layout_allowed(module: Any) -> bool:+ return _SWIGLU_ALL_LAYOUTS or _is_zimage_ff(module)+++def _get(module: Any, dotted: str) -> Any:+ for part in dotted.split("."):+ module = module[int(part)] if part.isdigit() else getattr(module, part)+ return module+++def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:+ """SwiGLU MLP with plain int8 weights: one act quant, one GEMM over [gate | value], fused SiLU-product quant."""+ import torch++ rec = self.__dict__.get(_SWIGLU_ATTR)+ lead = hidden_states.shape[:-1]+ x2d = hidden_states.reshape(-1, hidden_states.shape[-1])+ ok = (+ rec is not None+ and not args+ and not kwargs+ and x2d.shape[0] >= _MIN_ROWS+ and hidden_states.is_cuda+ )+ ok = ok and hidden_states.dtype == torch.bfloat16+ if ok:+ fused_in, parts, down_name, n, gate_col, value_col = rec+ ok = all(_get(self, nm) is mod for nm, mod in parts)+ if not ok:+ return type(self).forward(self, hidden_states, *args, **kwargs)+ down = _get(self, down_name)+ xq, xs = _act_quant(x2d, fused_in.weight)+ c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)+ q, s = _OP_HANDLE.swiglu(+ c, xs.reshape(-1), fused_in.weight.scale.flatten(), fused_in.bias, gate_col, value_col, n+ )+ y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)+ return y.reshape(*lead, y.shape[-1])+++def _prepare_swiglu(module: Any) -> bool:+ """Attach the (off-tree) record the SwiGLU forward reads; False keeps the stock forward."""+ from torch import nn++ if not _swiglu_layout_allowed(module):+ return False+ spec = _swiglu_spec(module)+ if spec is not None:+ in_name, _unused, down_name, n, gate_col, value_col = spec+ lin, down = _get(module, in_name), _get(module, down_name)+ if type(lin) is not nn.Linear or type(down) is not nn.Linear:+ return False+ if not (_plain_int8_weight(lin.weight) and _plain_int8_weight(down.weight)):+ return False+ module.__dict__[_SWIGLU_ATTR] = (+ lin,+ ((in_name, lin), (down_name, down)),+ down_name,+ n,+ gate_col,+ value_col,+ )+ return True+ split = _split_spec(module)+ if split is None:+ return False+ gate_name, value_name, down_name = split+ gate, value, down = _get(module, gate_name), _get(module, value_name), _get(module, down_name)+ if any(type(m) is not nn.Linear for m in (gate, value, down)):+ return False+ if not all(_plain_int8_weight(m.weight) for m in (gate, value, down)):+ return False+ from .diffusion_zimage_fused import _fuse_linears, _share_storage++ fused = _fuse_linears([gate, value])+ if fused is None or not _share_storage(fused, [gate, value]):+ return False+ n = gate.weight.shape[0]+ parts = ((gate_name, gate), (value_name, value), (down_name, down))+ module.__dict__[_SWIGLU_ATTR] = (fused, parts, down_name, n, 0, n)+ return True+++def _exact_linear(*modules: Any) -> bool:+ """Plain nn.Linear only: a subclass (ConvRotLinear) may transform the input the fused _int_mm would skip."""+ from torch import nn+ return all(type(m) is nn.Linear for m in modules)+++def _ff_eligible(module: Any) -> bool:+ try:+ from diffusers.models.activations import GELU+ from diffusers.models.attention import FeedForward+ except Exception: # noqa: BLE001+ return False+ if type(module) is not FeedForward:+ return False+ net = getattr(module, "net", None)+ if net is None or len(net) < 3:+ return False+ act, drop, down = net[0], net[1], net[2]+ if type(act) is not GELU or getattr(act, "approximate", None) != "tanh":+ return False+ if type(drop).__name__ != "Dropout" or not _exact_linear(act.proj, down):+ return False+ if any(type(extra).__name__ != "Dropout" for extra in net[3:]):+ return False+ return _plain_int8_weight(act.proj.weight) and _plain_int8_weight(down.weight)+++def _flux_single_eligible(module: Any) -> bool:+ if type(module).__name__ != "FluxSingleTransformerBlock":+ return False+ try:+ act = module.act_mlp+ if type(act).__name__ != "GELU" or getattr(act, "approximate", None) != "tanh":+ return False+ if getattr(module.attn, "heads", None) is None or not getattr(+ module.attn, "pre_only", False+ ):+ return False+ if not _exact_linear(module.proj_mlp, module.proj_out):+ return False+ return _plain_int8_weight(module.proj_mlp.weight) and _plain_int8_weight(+ module.proj_out.weight+ )+ except Exception: # noqa: BLE001+ return False+++def _flux_single_class_is_arch_patched(module: Any) -> bool:+ try:+ from . import diffusion_arch_patches as ap+ return getattr(type(module), "forward", None) is getattr(+ ap, "_flux_single_forward", object()+ )+ except Exception: # noqa: BLE001+ return False+++def resident_cuda_device(module: Any) -> Any:+ """The one CUDA device every parameter of ``module`` lives on (NVIDIA, not ROCm), else None."""+ try:+ import torch+ if getattr(torch.version, "hip", None):+ return None+ devices = {p.device for p in module.parameters()}+ except Exception: # noqa: BLE001+ return None+ if len(devices) != 1:+ return None+ dev = next(iter(devices))+ return dev if dev.type == "cuda" else None+++def run_on_first_call(module: Any, key: str, fn: Any) -> None:+ """Run ``fn(module)`` once, eagerly, right before ``module``'s first forward (after load-time placement moved+ the weights, before the regional compile traces the blocks and before any CUDA-graph capture). Idempotent per key."""+ hooks = module.__dict__.setdefault("_unsloth_first_call_hooks", {})+ if key in hooks:+ return++ def _pre_hook(mod, args, kwargs):+ handle = hooks.pop(key, None)+ if handle is not None:+ handle.remove()+ try:+ import torch++ # Studio renders under inference_mode, where detaching / viewing a torchao weight subclass raises+ # "Cannot set version_counter for inference tensor" and the fused SwiGLU / QKV silently stayed off.+ with torch.inference_mode(False), torch.no_grad():+ fn(mod)+ except Exception: # noqa: BLE001 - an optimisation: the stock path stays+ pass+ return None++ hooks[key] = module.register_forward_pre_hook(_pre_hook, with_kwargs = True)+++def cancel_first_call(module: Any, key: str) -> None:+ """Drop a ``run_on_first_call`` hook that has not fired yet (no-op if it has, or was never registered)."""+ hooks = getattr(module, "__dict__", {}).get("_unsloth_first_call_hooks")+ handle = hooks.pop(key, None) if hooks else None+ if handle is not None:+ handle.remove()+++def install(+ transformer: Any,+ logger: Any = None,+ offload_active: bool = False,+) -> int:+ """Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""+ if int8_fused_disabled() or transformer is None or offload_active:+ return 0+ if resident_cuda_device(transformer) is not None:+ return _finalize(transformer, logger)+ candidates = sum(+ 1+ for m in transformer.modules()+ if _ff_eligible(m) or _flux_single_eligible(m) or _swiglu_candidate(m)+ )+ if candidates:+ run_on_first_call(transformer, "int8_fused", lambda t: _finalize(t, logger))+ return candidates+++def _swiglu_candidate(module: Any) -> bool:+ return (+ _swiglu_spec(module) is not None or _split_spec(module) is not None+ ) and _swiglu_layout_allowed(module)+++def _swiglu_eligible(module: Any) -> bool:+ """The checks ``_prepare_swiglu`` makes, without fusing anything."""+ from torch import nn++ if not _swiglu_layout_allowed(module):+ return False+ try:+ spec = _swiglu_spec(module)+ if spec is not None:+ parts = (_get(module, spec[0]), _get(module, spec[2]))+ else:+ split = _split_spec(module)+ if split is None:+ return False+ parts = tuple(_get(module, name) for name in split)+ return all(type(m) is nn.Linear and _plain_int8_weight(m.weight) for m in parts)+ except Exception: # noqa: BLE001+ return False+++def _has_eligible(transformer: Any) -> bool:+ return any(+ _MARK in m.__dict__ or _ff_eligible(m) or _swiglu_eligible(m) or _flux_single_eligible(m)+ for m in transformer.modules()+ )+++def _finalize(transformer: Any, logger: Any = None) -> int:+ global _OP_HANDLE+ dev = resident_cuda_device(transformer)+ if dev is None:+ return 0+ # Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.+ if not _has_eligible(transformer):+ return 0+ import torch++ if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):+ return 0+ count = 0+ rearm = False+ with _LOCK:+ _OP_HANDLE = _op()+ for _name, module in transformer.named_modules():+ if _MARK in module.__dict__:+ count += 1+ continue+ if _ff_eligible(module):+ fn = _ff_forward+ elif _prepare_swiglu(module):+ fn = _swiglu_forward+ elif _flux_single_eligible(module):+ fn = _flux_single_forward+ module._unsloth_i8_addcmul = _flux_single_class_is_arch_patched(module)+ else:+ continue+ bound = types.MethodType(fn, module)+ slot = _hooked_forward_slot(module)+ if slot is not None:+ # A step cache (FBCache / MagCache) engaged before the speed layer: its hook owns the instance forward,+ # so the swap goes under it. Overwriting the wrapper would drop the skip logic and the tail residuals.+ ref, attr = slot+ stock_inner = _drop_compiled_inner(module)+ module.__dict__[_MARK] = (+ stock_inner if stock_inner is not None else getattr(ref, attr)+ )+ setattr(ref, attr, bound)+ rearm = rearm or stock_inner is not None+ else:+ module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)+ module.forward = bound+ count += 1+ if rearm:+ try:+ from .diffusion_cache import _compile_hooked_block_inners+ _compile_hooked_block_inners(transformer, logger)+ except Exception: # noqa: BLE001 - the fused inner still runs, eager+ pass+ if logger is not None and count:+ logger.info(+ "diffusion.int8_fused: %d int8 MLP region(s) run the fused dequant/activation/quant kernel",+ count,+ )+ return count+++def _hooked_forward_slot(module: Any) -> Optional[tuple]:+ """(fn_ref, attribute) holding the module's own forward in the innermost ``HookFunctionReference``, else None."""+ registry = getattr(module, "_diffusers_hook", None)+ fn_refs = getattr(registry, "_fn_refs", None)+ if not fn_refs or "forward" not in getattr(module, "__dict__", {}):+ return None+ ref = fn_refs[0]+ if getattr(ref, "original_forward", None) is not None:+ return ref, "original_forward"+ if getattr(ref, "forward", None) is not None:+ return ref, "forward"+ return None+++def _is_fused_forward(fn: Any, module: Any) -> bool:+ return getattr(fn, "__self__", None) is module and getattr(fn, "__func__", None) in (+ _ff_forward,+ _swiglu_forward,+ _flux_single_forward,+ )+++def _class_forward(module: Any) -> Any:+ return types.MethodType(type(module).forward, module)+++def _cache_hooks(module: Any) -> list:+ try:+ from .diffusion_cache import _CACHE_HOOK_NAMES+ except Exception: # noqa: BLE001+ return []+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}+ return [hooks[name] for name in _CACHE_HOOK_NAMES if name in hooks]+++def _drop_compiled_inner(module: Any) -> Any:+ """Drop a cache hook's compiled wrapper of the STOCK inner forward (deferred install); return that inner, else None."""+ stock = None+ for hook in _cache_hooks(module):+ inner = getattr(hook, "_unsloth_orig_inner", None)+ if inner is not None:+ hook._unsloth_orig_inner = None+ stock = inner+ return stock+++def _disarm_fused_inner(module: Any, prev: Any) -> None:+ """Uninstall under an armed cache hook: its compiled inner wraps the fused forward, point it back at stock."""+ for hook in _cache_hooks(module):+ inner = getattr(hook, "_unsloth_orig_inner", None)+ if inner is not None and _is_fused_forward(inner, module):+ stock = prev if prev is not _NO_PREV else _class_forward(module)+ hook._unsloth_orig_inner = None+ try:+ hook.fn_ref.original_forward = stock+ except Exception: # noqa: BLE001+ pass+++def uninstall(transformer: Any = None) -> None:+ """Restore the stock forwards under ``transformer`` (a dropped transformer needs nothing: the patch is per instance)."""+ if transformer is None:+ return+ # A deferred install still pending (weights not yet on the GPU) must not fire at the next forward.+ cancel_first_call(transformer, "int8_fused")+ with _LOCK:+ for module in transformer.modules():+ prev = module.__dict__.pop(_MARK, None)+ if prev is None:+ continue+ slot = _hooked_forward_slot(module)+ if slot is not None and _is_fused_forward(getattr(*slot), module):+ # Swapped under a hook, or a cache engaged after the swap: restore the hook's inner, keep its wrapper.+ setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))+ elif _is_fused_forward(module.__dict__.get("forward"), module):+ if prev is _NO_PREV:+ module.__dict__.pop("forward", None)+ else:+ module.forward = prev+ _disarm_fused_inner(module, prev)+ module.__dict__.pop("_unsloth_i8_addcmul", None)+ module.__dict__.pop(_SWIGLU_ATTR, None)+++def is_installed(module: Any) -> bool:+ return _MARK in getattr(module, "__dict__", {})diff --git a/studio/backend/core/inference/diffusion_qwenimage_rope.py b/studio/backend/core/inference/diffusion_qwenimage_rope.pynew file mode 100644index 00000000000..41a53a309d7--- /dev/null+++ b/studio/backend/core/inference/diffusion_qwenimage_rope.py@@ -0,0 +1,111 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Qwen-Image RoPE in real arithmetic (Inductor cannot lower the complex multiply): points+``ROPE_PER_DEVICE["cuda"]`` at ``diffusion_qwenimage21_rope``'s probed fma wrapper, bit-identical to the complex product.+Kill switch: ``UNSLOTH_DIFFUSION_QWEN_REAL_ROPE=0`` (``UNSLOTH_DIFFUSION_Q21_REAL_ROPE=0`` also disables it).+"""++from __future__ import annotations++import functools+import os+import threading+from typing import Any++QWEN_REAL_ROPE_ENV = "UNSLOTH_DIFFUSION_QWEN_REAL_ROPE"+_MODULE = "diffusers.models.transformers.transformer_qwenimage"+_FINGERPRINT = frozenset({"ba81dd3fc907c23a"}) # apply_rotary_emb_qwen, shared with Qwen-Image-2.1+_LOCK = threading.Lock()+_STOCK: dict = {}+++def disabled() -> bool:+ from .diffusion_qwenimage21_rope import real_rope_disabled+ return (os.environ.get(QWEN_REAL_ROPE_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ) or real_rope_disabled()+++def _patch_table(index: int, logger: Any = None) -> bool:+ import importlib++ import torch++ from . import diffusion_qwenimage21_rope as q21++ if not q21.inductor_addcmul_is_fma():+ return False+ mod = importlib.import_module(_MODULE)+ table = getattr(mod, "ROPE_PER_DEVICE", None)+ entry = table.get("cuda") if isinstance(table, dict) else None+ if entry is None:+ return False+ with _LOCK:+ stock = getattr(entry, "func", None)+ patched = getattr(stock, "__unsloth_real_rope__", False)+ if not patched and (+ stock is None+ or dict(getattr(entry, "keywords", {}) or {}) != {"use_real": False}+ or q21._digest(stock) not in _FINGERPRINT+ ):+ if logger is not None:+ logger.info("diffusion.qwenimage_rope: complex RoPE kept: RoPE table entry differs")+ return False+ # Per device, even once the table is patched: the wrapper keeps the complex path on an unprobed index.+ if index not in q21._FUSION:+ q21._FUSION[index] = q21.probe_fusion(torch.device("cuda", index))+ if q21._FUSION[index] is None:+ return False+ if patched:+ return True+ q21._NEEDS_EMULATE[0] = q21._addcmul_lowering()[1]+ wrapper = q21._WRAPPERS.get(stock)+ if wrapper is None:+ wrapper = q21._WRAPPERS[stock] = q21._make_rope(stock)+ _STOCK[id(table)] = (table, entry)+ table["cuda"] = functools.partial(wrapper, use_real = False)+ if logger is not None:+ logger.info(+ "diffusion.qwenimage_rope: RoPE runs in real arithmetic inside the compiled blocks"+ )+ return True+++def install(transformer: Any, logger: Any = None) -> bool:+ """Before the first compile; patches at the first forward if the DiT is not yet on the GPU."""+ if disabled():+ uninstall() # a previous load may have patched the process-global table+ return False+ if type(transformer).__name__ != "QwenImageTransformer2DModel":+ return False+ from .diffusion_int8_fused import resident_cuda_device, run_on_first_call++ def _finish(t: Any) -> bool:+ dev = resident_cuda_device(t)+ if dev is None:+ return False+ import torch++ return _patch_table(+ dev.index if dev.index is not None else torch.cuda.current_device(), logger+ )++ if resident_cuda_device(transformer) is not None:+ return _finish(transformer)+ run_on_first_call(transformer, "qwenimage_rope", _finish)+ return True+++def uninstall(transformer: Any = None) -> None:+ """Restore the stock table entry; drop ``transformer``'s pending first-forward install."""+ if transformer is not None:+ from .diffusion_int8_fused import cancel_first_call+ cancel_first_call(transformer, "qwenimage_rope")+ with _LOCK:+ for table, entry in list(_STOCK.values()):+ table["cuda"] = entry+ _STOCK.clear()diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 07f913d1e57..ff3b5c8f6c2 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -841,6 +841,24 @@ def _compile_repeated_blocks(install_real_rope(logger)except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "qwen-image-2.1 real rope", exc)+ # Before the compile: the regional compile traces whatever forward the blocks carry at the first call.+ try:+ from .diffusion_int8_fused import install as install_int8_fused+ install_int8_fused(transformer, logger, offload_active = offload_active)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "int8 fused mlp", exc)+ if type(transformer).__name__ == "QwenImageTransformer2DModel":+ try:+ from .diffusion_qwenimage_rope import install as install_qwen_real_rope+ install_qwen_real_rope(transformer, logger)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "qwen-image real rope", exc)+ if type(transformer).__name__ == "ZImageTransformer2DModel":+ try:+ from .diffusion_zimage_fused import install as install_zimage_fused+ install_zimage_fused(transformer, logger, offload_active = offload_active)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "z-image fused attention", exc)try:transformer.compile_repeated_blocks(**dit_kwargs)engaged = Truediff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.pynew file mode 100644index 00000000000..857f14876fa--- /dev/null+++ b/studio/backend/core/inference/diffusion_zimage_fused.py@@ -0,0 +1,340 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Z-Image-Turbo attention inside the regional compile: RoPE as ``addcmul`` in the card's probed fma form+(bit-identical to the complex multiply Inductor cannot lower) and one fused QKV GEMM (exact for int8:+per-row scales). ``to_q/to_k/to_v`` weights become views of the fused one; the fused Linear is used only+while they are still the modules it was built from, so LoRA / later wrappers fall back to stock.+Kill switch: ``UNSLOTH_DIFFUSION_ZIMAGE_FUSED=0``.+"""++from __future__ import annotations++import functools+import os+import threading+from typing import Any, Optional++ZIMAGE_FUSED_ENV = "UNSLOTH_DIFFUSION_ZIMAGE_FUSED"+_MODULE = "diffusers.models.transformers.transformer_z_image"+_FINGERPRINTS = {"ZSingleStreamAttnProcessor.__call__": frozenset({"c87f9c27d7df695f"})}+_QKV_ATTR = "_unsloth_fused_qkv"++_LOCK = threading.Lock()+_STATE: dict = {}+# device index -> (even lanes, odd lanes) fma form of this card's complex multiply, or None+_FUSION: dict = {}+++def zimage_fused_disabled() -> bool:+ return (os.environ.get(ZIMAGE_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def _stock_digest(cls: Any) -> Optional[str]:+ from .diffusion_qwenimage21_rope import _digest+ return _digest(getattr(cls, "__call__", None))+++def _real_rope(x: Any, freqs_cis: Any, fusion: tuple) -> Any:+ """``view_as_real(view_as_complex(x.float()) * freqs_cis.unsqueeze(2)).flatten(3).type_as(x)`` in real ops.+ ``x``: [B, S, H, D]; ``freqs_cis``: [B, S, D/2] complex64."""+ import torch++ xf = x.float()+ f = torch.view_as_real(freqs_cis).unsqueeze(2)+ cos = f[..., :1].expand(*f.shape[:-1], 2).flatten(-2)+ sin = torch.stack([-f[..., 1], f[..., 1]], dim = -1).flatten(-2)+ swapped = xf.unflatten(-1, (-1, 2)).flip(-1).flatten(-2)+ on_x = torch.addcmul(swapped * sin, xf, cos) if "x" in fusion else None+ on_s = torch.addcmul(xf * cos, swapped, sin) if "s" in fusion else None+ if on_s is None:+ out = on_x+ elif on_x is None:+ out = on_s+ else:+ even = torch.arange(x.shape[-1], device = x.device) % 2 == 0+ out = torch.where(even, on_x, on_s) if fusion[0] == "x" else torch.where(even, on_s, on_x)+ return out.type_as(x)+++def _qkv_intact(attn: Any) -> Any:+ """The fused Linear, if ``to_q/to_k/to_v`` are still the exact modules it was built from."""+ rec = attn.__dict__.get(_QKV_ATTR)+ if rec is None:+ return None+ lin, q, k, v = rec+ if attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:+ return None+ return lin+++def _make_call(stock: Any) -> Any:+ import torch++ def __call__(+ self,+ attn,+ hidden_states,+ encoder_hidden_states = None,+ attention_mask = None,+ freqs_cis = None,+ ):+ if not torch.compiler.is_compiling():+ return stock(+ self, attn, hidden_states, encoder_hidden_states, attention_mask, freqs_cis+ )+ from diffusers.models.attention_dispatch import dispatch_attention_fn++ fused = _qkv_intact(attn)+ if fused is not None:+ query, key, value = fused(hidden_states).chunk(3, dim = -1)+ else:+ query = attn.to_q(hidden_states)+ key = attn.to_k(hidden_states)+ value = attn.to_v(hidden_states)+ query = query.unflatten(-1, (attn.heads, -1))+ key = key.unflatten(-1, (attn.heads, -1))+ value = value.unflatten(-1, (attn.heads, -1))+ if attn.norm_q is not None:+ query = attn.norm_q(query)+ if attn.norm_k is not None:+ key = attn.norm_k(key)+ if freqs_cis is not None:+ fusion = _FUSION.get(query.device.index) if query.is_cuda else None+ if (+ fusion is not None+ and freqs_cis.dtype == torch.complex64+ and query.shape[-1] % 2 == 0+ ):+ query = _real_rope(query, freqs_cis, fusion)+ key = _real_rope(key, freqs_cis, fusion)+ else:+ query = _stock_rope(query, freqs_cis)+ key = _stock_rope(key, freqs_cis)+ dtype = query.dtype+ query, key = query.to(dtype), key.to(dtype)+ if attention_mask is not None and attention_mask.ndim == 2:+ attention_mask = attention_mask[:, None, None, :]+ hidden_states = dispatch_attention_fn(+ query,+ key,+ value,+ attn_mask = attention_mask,+ dropout_p = 0.0,+ is_causal = False,+ backend = self._attention_backend,+ parallel_config = self._parallel_config,+ )+ hidden_states = hidden_states.flatten(2, 3).to(dtype)+ output = attn.to_out[0](hidden_states)+ if len(attn.to_out) > 1:+ output = attn.to_out[1](output)+ return output++ functools.update_wrapper(__call__, stock, assigned = ("__name__", "__doc__"), updated = ())+ __call__.__unsloth_zimage_fused__ = True+ return __call__+++def _stock_rope(x_in: Any, freqs_cis: Any) -> Any:+ import torch+ with torch.amp.autocast("cuda", enabled = False):+ x = torch.view_as_complex(x_in.float().reshape(*x_in.shape[:-1], -1, 2))+ x_out = torch.view_as_real(x * freqs_cis.unsqueeze(2)).flatten(3)+ return x_out.type_as(x_in)+++def _fuse_linears(linears: list) -> Any:+ """One ``nn.Linear`` over the concatenated outputs, or None; torchao tensors are rebuilt from their data/attribute lists."""+ import torch+ from torch import nn++ ws = [lin.weight for lin in linears]+ if any(type(lin) is not nn.Linear for lin in linears):+ return None+ has_bias = [lin.bias is not None for lin in linears]+ if any(has_bias) and not all(has_bias):+ return None+ cls = type(ws[0])+ if any(type(w) is not cls for w in ws) or any(+ w.dtype != ws[0].dtype or w.device != ws[0].device for w in ws+ ):+ return None+ if any(w.shape[1] != ws[0].shape[1] for w in ws):+ return None+ if cls in (torch.Tensor, nn.Parameter):+ fused_w = torch.cat([w.detach() for w in ws], dim = 0)+ else:+ names = list(getattr(cls, "tensor_data_names", ()))+ attrs = list(getattr(cls, "tensor_attribute_names", ()))+ opt_t = list(getattr(cls, "optional_tensor_data_names", ()))+ opt_a = list(getattr(cls, "optional_tensor_attribute_names", ()))+ if not names:+ return None+ kwargs: dict = {}+ for name in names + opt_t:+ parts = [getattr(w, name, None) for w in ws]+ if all(p is None for p in parts):+ continue+ if any(+ p is None or p.dim() == 0 or p.shape[0] != w.shape[0] for p, w in zip(parts, ws)+ ):+ return None+ kwargs[name] = torch.cat(parts, dim = 0)+ for name in attrs + opt_a:+ first = getattr(ws[0], name, None)+ if any(getattr(w, name, None) != first for w in ws[1:]):+ return None+ kwargs[name] = first+ try:+ fused_w = cls(**kwargs)+ except Exception: # noqa: BLE001 - an unknown constructor keeps the stock projections+ return None+ out = nn.Linear(ws[0].shape[1], sum(w.shape[0] for w in ws), bias = False, device = "meta")+ out.weight = nn.Parameter(fused_w, requires_grad = False)+ if all(has_bias):+ out.bias = nn.Parameter(+ torch.cat([lin.bias.detach() for lin in linears]), requires_grad = False+ )+ return out+++def _share_storage(fused: Any, linears: list) -> bool:+ from torch import nn++ start = 0+ views = []+ try:+ whole = fused.weight.detach()+ for lin in linears:+ n = lin.weight.shape[0]+ view = whole[start : start + n]+ if tuple(view.shape) != tuple(lin.weight.shape) or type(view) is not type(+ lin.weight.detach()+ ):+ return False+ views.append(view)+ start += n+ for lin, view in zip(linears, views):+ lin.weight = nn.Parameter(view, requires_grad = False)+ except Exception: # noqa: BLE001+ return False+ return True+++def install(+ transformer: Any,+ logger: Any = None,+ offload_active: bool = False,+) -> dict:+ """Before the first compile; probe + QKV fusion wait for the first forward if the weights are not on the GPU."""+ if zimage_fused_disabled():+ uninstall() # a previous load may have patched the process-global class+ return {"real_rope": False, "fused_qkv": 0}+ if type(transformer).__name__ != "ZImageTransformer2DModel":+ return {"real_rope": False, "fused_qkv": 0}+ if not _patch_class(logger):+ return {"real_rope": False, "fused_qkv": 0}+ from .diffusion_int8_fused import resident_cuda_device, run_on_first_call++ if resident_cuda_device(transformer) is not None:+ return install_modules(transformer, logger, fuse_qkv = not offload_active)+ run_on_first_call(+ transformer,+ "zimage_fused",+ lambda t: install_modules(t, logger, fuse_qkv = not offload_active),+ )+ return {"real_rope": None, "fused_qkv": None}+++def _patch_class(logger: Any = None) -> bool:+ try:+ import importlib+ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor+ except Exception: # noqa: BLE001+ return False+ with _LOCK:+ current = cls.__dict__.get("__call__")+ if getattr(current, "__unsloth_zimage_fused__", False):+ return True+ digest = _stock_digest(cls)+ if digest not in _FINGERPRINTS["ZSingleStreamAttnProcessor.__call__"]:+ if logger is not None:+ logger.info(+ "diffusion.zimage_fused: stock attention kept: processor differs (%s)", digest+ )+ return False+ _STATE["stock"] = current+ cls.__call__ = _make_call(current)+ return True+++def install_modules(+ root: Any,+ logger: Any = None,+ fuse_qkv: bool = True,+) -> dict:+ result = {"real_rope": False, "fused_qkv": 0}+ if zimage_fused_disabled() or not _patch_class(logger):+ return result+ import importlib++ import torch++ from .diffusion_int8_fused import resident_cuda_device++ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor+ dev = resident_cuda_device(root)+ if dev is None:+ return result+ with _LOCK:+ index = dev.index if dev.index is not None else torch.cuda.current_device()+ if index not in _FUSION:+ try:+ from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion+ _FUSION[index] = (+ probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None+ )+ except Exception: # noqa: BLE001+ _FUSION[index] = None+ result["real_rope"] = _FUSION[index] is not None+ for _name, module in root.named_modules():+ if not fuse_qkv or type(getattr(module, "processor", None)) is not cls:+ continue+ if _QKV_ATTR in module.__dict__:+ result["fused_qkv"] += 1+ continue+ parts = [getattr(module, n, None) for n in ("to_q", "to_k", "to_v")]+ if any(p is None for p in parts):+ continue+ fused = _fuse_linears(parts)+ if fused is None or not _share_storage(fused, parts):+ continue+ module.__dict__[_QKV_ATTR] = (fused, *parts)+ result["fused_qkv"] += 1+ if logger is not None:+ logger.info(+ "diffusion.zimage_fused: real-arithmetic RoPE %s, fused QKV on %d attention module(s)",+ "on" if result["real_rope"] else "off (card's complex multiply not a known fma form)",+ result["fused_qkv"],+ )+ return result+++def uninstall(transformer: Any = None) -> None:+ """Restore the stock processor; drop the fused Linears (the per-row views stay valid) and pending install."""+ if transformer is not None:+ from .diffusion_int8_fused import cancel_first_call+ cancel_first_call(transformer, "zimage_fused")+ with _LOCK:+ if "stock" in _STATE:+ try:+ import importlib+ cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor+ if getattr(cls.__dict__.get("__call__"), "__unsloth_zimage_fused__", False):+ cls.__call__ = _STATE.pop("stock")+ except Exception: # noqa: BLE001+ pass+ if transformer is not None:+ for module in transformer.modules():+ module.__dict__.pop(_QKV_ATTR, None)diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.pynew file mode 100644index 00000000000..2f666a23328--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_fused.py@@ -0,0 +1,665 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for ``diffusion_int8_fused.py``. The kernel / model tests need CUDA, Triton and torchao's Int8Tensor."""++from __future__ import annotations++import pytest++from core.inference import diffusion_int8_fused as fused++torch = pytest.importorskip("torch")+++def _cuda_int8_ready() -> bool:+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ try:+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig # noqa: F401+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor # noqa: F401+ except Exception: # noqa: BLE001+ return False+ return fused._device_ok(torch.cuda.current_device())+++needs_cuda = pytest.mark.skipif(+ not _cuda_int8_ready(), reason = "needs CUDA (not ROCm), Triton and torchao Int8Tensor"+)+++def _require_int8tensor(module):+ """Skip when the torchao (<= 0.17) int8 config builds the legacy tensor: the fused path then keeps stock."""+ for m in module.modules():+ if isinstance(m, torch.nn.Linear) and type(m.weight).__name__ not in (+ "Parameter",+ "Tensor",+ ):+ if type(m.weight).__name__ != "Int8Tensor":+ pytest.skip(f"torchao int8 config builds {type(m.weight).__name__}, not Int8Tensor")+ return module+ return module++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)+ yield+++ "version, ok",+ [+ ("3.7.1", True),+ ("3.2.0", True),+ ("3.1.0", False),+ ("2.3.1", False),+ ("garbage", False),+ ("4.0", True),+ ],+)+def test_triton_version_gate(version, ok):+ assert fused._triton_version_ok(version) is ok+++ "value, off", [("0", True), ("off", True), ("false", True), ("1", False), ("", False)]+)+def test_kill_switch(monkeypatch, value, off):+ monkeypatch.setenv(fused.INT8_FUSED_ENV, value)+ assert fused.int8_fused_disabled() is off+++def test_install_noop_when_disabled(monkeypatch):+ monkeypatch.setenv(fused.INT8_FUSED_ENV, "0")+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8))+ assert fused.install(mod) == 0+ assert "forward" not in mod[0].__dict__+++def test_plain_tensor_is_not_eligible():+ assert fused._plain_int8_weight(torch.zeros(8, 8)) is False+ assert fused._plain_int8_weight(None) is False+++def test_install_noop_on_cpu_module():+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward++ ff = FeedForward(64, 64, activation_fn = "gelu-approximate")+ assert fused.install(ff) == 0+ assert "forward" not in ff.__dict__+++def _rand_inputs(+ m,+ n,+ *,+ ws_dtype = torch.float32,+ bias = True,+ seed = 0,+):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ c = torch.randint(-(2**20), 2**20, (m, n), generator = g, dtype = torch.int32).cuda()+ xs = (torch.rand(m, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16).float().cuda()+ ws = (torch.rand(n, generator = g) * 1e-4 + 1e-6).to(ws_dtype).cuda()+ b = (torch.randn(n, generator = g) * 0.1).to(torch.bfloat16).cuda() if bias else None+ return c, xs, ws, b+++@needs_cuda+ "m, n, ws_dtype, bias",+ [+ (4096, 12288, torch.float32, True), # Qwen-Image / FLUX.1 MLP+ (4101, 3000, torch.float32, True), # odd rows, N not a multiple of the chunk+ (512, 12288, torch.bfloat16, False), # bf16 weight scale, no bias+ (17, 8, torch.float32, True), # smallest eligible+ ],+)+def test_kernel_bit_exact_vs_eager_reference(m, n, ws_dtype, bias):+ c, xs, ws, b = _rand_inputs(m, n, ws_dtype = ws_dtype, bias = bias)+ q, s = fused._launch(c, xs, ws, b, None)+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)+ assert torch.equal(s, s_ref)+ assert torch.equal(q, q_ref)+++@needs_cuda+def test_kernel_small_activations_take_the_exact_amax_path():+ # Every pre-activation negative: max|gelu| comes from the negative lobe, not gelu(max y).+ c, xs, ws, b = _rand_inputs(256, 1024, bias = False)+ c = -c.abs() - 1+ q, s = fused._launch(c, xs, ws, b, None)+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)+++@needs_cuda[email protected]("transposed", [False, True])+def test_kernel_prefix_segment(transposed):+ bsz, seq, heads, hd, n = 2, 300, 4, 64, 1024+ c, xs, ws, b = _rand_inputs(bsz * seq, n)+ if transposed: # SDPA output layout [B, H, S, D] seen as [B, S, H, D]+ prefix = (+ (torch.randn(bsz, heads, seq, hd, device = "cuda") * 3).to(torch.bfloat16).transpose(1, 2)+ )+ else:+ prefix = (torch.randn(bsz, seq, heads, hd, device = "cuda") * 3).to(torch.bfloat16)+ q, s = fused._launch(c, xs, ws, b, prefix)+ q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, prefix)+ assert q.shape == (bsz * seq, heads * hd + n)+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)+++def _quantized_ff(+ dim = 256,+ inner = 1024,+ seed = 0,+):+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ torch.manual_seed(seed)+ ff = (+ FeedForward(dim, dim, mult = inner // dim, activation_fn = "gelu-approximate")+ .cuda()+ .to(torch.bfloat16)+ .eval()+ )+ for p in ff.parameters():+ p.data.normal_(0, 0.05)+ quantize_(ff, Int8DynamicActivationInt8WeightConfig())+ return _require_int8tensor(ff)+++@needs_cuda+def test_feedforward_bit_identical_to_stock_eager():+ ff = _quantized_ff()+ x = torch.randn(2, 300, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = ff(x)+ assert fused.install(ff) == 1+ out = ff(x)+ assert torch.equal(out, ref)+++@needs_cuda+def test_feedforward_small_m_keeps_stock_path():+ ff = _quantized_ff()+ x = torch.randn(1, 8, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = ff(x)+ fused.install(ff)+ out = ff(x)+ assert torch.equal(out, ref)+++@needs_cuda+def test_feedforward_compiles_without_graph_break():+ ff = _quantized_ff()+ x = torch.randn(1, 512, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ eager = ff(x)+ torch._dynamo.reset()+ stock_compiled = torch.compile(ff, fullgraph = True)(x)+ fused.install(ff)+ torch._dynamo.reset()+ out = torch.compile(ff, fullgraph = True)(x)+ _assert_within_compile_floor(out, stock_compiled, eager)+++@needs_cuda+def test_install_is_idempotent_and_holds_no_global_reference():+ import gc+ import weakref++ ff = _quantized_ff()+ assert fused.install(ff) == 1+ assert fused.install(ff) == 1+ ref = weakref.ref(ff)+ del ff+ gc.collect()+ assert ref() is None+++@needs_cuda+def test_uninstall_restores_stock_forward():+ ff = _quantized_ff()+ fused.install(ff)+ assert "forward" in ff.__dict__+ fused.uninstall(ff)+ assert "forward" not in ff.__dict__+ assert not fused.is_installed(ff)+++@needs_cuda+def test_flux_single_block_bit_identical_to_stock_eager():+ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ torch.manual_seed(0)+ blk = (+ tf.FluxSingleTransformerBlock(dim = 256, num_attention_heads = 4, attention_head_dim = 64)+ .cuda()+ .to(torch.bfloat16)+ .eval()+ )+ for p in blk.parameters():+ p.data.normal_(0, 0.05)+ quantize_(+ blk,+ Int8DynamicActivationInt8WeightConfig(),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and "norm" not in fqn,+ )+ _require_int8tensor(blk)+ hid = torch.randn(1, 200, 256, device = "cuda", dtype = torch.bfloat16)+ enc = torch.randn(1, 40, 256, device = "cuda", dtype = torch.bfloat16)+ temb = torch.randn(1, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = blk(hid, enc, temb)+ assert fused.install(blk) == 1+ out = blk(hid, enc, temb)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])+++@needs_cuda+ "m, n, gate_col, value_col", [(4352, 10240, 0, 10240), (300, 1024, 1024, 0), (17, 64, 0, 64)]+)+def test_swiglu_kernel_bit_exact_vs_eager_reference(m, n, gate_col, value_col):+ c, xs, ws, b = _rand_inputs(m, 2 * n, bias = False)+ q, s = fused._launch_swiglu(c, xs, ws, None, gate_col, value_col, n)+ q_ref, s_ref = fused.reference_dq_swiglu_quant(c, xs, ws, None, gate_col, value_col, n)+ assert torch.equal(s, s_ref) and torch.equal(q, q_ref)+++def _quantize(module):+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ for p in module.parameters():+ p.data.normal_(0, 0.05)+ quantize_(module, Int8DynamicActivationInt8WeightConfig())+ return _require_int8tensor(module)+++@needs_cuda[email protected]("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])+def test_swiglu_mlps_bit_identical_to_stock_eager(kind, monkeypatch):+ monkeypatch.setattr(fused, "_SWIGLU_ALL_LAYOUTS", True)+ torch.manual_seed(0)+ if kind == "diffusers_swiglu":+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward+ ff = FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)+ elif kind == "zimage":+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")+ ff = zmod.FeedForward(256, 512)+ elif kind == "flux2":+ f2 = pytest.importorskip("diffusers.models.transformers.transformer_flux2")+ ff = f2.Flux2FeedForward(256, inner_dim = 512)+ else:+ q21 = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")+ ff = q21.QwenImage21SwiGLUFeedForward(256, 512)+ ff = _quantize(ff.cuda().to(torch.bfloat16).eval())+ x = torch.randn(2, 150, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = ff(x)+ torch._dynamo.reset()+ stock_compiled = torch.compile(ff, fullgraph = True)(x)+ assert fused.install(ff) == 1+ out = ff(x)+ torch._dynamo.reset()+ compiled = torch.compile(ff, fullgraph = True)(x)+ assert torch.equal(out, ref)+ _assert_within_compile_floor(compiled, stock_compiled, ref)+++def _assert_within_compile_floor(compiled, stock_compiled, eager):+ # Inductor's own act-quant codegen is not eager-exact, so the bar is the stock compile's distance from eager.+ floor = (stock_compiled.float() - eager.float()).abs()+ ours = (compiled.float() - eager.float()).abs()+ assert ours.max().item() <= 1.5 * floor.max().item() + 1e-6+ assert ours.mean().item() <= 1.1 * floor.mean().item() + 1e-6+++@needs_cuda+def test_cpu_placed_model_is_swapped_at_the_first_forward():+ ff = _quantized_ff().cpu()+ assert fused.install(ff) == 1+ assert not fused.is_installed(ff)+ ff = ff.cuda()+ x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ff(x)+ assert fused.is_installed(ff)+ assert (+ "_unsloth_first_call_hooks" in ff.__dict__ and not ff.__dict__["_unsloth_first_call_hooks"]+ )+++@needs_cuda+def test_zimage_swiglu_deferred_swap_engages_under_inference_mode():+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")+ torch.manual_seed(0)+ ff = _quantize(zmod.FeedForward(256, 512).to(torch.bfloat16).eval())+ assert fused.install(ff) == 1+ ff = ff.cuda()+ x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.inference_mode():+ ref = type(ff).forward(ff, x)+ out = ff(x)+ assert fused.is_installed(ff)+ assert torch.equal(out, ref)+++def test_offload_skips_install():+ assert fused.install(torch.nn.Linear(8, 8), offload_active = True) == 0+++@needs_cuda[email protected]("kind", ["gelu", "swiglu"])+def test_convrot_linears_keep_the_stock_forward(kind, monkeypatch):+ # MiniMax-H3's ConvRotLinear rotates the input before the GEMM; the fused _int_mm would skip it.+ from core.inference.diffusion_convrot import _install_rotation++ if kind == "gelu":+ ff = _quantized_ff()+ lins = (ff.net[0].proj, ff.net[2])+ else:+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward++ torch.manual_seed(0)+ ff = _quantize(+ FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)+ .cuda()+ .to(torch.bfloat16)+ .eval()+ )+ lins = (ff.net[0].proj, ff.net[2])+ monkeypatch.setattr(+ fused, "_SWIGLU_ALL_LAYOUTS", True+ ) # the layout gate alone would already refuse diffusers SwiGLU+ for lin in lins:+ _install_rotation(lin, 16)+ assert fused.install(ff) == 0+ assert not fused.is_installed(ff)+++def _swiglu_module(kind):+ if kind == "diffusers_swiglu":+ FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward+ return FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)+ if kind == "zimage":+ return pytest.importorskip("diffusers.models.transformers.transformer_z_image").FeedForward(+ 256, 512+ )+ if kind == "flux2":+ return pytest.importorskip(+ "diffusers.models.transformers.transformer_flux2"+ ).Flux2FeedForward(256, inner_dim = 512)+ return pytest.importorskip(+ "diffusers.models.transformers.transformer_qwenimage21"+ ).QwenImage21SwiGLUFeedForward(256, 512)++[email protected]("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])+def test_swiglu_quality_gate_allows_zimage_only(kind):+ # See _SWIGLU_ALL_LAYOUTS.+ ff = _swiglu_module(kind)+ assert fused._swiglu_candidate(ff) is (kind == "zimage")+ assert fused._swiglu_layout_allowed(ff) is (kind == "zimage")+++@needs_cuda[email protected]("kind", ["zimage", "flux2", "qwenimage21"])+def test_swiglu_install_count_follows_the_quality_gate(kind):+ torch.manual_seed(0)+ ff = _quantize(_swiglu_module(kind).cuda().to(torch.bfloat16).eval())+ assert fused.install(ff) == (1 if kind == "zimage" else 0)+ assert fused.is_installed(ff) is (kind == "zimage")+++def test_ineligible_resident_model_never_probes_the_kernels(monkeypatch):+ # A bf16 / fp16 DiT already on the GPU has nothing to fuse: the Triton validation launch must not run.+ probed = []+ monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))+ monkeypatch.setattr(fused, "_device_ok", lambda index: probed.append(index) or True)+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.GELU(), torch.nn.Linear(8, 8))+ assert fused.install(mod) == 0+ assert probed == []+ assert "forward" not in mod[0].__dict__+++def test_uninstall_cancels_the_deferred_install(monkeypatch):+ mod = torch.nn.Sequential(torch.nn.Linear(8, 8))+ target = mod[0]+ monkeypatch.setattr(fused, "_ff_eligible", lambda m: m is target)+ finalized = []+ monkeypatch.setattr(fused, "_finalize", lambda t, logger = None: finalized.append(t) or 1)+ assert fused.install(mod) == 1+ assert "int8_fused" in mod.__dict__["_unsloth_first_call_hooks"]+ fused.uninstall(mod)+ mod(torch.randn(2, 8))+ assert finalized == []+ assert "int8_fused" not in mod.__dict__.get("_unsloth_first_call_hooks", {})+ assert not mod._forward_pre_hooks+++def _tiny_flux():+ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")+ torch.manual_seed(0)+ return tf.FluxTransformer2DModel(+ patch_size = 1,+ in_channels = 4,+ num_layers = 1,+ num_single_layers = 3,+ attention_head_dim = 16,+ num_attention_heads = 2,+ joint_attention_dim = 32,+ pooled_projection_dim = 32,+ axes_dims_rope = (4, 6, 6),+ ).eval()+++def _flux_inputs():+ g = torch.Generator().manual_seed(1)+ return dict(+ hidden_states = torch.randn(1, 16, 4, generator = g),+ encoder_hidden_states = torch.randn(1, 8, 32, generator = g),+ pooled_projections = torch.randn(1, 32, generator = g),+ timestep = torch.tensor([1.0]),+ img_ids = torch.zeros(16, 3),+ txt_ids = torch.zeros(8, 3),+ return_dict = False,+ )+++def _fake_cuda_install(monkeypatch, model):+ """Real _finalize swap on CPU (fused forwards fall back off CUDA); returns the list the swapped forward appends to."""+ calls = []+ real = fused._flux_single_forward++ def spy(self, *args, **kwargs):+ calls.append(self)+ return real(self, *args, **kwargs)++ monkeypatch.setattr(fused, "_flux_single_forward", spy)+ monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))+ monkeypatch.setattr(fused, "_device_ok", lambda index: True)+ monkeypatch.setattr(fused, "_op", lambda: None)+ monkeypatch.setattr(fused, "_ff_eligible", lambda m: False)+ monkeypatch.setattr(fused, "_prepare_swiglu", lambda m: False)+ monkeypatch.setattr(fused, "_has_eligible", lambda t: True)+ monkeypatch.setattr(+ fused, "_flux_single_eligible", lambda m: type(m).__name__ == "FluxSingleTransformerBlock"+ )+ assert fused.install(model) == len(model.single_transformer_blocks)+ return calls+++def _fbcache(model):+ hooks = pytest.importorskip("diffusers.hooks")+ model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))+++def _two_steps(model):+ outs = []+ with torch.no_grad(), model.cache_context("cond"):+ for _ in range(2):+ outs.append(model(**_flux_inputs())[0])+ return outs+++def test_fused_flux_single_keeps_fbcache_hooks_installed_before(monkeypatch):+ # Swap must go under the FBCache hooks, else the tail hook never records residuals and reuse reads None.+ import copy++ model = _tiny_flux()+ ref_model = copy.deepcopy(model)+ _fbcache(ref_model)+ ref = _two_steps(ref_model)++ _fbcache(model)+ wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]+ calls = _fake_cuda_install(monkeypatch, model)+ out = _two_steps(model)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])+ assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers+ assert len(calls) == len(model.single_transformer_blocks)++ # Turning the cache off splices the hook's inner forward back: the fused forward must survive it.+ model.disable_cache()+ for b in model.single_transformer_blocks:+ assert b.forward.__func__ is fused._flux_single_forward+ fused.uninstall(model)+ for b in model.single_transformer_blocks:+ assert not fused.is_installed(b)+ assert b.forward.__func__ is type(b).forward+++def test_fused_flux_single_then_fbcache_and_uninstall_keeps_hooks(monkeypatch):+ import copy++ model = _tiny_flux()+ ref_model = copy.deepcopy(model)+ _fbcache(ref_model)+ ref = _two_steps(ref_model)++ calls = _fake_cuda_install(monkeypatch, model)+ _fbcache(model)+ out = _two_steps(model)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])+ assert len(calls) == len(model.single_transformer_blocks)+ wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]+ fused.uninstall(model)+ assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers+ calls.clear()+ out = _two_steps(model)+ # Both caches now hold the first run's residuals; a fresh reference would round step 0 differently on some CPUs.+ ref = _two_steps(ref_model)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]) and not calls+++def test_fused_flux_single_rearms_a_compiled_cache_inner(monkeypatch):+ # Deferred swap must re-arm the hooks' compiled inner on the fused forward, not bypass it.+ import copy++ from core.inference import diffusion_cache++ model = _tiny_flux()+ ref_model = copy.deepcopy(model)+ _fbcache(ref_model)+ ref = _two_steps(ref_model)++ compiled = []++ def fake_compile(fn, **kwargs):+ def wrapper(*args, **kw):+ return fn(*args, **kw)++ wrapper.inner = fn+ compiled.append(fn)+ return wrapper++ monkeypatch.setattr(torch, "compile", fake_compile)+ _fbcache(model)+ for b in model.single_transformer_blocks:+ b._compiled_call_impl = b._call_impl # stands in for compile_repeated_blocks+ armed = diffusion_cache._compile_hooked_block_inners(model)+ assert armed == len(model.single_transformer_blocks)+ calls = _fake_cuda_install(monkeypatch, model)+ for b in model.single_transformer_blocks:+ inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward+ assert inner.inner.__func__ is fused._flux_single_forward+ out = _two_steps(model)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])+ assert len(calls) == len(model.single_transformer_blocks)++ fused.uninstall(model)+ for b in model.single_transformer_blocks:+ inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward+ assert inner.__func__ is type(b).forward+++@needs_cuda+def test_int8_flux_under_fbcache_renders_through_the_fused_kernel():+ import copy++ tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")+ hooks = pytest.importorskip("diffusers.hooks")+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ torch.manual_seed(0)+ model = (+ tf.FluxTransformer2DModel(+ patch_size = 1,+ in_channels = 16,+ num_layers = 1,+ num_single_layers = 2,+ attention_head_dim = 64,+ num_attention_heads = 4,+ joint_attention_dim = 64,+ pooled_projection_dim = 64,+ axes_dims_rope = (16, 24, 24),+ )+ .cuda()+ .to(torch.bfloat16)+ .eval()+ )+ for p in model.parameters():+ p.data.normal_(0, 0.05)+ quantize_(+ model,+ Int8DynamicActivationInt8WeightConfig(),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear)+ and "single_transformer_blocks" in fqn+ and "norm" not in fqn,+ )+ _require_int8tensor(model)+ ref_model = copy.deepcopy(model)+ g = torch.Generator(device = "cuda").manual_seed(1)+ kwargs = dict(+ hidden_states = torch.randn(1, 256, 16, device = "cuda", dtype = torch.bfloat16, generator = g),+ encoder_hidden_states = torch.randn(+ 1, 64, 64, device = "cuda", dtype = torch.bfloat16, generator = g+ ),+ pooled_projections = torch.randn(1, 64, device = "cuda", dtype = torch.bfloat16, generator = g),+ timestep = torch.tensor([1.0], device = "cuda"),+ img_ids = torch.zeros(256, 3, device = "cuda"),+ txt_ids = torch.zeros(64, 3, device = "cuda"),+ return_dict = False,+ )++ def run(m):+ outs = []+ with torch.no_grad(), m.cache_context("cond"):+ for _ in range(2):+ outs.append(m(**kwargs)[0])+ return outs++ ref_model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))+ ref = run(ref_model)+ model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))+ assert fused.install(model) == 2+ out = run(model)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])+ assert all(fused.is_installed(b) for b in model.single_transformer_blocks)diff --git a/studio/backend/tests/test_diffusion_qwenimage_rope.py b/studio/backend/tests/test_diffusion_qwenimage_rope.pynew file mode 100644index 00000000000..ff766fb4d8a--- /dev/null+++ b/studio/backend/tests/test_diffusion_qwenimage_rope.py@@ -0,0 +1,97 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for ``diffusion_qwenimage_rope.py``: the Qwen-Image RoPE table entry swap."""++from __future__ import annotations++import pytest++from core.inference import diffusion_qwenimage21_rope as q21+from core.inference import diffusion_qwenimage_rope as qr++torch = pytest.importorskip("torch")+qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage")++needs_cuda = pytest.mark.skipif(+ not (+ torch.cuda.is_available()+ and not getattr(torch.version, "hip", None)+ and q21.inductor_addcmul_is_fma()+ ),+ reason = "needs CUDA and inductor's fma addcmul lowering",+)++[email protected](autouse = True)+def _restore(monkeypatch):+ monkeypatch.delenv(qr.QWEN_REAL_ROPE_ENV, raising = False)+ yield+ qr.uninstall()+++def test_stock_entry_fingerprint_matches():+ entry = qmod.ROPE_PER_DEVICE["cuda"]+ assert q21._digest(entry.func) in qr._FINGERPRINT+ assert entry.keywords == {"use_real": False}+++def test_disabled_and_other_models_are_noops(monkeypatch):+ assert qr.install(torch.nn.Linear(4, 4)) is False+ monkeypatch.setenv(qr.QWEN_REAL_ROPE_ENV, "0")+ assert qr.disabled()+++@needs_cuda+def test_compiled_rope_bit_identical_to_complex_product():+ index = torch.cuda.current_device()+ stock = qmod.ROPE_PER_DEVICE["cuda"]+ assert qr._patch_table(index)+ patched = qmod.ROPE_PER_DEVICE["cuda"]+ assert patched is not stock+ g = torch.Generator(device = "cpu").manual_seed(0)+ x = torch.randn(2, 300, 24, 128, generator = g).cuda().to(torch.bfloat16)+ ang = torch.rand(300, 64, generator = g) * 6.28+ freqs = torch.polar(torch.ones_like(ang), ang).to(torch.complex64).cuda()+ ref = stock(x, freqs)+ torch._dynamo.reset()+ out = torch.compile(lambda a, f: qmod.ROPE_PER_DEVICE["cuda"](a, f))(x, freqs)+ assert torch.equal(out, ref)+ # Eager calls keep the stock complex path.+ assert torch.equal(patched(x, freqs), ref)+ qr.uninstall()+ assert qmod.ROPE_PER_DEVICE["cuda"] is stock+++def _fake_probe(monkeypatch):+ probed = []+ monkeypatch.setattr(q21, "inductor_addcmul_is_fma", lambda: True)+ monkeypatch.setattr(q21, "_addcmul_lowering", lambda: (True, False))+ monkeypatch.setattr(q21, "_FUSION", {})+ monkeypatch.setattr(q21, "probe_fusion", lambda dev: probed.append(dev.index) or ("x", "x"))+ return probed+++def test_every_device_is_probed_once_the_table_is_patched(monkeypatch):+ probed = _fake_probe(monkeypatch)+ assert qr._patch_table(0)+ assert qr._patch_table(1)+ assert probed == [0, 1]+ assert q21._FUSION == {0: ("x", "x"), 1: ("x", "x")}+++class QwenImageTransformer2DModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++[email protected]("env", [qr.QWEN_REAL_ROPE_ENV, q21.REAL_ROPE_ENV])+def test_kill_switch_on_a_later_load_restores_the_table(monkeypatch, env):+ _fake_probe(monkeypatch)+ stock = qmod.ROPE_PER_DEVICE["cuda"]+ assert qr._patch_table(0)+ assert qmod.ROPE_PER_DEVICE["cuda"] is not stock+ monkeypatch.setenv(env, "0")+ assert qr.install(QwenImageTransformer2DModel()) is False+ assert qmod.ROPE_PER_DEVICE["cuda"] is stockdiff --git a/studio/backend/tests/test_diffusion_zimage_fused.py b/studio/backend/tests/test_diffusion_zimage_fused.pynew file mode 100644index 00000000000..3bf49476f0b--- /dev/null+++ b/studio/backend/tests/test_diffusion_zimage_fused.py@@ -0,0 +1,200 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for ``diffusion_zimage_fused.py``; the bit-identity tests need CUDA and inductor's fma addcmul lowering."""++from __future__ import annotations++import pytest++from core.inference import diffusion_zimage_fused as zf++torch = pytest.importorskip("torch")+zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")+++def _ready() -> bool:+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ from core.inference.diffusion_qwenimage21_rope import inductor_addcmul_is_fma+ return inductor_addcmul_is_fma()+++needs_cuda = pytest.mark.skipif(+ not _ready(), reason = "needs CUDA (not ROCm) and inductor's fma addcmul lowering"+)++[email protected](autouse = True)+def _restore(monkeypatch):+ monkeypatch.delenv(zf.ZIMAGE_FUSED_ENV, raising = False)+ yield+ zf.uninstall()+++def test_stock_processor_fingerprint_matches_installed_diffusers():+ assert (+ zf._stock_digest(zmod.ZSingleStreamAttnProcessor)+ in zf._FINGERPRINTS["ZSingleStreamAttnProcessor.__call__"]+ )+++def test_disabled_and_wrong_model_are_noops(monkeypatch):+ assert zf.install(torch.nn.Linear(4, 4)) == {"real_rope": False, "fused_qkv": 0}+ monkeypatch.setenv(zf.ZIMAGE_FUSED_ENV, "0")+ assert zf.install_modules(torch.nn.Linear(4, 4)) == {"real_rope": False, "fused_qkv": 0}+++def test_fuse_linears_rejects_mismatched_parts():+ a, b = torch.nn.Linear(8, 8, bias = False), torch.nn.Linear(8, 8, bias = True)+ assert zf._fuse_linears([a, b]) is None+ assert zf._fuse_linears([a, torch.nn.Linear(4, 8, bias = False)]) is None+++def test_fuse_linears_plain_bf16_matches_and_shares_storage():+ parts = [torch.nn.Linear(16, 8, bias = False) for _ in range(3)]+ x = torch.randn(5, 16)+ # One GEMM over the stacked weights: split GEMMs can round differently (AVX512 BLAS, AMD EPYC 9B45).+ ref = torch.nn.functional.linear(x, torch.cat([p.weight for p in parts]))+ fused = zf._fuse_linears(parts)+ assert zf._share_storage(fused, parts)+ assert torch.equal(fused(x), ref)+ assert (+ parts[1].weight.data_ptr() == fused.weight.data_ptr() + 8 * 16 * fused.weight.element_size()+ )+++def _block(quant: str):+ torch.manual_seed(0)+ blk = (+ zmod.ZImageTransformerBlock(0, 384, 4, 4, 1e-5, True, modulation = True)+ .cuda()+ .to(torch.bfloat16)+ .eval()+ )+ for p in blk.parameters():+ p.data.normal_(0, 0.05)+ if quant == "int8":+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_+ quantize_(+ blk,+ Int8DynamicActivationInt8WeightConfig(),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and "adaLN" not in fqn,+ )+ return blk+++def _inputs(seq = 96):+ g = torch.Generator(device = "cpu").manual_seed(1)+ x = torch.randn(1, seq, 384, generator = g).cuda().to(torch.bfloat16)+ mask = torch.ones(1, seq, dtype = torch.bool).cuda()+ ang = torch.rand(1, seq, 48, generator = g) * 6.28+ freqs = torch.polar(torch.ones_like(ang), ang).to(torch.complex64).cuda()+ adaln = torch.randn(1, 256, generator = g).cuda().to(torch.bfloat16)+ return x, mask, freqs, adaln+++@needs_cuda[email protected]("quant", ["bf16", "int8"])+def test_compiled_block_stays_within_the_compile_floor(quant):+ # Not bit-identical to stock compiled (fusion boundaries move bf16 roundings): bar = stock compile's distance from eager.+ if quant == "int8":+ pytest.importorskip("torchao.quantization")+ blk = _block(quant)+ if quant == "int8" and type(blk.attention.to_q.weight).__name__ != "Int8Tensor":+ pytest.skip(+ "this torchao's int8 config builds the legacy tensor, which keeps the stock projections"+ )+ x, mask, freqs, adaln = _inputs()+ with torch.no_grad():+ eager = blk(x, mask, freqs, adaln)+ torch._dynamo.reset()+ ref = torch.compile(blk)(x, mask, freqs, adaln)+ res = zf.install_modules(blk)+ assert res["fused_qkv"] == 1+ torch._dynamo.reset()+ out = torch.compile(blk)(x, mask, freqs, adaln)+ floor = (ref != eager).sum().item()+ assert (out != eager).sum().item() <= 1.1 * floor + 16+ assert (out.float() - eager.float()).abs().max().item() <= 2 * (+ ref.float() - eager.float()+ ).abs().max().item() + 1e-6+++@needs_cuda+def test_eager_calls_keep_the_stock_processor():+ blk = _block("bf16")+ x, mask, freqs, adaln = _inputs()+ with torch.no_grad():+ ref = blk(x, mask, freqs, adaln)+ zf.install_modules(blk)+ out = blk(x, mask, freqs, adaln)+ assert torch.equal(out, ref)+++@needs_cuda+def test_wrapped_projection_falls_back_to_stock_projections():+ blk = _block("bf16")+ zf.install_modules(blk)+ attn = blk.attention+ assert zf._qkv_intact(attn) is not None+ attn.to_k = torch.nn.Sequential(attn.to_k) # stand-in for a LoRA wrapper+ assert zf._qkv_intact(attn) is None+++class ZImageTransformer2DModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)+++def test_kill_switch_on_a_later_load_restores_the_processor(monkeypatch):+ cls = zmod.ZSingleStreamAttnProcessor+ stock = cls.__dict__["__call__"]+ zf.install(ZImageTransformer2DModel())+ assert getattr(cls.__dict__["__call__"], "__unsloth_zimage_fused__", False)+ monkeypatch.setenv(zf.ZIMAGE_FUSED_ENV, "0")+ assert zf.install(ZImageTransformer2DModel()) == {"real_rope": False, "fused_qkv": 0}+ assert cls.__dict__["__call__"] is stock+++def test_model_unload_restores_the_process_global_patches(monkeypatch):+ from core.inference import diffusion+ from core.inference import diffusion_qwenimage21_rope as q21+ from core.inference import diffusion_qwenimage_rope as qr++ qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage")+ monkeypatch.setattr(q21, "inductor_addcmul_is_fma", lambda: True)+ monkeypatch.setattr(q21, "_addcmul_lowering", lambda: (True, False))+ monkeypatch.setattr(q21, "_FUSION", {})+ monkeypatch.setattr(q21, "probe_fusion", lambda dev: ("x", "x"))+ rope_stock = qmod.ROPE_PER_DEVICE["cuda"]+ call_stock = zmod.ZSingleStreamAttnProcessor.__dict__["__call__"]+ try:+ assert qr._patch_table(0) and zf._patch_class()+ assert qmod.ROPE_PER_DEVICE["cuda"] is not rope_stock+ assert zmod.ZSingleStreamAttnProcessor.__dict__["__call__"] is not call_stock+ for name in ("clear_gpu_cache", "release_pinned_host_memory", "reclaim_host_memory"):+ monkeypatch.setattr(diffusion, name, lambda *a, **k: None)+ backend = diffusion.DiffusionBackend()+ backend._state = diffusion._LoadState(object(), None, "r", "b", "cpu", "float32", False)+ backend._unload_locked()+ assert backend._state is None+ assert qmod.ROPE_PER_DEVICE["cuda"] is rope_stock+ assert zmod.ZSingleStreamAttnProcessor.__dict__["__call__"] is call_stock+ finally:+ qr.uninstall()+++def test_uninstall_cancels_the_deferred_module_install(monkeypatch):+ fired = []+ monkeypatch.setattr(zf, "install_modules", lambda t, *a, **k: fired.append(t))+ model = ZImageTransformer2DModel()+ zf.install(model)+ assert "zimage_fused" in model.__dict__["_unsloth_first_call_hooks"]+ zf.uninstall(model)+ model(torch.randn(2, 4))+ assert fired == [] and not model._forward_pre_hooks