Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: shared fused ConvRot act-quant kernel (L4 7.8% faster) and a G4 int8 GEMM tile table (#12693)
Daniel Hanb90fd2075b93
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.pynew file mode 100644index 00000000000..f79f459e0ac--- /dev/null+++ b/studio/backend/core/inference/diffusion_convrot_quant.py@@ -0,0 +1,478 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ConvRot rotation and the int8 per-row activation quant as ONE kernel (``rotq_i8``), for every int8 ConvRot Linear.++The stock path is two kernels: ``ConvRotLinear.forward``'s block-Hadamard rotation as a bf16 GEMM (writes the rotated+activation to HBM), then torchao's per-row act quant (reads it back, writes int8 + one scale per row). ``rotq_i8`` holds+whole activation rows of the ``[M * K / G, G]`` group view per tile, runs the rotation GEMM in the stock K order with+the stock bf16 rounding, closes the per-row amax in the epilogue and writes the int8 codes and the row scale: one read+of x, no rotated intermediate.++NUMERICS: codes and scales are bit-identical to the stock rotation + torchao quant (v1+``_int8_symm_per_token_reduced_range_quant``; v2 ``Int8Tensor.from_hp(PerRow)``: bf16 scale, reciprocal and product on+torchao <= 0.17, fp32 on >= 0.18). Every device is probed against that path before use (outliers, a zero row, ragged+M, every K the tile admits); a miss keeps the stock path on that device.++One home for the kernel: ``diffusion_int8_gemm`` uses it for every ConvRot Linear it swaps (fused-dequant GEMM on+sm80 / sm89 / sm120, cuBLAS ``_int_mm`` + torchao's epilogue on the other measured archs), and ``convrot_act_quant``+gives any other int8 consumer of a rotated activation the same codes.++Per-arch gate (``_ROTQ_CONFIG``): measured archs only; ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=1`` (or+``UNSLOTH_DIFFUSION_INT8_GEMM=1``) also enables it on an unmeasured sm80+ part, still probe-gated. Kill switch+``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``; ``UNSLOTH_DIFFUSION_INT8_GEMM=0`` turns off the whole int8 Linear swap.+"""++from __future__ import annotations++import os+import types+from functools import lru_cache+from typing import Any, Optional++INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"+_OP_NAMESPACE = "unsloth_studio"+_ROTQ_OP_NAME = "convrot_act_quant_int8"++# (BLOCK_M group rows, BLOCK_K, num_warps, num_stages) per (major, minor); measured end to end. Absent = stock.+_ROTQ_CONFIG = {+ (8, 0): (128, 32, 8, 3), # A100+ (8, 9): (128, 32, 8, 3), # L4+ (12, 0): (128, 32, 8, 4), # RTX PRO 6000+}+_ROTQ_FALLBACK = (128, 32, 8, 3)+_ROTQ_GROUPS = (256,)+# Per-K tiles: (major, minor) -> ((K_lo, K_hi, tile), ...), first match wins, bounds inclusive; a tile whose BLOCK_M+# cannot hold one whole row at that K is skipped. K 10240 stays on the default (a 64-row tile wastes 38% of it there).+_ROTQ_NARROW = (64, 32, 4, 3)+_ROTQ_K_TILES: dict = {+ cap: ((256, 8192, _ROTQ_NARROW), (11264, 16384, _ROTQ_NARROW))+ for cap in ((8, 0), (8, 9), (12, 0))+}++# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).+_ROTQ_QPARAMS = {+ "v1": (-127.0, 127.0, 127.0, 1e-5),+ "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),+}+# device index -> probed rotq tile (None = stock), and the per-K rules whose tile passed the probe there.+_ROTQ_DEVICE: dict = {}+_ROTQ_DEVICE_K: dict = {}+_ROTQ_CALLS = [0]+# Set once a device passed the probe; read by traced forwards (dynamo must not trace into the lru_cache'd registration).+_ROTQ_HANDLE: Any = None+++def _env_mode(name: str) -> str:+ raw = (os.environ.get(name) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return "off"+ if raw in ("1", "on", "true", "yes", "force"):+ return "force"+ return "auto"+++def rotquant_enabled() -> bool:+ """False only under ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""+ return _env_mode(INT8_ROTQUANT_ENV) != "off"+++def rotquant_mode() -> str:+ """'off', 'force' or 'auto'. The int8 GEMM switch is the master: its ``0`` turns the whole swap off and its ``1``+ forces this lever too."""+ gemm = _env_mode("UNSLOTH_DIFFUSION_INT8_GEMM")+ own = _env_mode(INT8_ROTQUANT_ENV)+ if gemm == "off" or own == "off":+ return "off"+ if gemm == "force" or own == "force":+ return "force"+ return "auto"+++def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:+ """The fused rotation tile for this compute capability, or None (stock rotation)."""+ mode = rotquant_mode() if mode is None else mode+ if mode == "off" or capability is None or not rotquant_enabled():+ return None+ cap = (int(capability[0]), int(capability[1]))+ cfg = _ROTQ_CONFIG.get(cap)+ if cfg is None and mode == "force" and cap >= (8, 0):+ return _ROTQ_FALLBACK+ return cfg+++def rotq_k_tiles(capability: Optional[tuple]) -> tuple:+ """This arch's per-K rules, or () (``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0`` keeps every K on the default tile)."""+ if capability is None or _env_mode("UNSLOTH_DIFFUSION_INT8_GEMM_TILES") == "off":+ return ()+ return tuple(_ROTQ_K_TILES.get((int(capability[0]), int(capability[1])), ()))+++def _triton_ok() -> bool:+ try:+ import re++ import triton++ match = re.match(r"(\d+)\.(\d+)", str(triton.__version__))+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= (3, 2)+ except Exception: # noqa: BLE001+ return False+++@lru_cache(maxsize = 1)+def _kernels() -> Optional[types.SimpleNamespace]:+ try:+ import triton+ import triton.language as tl+ from triton.language.extra import libdevice+ except Exception: # noqa: BLE001 - no Triton means the stock path+ return None++ @triton.jit+ def _rbf16(x):+ return x.to(tl.bfloat16).to(tl.float32)++ @triton.jit+ def rotq_i8(+ x_ptr,+ h_ptr,+ q_ptr,+ s_ptr,+ M,+ NG,+ G: tl.constexpr,+ BLOCK_M: tl.constexpr,+ BLOCK_K: tl.constexpr,+ ROWS: tl.constexpr,+ ROWS_P2: tl.constexpr,+ QMIN: tl.constexpr,+ QMAX: tl.constexpr,+ DIV: tl.constexpr,+ EPS: tl.constexpr,+ FP32_SCALE: tl.constexpr,+ ):+ # one program = ROWS whole activation rows, so the per-row amax closes inside the tile+ pid = tl.program_id(0)+ t = tl.arange(0, BLOCK_M)+ lr = t // NG+ row = pid.to(tl.int64) * ROWS + lr+ live = (lr < ROWS) & (row < M)+ rg = tl.where(live, row * NG + (t % NG), 0)+ offs_k = tl.arange(0, BLOCK_K)+ offs_n = tl.arange(0, G)+ acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)+ a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]+ h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]+ for _k in range(0, G // BLOCK_K):+ a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)+ acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)+ a_ptrs += BLOCK_K+ h_ptrs += BLOCK_K * G+ z = _rbf16(acc)+ # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18+ # Int8Tensor) keeps the reciprocal and the product in fp32.+ j = tl.arange(0, ROWS_P2)+ sel = lr[:, None] == j[None, :]+ amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))+ inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)+ if not FP32_SCALE:+ inv = _rbf16(inv)+ inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)+ p = z * inv_t[:, None]+ if not FP32_SCALE:+ p = _rbf16(p)+ p = libdevice.rint(p)+ p = tl.minimum(tl.maximum(p, QMIN), QMAX)+ tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])+ srow = pid.to(tl.int64) * ROWS + j+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))++ return types.SimpleNamespace(rotq_i8 = rotq_i8, triton = triton)+++@lru_cache(maxsize = 1)+def _v2_act_scale_fp32() -> bool:+ """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""+ try:+ import torch+ from torchao.quantization.granularity import PerRow+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor++ x = torch.ones(1, 16, dtype = torch.bfloat16)+ return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32+ except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit+ return False+++def _rotq_scale_fp32(kind: str) -> bool:+ return kind == "v2" and _v2_act_scale_fp32()+++def act_quant_v1(x2d: Any) -> tuple:+ """torchao 0.17's ``_int8_symm_per_token_reduced_range_quant`` minus the tensor-subclass wrapper."""+ import torch+ from torchao.quantization.quant_primitives import (+ MappingType,+ choose_qparams_affine,+ quantize_affine,+ )++ block = (1, x2d.shape[-1])+ scale, zero_point = choose_qparams_affine(+ x2d,+ MappingType.SYMMETRIC,+ block,+ torch.int8,+ -127,+ 127,+ 1e-5,+ torch.float32 if x2d.dtype == torch.float16 else None,+ None,+ )+ q = quantize_affine(x2d, block, scale, zero_point, torch.int8, -127, 127)+ return q, scale+++def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:+ """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""+ import torch+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation++ xr = rotate_convrot_activation(+ x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group+ )+ if kind == "v1":+ q, scale = act_quant_v1(xr)+ else:+ from torchao.quantization.granularity import PerRow+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor++ t = Int8Tensor.from_hp(xr, PerRow())+ q, scale = t.qdata, t.scale+ if _rotq_scale_fp32(kind):+ return q, scale.reshape(-1).to(torch.float32)+ return q, scale.reshape(-1).to(torch.bfloat16)+++def _rotq_rows(k: int, group: int, cfg: tuple) -> int:+ """Whole activation rows per program (0: this K does not fit one tile)."""+ return cfg[0] // (k // group)+++def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:+ import torch+ from .diffusion_convrot import build_convrot_hadamard++ kern = _kernels()+ triton = kern.triton+ m, k = x2d.shape+ bm, bk, warps, stages = cfg+ rows = _rotq_rows(k, group, cfg)+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]+ h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)+ q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)+ fp32_scale = _rotq_scale_fp32(kind)+ s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)+ if m == 0:+ return q, s+ with torch.cuda.device(x2d.device):+ kern.rotq_i8[(triton.cdiv(m, rows),)](+ x2d,+ h,+ q,+ s,+ m,+ k // group,+ G = group,+ BLOCK_M = bm,+ BLOCK_K = bk,+ ROWS = rows,+ ROWS_P2 = max(triton.next_power_of_2(rows), 2),+ QMIN = qmin,+ QMAX = qmax,+ DIV = div,+ EPS = eps,+ FP32_SCALE = fp32_scale,+ num_warps = warps,+ num_stages = stages,+ enable_fp_fusion = False,+ )+ return q, s+++def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:+ """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""+ import torch++ if (+ cfg is None+ or group not in _ROTQ_GROUPS+ or x2d.dim() != 2+ or x2d.dtype != torch.bfloat16+ or not x2d.is_cuda+ ):+ return False+ k = x2d.shape[1]+ return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1+++def rotq_tile_for(+ index: Any,+ k: int,+ group: int = 256,+) -> Optional[tuple]:+ """The probed tile this device quantizes a K-wide activation with: a per-K rule, else the arch default."""+ cfg = _ROTQ_DEVICE.get(index)+ if cfg is None:+ return None+ for k_lo, k_hi, tile in _ROTQ_DEVICE_K.get(index, ()):+ if k_lo <= k <= k_hi and k % group == 0 and _rotq_rows(k, group, tile) >= 1:+ return tile+ return cfg+++def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:+ """Op body: the probed tile for this device and K, the stock rotation + quant if anything does not fit."""+ _ROTQ_CALLS[0] += 1+ kind = "v2" if v2 else "v1"+ x2d = x2d if x2d.is_contiguous() else x2d.contiguous()+ index = x2d.device.index+ cfg = rotq_tile_for(index, x2d.shape[-1], group) if x2d.dim() == 2 else None+ if rotquant_supported(x2d, group, cfg):+ try:+ return _rotq_launch(x2d, group, kind, cfg)+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math+ if cfg != _ROTQ_DEVICE.get(index) and _ROTQ_DEVICE_K.get(index):+ _ROTQ_DEVICE_K[index] = () # drop the per-K tiles, keep the probed default+ return _rotq_run(x2d, group, v2)+ _ROTQ_DEVICE[index] = None+ return rotquant_reference(x2d, group, kind)+++@lru_cache(maxsize = 1)+def _rotq_op() -> Any:+ """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _ROTQ_OP_NAME):+ return getattr(ns, _ROTQ_OP_NAME)+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return None+ try:++ @custom_op(+ f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",+ mutates_args = (),+ schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",+ )+ def _convrot_act_quant_int8(x, group, v2):+ return _rotq_run(x, group, v2)++ @_convrot_act_quant_int8.register_fake+ def _(x, group, v2):+ return (+ x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),+ x.new_empty(+ (x.shape[0],),+ dtype = torch.float32+ if _rotq_scale_fp32("v2" if v2 else "v1")+ else torch.bfloat16,+ ),+ )+ except Exception: # noqa: BLE001+ return None+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)+++def rotquant_device_config(index: int) -> Optional[tuple]:+ """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged+ M). None = stock. A passing device also publishes the op handle the traced forwards call."""+ global _ROTQ_HANDLE+ if index in _ROTQ_DEVICE:+ return _ROTQ_DEVICE[index]+ cfg = None+ rules: tuple = ()+ try:+ import torch+ if (+ torch.cuda.is_available()+ and not getattr(torch.version, "hip", None)+ and _triton_ok()+ and _kernels() is not None+ and _rotq_op() is not None+ ):+ cap = torch.cuda.get_device_capability(index)+ want = rotquant_config(cap)+ for cand in (want, _ROTQ_FALLBACK) if want is not None else ():+ if _rotq_probe(index, cand):+ cfg = cand+ break+ if cfg is not None:+ ok: dict = {}+ for rule in rotq_k_tiles(cap):+ if rule[2] not in ok:+ ok[rule[2]] = rule[2] == cfg or _rotq_probe(index, rule[2])+ rules = tuple(rule for rule in rotq_k_tiles(cap) if ok[rule[2]])+ except Exception: # noqa: BLE001+ cfg = None+ _ROTQ_DEVICE_K[index] = rules if cfg is not None else ()+ _ROTQ_DEVICE[index] = cfg+ if cfg is not None:+ _ROTQ_HANDLE = _rotq_op()+ return cfg+++# (M, K): ragged M, one-group rows, 21 groups (a ragged row count per tile), 56 and 128 groups (two rows / one row).+_ROTQ_PROBE_SHAPES = ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128))+++def _rotq_probe(index: int, cfg: tuple) -> bool:+ import torch++ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(2)+ try:+ for m, k in _ROTQ_PROBE_SHAPES:+ if _rotq_rows(k, 256, cfg) < 1:+ continue+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)+ x[:, :3] *= 60+ x[1] = 0+ x[2, 11] = 3e4+ x = x.to(torch.bfloat16).to(dev)+ for kind in ("v1", "v2"):+ q, s = _rotq_launch(x, 256, kind, cfg)+ rq, rs = rotquant_reference(x, 256, kind)+ if not (torch.equal(q, rq) and torch.equal(s, rs)):+ return False+ torch.cuda.synchronize(dev)+ return True+ except Exception: # noqa: BLE001+ return False+++def rotquant_call_count() -> int:+ return _ROTQ_CALLS[0]+++def convrot_act_quant(x2d: Any, group: int, v2: bool) -> Optional[tuple]:+ """``(int8 [M, K], scale [M])`` of ``x2d @ blockdiag(H_group)`` quantized like torchao (v2 = ``Int8Tensor``, else+ the v1 per-token quant), from the fused kernel; None when this device / shape / group is not covered, so the caller+ keeps its own rotation + quant. Safe inside a traced forward: no host sync, the probe ran at install time."""+ import torch++ if (+ _ROTQ_HANDLE is None+ or x2d.dim() != 2+ or x2d.dtype != torch.bfloat16+ or not x2d.is_cuda+ or not rotquant_supported(x2d, group, rotq_tile_for(x2d.device.index, x2d.shape[-1], group))+ ):+ return None+ return _ROTQ_HANDLE(x2d, group, v2)diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 31d338aaf8c..d60673ff69c 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -17,15 +17,20 @@ read bf16 and Inductor re-tiles them, a rounding-order change smaller than compiPer-arch gate: sm80 / sm89 / sm120 on (measured), everything else stock. sm75: Triton cannot lower the int8 dot;sm100: Triton int8 is slower than cuBLAS. ROCm (weight-only int8 there) and CPU never reach it.+Tiles: one default per arch plus a per-shape table (``_SHAPE_TILES``) for the shapes where another tile measured+faster; every tile is probed bit-exact per device, and a tile that fails falls back to the arch default. All tiles+accumulate in int32, so the output does not depend on the tile.Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unmeasured arch (still probe-gated).+``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0`` keeps the arch default tile for every shape.-ConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;-kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.+ConvRot Linears (MiniMax-H3, Qwen-Image-2.1 ConvRot) run ``ConvRotLinear.forward``'s own rotation first, so eager+output stays bit-identical; kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.-Rotated Linears on sm120 / sm80 (group 256) run the rotation and the act quant as ONE kernel (``rotq_i8``): the same-rotation GEMM in the same K order, then torchao's per-row quant with every bf16 rounding kept, so codes and scales are-bit-identical (probed per device). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.+Rotated Linears on the archs ``diffusion_convrot_quant`` covers run the rotation and the act quant as ONE kernel+(``rotq_i8``, codes and scales bit-identical, probed per device). Where the fused GEMM is off but that kernel is on+(sm100), only the rotated Linears are swapped and their GEMM is cuBLAS ``_int_mm`` with torchao's epilogue, op for+op. Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every@@ -43,7 +48,7 @@ from typing import Any, OptionalINT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"-INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"+INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"_MIN_TRITON = (3, 2)# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge._MIN_ROWS = 17@@ -51,7 +56,6 @@ _K_ALIGN = 64_N_ALIGN = 16_OP_NAMESPACE = "unsloth_studio"_OP_NAME = "int8_mm_dequant"-_ROTQ_OP_NAME = "convrot_act_quant_int8"_REC = "_unsloth_i8_gemm"_MARK = "_unsloth_i8_gemm_prev"_NO_PREV = object()@@ -76,13 +80,17 @@ _ARCH_CONFIG = {# When the arch tile does not fit this part's shared memory._FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)-# rotq_i8 (BLOCK_M group rows, BLOCK_K, num_warps, num_stages); measured end to end on these archs only.-_ROTQ_CONFIG = {- (8, 0): (128, 32, 8, 3),- (12, 0): (128, 32, 8, 4),+# Per-shape tiles: (major, minor) -> ((N_lo, N_hi, K_lo, K_hi, tile), ...), first match wins, bounds inclusive,+# M >= _SHAPE_MIN_M only. No L4 rule on purpose: per-GEMM wins there cost s/step end to end (power-capped clocks).+_SHAPE_TILES: dict = {+ (12, 0): (+ # MiniMax-H3 fused QKV / FFN in+ (16384, 32768, 4096, 6144, (128, 128, 64, 32, 4, 4)),+ # deep K: H3 out / FFN down, Qwen-Image-2.1 MLP out, Z-Image w2+ (1, 8192, 7168, 16384, (256, 128, 128, 8, 8, 3)),+ ),}-_ROTQ_FALLBACK = (128, 32, 8, 3)-_ROTQ_GROUPS = (256,)+_SHAPE_MIN_M = 1024def int8_gemm_mode() -> str:@@ -105,25 +113,11 @@ def streamed_enabled() -> bool:return raw not in ("0", "off", "false", "no")-def rotquant_enabled() -> bool:- """ConvRot Linears quantize their activation with the fused rotation kernel unless- ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""- raw = (os.environ.get(INT8_ROTQUANT_ENV) or "").strip().lower()+def tiles_enabled() -> bool:+ raw = (os.environ.get(INT8_GEMM_TILES_ENV) or "").strip().lower()return raw not in ("0", "off", "false", "no")-def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:- """The fused rotation tile for this compute capability, or None (stock rotation)."""- mode = int8_gemm_mode() if mode is None else mode- if mode == "off" or capability is None or not rotquant_enabled():- return None- cap = (int(capability[0]), int(capability[1]))- cfg = _ROTQ_CONFIG.get(cap)- if cfg is None and mode == "force" and cap >= (8, 0):- return _ROTQ_FALLBACK- return cfg--def _triton_version_ok(version: Optional[str] = None) -> bool:if version is None:try:@@ -235,69 +229,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)- from triton.language.extra import libdevice-- @triton.jit- def rotq_i8(- x_ptr,- h_ptr,- q_ptr,- s_ptr,- M,- NG,- G: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_K: tl.constexpr,- ROWS: tl.constexpr,- ROWS_P2: tl.constexpr,- QMIN: tl.constexpr,- QMAX: tl.constexpr,- DIV: tl.constexpr,- EPS: tl.constexpr,- FP32_SCALE: tl.constexpr,- ):- # one program = ROWS whole activation rows, so the per-row amax closes inside the tile- pid = tl.program_id(0)- t = tl.arange(0, BLOCK_M)- lr = t // NG- row = pid.to(tl.int64) * ROWS + lr- live = (lr < ROWS) & (row < M)- rg = tl.where(live, row * NG + (t % NG), 0)- offs_k = tl.arange(0, BLOCK_K)- offs_n = tl.arange(0, G)- acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)- a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]- h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]- for _k in range(0, G // BLOCK_K):- a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)- acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)- a_ptrs += BLOCK_K- h_ptrs += BLOCK_K * G- z = _rbf16(acc)- # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18- # Int8Tensor) keeps the reciprocal and the product in fp32.- j = tl.arange(0, ROWS_P2)- sel = lr[:, None] == j[None, :]- amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)- s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))- inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)- if not FP32_SCALE:- inv = _rbf16(inv)- inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)- p = z * inv_t[:, None]- if not FP32_SCALE:- p = _rbf16(p)- p = libdevice.rint(p)- p = tl.minimum(tl.maximum(p, QMIN), QMAX)- tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])- srow = pid.to(tl.int64) * ROWS + j- tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))-- return types.SimpleNamespace(i8mm_dq = i8mm_dq, rotq_i8 = rotq_i8)+ return types.SimpleNamespace(i8mm_dq = i8mm_dq)# device index -> probed tile (None = stock)._DEVICE_CFG: dict = {}+# device index -> the per-shape rules whose tile passed the probe on that device.+_DEVICE_TILES: dict = {}def reference(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:@@ -368,16 +306,32 @@ def _aligned(a: Any, w: Any) -> bool:)+def tile_for(index: Any, m: int, n: int, k: int) -> Optional[tuple]:+ """The probed tile this device runs an ``[m, k] x [n, k]`` GEMM with: a per-shape rule, else the arch default."""+ cfg = _DEVICE_CFG.get(index)+ if cfg is None or m < _SHAPE_MIN_M:+ return cfg+ for n_lo, n_hi, k_lo, k_hi, tile in _DEVICE_TILES.get(index, ()):+ if n_lo <= n <= n_hi and k_lo <= k <= k_hi:+ return tile+ return cfg++def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:- """Op body: the probed tile for this device, else (launch failure, ``_aligned`` false) the stock epilogue."""+ """Op body: the probed tile for this device and shape, else (launch failure, ``_aligned`` false) the stock+ epilogue."""_CALLS[0] += 1a = a if a.stride(-1) == 1 else a.contiguous()- cfg = _DEVICE_CFG.get(a.device.index)+ index = a.device.index+ cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):try:return _launch(a, w, xs, ws, bias, cfg)except Exception: # noqa: BLE001 - a failed launch keeps the stock math- _DEVICE_CFG[a.device.index] = None+ if cfg != _DEVICE_CFG.get(index) and _DEVICE_TILES.get(index):+ _DEVICE_TILES[index] = () # drop the shape tiles, keep the probed default+ return _run(a, w, xs, ws, bias)+ _DEVICE_CFG[index] = Nonereturn reference(a, w, xs, ws, bias)@@ -412,12 +366,21 @@ def _op() -> Any:return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant+def shape_tiles(capability: Optional[tuple]) -> tuple:+ """This arch's per-shape rules, or () (``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0``, unmeasured arch)."""+ if capability is None or not tiles_enabled():+ return ()+ return tuple(_SHAPE_TILES.get((int(capability[0]), int(capability[1])), ()))++def device_config(index: int) -> Optional[tuple]:"""Probe once per device: arch gate, Triton, op registration, then a launch that must match the eager torchao- epilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path."""+ epilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path. Each+ per-shape tile is probed the same way; one that fails leaves its shapes on the default tile."""if index in _DEVICE_CFG:return _DEVICE_CFG[index]cfg = None+ rules: tuple = ()try:import torchif (@@ -427,13 +390,22 @@ def device_config(index: int) -> Optional[tuple]:and _kernels() is not Noneand _op() is not None):- want = arch_config(torch.cuda.get_device_capability(index))+ cap = torch.cuda.get_device_capability(index)+ want = arch_config(cap)for cand in (want, _FALLBACK_CONFIG) if want is not None else ():if cand is not None and _probe(index, cand):cfg = candbreak+ if cfg is not None:+ ok: dict = {}+ for rule in shape_tiles(cap):+ tile = rule[4]+ if tile not in ok:+ ok[tile] = tile == cfg or _probe(index, tile)+ rules = tuple(rule for rule in shape_tiles(cap) if ok[rule[4]])except Exception: # noqa: BLE001cfg = None+ _DEVICE_TILES[index] = rules if cfg is not None else ()_DEVICE_CFG[index] = cfgreturn cfg@@ -475,224 +447,6 @@ def _probe(index: int, cfg: tuple) -> bool:return False-# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).-_ROTQ_QPARAMS = {- "v1": (-127.0, 127.0, 127.0, 1e-5),- "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),-}-# device index -> probed rotq tile (None = stock).-_ROTQ_DEVICE: dict = {}-_ROTQ_CALLS = [0]-_ROTQ_HANDLE: Any = None---@lru_cache(maxsize = 1)-def _v2_act_scale_fp32() -> bool:- """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""- try:- import torch- from torchao.quantization.granularity import PerRow- from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor-- x = torch.ones(1, 16, dtype = torch.bfloat16)- return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32- except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit- return False---def _rotq_scale_fp32(kind: str) -> bool:- return kind == "v2" and _v2_act_scale_fp32()---def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:- """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""- import torch- from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation-- xr = rotate_convrot_activation(- x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group- )- if kind == "v1":- q, scale = _act_quant_v1(xr)- else:- from torchao.quantization.granularity import PerRow- from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor-- t = Int8Tensor.from_hp(xr, PerRow())- q, scale = t.qdata, t.scale- if _rotq_scale_fp32(kind):- return q, scale.reshape(-1).to(torch.float32)- return q, scale.reshape(-1).to(torch.bfloat16)---def _rotq_rows(k: int, group: int, cfg: tuple) -> int:- """Whole activation rows per program (0: this K does not fit one tile)."""- return cfg[0] // (k // group)---def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:- import torch- import triton- from .diffusion_convrot import build_convrot_hadamard-- kern = _kernels()- m, k = x2d.shape- bm, bk, warps, stages = cfg- rows = _rotq_rows(k, group, cfg)- qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]- h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)- q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)- fp32_scale = _rotq_scale_fp32(kind)- s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)- with torch.cuda.device(x2d.device):- kern.rotq_i8[(triton.cdiv(m, rows),)](- x2d,- h,- q,- s,- m,- k // group,- G = group,- BLOCK_M = bm,- BLOCK_K = bk,- ROWS = rows,- ROWS_P2 = max(triton.next_power_of_2(rows), 2),- QMIN = qmin,- QMAX = qmax,- DIV = div,- EPS = eps,- FP32_SCALE = fp32_scale,- num_warps = warps,- num_stages = stages,- enable_fp_fusion = False,- )- return q, s---def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:- """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""- import torch-- if (- cfg is None- or group not in _ROTQ_GROUPS- or x2d.dim() != 2- or x2d.dtype != torch.bfloat16- or not x2d.is_cuda- ):- return False- k = x2d.shape[1]- return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1---def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:- """Op body: the probed tile for this device, the stock rotation + quant if anything does not fit."""- _ROTQ_CALLS[0] += 1- kind = "v2" if v2 else "v1"- x2d = x2d if x2d.is_contiguous() else x2d.contiguous()- cfg = _ROTQ_DEVICE.get(x2d.device.index)- if rotquant_supported(x2d, group, cfg):- try:- return _rotq_launch(x2d, group, kind, cfg)- except Exception: # noqa: BLE001 - a failed launch keeps the stock math- _ROTQ_DEVICE[x2d.device.index] = None- return rotquant_reference(x2d, group, kind)---@lru_cache(maxsize = 1)-def _rotq_op() -> Any:- """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""- try:- import torch- except Exception: # noqa: BLE001- return None- ns = getattr(torch.ops, _OP_NAMESPACE, None)- if ns is not None and hasattr(ns, _ROTQ_OP_NAME):- return getattr(ns, _ROTQ_OP_NAME)- custom_op = getattr(getattr(torch, "library", None), "custom_op", None)- if custom_op is None:- return None- try:-- @custom_op(- f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",- mutates_args = (),- schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",- )- def _convrot_act_quant_int8(x, group, v2):- return _rotq_run(x, group, v2)-- @_convrot_act_quant_int8.register_fake- def _(x, group, v2):- return (- x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),- x.new_empty(- (x.shape[0],),- dtype = torch.float32- if _rotq_scale_fp32("v2" if v2 else "v1")- else torch.bfloat16,- ),- )- except Exception: # noqa: BLE001- return None- return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)---def rotquant_device_config(index: int) -> Optional[tuple]:- """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged- M). None = stock."""- if index in _ROTQ_DEVICE:- return _ROTQ_DEVICE[index]- cfg = None- try:- import torch- if (- torch.cuda.is_available()- and not getattr(torch.version, "hip", None)- and _triton_version_ok()- and _kernels() is not None- and _rotq_op() is not None- ):- want = rotquant_config(torch.cuda.get_device_capability(index))- for cand in (want, _ROTQ_FALLBACK) if want is not None else ():- if _rotq_probe(index, cand):- cfg = cand- break- except Exception: # noqa: BLE001- cfg = None- _ROTQ_DEVICE[index] = cfg- return cfg---def _rotq_probe(index: int, cfg: tuple) -> bool:- import torch-- dev = torch.device("cuda", index)- g = torch.Generator(device = "cpu").manual_seed(2)- try:- for m, k in ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128)):- if _rotq_rows(k, 256, cfg) < 1:- continue- x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)- x[:, :3] *= 60- x[1] = 0- x[2, 11] = 3e4- x = x.to(torch.bfloat16).to(dev)- for kind in ("v1", "v2"):- q, s = _rotq_launch(x, 256, kind, cfg)- rq, rs = rotquant_reference(x, 256, kind)- if not (torch.equal(q, rq) and torch.equal(s, rs)):- return False- torch.cuda.synchronize(dev)- return True- except Exception: # noqa: BLE001- return False---def rotquant_call_count() -> int:- return _ROTQ_CALLS[0]--def tie_operands(device: Any,rows: int = 32,@@ -777,28 +531,8 @@ def _v2_parts(w: Any, check_zero_point: bool = True) -> Optional[tuple]:def _act_quant_v1(x2d: Any) -> tuple:- """torchao 0.17's ``_int8_symm_per_token_reduced_range_quant`` minus the tensor-subclass wrapper."""- import torch- from torchao.quantization.quant_primitives import (- MappingType,- choose_qparams_affine,- quantize_affine,- )-- block = (1, x2d.shape[-1])- scale, zero_point = choose_qparams_affine(- x2d,- MappingType.SYMMETRIC,- block,- torch.int8,- -127,- 127,- 1e-5,- torch.float32 if x2d.dtype == torch.float16 else None,- None,- )- q = quantize_affine(x2d, block, scale, zero_point, torch.int8, -127, 127)- return q, scale+ from .diffusion_convrot_quant import act_quant_v1+ return act_quant_v1(x2d)def _act_quant_v2(x2d: Any, weight: Any) -> tuple:@@ -828,14 +562,28 @@ def _v1_act_quant_matches(index: int) -> bool:return False+def stock_gemm(xq: Any, wq: Any, xs: Any, ws: Any, bias: Any) -> Any:+ """torchao's int8 Linear after its act quant, op for op: cuBLAS ``_int_mm``, ``* x_scale`` in the scale's dtype,+ bf16, ``* w_scale``, ``+ bias``, bf16 (``reference`` minus the small-M padding; callers keep M >= 17). torchao 0.17's+ zero-point pass subtracts an exact zero for this symmetric quant and is skipped."""+ import torch++ y = (torch._int_mm(xq, wq.t()) * xs.reshape(-1, 1)).to(torch.bfloat16)+ y = y * ws.reshape(-1)+ if bias is not None:+ y = y + bias+ return y.to(torch.bfloat16)++def _linear_forward(self: Any, x: Any) -> Any:- """``nn.Linear.forward`` for a torchao int8 dynamic weight: torchao's activation quant, then the fused GEMM."""+ """``nn.Linear.forward`` for a torchao int8 dynamic weight: torchao's activation quant (fused with the ConvRot+ rotation where that kernel is on), then the fused-dequant GEMM, or cuBLAS + torchao's epilogue where it is not."""import torchrec = self.__dict__.get(_REC)if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:return type(self).forward(self, x)- kind, group, rotq, weight = rec+ kind, group, rotq, weight, fused = recif self.weight is not weight: # weight replaced since install (reload / LoRA bake): stockreturn type(self).forward(self, x)# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.@@ -849,12 +597,15 @@ def _linear_forward(self: Any, x: Any) -> Any:lead = x.shape[:-1]if x.numel() < _MIN_ROWS * x.shape[-1]:return type(self).forward(self, x)- if (- rotq- and _ROTQ_HANDLE is not None- and rotquant_supported(x.reshape(-1, x.shape[-1]), group, _ROTQ_DEVICE.get(x.device.index))- ):- xq, xs = _ROTQ_HANDLE(x.reshape(-1, x.shape[-1]), group, kind == "v2")+ q = None+ if rotq:+ from .diffusion_convrot_quant import convrot_act_quant+ q = convrot_act_quant(x.reshape(-1, x.shape[-1]), group, kind == "v2")+ if q is not None:+ xq, xs = q+ elif not fused:+ # rotation kernel not covering this call and no fused GEMM: the module's own stock forward+ return type(self).forward(self, x)else:if group is not None:from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation@@ -866,7 +617,10 @@ def _linear_forward(self: Any, x: Any) -> Any:xq, xs = _act_quant_v1(x2d)else:xq, xs = _act_quant_v2(x2d, weight)- y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)+ if fused:+ y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)+ else:+ y = stock_gemm(xq, wq, xs, ws, self.bias)return y.reshape(*lead, y.shape[-1])@@ -937,13 +691,23 @@ def _eligible(module: Any) -> Optional[tuple]:return None-def candidates(transformer: Any) -> int:+def candidates(transformer: Any, rotated_only: bool = False) -> int:try:- return sum(1 for m in transformer.modules() if _eligible(m) is not None)+ recs = (_eligible(m) for m in transformer.modules())+ return sum(1 for r in recs if r is not None and (r[1] is not None or not rotated_only))except Exception: # noqa: BLE001return 0+def _gates(capability: Optional[tuple]) -> tuple:+ """(fused GEMM on this arch, rotation kernel on this arch), before any probe."""+ from .diffusion_convrot_quant import rotquant_config++ gemm = arch_config(capability) is not None+ rotq = convrot_enabled() and rotquant_config(capability) is not None+ return gemm, rotq++def install(transformer: Any,logger: Any = None,@@ -974,11 +738,12 @@ def install(return 0if not _triton_version_ok() or _kernels() is None:return 0- if arch_config(torch.cuda.get_device_capability(torch.cuda.current_device())) is None:+ gemm, rotq = _gates(torch.cuda.get_device_capability(torch.cuda.current_device()))+ if not (gemm or rotq):return 0except Exception: # noqa: BLE001return 0- n = candidates(transformer)+ n = candidates(transformer, rotated_only = not gemm)if n:run_on_first_call(transformer, "int8_gemm", lambda t: _finalize(t, logger))return n@@ -1002,7 +767,7 @@ def _swap(logger: Any = None,device: Any = None,) -> int:- global _OP_HANDLE, _ROTQ_HANDLE+ global _OP_HANDLEfrom .diffusion_int8_fused import resident_cuda_deviceimport torch@@ -1026,31 +791,45 @@ def _swap(index = dev.index if dev.index is not None else torch.cuda.current_device()recs = [(m, _eligible(m)) for m in transformer.modules()]recs = [(m, r) for m, r in recs if r is not None]- if not recs or device_config(index) is None:+ if not recs:return 0+ from .diffusion_convrot_quant import rotquant_device_config++ gemm = device_config(index) is not None+ rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None+ if not gemm:+ # no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)+ recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):recs = [(m, r) for m, r in recs if r[0] != "v1"]- rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None+ if not recs:+ return 0count = 0with _LOCK:- _OP_HANDLE = _op()- if _OP_HANDLE is None:- return 0- if rotq:- _ROTQ_HANDLE = _rotq_op()+ if gemm:+ _OP_HANDLE = _op()+ if _OP_HANDLE is None:+ return 0for module, rec in recs:if _MARK in module.__dict__:count += 1continue- # kind, rotation group, fused rotation + act quant on, the Parameter; no payload alias- module.__dict__[_REC] = (rec[0], rec[1], bool(rotq and rec[1] is not None), rec[3])+ # kind, rotation group, fused rotation + act quant on, the Parameter (no payload alias), fused GEMM on+ module.__dict__[_REC] = (+ rec[0],+ rec[1],+ bool(rotq and rec[1] is not None),+ rec[3],+ gemm,+ )module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)count += 1if logger is not None and count:logger.info(- "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d%s",+ "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s",count,+ "run the fused-dequant GEMM (bf16 out)" if gemm else "keep cuBLAS",*torch.cuda.get_device_capability(index),", ConvRot rotation fused into the act quant" if rotq else "",)diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex c5b1abc7a68..0db6c827680 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport pytest+from core.inference import diffusion_convrot_quant as cqfrom core.inference import diffusion_int8_gemm as g8torch = pytest.importorskip("torch")@@ -16,12 +17,17 @@ torch = pytest.importorskip("torch")@pytest.fixture(autouse = True)def _clean(monkeypatch):monkeypatch.delenv(g8.INT8_GEMM_ENV, raising = False)- monkeypatch.delenv(g8.INT8_ROTQUANT_ENV, raising = False)+ monkeypatch.delenv(cq.INT8_ROTQUANT_ENV, raising = False)+ monkeypatch.delenv(g8.INT8_GEMM_TILES_ENV, raising = False)g8._DEVICE_CFG.clear()- g8._ROTQ_DEVICE.clear()+ g8._DEVICE_TILES.clear()+ cq._ROTQ_DEVICE.clear()+ cq._ROTQ_DEVICE_K.clear()yieldg8._DEVICE_CFG.clear()- g8._ROTQ_DEVICE.clear()+ g8._DEVICE_TILES.clear()+ cq._ROTQ_DEVICE.clear()+ cq._ROTQ_DEVICE_K.clear()@pytest.mark.parametrize(@@ -608,32 +614,32 @@ def test_streamed_kill_switch(forced, monkeypatch):[((8, 0), True), # A100: measured end to end (streamed H3)((8, 6), False),- ((8, 9), False), # L4: H3 Diffusers does not load there (host RAM floor), never measured+ ((8, 9), True), # L4((9, 0), False),((10, 0), False),((12, 0), True),],)def test_rotquant_arch_gate(cap, on):- assert (g8.rotquant_config(cap, "auto") is not None) is on+ assert (cq.rotquant_config(cap, "auto") is not None) is ondef test_rotquant_kill_switch_and_force(monkeypatch):assert (- g8.rotquant_config((12, 0), "off") is None+ cq.rotquant_config((12, 0), "off") is None) # the fused GEMM's own kill switch also turns it off- assert g8.rotquant_config((10, 0), "force") is not None- assert g8.rotquant_config((7, 5), "force") is None- assert g8.rotquant_config((8, 0), "off") is None- monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")- assert g8.rotquant_config((12, 0), "auto") is None- assert g8.rotquant_config((8, 0), "auto") is None- assert g8.rotquant_config((10, 0), "force") is None+ assert cq.rotquant_config((10, 0), "force") is not None+ assert cq.rotquant_config((7, 5), "force") is None+ assert cq.rotquant_config((8, 0), "off") is None+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "0")+ assert cq.rotquant_config((12, 0), "auto") is None+ assert cq.rotquant_config((8, 0), "auto") is None+ assert cq.rotquant_config((10, 0), "force") is None@pytest.fixturedef forced_rotq(forced):- cfg = g8.rotquant_device_config(torch.cuda.current_device())+ cfg = cq.rotquant_device_config(torch.cuda.current_device())if cfg is None:pytest.skip("fused rotation probe refused this device")return cfg@@ -654,10 +660,10 @@ def _act(m, k, seed):@pytest.mark.parametrize("kind", ["v1", "v2"])def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq, m, k, kind):x = _act(m, k, m + k)- q, s = g8._rotq_op()(x, 256, kind == "v2")- rq, rs = g8.rotquant_reference(x, 256, kind)+ q, s = cq._rotq_op()(x, 256, kind == "v2")+ rq, rs = cq.rotquant_reference(x, 256, kind)# torchao's own activation-scale dtype: bf16, except v2 on torchao >= 0.18 (fp32)- want = torch.float32 if kind == "v2" and g8._v2_act_scale_fp32() else torch.bfloat16+ want = torch.float32 if kind == "v2" and cq._v2_act_scale_fp32() else torch.bfloat16assert q.dtype == torch.int8 and s.dtype == want and s.shape == (m,)assert torch.equal(q, rq) and torch.equal(s, rs)@@ -665,7 +671,7 @@ def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq@needs_cudadef test_rotquant_probe_accepts_this_torchao(forced):# with 0.17's bf16 roundings on 0.18, ~4% of codes differed and the probe refused every device- assert g8.rotquant_device_config(torch.cuda.current_device()) is not None+ assert cq.rotquant_device_config(torch.cuda.current_device()) is not None@needs_cuda@@ -673,24 +679,24 @@ def test_rotquant_fake_op_matches_the_real_scale_dtype(forced_rotq):from torch._subclasses.fake_tensor import FakeTensorModex = _act(40, 768, 9)for v2 in (False, True):- _, s = g8._rotq_op()(x, 256, v2)+ _, s = cq._rotq_op()(x, 256, v2)with FakeTensorMode() as mode:- _, fs = g8._rotq_op()(mode.from_tensor(x), 256, v2)+ _, fs = cq._rotq_op()(mode.from_tensor(x), 256, v2)assert fs.dtype == s.dtype@needs_cudadef test_rotquant_unsupported_shapes_take_the_stock_math(forced_rotq):x = _act(40, 256 * 129, 3) # more groups than one tile holds- assert not g8.rotquant_supported(x, 256, forced_rotq)- assert not g8.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)- assert not g8.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)- q, s = g8._rotq_op()(x, 256, True)- rq, rs = g8.rotquant_reference(x, 256, "v2")+ assert not cq.rotquant_supported(x, 256, forced_rotq)+ assert not cq.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)+ assert not cq.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)+ q, s = cq._rotq_op()(x, 256, True)+ rq, rs = cq.rotquant_reference(x, 256, "v2")assert torch.equal(q, rq) and torch.equal(s, rs)xt = _act(1024, 300, 6).t() # non-contiguous input: made contiguous, same result- q, s = g8._rotq_op()(xt, 256, True)- rq, rs = g8.rotquant_reference(xt.contiguous(), 256, "v2")+ q, s = cq._rotq_op()(xt, 256, True)+ rq, rs = cq.rotquant_reference(xt.contiguous(), 256, "v2")assert torch.equal(q, rq) and torch.equal(s, rs)@@ -714,9 +720,9 @@ def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(x * 0.5) # made outside inference_mode like x, so a recompile here would be the op's own guardswith torch.inference_mode():- before = g8.rotquant_call_count()+ before = cq.rotquant_call_count()assert torch.equal(fused(x), stock(x))- assert g8.rotquant_call_count() == before + 1+ assert cq.rotquant_call_count() == before + 1counters.clear()torch._dynamo.reset()with torch._inductor.config.patch(emulate_precision_casts = True):@@ -732,22 +738,246 @@ def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(@needs_cudadef test_rotquant_kill_switch_keeps_the_stock_rotation(forced, monkeypatch):monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)- monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "0")stock = _convrot(_int8_linear(1024, 768, False, 2))fused = _convrot(_int8_linear(1024, 768, False, 2))holder = torch.nn.Sequential(fused)assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is Falsex = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)with torch.inference_mode():- before, gemm = g8.rotquant_call_count(), g8.call_count()+ before, gemm = cq.rotquant_call_count(), g8.call_count()assert torch.equal(fused(x), stock(x))- assert g8.rotquant_call_count() == before and g8.call_count() == gemm + 1+ assert cq.rotquant_call_count() == before and g8.call_count() == gemm + 1g8.uninstall(holder)@needs_cudadef test_rotquant_probe_refusal_keeps_the_stock_rotation(forced, monkeypatch):monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)- monkeypatch.setattr(g8, "_rotq_probe", lambda index, cfg: False)+ monkeypatch.setattr(cq, "_rotq_probe", lambda index, cfg: False)fused = _convrot(_int8_linear(1024, 768, False, 2))assert g8.install(torch.nn.Sequential(fused)) == 1 and fused.__dict__[g8._REC][2] is False+++# ---------------------------------------------------------------- per-shape tiles+++def test_tile_for_picks_the_shape_rule_then_the_arch_default(monkeypatch):+ default, wide = (128, 128, 64, 8, 4, 4), (128, 256, 64, 8, 8, 3)+ g8._DEVICE_CFG[0] = default+ g8._DEVICE_TILES[0] = ((16384, 65536, 4096, 8192, wide),)+ assert g8.tile_for(0, 4096, 21504, 5376) == wide+ assert g8.tile_for(0, 4096, 5376, 5376) == default # N outside the rule+ assert g8.tile_for(0, 4096, 21504, 14336) == default # K outside the rule+ assert g8.tile_for(0, g8._SHAPE_MIN_M - 1, 21504, 5376) == default # small M keeps the default+ assert g8.tile_for(1, 4096, 21504, 5376) is None # unprobed device: stock+++def test_tiles_kill_switch_and_unmeasured_arch(monkeypatch):+ monkeypatch.setitem(g8._SHAPE_TILES, (12, 0), ((1, 2, 3, 4, (64, 64, 64, 8, 4, 3)),))+ assert g8.shape_tiles((12, 0))+ assert g8.shape_tiles((8, 6)) == ()+ monkeypatch.setenv(g8.INT8_GEMM_TILES_ENV, "0")+ assert g8.shape_tiles((12, 0)) == ()+++def test_shape_tile_launch_failure_falls_back_to_the_default(monkeypatch):+ default, wide = (128, 128, 64, 8, 4, 4), (128, 256, 64, 8, 8, 3)+ g8._DEVICE_CFG[None] = default # CPU tensors report device index None+ g8._DEVICE_TILES[None] = ((1, 1 << 20, 1, 1 << 20, wide),)+ used = []++ def launch(a, w, xs, ws, bias, cfg):+ used.append(cfg)+ if cfg == wide:+ raise RuntimeError("out of shared memory")+ return "fused"++ monkeypatch.setattr(g8, "_launch", launch)+ monkeypatch.setattr(g8, "reference", lambda *a: "stock")+ a = torch.zeros(2048, 1024, dtype = torch.int8)+ w = torch.zeros(1024, 1024, dtype = torch.int8)+ xs = torch.ones(2048, dtype = torch.bfloat16)+ ws = torch.ones(1024, dtype = torch.bfloat16)+ assert g8._run(a, w, xs, ws, None) == "fused"+ assert used == [wide, default]+ assert g8._DEVICE_TILES[None] == () and g8._DEVICE_CFG[None] == default+ assert g8._run(a, w, xs, ws, None) == "fused" and used[-1] == default+++@needs_cuda+def test_failed_shape_tile_probe_keeps_its_shapes_on_the_default(monkeypatch):+ cap = torch.cuda.get_device_capability()+ bad, good = (64, 64, 64, 8, 4, 3), (128, 64, 64, 8, 4, 3)+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")+ monkeypatch.setitem(+ g8._SHAPE_TILES, cap, ((1, 4096, 1, 4096, bad), (4097, 1 << 20, 1, 1 << 20, good))+ )+ real = g8._probe+ monkeypatch.setattr(g8, "_probe", lambda index, cfg: cfg != bad and real(index, cfg))+ cfg = g8.device_config(torch.cuda.current_device())+ if cfg is None:+ pytest.skip("int8 GEMM probe refused this device")+ rules = g8._DEVICE_TILES[torch.cuda.current_device()]+ assert [r[4] for r in rules] == [good]+ assert g8.tile_for(torch.cuda.current_device(), 4096, 1024, 1024) == cfg+++def _shipped_tiles():+ return sorted({rule[4] for rules in g8._SHAPE_TILES.values() for rule in rules})+++@needs_cuda[email protected]("tile", _shipped_tiles() or [None])[email protected]("ws32", [False, True])+def test_every_shipped_shape_tile_is_bit_exact(forced, tile, ws32):+ """Every tile in the shape table, at a shape it is picked for and ragged neighbours: int32 accumulation makes the+ output independent of the tile, so each must equal the eager torchao epilogue bit for bit (and the tie operands)."""+ if tile is None:+ pytest.skip("no per-shape tiles shipped")+ g = torch.Generator().manual_seed(7)+ for m, n, k in ((1100, 1024, 1024), (2061, 2064, 1088), (17, 512, 256)):+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).cuda()+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).cuda()+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(torch.bfloat16).cuda()+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(torch.bfloat16).cuda()+ ws = ws.float() if ws32 else ws+ try:+ out = g8._launch(a, w, xs, ws, None, tile)+ except Exception as exc: # noqa: BLE001 - a tile that does not fit this part is dropped by the probe+ if g8._probe(torch.cuda.current_device(), tile):+ raise+ pytest.skip(+ f"tile {tile} does not fit this part ({type(exc).__name__}); probe drops it"+ )+ assert torch.equal(out, g8.reference(a, w, xs, ws, None))+ a, w = g8.tie_operands(torch.device("cuda"))+ ones = torch.ones(a.shape[0], device = "cuda", dtype = torch.bfloat16)+ wones = torch.ones(w.shape[0], device = "cuda", dtype = torch.bfloat16)+ assert torch.equal(+ g8._launch(a, w, ones, wones, None, tile), g8.reference(a, w, ones, wones, None)+ )+++# ---------------------------------------------------------------- fused rotation without the fused GEMM+++def rotq_only(monkeypatch):+ """The fused rotation forced on, the fused GEMM forced off (what sm100 and other unfused archs run)."""+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "1")+ monkeypatch.setattr(g8, "device_config", lambda index: None)+ monkeypatch.setattr(g8, "arch_config", lambda cap, mode = None: None)+ cfg = cq.rotquant_device_config(torch.cuda.current_device())+ if cfg is None:+ pytest.skip("fused rotation probe refused this device")+ return cfg+++def test_rotquant_mode_follows_its_own_switch_and_the_gemm_master(monkeypatch):+ assert cq.rotquant_mode() == "auto"+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "1")+ assert cq.rotquant_mode() == "force" and cq.rotquant_config((9, 0)) == cq._ROTQ_FALLBACK+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "0") # the whole int8 swap off+ assert cq.rotquant_mode() == "off" and cq.rotquant_config((12, 0)) is None+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")+ monkeypatch.delenv(cq.INT8_ROTQUANT_ENV)+ assert cq.rotquant_mode() == "force"+++@needs_cuda[email protected]("version", [None, 2])[email protected]("bias", [False, True])+def test_rotated_linear_without_the_fused_gemm_is_bit_identical(+ rotq_only, monkeypatch, version, bias+):+ """Fused GEMM off, rotation kernel on: only the ConvRot Linear is swapped; rotq + cuBLAS _int_mm + torchao's+ epilogue equals ConvRotLinear.forward eager, compiled fullgraph keeps one graph, the plain Linear stays stock."""+ from torch._dynamo.utils import counters++ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ stock = _convrot(_int8_linear(1024, 768, bias, version))+ fused = _convrot(_int8_linear(1024, 768, bias, version))+ plain = _int8_linear(1024, 768, bias, version)+ holder = torch.nn.Sequential(fused, torch.nn.Linear(768, 1024).cuda().to(torch.bfloat16), plain)+ assert g8.install(holder) == 1+ assert g8.is_installed(fused) and not g8.is_installed(plain)+ assert fused.__dict__[g8._REC][2] is True and fused.__dict__[g8._REC][4] is False+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3+ x[0, 7, :5] *= 200+ with torch.inference_mode():+ before, gemm = cq.rotquant_call_count(), g8.call_count()+ assert torch.equal(fused(x), stock(x))+ assert cq.rotquant_call_count() == before + 1 and g8.call_count() == gemm+ counters.clear()+ torch._dynamo.reset()+ with torch._inductor.config.patch(emulate_precision_casts = True):+ compiled = torch.compile(fused, fullgraph = True)+ out = compiled(x)+ assert not counters["graph_break"]+ assert counters["stats"]["unique_graphs"] == 1+ # codes / scales come from the eager-exact op; the epilogue's bf16 roundings are emulated+ assert torch.equal(out, stock(x))+ g8.uninstall(holder)+++@needs_cuda+def test_rotated_linear_without_either_kernel_is_left_alone(monkeypatch):+ monkeypatch.setattr(g8, "device_config", lambda index: None)+ monkeypatch.setattr(cq, "rotquant_device_config", lambda index: None)+ fused = _convrot(_int8_linear(1024, 768, False, 2))+ assert g8.install(torch.nn.Sequential(fused)) == 0 and not g8.is_installed(fused)+++@needs_cuda+def test_stock_gemm_matches_torchao_linear_bit_for_bit(rotq_only):+ for version in (None, 2):+ lin = _int8_linear(1024, 768, True, version)+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16) * 2+ parts = g8._v1_parts(lin.weight) or g8._v2_parts(lin.weight)+ kind = "v1" if g8._v1_parts(lin.weight) else "v2"+ xq, xs = g8._act_quant_v1(x) if kind == "v1" else g8._act_quant_v2(x, lin.weight)+ with torch.inference_mode():+ assert torch.equal(g8.stock_gemm(xq, parts[0], xs, parts[1], lin.bias), lin(x))+++@needs_cuda+def test_convrot_act_quant_api(forced_rotq):+ x = _act(300, 1024, 11)+ q, s = cq.convrot_act_quant(x, 256, True)+ rq, rs = cq.rotquant_reference(x, 256, "v2")+ assert torch.equal(q, rq) and torch.equal(s, rs)+ assert cq.convrot_act_quant(x, 64, True) is None # group the kernel does not cover+ assert cq.convrot_act_quant(x.half(), 256, True) is None+ assert cq.convrot_act_quant(x.cpu(), 256, True) is None+++def test_rotq_tile_for_picks_the_k_rule_only_where_a_row_fits(monkeypatch):+ default, narrow = (128, 32, 8, 3), (64, 32, 4, 3)+ cq._ROTQ_DEVICE[0] = default+ cq._ROTQ_DEVICE_K[0] = ((5000, 20000, narrow),)+ assert cq.rotq_tile_for(0, 5376) == narrow+ assert cq.rotq_tile_for(0, 4096) == default # outside the rule+ assert cq.rotq_tile_for(0, 256 * 70) == default # 70 groups do not fit a 64-row tile+ assert cq.rotq_tile_for(1, 5376) is None+ monkeypatch.setitem(cq._ROTQ_K_TILES, (8, 0), ((1, 2, narrow),))+ assert cq.rotq_k_tiles((8, 0))+ monkeypatch.setenv(g8.INT8_GEMM_TILES_ENV, "0")+ assert cq.rotq_k_tiles((8, 0)) == ()+++@needs_cuda+def test_every_shipped_rotq_k_tile_is_bit_exact(forced):+ tiles = sorted({rule[2] for rules in cq._ROTQ_K_TILES.values() for rule in rules})+ if not tiles:+ pytest.skip("no per-K rotq tiles shipped")+ for tile in tiles:+ assert cq._rotq_probe(torch.cuda.current_device(), tile), tile+ for m, k in ((1037, 5376), (300, 4096), (129, 14336), (77, 10240)):+ if cq._rotq_rows(k, 256, tile) < 1:+ continue+ x = _act(m, k, m * k)+ for kind in ("v1", "v2"):+ q, s = cq._rotq_launch(x, 256, kind, tile)+ rq, rs = cq.rotquant_reference(x, 256, kind)+ assert torch.equal(q, rq) and torch.equal(s, rs), (tile, m, k, kind)