Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: shared fused ConvRot act-quant kernel (L4 7.8% faster) and a G4 int8 GEMM tile table (#12693)

Daniel Hanb90fd2075b93
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
new file mode 100644
index 00000000000..f79f459e0ac
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -0,0 +1,478 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ConvRot rotation and the int8 per-row activation quant as ONE kernel (``rotq_i8``), for every int8 ConvRot Linear.
+
+The stock path is two kernels: ``ConvRotLinear.forward``'s block-Hadamard rotation as a bf16 GEMM (writes the rotated
+activation to HBM), then torchao's per-row act quant (reads it back, writes int8 + one scale per row). ``rotq_i8`` holds
+whole activation rows of the ``[M * K / G, G]`` group view per tile, runs the rotation GEMM in the stock K order with
+the stock bf16 rounding, closes the per-row amax in the epilogue and writes the int8 codes and the row scale: one read
+of x, no rotated intermediate.
+
+NUMERICS: codes and scales are bit-identical to the stock rotation + torchao quant (v1
+``_int8_symm_per_token_reduced_range_quant``; v2 ``Int8Tensor.from_hp(PerRow)``: bf16 scale, reciprocal and product on
+torchao <= 0.17, fp32 on >= 0.18). Every device is probed against that path before use (outliers, a zero row, ragged
+M, every K the tile admits); a miss keeps the stock path on that device.
+
+One home for the kernel: ``diffusion_int8_gemm`` uses it for every ConvRot Linear it swaps (fused-dequant GEMM on
+sm80 / sm89 / sm120, cuBLAS ``_int_mm`` + torchao's epilogue on the other measured archs), and ``convrot_act_quant``
+gives any other int8 consumer of a rotated activation the same codes.
+
+Per-arch gate (``_ROTQ_CONFIG``): measured archs only; ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=1`` (or
+``UNSLOTH_DIFFUSION_INT8_GEMM=1``) also enables it on an unmeasured sm80+ part, still probe-gated. Kill switch
+``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``; ``UNSLOTH_DIFFUSION_INT8_GEMM=0`` turns off the whole int8 Linear swap.
+"""
+
+from __future__ import annotations
+
+import os
+import types
+from functools import lru_cache
+from typing import Any, Optional
+
+INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"
+_OP_NAMESPACE = "unsloth_studio"
+_ROTQ_OP_NAME = "convrot_act_quant_int8"
+
+# (BLOCK_M group rows, BLOCK_K, num_warps, num_stages) per (major, minor); measured end to end. Absent = stock.
+_ROTQ_CONFIG = {
+ (8, 0): (128, 32, 8, 3), # A100
+ (8, 9): (128, 32, 8, 3), # L4
+ (12, 0): (128, 32, 8, 4), # RTX PRO 6000
+}
+_ROTQ_FALLBACK = (128, 32, 8, 3)
+_ROTQ_GROUPS = (256,)
+# Per-K tiles: (major, minor) -> ((K_lo, K_hi, tile), ...), first match wins, bounds inclusive; a tile whose BLOCK_M
+# cannot hold one whole row at that K is skipped. K 10240 stays on the default (a 64-row tile wastes 38% of it there).
+_ROTQ_NARROW = (64, 32, 4, 3)
+_ROTQ_K_TILES: dict = {
+ cap: ((256, 8192, _ROTQ_NARROW), (11264, 16384, _ROTQ_NARROW))
+ for cap in ((8, 0), (8, 9), (12, 0))
+}
+
+# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).
+_ROTQ_QPARAMS = {
+ "v1": (-127.0, 127.0, 127.0, 1e-5),
+ "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),
+}
+# device index -> probed rotq tile (None = stock), and the per-K rules whose tile passed the probe there.
+_ROTQ_DEVICE: dict = {}
+_ROTQ_DEVICE_K: dict = {}
+_ROTQ_CALLS = [0]
+# Set once a device passed the probe; read by traced forwards (dynamo must not trace into the lru_cache'd registration).
+_ROTQ_HANDLE: Any = None
+
+
+def _env_mode(name: str) -> str:
+ raw = (os.environ.get(name) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw in ("1", "on", "true", "yes", "force"):
+ return "force"
+ return "auto"
+
+
+def rotquant_enabled() -> bool:
+ """False only under ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""
+ return _env_mode(INT8_ROTQUANT_ENV) != "off"
+
+
+def rotquant_mode() -> str:
+ """'off', 'force' or 'auto'. The int8 GEMM switch is the master: its ``0`` turns the whole swap off and its ``1``
+ forces this lever too."""
+ gemm = _env_mode("UNSLOTH_DIFFUSION_INT8_GEMM")
+ own = _env_mode(INT8_ROTQUANT_ENV)
+ if gemm == "off" or own == "off":
+ return "off"
+ if gemm == "force" or own == "force":
+ return "force"
+ return "auto"
+
+
+def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:
+ """The fused rotation tile for this compute capability, or None (stock rotation)."""
+ mode = rotquant_mode() if mode is None else mode
+ if mode == "off" or capability is None or not rotquant_enabled():
+ return None
+ cap = (int(capability[0]), int(capability[1]))
+ cfg = _ROTQ_CONFIG.get(cap)
+ if cfg is None and mode == "force" and cap >= (8, 0):
+ return _ROTQ_FALLBACK
+ return cfg
+
+
+def rotq_k_tiles(capability: Optional[tuple]) -> tuple:
+ """This arch's per-K rules, or () (``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0`` keeps every K on the default tile)."""
+ if capability is None or _env_mode("UNSLOTH_DIFFUSION_INT8_GEMM_TILES") == "off":
+ return ()
+ return tuple(_ROTQ_K_TILES.get((int(capability[0]), int(capability[1])), ()))
+
+
+def _triton_ok() -> bool:
+ try:
+ import re
+
+ import triton
+
+ match = re.match(r"(\d+)\.(\d+)", str(triton.__version__))
+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= (3, 2)
+ except Exception: # noqa: BLE001
+ return False
+
+
+@lru_cache(maxsize = 1)
+def _kernels() -> Optional[types.SimpleNamespace]:
+ try:
+ import triton
+ import triton.language as tl
+ from triton.language.extra import libdevice
+ except Exception: # noqa: BLE001 - no Triton means the stock path
+ return None
+
+ @triton.jit
+ def _rbf16(x):
+ return x.to(tl.bfloat16).to(tl.float32)
+
+ @triton.jit
+ def rotq_i8(
+ x_ptr,
+ h_ptr,
+ q_ptr,
+ s_ptr,
+ M,
+ NG,
+ G: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ ROWS: tl.constexpr,
+ ROWS_P2: tl.constexpr,
+ QMIN: tl.constexpr,
+ QMAX: tl.constexpr,
+ DIV: tl.constexpr,
+ EPS: tl.constexpr,
+ FP32_SCALE: tl.constexpr,
+ ):
+ # one program = ROWS whole activation rows, so the per-row amax closes inside the tile
+ pid = tl.program_id(0)
+ t = tl.arange(0, BLOCK_M)
+ lr = t // NG
+ row = pid.to(tl.int64) * ROWS + lr
+ live = (lr < ROWS) & (row < M)
+ rg = tl.where(live, row * NG + (t % NG), 0)
+ offs_k = tl.arange(0, BLOCK_K)
+ offs_n = tl.arange(0, G)
+ acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)
+ a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]
+ h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]
+ for _k in range(0, G // BLOCK_K):
+ a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)
+ acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)
+ a_ptrs += BLOCK_K
+ h_ptrs += BLOCK_K * G
+ z = _rbf16(acc)
+ # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18
+ # Int8Tensor) keeps the reciprocal and the product in fp32.
+ j = tl.arange(0, ROWS_P2)
+ sel = lr[:, None] == j[None, :]
+ amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)
+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))
+ inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)
+ if not FP32_SCALE:
+ inv = _rbf16(inv)
+ inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)
+ p = z * inv_t[:, None]
+ if not FP32_SCALE:
+ p = _rbf16(p)
+ p = libdevice.rint(p)
+ p = tl.minimum(tl.maximum(p, QMIN), QMAX)
+ tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])
+ srow = pid.to(tl.int64) * ROWS + j
+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))
+
+ return types.SimpleNamespace(rotq_i8 = rotq_i8, triton = triton)
+
+
+@lru_cache(maxsize = 1)
+def _v2_act_scale_fp32() -> bool:
+ """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""
+ try:
+ import torch
+ from torchao.quantization.granularity import PerRow
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
+
+ x = torch.ones(1, 16, dtype = torch.bfloat16)
+ return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32
+ except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit
+ return False
+
+
+def _rotq_scale_fp32(kind: str) -> bool:
+ return kind == "v2" and _v2_act_scale_fp32()
+
+
+def act_quant_v1(x2d: Any) -> tuple:
+ """torchao 0.17's ``_int8_symm_per_token_reduced_range_quant`` minus the tensor-subclass wrapper."""
+ import torch
+ from torchao.quantization.quant_primitives import (
+ MappingType,
+ choose_qparams_affine,
+ quantize_affine,
+ )
+
+ block = (1, x2d.shape[-1])
+ scale, zero_point = choose_qparams_affine(
+ x2d,
+ MappingType.SYMMETRIC,
+ block,
+ torch.int8,
+ -127,
+ 127,
+ 1e-5,
+ torch.float32 if x2d.dtype == torch.float16 else None,
+ None,
+ )
+ q = quantize_affine(x2d, block, scale, zero_point, torch.int8, -127, 127)
+ return q, scale
+
+
+def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:
+ """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""
+ import torch
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+
+ xr = rotate_convrot_activation(
+ x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group
+ )
+ if kind == "v1":
+ q, scale = act_quant_v1(xr)
+ else:
+ from torchao.quantization.granularity import PerRow
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
+
+ t = Int8Tensor.from_hp(xr, PerRow())
+ q, scale = t.qdata, t.scale
+ if _rotq_scale_fp32(kind):
+ return q, scale.reshape(-1).to(torch.float32)
+ return q, scale.reshape(-1).to(torch.bfloat16)
+
+
+def _rotq_rows(k: int, group: int, cfg: tuple) -> int:
+ """Whole activation rows per program (0: this K does not fit one tile)."""
+ return cfg[0] // (k // group)
+
+
+def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:
+ import torch
+ from .diffusion_convrot import build_convrot_hadamard
+
+ kern = _kernels()
+ triton = kern.triton
+ m, k = x2d.shape
+ bm, bk, warps, stages = cfg
+ rows = _rotq_rows(k, group, cfg)
+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]
+ h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)
+ q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)
+ fp32_scale = _rotq_scale_fp32(kind)
+ s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)
+ if m == 0:
+ return q, s
+ with torch.cuda.device(x2d.device):
+ kern.rotq_i8[(triton.cdiv(m, rows),)](
+ x2d,
+ h,
+ q,
+ s,
+ m,
+ k // group,
+ G = group,
+ BLOCK_M = bm,
+ BLOCK_K = bk,
+ ROWS = rows,
+ ROWS_P2 = max(triton.next_power_of_2(rows), 2),
+ QMIN = qmin,
+ QMAX = qmax,
+ DIV = div,
+ EPS = eps,
+ FP32_SCALE = fp32_scale,
+ num_warps = warps,
+ num_stages = stages,
+ enable_fp_fusion = False,
+ )
+ return q, s
+
+
+def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:
+ """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""
+ import torch
+
+ if (
+ cfg is None
+ or group not in _ROTQ_GROUPS
+ or x2d.dim() != 2
+ or x2d.dtype != torch.bfloat16
+ or not x2d.is_cuda
+ ):
+ return False
+ k = x2d.shape[1]
+ return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1
+
+
+def rotq_tile_for(
+ index: Any,
+ k: int,
+ group: int = 256,
+) -> Optional[tuple]:
+ """The probed tile this device quantizes a K-wide activation with: a per-K rule, else the arch default."""
+ cfg = _ROTQ_DEVICE.get(index)
+ if cfg is None:
+ return None
+ for k_lo, k_hi, tile in _ROTQ_DEVICE_K.get(index, ()):
+ if k_lo <= k <= k_hi and k % group == 0 and _rotq_rows(k, group, tile) >= 1:
+ return tile
+ return cfg
+
+
+def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:
+ """Op body: the probed tile for this device and K, the stock rotation + quant if anything does not fit."""
+ _ROTQ_CALLS[0] += 1
+ kind = "v2" if v2 else "v1"
+ x2d = x2d if x2d.is_contiguous() else x2d.contiguous()
+ index = x2d.device.index
+ cfg = rotq_tile_for(index, x2d.shape[-1], group) if x2d.dim() == 2 else None
+ if rotquant_supported(x2d, group, cfg):
+ try:
+ return _rotq_launch(x2d, group, kind, cfg)
+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math
+ if cfg != _ROTQ_DEVICE.get(index) and _ROTQ_DEVICE_K.get(index):
+ _ROTQ_DEVICE_K[index] = () # drop the per-K tiles, keep the probed default
+ return _rotq_run(x2d, group, v2)
+ _ROTQ_DEVICE[index] = None
+ return rotquant_reference(x2d, group, kind)
+
+
+@lru_cache(maxsize = 1)
+def _rotq_op() -> Any:
+ """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _ROTQ_OP_NAME):
+ return getattr(ns, _ROTQ_OP_NAME)
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return None
+ try:
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",
+ mutates_args = (),
+ schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",
+ )
+ def _convrot_act_quant_int8(x, group, v2):
+ return _rotq_run(x, group, v2)
+
+ @_convrot_act_quant_int8.register_fake
+ def _(x, group, v2):
+ return (
+ x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),
+ x.new_empty(
+ (x.shape[0],),
+ dtype = torch.float32
+ if _rotq_scale_fp32("v2" if v2 else "v1")
+ else torch.bfloat16,
+ ),
+ )
+ except Exception: # noqa: BLE001
+ return None
+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)
+
+
+def rotquant_device_config(index: int) -> Optional[tuple]:
+ """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged
+ M). None = stock. A passing device also publishes the op handle the traced forwards call."""
+ global _ROTQ_HANDLE
+ if index in _ROTQ_DEVICE:
+ return _ROTQ_DEVICE[index]
+ cfg = None
+ rules: tuple = ()
+ try:
+ import torch
+ if (
+ torch.cuda.is_available()
+ and not getattr(torch.version, "hip", None)
+ and _triton_ok()
+ and _kernels() is not None
+ and _rotq_op() is not None
+ ):
+ cap = torch.cuda.get_device_capability(index)
+ want = rotquant_config(cap)
+ for cand in (want, _ROTQ_FALLBACK) if want is not None else ():
+ if _rotq_probe(index, cand):
+ cfg = cand
+ break
+ if cfg is not None:
+ ok: dict = {}
+ for rule in rotq_k_tiles(cap):
+ if rule[2] not in ok:
+ ok[rule[2]] = rule[2] == cfg or _rotq_probe(index, rule[2])
+ rules = tuple(rule for rule in rotq_k_tiles(cap) if ok[rule[2]])
+ except Exception: # noqa: BLE001
+ cfg = None
+ _ROTQ_DEVICE_K[index] = rules if cfg is not None else ()
+ _ROTQ_DEVICE[index] = cfg
+ if cfg is not None:
+ _ROTQ_HANDLE = _rotq_op()
+ return cfg
+
+
+# (M, K): ragged M, one-group rows, 21 groups (a ragged row count per tile), 56 and 128 groups (two rows / one row).
+_ROTQ_PROBE_SHAPES = ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128))
+
+
+def _rotq_probe(index: int, cfg: tuple) -> bool:
+ import torch
+
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(2)
+ try:
+ for m, k in _ROTQ_PROBE_SHAPES:
+ if _rotq_rows(k, 256, cfg) < 1:
+ continue
+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
+ x[:, :3] *= 60
+ x[1] = 0
+ x[2, 11] = 3e4
+ x = x.to(torch.bfloat16).to(dev)
+ for kind in ("v1", "v2"):
+ q, s = _rotq_launch(x, 256, kind, cfg)
+ rq, rs = rotquant_reference(x, 256, kind)
+ if not (torch.equal(q, rq) and torch.equal(s, rs)):
+ return False
+ torch.cuda.synchronize(dev)
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def rotquant_call_count() -> int:
+ return _ROTQ_CALLS[0]
+
+
+def convrot_act_quant(x2d: Any, group: int, v2: bool) -> Optional[tuple]:
+ """``(int8 [M, K], scale [M])`` of ``x2d @ blockdiag(H_group)`` quantized like torchao (v2 = ``Int8Tensor``, else
+ the v1 per-token quant), from the fused kernel; None when this device / shape / group is not covered, so the caller
+ keeps its own rotation + quant. Safe inside a traced forward: no host sync, the probe ran at install time."""
+ import torch
+
+ if (
+ _ROTQ_HANDLE is None
+ or x2d.dim() != 2
+ or x2d.dtype != torch.bfloat16
+ or not x2d.is_cuda
+ or not rotquant_supported(x2d, group, rotq_tile_for(x2d.device.index, x2d.shape[-1], group))
+ ):
+ return None
+ return _ROTQ_HANDLE(x2d, group, v2)
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 31d338aaf8c..d60673ff69c 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -17,15 +17,20 @@ read bf16 and Inductor re-tiles them, a rounding-order change smaller than compi
Per-arch gate: sm80 / sm89 / sm120 on (measured), everything else stock. sm75: Triton cannot lower the int8 dot;
sm100: Triton int8 is slower than cuBLAS. ROCm (weight-only int8 there) and CPU never reach it.
+Tiles: one default per arch plus a per-shape table (``_SHAPE_TILES``) for the shapes where another tile measured
+faster; every tile is probed bit-exact per device, and a tile that fails falls back to the arch default. All tiles
+accumulate in int32, so the output does not depend on the tile.
Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unmeasured arch (still probe-gated).
+``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0`` keeps the arch default tile for every shape.
-ConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;
-kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.
+ConvRot Linears (MiniMax-H3, Qwen-Image-2.1 ConvRot) run ``ConvRotLinear.forward``'s own rotation first, so eager
+output stays bit-identical; kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.
-Rotated Linears on sm120 / sm80 (group 256) run the rotation and the act quant as ONE kernel (``rotq_i8``): the same
-rotation GEMM in the same K order, then torchao's per-row quant with every bf16 rounding kept, so codes and scales are
-bit-identical (probed per device). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.
+Rotated Linears on the archs ``diffusion_convrot_quant`` covers run the rotation and the act quant as ONE kernel
+(``rotq_i8``, codes and scales bit-identical, probed per device). Where the fused GEMM is off but that kernel is on
+(sm100), only the rotated Linears are swapped and their GEMM is cuBLAS ``_int_mm`` with torchao's epilogue, op for
+op. Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.
A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's
``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every
@@ -43,7 +48,7 @@ from typing import Any, Optional
INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"
INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
-INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"
+INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
@@ -51,7 +56,6 @@ _K_ALIGN = 64
_N_ALIGN = 16
_OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_mm_dequant"
-_ROTQ_OP_NAME = "convrot_act_quant_int8"
_REC = "_unsloth_i8_gemm"
_MARK = "_unsloth_i8_gemm_prev"
_NO_PREV = object()
@@ -76,13 +80,17 @@ _ARCH_CONFIG = {
# When the arch tile does not fit this part's shared memory.
_FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)
-# rotq_i8 (BLOCK_M group rows, BLOCK_K, num_warps, num_stages); measured end to end on these archs only.
-_ROTQ_CONFIG = {
- (8, 0): (128, 32, 8, 3),
- (12, 0): (128, 32, 8, 4),
+# Per-shape tiles: (major, minor) -> ((N_lo, N_hi, K_lo, K_hi, tile), ...), first match wins, bounds inclusive,
+# M >= _SHAPE_MIN_M only. No L4 rule on purpose: per-GEMM wins there cost s/step end to end (power-capped clocks).
+_SHAPE_TILES: dict = {
+ (12, 0): (
+ # MiniMax-H3 fused QKV / FFN in
+ (16384, 32768, 4096, 6144, (128, 128, 64, 32, 4, 4)),
+ # deep K: H3 out / FFN down, Qwen-Image-2.1 MLP out, Z-Image w2
+ (1, 8192, 7168, 16384, (256, 128, 128, 8, 8, 3)),
+ ),
}
-_ROTQ_FALLBACK = (128, 32, 8, 3)
-_ROTQ_GROUPS = (256,)
+_SHAPE_MIN_M = 1024
def int8_gemm_mode() -> str:
@@ -105,25 +113,11 @@ def streamed_enabled() -> bool:
return raw not in ("0", "off", "false", "no")
-def rotquant_enabled() -> bool:
- """ConvRot Linears quantize their activation with the fused rotation kernel unless
- ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""
- raw = (os.environ.get(INT8_ROTQUANT_ENV) or "").strip().lower()
+def tiles_enabled() -> bool:
+ raw = (os.environ.get(INT8_GEMM_TILES_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
-def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:
- """The fused rotation tile for this compute capability, or None (stock rotation)."""
- mode = int8_gemm_mode() if mode is None else mode
- if mode == "off" or capability is None or not rotquant_enabled():
- return None
- cap = (int(capability[0]), int(capability[1]))
- cfg = _ROTQ_CONFIG.get(cap)
- if cfg is None and mode == "force" and cap >= (8, 0):
- return _ROTQ_FALLBACK
- return cfg
-
-
def _triton_version_ok(version: Optional[str] = None) -> bool:
if version is None:
try:
@@ -235,69 +229,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)
- from triton.language.extra import libdevice
-
- @triton.jit
- def rotq_i8(
- x_ptr,
- h_ptr,
- q_ptr,
- s_ptr,
- M,
- NG,
- G: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_K: tl.constexpr,
- ROWS: tl.constexpr,
- ROWS_P2: tl.constexpr,
- QMIN: tl.constexpr,
- QMAX: tl.constexpr,
- DIV: tl.constexpr,
- EPS: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- ):
- # one program = ROWS whole activation rows, so the per-row amax closes inside the tile
- pid = tl.program_id(0)
- t = tl.arange(0, BLOCK_M)
- lr = t // NG
- row = pid.to(tl.int64) * ROWS + lr
- live = (lr < ROWS) & (row < M)
- rg = tl.where(live, row * NG + (t % NG), 0)
- offs_k = tl.arange(0, BLOCK_K)
- offs_n = tl.arange(0, G)
- acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)
- a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]
- h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]
- for _k in range(0, G // BLOCK_K):
- a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)
- acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)
- a_ptrs += BLOCK_K
- h_ptrs += BLOCK_K * G
- z = _rbf16(acc)
- # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18
- # Int8Tensor) keeps the reciprocal and the product in fp32.
- j = tl.arange(0, ROWS_P2)
- sel = lr[:, None] == j[None, :]
- amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)
- s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))
- inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)
- if not FP32_SCALE:
- inv = _rbf16(inv)
- inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)
- p = z * inv_t[:, None]
- if not FP32_SCALE:
- p = _rbf16(p)
- p = libdevice.rint(p)
- p = tl.minimum(tl.maximum(p, QMIN), QMAX)
- tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])
- srow = pid.to(tl.int64) * ROWS + j
- tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))
-
- return types.SimpleNamespace(i8mm_dq = i8mm_dq, rotq_i8 = rotq_i8)
+ return types.SimpleNamespace(i8mm_dq = i8mm_dq)
# device index -> probed tile (None = stock).
_DEVICE_CFG: dict = {}
+# device index -> the per-shape rules whose tile passed the probe on that device.
+_DEVICE_TILES: dict = {}
def reference(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
@@ -368,16 +306,32 @@ def _aligned(a: Any, w: Any) -> bool:
)
+def tile_for(index: Any, m: int, n: int, k: int) -> Optional[tuple]:
+ """The probed tile this device runs an ``[m, k] x [n, k]`` GEMM with: a per-shape rule, else the arch default."""
+ cfg = _DEVICE_CFG.get(index)
+ if cfg is None or m < _SHAPE_MIN_M:
+ return cfg
+ for n_lo, n_hi, k_lo, k_hi, tile in _DEVICE_TILES.get(index, ()):
+ if n_lo <= n <= n_hi and k_lo <= k <= k_hi:
+ return tile
+ return cfg
+
+
def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
- """Op body: the probed tile for this device, else (launch failure, ``_aligned`` false) the stock epilogue."""
+ """Op body: the probed tile for this device and shape, else (launch failure, ``_aligned`` false) the stock
+ epilogue."""
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
- cfg = _DEVICE_CFG.get(a.device.index)
+ index = a.device.index
+ cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])
if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):
try:
return _launch(a, w, xs, ws, bias, cfg)
except Exception: # noqa: BLE001 - a failed launch keeps the stock math
- _DEVICE_CFG[a.device.index] = None
+ if cfg != _DEVICE_CFG.get(index) and _DEVICE_TILES.get(index):
+ _DEVICE_TILES[index] = () # drop the shape tiles, keep the probed default
+ return _run(a, w, xs, ws, bias)
+ _DEVICE_CFG[index] = None
return reference(a, w, xs, ws, bias)
@@ -412,12 +366,21 @@ def _op() -> Any:
return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant
+def shape_tiles(capability: Optional[tuple]) -> tuple:
+ """This arch's per-shape rules, or () (``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0``, unmeasured arch)."""
+ if capability is None or not tiles_enabled():
+ return ()
+ return tuple(_SHAPE_TILES.get((int(capability[0]), int(capability[1])), ()))
+
+
def device_config(index: int) -> Optional[tuple]:
"""Probe once per device: arch gate, Triton, op registration, then a launch that must match the eager torchao
- epilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path."""
+ epilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path. Each
+ per-shape tile is probed the same way; one that fails leaves its shapes on the default tile."""
if index in _DEVICE_CFG:
return _DEVICE_CFG[index]
cfg = None
+ rules: tuple = ()
try:
import torch
if (
@@ -427,13 +390,22 @@ def device_config(index: int) -> Optional[tuple]:
and _kernels() is not None
and _op() is not None
):
- want = arch_config(torch.cuda.get_device_capability(index))
+ cap = torch.cuda.get_device_capability(index)
+ want = arch_config(cap)
for cand in (want, _FALLBACK_CONFIG) if want is not None else ():
if cand is not None and _probe(index, cand):
cfg = cand
break
+ if cfg is not None:
+ ok: dict = {}
+ for rule in shape_tiles(cap):
+ tile = rule[4]
+ if tile not in ok:
+ ok[tile] = tile == cfg or _probe(index, tile)
+ rules = tuple(rule for rule in shape_tiles(cap) if ok[rule[4]])
except Exception: # noqa: BLE001
cfg = None
+ _DEVICE_TILES[index] = rules if cfg is not None else ()
_DEVICE_CFG[index] = cfg
return cfg
@@ -475,224 +447,6 @@ def _probe(index: int, cfg: tuple) -> bool:
return False
-# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).
-_ROTQ_QPARAMS = {
- "v1": (-127.0, 127.0, 127.0, 1e-5),
- "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),
-}
-# device index -> probed rotq tile (None = stock).
-_ROTQ_DEVICE: dict = {}
-_ROTQ_CALLS = [0]
-_ROTQ_HANDLE: Any = None
-
-
-@lru_cache(maxsize = 1)
-def _v2_act_scale_fp32() -> bool:
- """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""
- try:
- import torch
- from torchao.quantization.granularity import PerRow
- from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
-
- x = torch.ones(1, 16, dtype = torch.bfloat16)
- return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32
- except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit
- return False
-
-
-def _rotq_scale_fp32(kind: str) -> bool:
- return kind == "v2" and _v2_act_scale_fp32()
-
-
-def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:
- """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""
- import torch
- from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
-
- xr = rotate_convrot_activation(
- x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group
- )
- if kind == "v1":
- q, scale = _act_quant_v1(xr)
- else:
- from torchao.quantization.granularity import PerRow
- from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
-
- t = Int8Tensor.from_hp(xr, PerRow())
- q, scale = t.qdata, t.scale
- if _rotq_scale_fp32(kind):
- return q, scale.reshape(-1).to(torch.float32)
- return q, scale.reshape(-1).to(torch.bfloat16)
-
-
-def _rotq_rows(k: int, group: int, cfg: tuple) -> int:
- """Whole activation rows per program (0: this K does not fit one tile)."""
- return cfg[0] // (k // group)
-
-
-def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:
- import torch
- import triton
- from .diffusion_convrot import build_convrot_hadamard
-
- kern = _kernels()
- m, k = x2d.shape
- bm, bk, warps, stages = cfg
- rows = _rotq_rows(k, group, cfg)
- qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]
- h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)
- q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)
- fp32_scale = _rotq_scale_fp32(kind)
- s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)
- with torch.cuda.device(x2d.device):
- kern.rotq_i8[(triton.cdiv(m, rows),)](
- x2d,
- h,
- q,
- s,
- m,
- k // group,
- G = group,
- BLOCK_M = bm,
- BLOCK_K = bk,
- ROWS = rows,
- ROWS_P2 = max(triton.next_power_of_2(rows), 2),
- QMIN = qmin,
- QMAX = qmax,
- DIV = div,
- EPS = eps,
- FP32_SCALE = fp32_scale,
- num_warps = warps,
- num_stages = stages,
- enable_fp_fusion = False,
- )
- return q, s
-
-
-def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:
- """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""
- import torch
-
- if (
- cfg is None
- or group not in _ROTQ_GROUPS
- or x2d.dim() != 2
- or x2d.dtype != torch.bfloat16
- or not x2d.is_cuda
- ):
- return False
- k = x2d.shape[1]
- return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1
-
-
-def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:
- """Op body: the probed tile for this device, the stock rotation + quant if anything does not fit."""
- _ROTQ_CALLS[0] += 1
- kind = "v2" if v2 else "v1"
- x2d = x2d if x2d.is_contiguous() else x2d.contiguous()
- cfg = _ROTQ_DEVICE.get(x2d.device.index)
- if rotquant_supported(x2d, group, cfg):
- try:
- return _rotq_launch(x2d, group, kind, cfg)
- except Exception: # noqa: BLE001 - a failed launch keeps the stock math
- _ROTQ_DEVICE[x2d.device.index] = None
- return rotquant_reference(x2d, group, kind)
-
-
-@lru_cache(maxsize = 1)
-def _rotq_op() -> Any:
- """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""
- try:
- import torch
- except Exception: # noqa: BLE001
- return None
- ns = getattr(torch.ops, _OP_NAMESPACE, None)
- if ns is not None and hasattr(ns, _ROTQ_OP_NAME):
- return getattr(ns, _ROTQ_OP_NAME)
- custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
- if custom_op is None:
- return None
- try:
-
- @custom_op(
- f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",
- mutates_args = (),
- schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",
- )
- def _convrot_act_quant_int8(x, group, v2):
- return _rotq_run(x, group, v2)
-
- @_convrot_act_quant_int8.register_fake
- def _(x, group, v2):
- return (
- x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),
- x.new_empty(
- (x.shape[0],),
- dtype = torch.float32
- if _rotq_scale_fp32("v2" if v2 else "v1")
- else torch.bfloat16,
- ),
- )
- except Exception: # noqa: BLE001
- return None
- return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)
-
-
-def rotquant_device_config(index: int) -> Optional[tuple]:
- """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged
- M). None = stock."""
- if index in _ROTQ_DEVICE:
- return _ROTQ_DEVICE[index]
- cfg = None
- try:
- import torch
- if (
- torch.cuda.is_available()
- and not getattr(torch.version, "hip", None)
- and _triton_version_ok()
- and _kernels() is not None
- and _rotq_op() is not None
- ):
- want = rotquant_config(torch.cuda.get_device_capability(index))
- for cand in (want, _ROTQ_FALLBACK) if want is not None else ():
- if _rotq_probe(index, cand):
- cfg = cand
- break
- except Exception: # noqa: BLE001
- cfg = None
- _ROTQ_DEVICE[index] = cfg
- return cfg
-
-
-def _rotq_probe(index: int, cfg: tuple) -> bool:
- import torch
-
- dev = torch.device("cuda", index)
- g = torch.Generator(device = "cpu").manual_seed(2)
- try:
- for m, k in ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128)):
- if _rotq_rows(k, 256, cfg) < 1:
- continue
- x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
- x[:, :3] *= 60
- x[1] = 0
- x[2, 11] = 3e4
- x = x.to(torch.bfloat16).to(dev)
- for kind in ("v1", "v2"):
- q, s = _rotq_launch(x, 256, kind, cfg)
- rq, rs = rotquant_reference(x, 256, kind)
- if not (torch.equal(q, rq) and torch.equal(s, rs)):
- return False
- torch.cuda.synchronize(dev)
- return True
- except Exception: # noqa: BLE001
- return False
-
-
-def rotquant_call_count() -> int:
- return _ROTQ_CALLS[0]
-
-
def tie_operands(
device: Any,
rows: int = 32,
@@ -777,28 +531,8 @@ def _v2_parts(w: Any, check_zero_point: bool = True) -> Optional[tuple]:
def _act_quant_v1(x2d: Any) -> tuple:
- """torchao 0.17's ``_int8_symm_per_token_reduced_range_quant`` minus the tensor-subclass wrapper."""
- import torch
- from torchao.quantization.quant_primitives import (
- MappingType,
- choose_qparams_affine,
- quantize_affine,
- )
-
- block = (1, x2d.shape[-1])
- scale, zero_point = choose_qparams_affine(
- x2d,
- MappingType.SYMMETRIC,
- block,
- torch.int8,
- -127,
- 127,
- 1e-5,
- torch.float32 if x2d.dtype == torch.float16 else None,
- None,
- )
- q = quantize_affine(x2d, block, scale, zero_point, torch.int8, -127, 127)
- return q, scale
+ from .diffusion_convrot_quant import act_quant_v1
+ return act_quant_v1(x2d)
def _act_quant_v2(x2d: Any, weight: Any) -> tuple:
@@ -828,14 +562,28 @@ def _v1_act_quant_matches(index: int) -> bool:
return False
+def stock_gemm(xq: Any, wq: Any, xs: Any, ws: Any, bias: Any) -> Any:
+ """torchao's int8 Linear after its act quant, op for op: cuBLAS ``_int_mm``, ``* x_scale`` in the scale's dtype,
+ bf16, ``* w_scale``, ``+ bias``, bf16 (``reference`` minus the small-M padding; callers keep M >= 17). torchao 0.17's
+ zero-point pass subtracts an exact zero for this symmetric quant and is skipped."""
+ import torch
+
+ y = (torch._int_mm(xq, wq.t()) * xs.reshape(-1, 1)).to(torch.bfloat16)
+ y = y * ws.reshape(-1)
+ if bias is not None:
+ y = y + bias
+ return y.to(torch.bfloat16)
+
+
def _linear_forward(self: Any, x: Any) -> Any:
- """``nn.Linear.forward`` for a torchao int8 dynamic weight: torchao's activation quant, then the fused GEMM."""
+ """``nn.Linear.forward`` for a torchao int8 dynamic weight: torchao's activation quant (fused with the ConvRot
+ rotation where that kernel is on), then the fused-dequant GEMM, or cuBLAS + torchao's epilogue where it is not."""
import torch
rec = self.__dict__.get(_REC)
if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:
return type(self).forward(self, x)
- kind, group, rotq, weight = rec
+ kind, group, rotq, weight, fused = rec
if self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock
return type(self).forward(self, x)
# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.
@@ -849,12 +597,15 @@ def _linear_forward(self: Any, x: Any) -> Any:
lead = x.shape[:-1]
if x.numel() < _MIN_ROWS * x.shape[-1]:
return type(self).forward(self, x)
- if (
- rotq
- and _ROTQ_HANDLE is not None
- and rotquant_supported(x.reshape(-1, x.shape[-1]), group, _ROTQ_DEVICE.get(x.device.index))
- ):
- xq, xs = _ROTQ_HANDLE(x.reshape(-1, x.shape[-1]), group, kind == "v2")
+ q = None
+ if rotq:
+ from .diffusion_convrot_quant import convrot_act_quant
+ q = convrot_act_quant(x.reshape(-1, x.shape[-1]), group, kind == "v2")
+ if q is not None:
+ xq, xs = q
+ elif not fused:
+ # rotation kernel not covering this call and no fused GEMM: the module's own stock forward
+ return type(self).forward(self, x)
else:
if group is not None:
from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
@@ -866,7 +617,10 @@ def _linear_forward(self: Any, x: Any) -> Any:
xq, xs = _act_quant_v1(x2d)
else:
xq, xs = _act_quant_v2(x2d, weight)
- y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
+ if fused:
+ y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
+ else:
+ y = stock_gemm(xq, wq, xs, ws, self.bias)
return y.reshape(*lead, y.shape[-1])
@@ -937,13 +691,23 @@ def _eligible(module: Any) -> Optional[tuple]:
return None
-def candidates(transformer: Any) -> int:
+def candidates(transformer: Any, rotated_only: bool = False) -> int:
try:
- return sum(1 for m in transformer.modules() if _eligible(m) is not None)
+ recs = (_eligible(m) for m in transformer.modules())
+ return sum(1 for r in recs if r is not None and (r[1] is not None or not rotated_only))
except Exception: # noqa: BLE001
return 0
+def _gates(capability: Optional[tuple]) -> tuple:
+ """(fused GEMM on this arch, rotation kernel on this arch), before any probe."""
+ from .diffusion_convrot_quant import rotquant_config
+
+ gemm = arch_config(capability) is not None
+ rotq = convrot_enabled() and rotquant_config(capability) is not None
+ return gemm, rotq
+
+
def install(
transformer: Any,
logger: Any = None,
@@ -974,11 +738,12 @@ def install(
return 0
if not _triton_version_ok() or _kernels() is None:
return 0
- if arch_config(torch.cuda.get_device_capability(torch.cuda.current_device())) is None:
+ gemm, rotq = _gates(torch.cuda.get_device_capability(torch.cuda.current_device()))
+ if not (gemm or rotq):
return 0
except Exception: # noqa: BLE001
return 0
- n = candidates(transformer)
+ n = candidates(transformer, rotated_only = not gemm)
if n:
run_on_first_call(transformer, "int8_gemm", lambda t: _finalize(t, logger))
return n
@@ -1002,7 +767,7 @@ def _swap(
logger: Any = None,
device: Any = None,
) -> int:
- global _OP_HANDLE, _ROTQ_HANDLE
+ global _OP_HANDLE
from .diffusion_int8_fused import resident_cuda_device
import torch
@@ -1026,31 +791,45 @@ def _swap(
index = dev.index if dev.index is not None else torch.cuda.current_device()
recs = [(m, _eligible(m)) for m in transformer.modules()]
recs = [(m, r) for m, r in recs if r is not None]
- if not recs or device_config(index) is None:
+ if not recs:
return 0
+ from .diffusion_convrot_quant import rotquant_device_config
+
+ gemm = device_config(index) is not None
+ rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None
+ if not gemm:
+ # no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)
+ recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []
if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):
recs = [(m, r) for m, r in recs if r[0] != "v1"]
- rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None
+ if not recs:
+ return 0
count = 0
with _LOCK:
- _OP_HANDLE = _op()
- if _OP_HANDLE is None:
- return 0
- if rotq:
- _ROTQ_HANDLE = _rotq_op()
+ if gemm:
+ _OP_HANDLE = _op()
+ if _OP_HANDLE is None:
+ return 0
for module, rec in recs:
if _MARK in module.__dict__:
count += 1
continue
- # kind, rotation group, fused rotation + act quant on, the Parameter; no payload alias
- module.__dict__[_REC] = (rec[0], rec[1], bool(rotq and rec[1] is not None), rec[3])
+ # kind, rotation group, fused rotation + act quant on, the Parameter (no payload alias), fused GEMM on
+ module.__dict__[_REC] = (
+ rec[0],
+ rec[1],
+ bool(rotq and rec[1] is not None),
+ rec[3],
+ gemm,
+ )
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
count += 1
if logger is not None and count:
logger.info(
- "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d%s",
+ "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s",
count,
+ "run the fused-dequant GEMM (bf16 out)" if gemm else "keep cuBLAS",
*torch.cuda.get_device_capability(index),
", ConvRot rotation fused into the act quant" if rotq else "",
)
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index c5b1abc7a68..0db6c827680 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import pytest
+from core.inference import diffusion_convrot_quant as cq
from core.inference import diffusion_int8_gemm as g8
torch = pytest.importorskip("torch")
@@ -16,12 +17,17 @@ torch = pytest.importorskip("torch")
@pytest.fixture(autouse = True)
def _clean(monkeypatch):
monkeypatch.delenv(g8.INT8_GEMM_ENV, raising = False)
- monkeypatch.delenv(g8.INT8_ROTQUANT_ENV, raising = False)
+ monkeypatch.delenv(cq.INT8_ROTQUANT_ENV, raising = False)
+ monkeypatch.delenv(g8.INT8_GEMM_TILES_ENV, raising = False)
g8._DEVICE_CFG.clear()
- g8._ROTQ_DEVICE.clear()
+ g8._DEVICE_TILES.clear()
+ cq._ROTQ_DEVICE.clear()
+ cq._ROTQ_DEVICE_K.clear()
yield
g8._DEVICE_CFG.clear()
- g8._ROTQ_DEVICE.clear()
+ g8._DEVICE_TILES.clear()
+ cq._ROTQ_DEVICE.clear()
+ cq._ROTQ_DEVICE_K.clear()
@pytest.mark.parametrize(
@@ -608,32 +614,32 @@ def test_streamed_kill_switch(forced, monkeypatch):
[
((8, 0), True), # A100: measured end to end (streamed H3)
((8, 6), False),
- ((8, 9), False), # L4: H3 Diffusers does not load there (host RAM floor), never measured
+ ((8, 9), True), # L4
((9, 0), False),
((10, 0), False),
((12, 0), True),
],
)
def test_rotquant_arch_gate(cap, on):
- assert (g8.rotquant_config(cap, "auto") is not None) is on
+ assert (cq.rotquant_config(cap, "auto") is not None) is on
def test_rotquant_kill_switch_and_force(monkeypatch):
assert (
- g8.rotquant_config((12, 0), "off") is None
+ cq.rotquant_config((12, 0), "off") is None
) # the fused GEMM's own kill switch also turns it off
- assert g8.rotquant_config((10, 0), "force") is not None
- assert g8.rotquant_config((7, 5), "force") is None
- assert g8.rotquant_config((8, 0), "off") is None
- monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")
- assert g8.rotquant_config((12, 0), "auto") is None
- assert g8.rotquant_config((8, 0), "auto") is None
- assert g8.rotquant_config((10, 0), "force") is None
+ assert cq.rotquant_config((10, 0), "force") is not None
+ assert cq.rotquant_config((7, 5), "force") is None
+ assert cq.rotquant_config((8, 0), "off") is None
+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "0")
+ assert cq.rotquant_config((12, 0), "auto") is None
+ assert cq.rotquant_config((8, 0), "auto") is None
+ assert cq.rotquant_config((10, 0), "force") is None
@pytest.fixture
def forced_rotq(forced):
- cfg = g8.rotquant_device_config(torch.cuda.current_device())
+ cfg = cq.rotquant_device_config(torch.cuda.current_device())
if cfg is None:
pytest.skip("fused rotation probe refused this device")
return cfg
@@ -654,10 +660,10 @@ def _act(m, k, seed):
@pytest.mark.parametrize("kind", ["v1", "v2"])
def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq, m, k, kind):
x = _act(m, k, m + k)
- q, s = g8._rotq_op()(x, 256, kind == "v2")
- rq, rs = g8.rotquant_reference(x, 256, kind)
+ q, s = cq._rotq_op()(x, 256, kind == "v2")
+ rq, rs = cq.rotquant_reference(x, 256, kind)
# torchao's own activation-scale dtype: bf16, except v2 on torchao >= 0.18 (fp32)
- want = torch.float32 if kind == "v2" and g8._v2_act_scale_fp32() else torch.bfloat16
+ want = torch.float32 if kind == "v2" and cq._v2_act_scale_fp32() else torch.bfloat16
assert q.dtype == torch.int8 and s.dtype == want and s.shape == (m,)
assert torch.equal(q, rq) and torch.equal(s, rs)
@@ -665,7 +671,7 @@ def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq
@needs_cuda
def test_rotquant_probe_accepts_this_torchao(forced):
# with 0.17's bf16 roundings on 0.18, ~4% of codes differed and the probe refused every device
- assert g8.rotquant_device_config(torch.cuda.current_device()) is not None
+ assert cq.rotquant_device_config(torch.cuda.current_device()) is not None
@needs_cuda
@@ -673,24 +679,24 @@ def test_rotquant_fake_op_matches_the_real_scale_dtype(forced_rotq):
from torch._subclasses.fake_tensor import FakeTensorMode
x = _act(40, 768, 9)
for v2 in (False, True):
- _, s = g8._rotq_op()(x, 256, v2)
+ _, s = cq._rotq_op()(x, 256, v2)
with FakeTensorMode() as mode:
- _, fs = g8._rotq_op()(mode.from_tensor(x), 256, v2)
+ _, fs = cq._rotq_op()(mode.from_tensor(x), 256, v2)
assert fs.dtype == s.dtype
@needs_cuda
def test_rotquant_unsupported_shapes_take_the_stock_math(forced_rotq):
x = _act(40, 256 * 129, 3) # more groups than one tile holds
- assert not g8.rotquant_supported(x, 256, forced_rotq)
- assert not g8.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)
- assert not g8.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)
- q, s = g8._rotq_op()(x, 256, True)
- rq, rs = g8.rotquant_reference(x, 256, "v2")
+ assert not cq.rotquant_supported(x, 256, forced_rotq)
+ assert not cq.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)
+ assert not cq.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)
+ q, s = cq._rotq_op()(x, 256, True)
+ rq, rs = cq.rotquant_reference(x, 256, "v2")
assert torch.equal(q, rq) and torch.equal(s, rs)
xt = _act(1024, 300, 6).t() # non-contiguous input: made contiguous, same result
- q, s = g8._rotq_op()(xt, 256, True)
- rq, rs = g8.rotquant_reference(xt.contiguous(), 256, "v2")
+ q, s = cq._rotq_op()(xt, 256, True)
+ rq, rs = cq.rotquant_reference(xt.contiguous(), 256, "v2")
assert torch.equal(q, rq) and torch.equal(s, rs)
@@ -714,9 +720,9 @@ def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(
x * 0.5
) # made outside inference_mode like x, so a recompile here would be the op's own guards
with torch.inference_mode():
- before = g8.rotquant_call_count()
+ before = cq.rotquant_call_count()
assert torch.equal(fused(x), stock(x))
- assert g8.rotquant_call_count() == before + 1
+ assert cq.rotquant_call_count() == before + 1
counters.clear()
torch._dynamo.reset()
with torch._inductor.config.patch(emulate_precision_casts = True):
@@ -732,22 +738,246 @@ def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(
@needs_cuda
def test_rotquant_kill_switch_keeps_the_stock_rotation(forced, monkeypatch):
monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
- monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")
+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "0")
stock = _convrot(_int8_linear(1024, 768, False, 2))
fused = _convrot(_int8_linear(1024, 768, False, 2))
holder = torch.nn.Sequential(fused)
assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is False
x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)
with torch.inference_mode():
- before, gemm = g8.rotquant_call_count(), g8.call_count()
+ before, gemm = cq.rotquant_call_count(), g8.call_count()
assert torch.equal(fused(x), stock(x))
- assert g8.rotquant_call_count() == before and g8.call_count() == gemm + 1
+ assert cq.rotquant_call_count() == before and g8.call_count() == gemm + 1
g8.uninstall(holder)
@needs_cuda
def test_rotquant_probe_refusal_keeps_the_stock_rotation(forced, monkeypatch):
monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
- monkeypatch.setattr(g8, "_rotq_probe", lambda index, cfg: False)
+ monkeypatch.setattr(cq, "_rotq_probe", lambda index, cfg: False)
fused = _convrot(_int8_linear(1024, 768, False, 2))
assert g8.install(torch.nn.Sequential(fused)) == 1 and fused.__dict__[g8._REC][2] is False
+
+
+# ---------------------------------------------------------------- per-shape tiles
+
+
+def test_tile_for_picks_the_shape_rule_then_the_arch_default(monkeypatch):
+ default, wide = (128, 128, 64, 8, 4, 4), (128, 256, 64, 8, 8, 3)
+ g8._DEVICE_CFG[0] = default
+ g8._DEVICE_TILES[0] = ((16384, 65536, 4096, 8192, wide),)
+ assert g8.tile_for(0, 4096, 21504, 5376) == wide
+ assert g8.tile_for(0, 4096, 5376, 5376) == default # N outside the rule
+ assert g8.tile_for(0, 4096, 21504, 14336) == default # K outside the rule
+ assert g8.tile_for(0, g8._SHAPE_MIN_M - 1, 21504, 5376) == default # small M keeps the default
+ assert g8.tile_for(1, 4096, 21504, 5376) is None # unprobed device: stock
+
+
+def test_tiles_kill_switch_and_unmeasured_arch(monkeypatch):
+ monkeypatch.setitem(g8._SHAPE_TILES, (12, 0), ((1, 2, 3, 4, (64, 64, 64, 8, 4, 3)),))
+ assert g8.shape_tiles((12, 0))
+ assert g8.shape_tiles((8, 6)) == ()
+ monkeypatch.setenv(g8.INT8_GEMM_TILES_ENV, "0")
+ assert g8.shape_tiles((12, 0)) == ()
+
+
+def test_shape_tile_launch_failure_falls_back_to_the_default(monkeypatch):
+ default, wide = (128, 128, 64, 8, 4, 4), (128, 256, 64, 8, 8, 3)
+ g8._DEVICE_CFG[None] = default # CPU tensors report device index None
+ g8._DEVICE_TILES[None] = ((1, 1 << 20, 1, 1 << 20, wide),)
+ used = []
+
+ def launch(a, w, xs, ws, bias, cfg):
+ used.append(cfg)
+ if cfg == wide:
+ raise RuntimeError("out of shared memory")
+ return "fused"
+
+ monkeypatch.setattr(g8, "_launch", launch)
+ monkeypatch.setattr(g8, "reference", lambda *a: "stock")
+ a = torch.zeros(2048, 1024, dtype = torch.int8)
+ w = torch.zeros(1024, 1024, dtype = torch.int8)
+ xs = torch.ones(2048, dtype = torch.bfloat16)
+ ws = torch.ones(1024, dtype = torch.bfloat16)
+ assert g8._run(a, w, xs, ws, None) == "fused"
+ assert used == [wide, default]
+ assert g8._DEVICE_TILES[None] == () and g8._DEVICE_CFG[None] == default
+ assert g8._run(a, w, xs, ws, None) == "fused" and used[-1] == default
+
+
+@needs_cuda
+def test_failed_shape_tile_probe_keeps_its_shapes_on_the_default(monkeypatch):
+ cap = torch.cuda.get_device_capability()
+ bad, good = (64, 64, 64, 8, 4, 3), (128, 64, 64, 8, 4, 3)
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")
+ monkeypatch.setitem(
+ g8._SHAPE_TILES, cap, ((1, 4096, 1, 4096, bad), (4097, 1 << 20, 1, 1 << 20, good))
+ )
+ real = g8._probe
+ monkeypatch.setattr(g8, "_probe", lambda index, cfg: cfg != bad and real(index, cfg))
+ cfg = g8.device_config(torch.cuda.current_device())
+ if cfg is None:
+ pytest.skip("int8 GEMM probe refused this device")
+ rules = g8._DEVICE_TILES[torch.cuda.current_device()]
+ assert [r[4] for r in rules] == [good]
+ assert g8.tile_for(torch.cuda.current_device(), 4096, 1024, 1024) == cfg
+
+
+def _shipped_tiles():
+ return sorted({rule[4] for rules in g8._SHAPE_TILES.values() for rule in rules})
+
+
+@needs_cuda
[email protected]("tile", _shipped_tiles() or [None])
[email protected]("ws32", [False, True])
+def test_every_shipped_shape_tile_is_bit_exact(forced, tile, ws32):
+ """Every tile in the shape table, at a shape it is picked for and ragged neighbours: int32 accumulation makes the
+ output independent of the tile, so each must equal the eager torchao epilogue bit for bit (and the tie operands)."""
+ if tile is None:
+ pytest.skip("no per-shape tiles shipped")
+ g = torch.Generator().manual_seed(7)
+ for m, n, k in ((1100, 1024, 1024), (2061, 2064, 1088), (17, 512, 256)):
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).cuda()
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).cuda()
+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(torch.bfloat16).cuda()
+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(torch.bfloat16).cuda()
+ ws = ws.float() if ws32 else ws
+ try:
+ out = g8._launch(a, w, xs, ws, None, tile)
+ except Exception as exc: # noqa: BLE001 - a tile that does not fit this part is dropped by the probe
+ if g8._probe(torch.cuda.current_device(), tile):
+ raise
+ pytest.skip(
+ f"tile {tile} does not fit this part ({type(exc).__name__}); probe drops it"
+ )
+ assert torch.equal(out, g8.reference(a, w, xs, ws, None))
+ a, w = g8.tie_operands(torch.device("cuda"))
+ ones = torch.ones(a.shape[0], device = "cuda", dtype = torch.bfloat16)
+ wones = torch.ones(w.shape[0], device = "cuda", dtype = torch.bfloat16)
+ assert torch.equal(
+ g8._launch(a, w, ones, wones, None, tile), g8.reference(a, w, ones, wones, None)
+ )
+
+
+# ---------------------------------------------------------------- fused rotation without the fused GEMM
+
+
+def rotq_only(monkeypatch):
+ """The fused rotation forced on, the fused GEMM forced off (what sm100 and other unfused archs run)."""
+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "1")
+ monkeypatch.setattr(g8, "device_config", lambda index: None)
+ monkeypatch.setattr(g8, "arch_config", lambda cap, mode = None: None)
+ cfg = cq.rotquant_device_config(torch.cuda.current_device())
+ if cfg is None:
+ pytest.skip("fused rotation probe refused this device")
+ return cfg
+
+
+def test_rotquant_mode_follows_its_own_switch_and_the_gemm_master(monkeypatch):
+ assert cq.rotquant_mode() == "auto"
+ monkeypatch.setenv(cq.INT8_ROTQUANT_ENV, "1")
+ assert cq.rotquant_mode() == "force" and cq.rotquant_config((9, 0)) == cq._ROTQ_FALLBACK
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "0") # the whole int8 swap off
+ assert cq.rotquant_mode() == "off" and cq.rotquant_config((12, 0)) is None
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")
+ monkeypatch.delenv(cq.INT8_ROTQUANT_ENV)
+ assert cq.rotquant_mode() == "force"
+
+
+@needs_cuda
[email protected]("version", [None, 2])
[email protected]("bias", [False, True])
+def test_rotated_linear_without_the_fused_gemm_is_bit_identical(
+ rotq_only, monkeypatch, version, bias
+):
+ """Fused GEMM off, rotation kernel on: only the ConvRot Linear is swapped; rotq + cuBLAS _int_mm + torchao's
+ epilogue equals ConvRotLinear.forward eager, compiled fullgraph keeps one graph, the plain Linear stays stock."""
+ from torch._dynamo.utils import counters
+
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ stock = _convrot(_int8_linear(1024, 768, bias, version))
+ fused = _convrot(_int8_linear(1024, 768, bias, version))
+ plain = _int8_linear(1024, 768, bias, version)
+ holder = torch.nn.Sequential(fused, torch.nn.Linear(768, 1024).cuda().to(torch.bfloat16), plain)
+ assert g8.install(holder) == 1
+ assert g8.is_installed(fused) and not g8.is_installed(plain)
+ assert fused.__dict__[g8._REC][2] is True and fused.__dict__[g8._REC][4] is False
+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3
+ x[0, 7, :5] *= 200
+ with torch.inference_mode():
+ before, gemm = cq.rotquant_call_count(), g8.call_count()
+ assert torch.equal(fused(x), stock(x))
+ assert cq.rotquant_call_count() == before + 1 and g8.call_count() == gemm
+ counters.clear()
+ torch._dynamo.reset()
+ with torch._inductor.config.patch(emulate_precision_casts = True):
+ compiled = torch.compile(fused, fullgraph = True)
+ out = compiled(x)
+ assert not counters["graph_break"]
+ assert counters["stats"]["unique_graphs"] == 1
+ # codes / scales come from the eager-exact op; the epilogue's bf16 roundings are emulated
+ assert torch.equal(out, stock(x))
+ g8.uninstall(holder)
+
+
+@needs_cuda
+def test_rotated_linear_without_either_kernel_is_left_alone(monkeypatch):
+ monkeypatch.setattr(g8, "device_config", lambda index: None)
+ monkeypatch.setattr(cq, "rotquant_device_config", lambda index: None)
+ fused = _convrot(_int8_linear(1024, 768, False, 2))
+ assert g8.install(torch.nn.Sequential(fused)) == 0 and not g8.is_installed(fused)
+
+
+@needs_cuda
+def test_stock_gemm_matches_torchao_linear_bit_for_bit(rotq_only):
+ for version in (None, 2):
+ lin = _int8_linear(1024, 768, True, version)
+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16) * 2
+ parts = g8._v1_parts(lin.weight) or g8._v2_parts(lin.weight)
+ kind = "v1" if g8._v1_parts(lin.weight) else "v2"
+ xq, xs = g8._act_quant_v1(x) if kind == "v1" else g8._act_quant_v2(x, lin.weight)
+ with torch.inference_mode():
+ assert torch.equal(g8.stock_gemm(xq, parts[0], xs, parts[1], lin.bias), lin(x))
+
+
+@needs_cuda
+def test_convrot_act_quant_api(forced_rotq):
+ x = _act(300, 1024, 11)
+ q, s = cq.convrot_act_quant(x, 256, True)
+ rq, rs = cq.rotquant_reference(x, 256, "v2")
+ assert torch.equal(q, rq) and torch.equal(s, rs)
+ assert cq.convrot_act_quant(x, 64, True) is None # group the kernel does not cover
+ assert cq.convrot_act_quant(x.half(), 256, True) is None
+ assert cq.convrot_act_quant(x.cpu(), 256, True) is None
+
+
+def test_rotq_tile_for_picks_the_k_rule_only_where_a_row_fits(monkeypatch):
+ default, narrow = (128, 32, 8, 3), (64, 32, 4, 3)
+ cq._ROTQ_DEVICE[0] = default
+ cq._ROTQ_DEVICE_K[0] = ((5000, 20000, narrow),)
+ assert cq.rotq_tile_for(0, 5376) == narrow
+ assert cq.rotq_tile_for(0, 4096) == default # outside the rule
+ assert cq.rotq_tile_for(0, 256 * 70) == default # 70 groups do not fit a 64-row tile
+ assert cq.rotq_tile_for(1, 5376) is None
+ monkeypatch.setitem(cq._ROTQ_K_TILES, (8, 0), ((1, 2, narrow),))
+ assert cq.rotq_k_tiles((8, 0))
+ monkeypatch.setenv(g8.INT8_GEMM_TILES_ENV, "0")
+ assert cq.rotq_k_tiles((8, 0)) == ()
+
+
+@needs_cuda
+def test_every_shipped_rotq_k_tile_is_bit_exact(forced):
+ tiles = sorted({rule[2] for rules in cq._ROTQ_K_TILES.values() for rule in rules})
+ if not tiles:
+ pytest.skip("no per-K rotq tiles shipped")
+ for tile in tiles:
+ assert cq._rotq_probe(torch.cuda.current_device(), tile), tile
+ for m, k in ((1037, 5376), (300, 4096), (129, 14336), (77, 10240)):
+ if cq._rotq_rows(k, 256, tile) < 1:
+ continue
+ x = _act(m, k, m * k)
+ for kind in ("v1", "v2"):
+ q, s = cq._rotq_launch(x, 256, kind, tile)
+ rq, rs = cq.rotquant_reference(x, 256, kind)
+ assert torch.equal(q, rq) and torch.equal(s, rs), (tile, m, k, kind)