Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fuse MiniMax-H3's ConvRot rotation into the int8 activation quant (#12517)

Daniel Han34b87cb6a32a
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 865da207e09..31d338aaf8c 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -23,6 +23,10 @@ Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unm
ConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;
kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.
+Rotated Linears on sm120 / sm80 (group 256) run the rotation and the act quant as ONE kernel (``rotq_i8``): the same
+rotation GEMM in the same K order, then torchao's per-row quant with every bf16 rounding kept, so codes and scales are
+bit-identical (probed per device). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.
+
A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's
``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every
call sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``.
@@ -39,6 +43,7 @@ from typing import Any, Optional
INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"
INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
+INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
@@ -46,6 +51,7 @@ _K_ALIGN = 64
_N_ALIGN = 16
_OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_mm_dequant"
+_ROTQ_OP_NAME = "convrot_act_quant_int8"
_REC = "_unsloth_i8_gemm"
_MARK = "_unsloth_i8_gemm_prev"
_NO_PREV = object()
@@ -70,6 +76,14 @@ _ARCH_CONFIG = {
# When the arch tile does not fit this part's shared memory.
_FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)
+# rotq_i8 (BLOCK_M group rows, BLOCK_K, num_warps, num_stages); measured end to end on these archs only.
+_ROTQ_CONFIG = {
+ (8, 0): (128, 32, 8, 3),
+ (12, 0): (128, 32, 8, 4),
+}
+_ROTQ_FALLBACK = (128, 32, 8, 3)
+_ROTQ_GROUPS = (256,)
+
def int8_gemm_mode() -> str:
"""'off', 'force' or 'auto' from the environment."""
@@ -91,6 +105,25 @@ def streamed_enabled() -> bool:
return raw not in ("0", "off", "false", "no")
+def rotquant_enabled() -> bool:
+ """ConvRot Linears quantize their activation with the fused rotation kernel unless
+ ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""
+ raw = (os.environ.get(INT8_ROTQUANT_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
+def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:
+ """The fused rotation tile for this compute capability, or None (stock rotation)."""
+ mode = int8_gemm_mode() if mode is None else mode
+ if mode == "off" or capability is None or not rotquant_enabled():
+ return None
+ cap = (int(capability[0]), int(capability[1]))
+ cfg = _ROTQ_CONFIG.get(cap)
+ if cfg is None and mode == "force" and cap >= (8, 0):
+ return _ROTQ_FALLBACK
+ return cfg
+
+
def _triton_version_ok(version: Optional[str] = None) -> bool:
if version is None:
try:
@@ -202,7 +235,65 @@ def _kernels() -> Optional[types.SimpleNamespace]:
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)
- return types.SimpleNamespace(i8mm_dq = i8mm_dq)
+ from triton.language.extra import libdevice
+
+ @triton.jit
+ def rotq_i8(
+ x_ptr,
+ h_ptr,
+ q_ptr,
+ s_ptr,
+ M,
+ NG,
+ G: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ ROWS: tl.constexpr,
+ ROWS_P2: tl.constexpr,
+ QMIN: tl.constexpr,
+ QMAX: tl.constexpr,
+ DIV: tl.constexpr,
+ EPS: tl.constexpr,
+ FP32_SCALE: tl.constexpr,
+ ):
+ # one program = ROWS whole activation rows, so the per-row amax closes inside the tile
+ pid = tl.program_id(0)
+ t = tl.arange(0, BLOCK_M)
+ lr = t // NG
+ row = pid.to(tl.int64) * ROWS + lr
+ live = (lr < ROWS) & (row < M)
+ rg = tl.where(live, row * NG + (t % NG), 0)
+ offs_k = tl.arange(0, BLOCK_K)
+ offs_n = tl.arange(0, G)
+ acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)
+ a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]
+ h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]
+ for _k in range(0, G // BLOCK_K):
+ a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)
+ acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)
+ a_ptrs += BLOCK_K
+ h_ptrs += BLOCK_K * G
+ z = _rbf16(acc)
+ # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18
+ # Int8Tensor) keeps the reciprocal and the product in fp32.
+ j = tl.arange(0, ROWS_P2)
+ sel = lr[:, None] == j[None, :]
+ amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)
+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))
+ inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)
+ if not FP32_SCALE:
+ inv = _rbf16(inv)
+ inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)
+ p = z * inv_t[:, None]
+ if not FP32_SCALE:
+ p = _rbf16(p)
+ p = libdevice.rint(p)
+ p = tl.minimum(tl.maximum(p, QMIN), QMAX)
+ tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])
+ srow = pid.to(tl.int64) * ROWS + j
+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))
+
+ return types.SimpleNamespace(i8mm_dq = i8mm_dq, rotq_i8 = rotq_i8)
# device index -> probed tile (None = stock).
@@ -384,6 +475,224 @@ def _probe(index: int, cfg: tuple) -> bool:
return False
+# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).
+_ROTQ_QPARAMS = {
+ "v1": (-127.0, 127.0, 127.0, 1e-5),
+ "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),
+}
+# device index -> probed rotq tile (None = stock).
+_ROTQ_DEVICE: dict = {}
+_ROTQ_CALLS = [0]
+_ROTQ_HANDLE: Any = None
+
+
+@lru_cache(maxsize = 1)
+def _v2_act_scale_fp32() -> bool:
+ """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""
+ try:
+ import torch
+ from torchao.quantization.granularity import PerRow
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
+
+ x = torch.ones(1, 16, dtype = torch.bfloat16)
+ return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32
+ except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit
+ return False
+
+
+def _rotq_scale_fp32(kind: str) -> bool:
+ return kind == "v2" and _v2_act_scale_fp32()
+
+
+def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:
+ """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""
+ import torch
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+
+ xr = rotate_convrot_activation(
+ x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group
+ )
+ if kind == "v1":
+ q, scale = _act_quant_v1(xr)
+ else:
+ from torchao.quantization.granularity import PerRow
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor
+
+ t = Int8Tensor.from_hp(xr, PerRow())
+ q, scale = t.qdata, t.scale
+ if _rotq_scale_fp32(kind):
+ return q, scale.reshape(-1).to(torch.float32)
+ return q, scale.reshape(-1).to(torch.bfloat16)
+
+
+def _rotq_rows(k: int, group: int, cfg: tuple) -> int:
+ """Whole activation rows per program (0: this K does not fit one tile)."""
+ return cfg[0] // (k // group)
+
+
+def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:
+ import torch
+ import triton
+ from .diffusion_convrot import build_convrot_hadamard
+
+ kern = _kernels()
+ m, k = x2d.shape
+ bm, bk, warps, stages = cfg
+ rows = _rotq_rows(k, group, cfg)
+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]
+ h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)
+ q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)
+ fp32_scale = _rotq_scale_fp32(kind)
+ s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)
+ with torch.cuda.device(x2d.device):
+ kern.rotq_i8[(triton.cdiv(m, rows),)](
+ x2d,
+ h,
+ q,
+ s,
+ m,
+ k // group,
+ G = group,
+ BLOCK_M = bm,
+ BLOCK_K = bk,
+ ROWS = rows,
+ ROWS_P2 = max(triton.next_power_of_2(rows), 2),
+ QMIN = qmin,
+ QMAX = qmax,
+ DIV = div,
+ EPS = eps,
+ FP32_SCALE = fp32_scale,
+ num_warps = warps,
+ num_stages = stages,
+ enable_fp_fusion = False,
+ )
+ return q, s
+
+
+def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:
+ """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""
+ import torch
+
+ if (
+ cfg is None
+ or group not in _ROTQ_GROUPS
+ or x2d.dim() != 2
+ or x2d.dtype != torch.bfloat16
+ or not x2d.is_cuda
+ ):
+ return False
+ k = x2d.shape[1]
+ return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1
+
+
+def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:
+ """Op body: the probed tile for this device, the stock rotation + quant if anything does not fit."""
+ _ROTQ_CALLS[0] += 1
+ kind = "v2" if v2 else "v1"
+ x2d = x2d if x2d.is_contiguous() else x2d.contiguous()
+ cfg = _ROTQ_DEVICE.get(x2d.device.index)
+ if rotquant_supported(x2d, group, cfg):
+ try:
+ return _rotq_launch(x2d, group, kind, cfg)
+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math
+ _ROTQ_DEVICE[x2d.device.index] = None
+ return rotquant_reference(x2d, group, kind)
+
+
+@lru_cache(maxsize = 1)
+def _rotq_op() -> Any:
+ """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _ROTQ_OP_NAME):
+ return getattr(ns, _ROTQ_OP_NAME)
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return None
+ try:
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",
+ mutates_args = (),
+ schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",
+ )
+ def _convrot_act_quant_int8(x, group, v2):
+ return _rotq_run(x, group, v2)
+
+ @_convrot_act_quant_int8.register_fake
+ def _(x, group, v2):
+ return (
+ x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),
+ x.new_empty(
+ (x.shape[0],),
+ dtype = torch.float32
+ if _rotq_scale_fp32("v2" if v2 else "v1")
+ else torch.bfloat16,
+ ),
+ )
+ except Exception: # noqa: BLE001
+ return None
+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)
+
+
+def rotquant_device_config(index: int) -> Optional[tuple]:
+ """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged
+ M). None = stock."""
+ if index in _ROTQ_DEVICE:
+ return _ROTQ_DEVICE[index]
+ cfg = None
+ try:
+ import torch
+ if (
+ torch.cuda.is_available()
+ and not getattr(torch.version, "hip", None)
+ and _triton_version_ok()
+ and _kernels() is not None
+ and _rotq_op() is not None
+ ):
+ want = rotquant_config(torch.cuda.get_device_capability(index))
+ for cand in (want, _ROTQ_FALLBACK) if want is not None else ():
+ if _rotq_probe(index, cand):
+ cfg = cand
+ break
+ except Exception: # noqa: BLE001
+ cfg = None
+ _ROTQ_DEVICE[index] = cfg
+ return cfg
+
+
+def _rotq_probe(index: int, cfg: tuple) -> bool:
+ import torch
+
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(2)
+ try:
+ for m, k in ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128)):
+ if _rotq_rows(k, 256, cfg) < 1:
+ continue
+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
+ x[:, :3] *= 60
+ x[1] = 0
+ x[2, 11] = 3e4
+ x = x.to(torch.bfloat16).to(dev)
+ for kind in ("v1", "v2"):
+ q, s = _rotq_launch(x, 256, kind, cfg)
+ rq, rs = rotquant_reference(x, 256, kind)
+ if not (torch.equal(q, rq) and torch.equal(s, rs)):
+ return False
+ torch.cuda.synchronize(dev)
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def rotquant_call_count() -> int:
+ return _ROTQ_CALLS[0]
+
+
def tie_operands(
device: Any,
rows: int = 32,
@@ -526,7 +835,7 @@ def _linear_forward(self: Any, x: Any) -> Any:
rec = self.__dict__.get(_REC)
if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:
return type(self).forward(self, x)
- kind, group, _ws, weight = rec
+ kind, group, rotq, weight = rec
if self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock
return type(self).forward(self, x)
# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.
@@ -540,16 +849,23 @@ def _linear_forward(self: Any, x: Any) -> Any:
lead = x.shape[:-1]
if x.numel() < _MIN_ROWS * x.shape[-1]:
return type(self).forward(self, x)
- if group is not None:
- from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
- x = rotate_convrot_activation(
- x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
- )
- x2d = x.reshape(-1, x.shape[-1])
- if kind == "v1":
- xq, xs = _act_quant_v1(x2d)
+ if (
+ rotq
+ and _ROTQ_HANDLE is not None
+ and rotquant_supported(x.reshape(-1, x.shape[-1]), group, _ROTQ_DEVICE.get(x.device.index))
+ ):
+ xq, xs = _ROTQ_HANDLE(x.reshape(-1, x.shape[-1]), group, kind == "v2")
else:
- xq, xs = _act_quant_v2(x2d, weight)
+ if group is not None:
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+ x = rotate_convrot_activation(
+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
+ )
+ x2d = x.reshape(-1, x.shape[-1])
+ if kind == "v1":
+ xq, xs = _act_quant_v1(x2d)
+ else:
+ xq, xs = _act_quant_v2(x2d, weight)
y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
return y.reshape(*lead, y.shape[-1])
@@ -686,7 +1002,7 @@ def _swap(
logger: Any = None,
device: Any = None,
) -> int:
- global _OP_HANDLE
+ global _OP_HANDLE, _ROTQ_HANDLE
from .diffusion_int8_fused import resident_cuda_device
import torch
@@ -714,29 +1030,29 @@ def _swap(
return 0
if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):
recs = [(m, r) for m, r in recs if r[0] != "v1"]
+ rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None
count = 0
with _LOCK:
_OP_HANDLE = _op()
if _OP_HANDLE is None:
return 0
+ if rotq:
+ _ROTQ_HANDLE = _rotq_op()
for module, rec in recs:
if _MARK in module.__dict__:
count += 1
continue
- module.__dict__[_REC] = (
- rec[0],
- rec[1],
- None,
- rec[3],
- ) # kind, rotation, the Parameter; no payload alias
+ # kind, rotation group, fused rotation + act quant on, the Parameter; no payload alias
+ module.__dict__[_REC] = (rec[0], rec[1], bool(rotq and rec[1] is not None), rec[3])
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
count += 1
if logger is not None and count:
logger.info(
- "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d",
+ "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d%s",
count,
*torch.cuda.get_device_capability(index),
+ ", ConvRot rotation fused into the act quant" if rotq else "",
)
return count
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index 82e7c45a12a..2100e002b3a 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -16,9 +16,12 @@ torch = pytest.importorskip("torch")
@pytest.fixture(autouse = True)
def _clean(monkeypatch):
monkeypatch.delenv(g8.INT8_GEMM_ENV, raising = False)
+ monkeypatch.delenv(g8.INT8_ROTQUANT_ENV, raising = False)
g8._DEVICE_CFG.clear()
+ g8._ROTQ_DEVICE.clear()
yield
g8._DEVICE_CFG.clear()
+ g8._ROTQ_DEVICE.clear()
@pytest.mark.parametrize(
@@ -452,7 +455,11 @@ def test_convrot_linear_swap_is_bit_identical_eager_and_compiled(forced, monkeyp
with torch._inductor.config.patch(emulate_precision_casts = True):
out = torch.compile(fused, fullgraph = True)(x)
assert not counters["graph_break"]
- assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))
+ if fused.__dict__[g8._REC][2]:
+ # the opaque fused op is eager-exact; Inductor's compiled act quant is not on every torch
+ assert torch.equal(out, stock(x))
+ else:
+ assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))
g8.uninstall(holder)
assert not g8.is_installed(fused)
@@ -517,3 +524,153 @@ def test_streamed_kill_switch(forced, monkeypatch):
monkeypatch.setenv(g8.INT8_GEMM_STREAMED_ENV, "0")
holder = torch.nn.Sequential(_int8_linear(1024, 768, False, None))
assert g8.install(holder, device = "cuda") == 0 and not g8.is_installed(holder[0])
+
+
+ "cap, on",
+ [
+ ((8, 0), True), # A100: measured end to end (streamed H3)
+ ((8, 6), False),
+ ((8, 9), False), # L4: H3 Diffusers does not load there (host RAM floor), never measured
+ ((9, 0), False),
+ ((10, 0), False),
+ ((12, 0), True),
+ ],
+)
+def test_rotquant_arch_gate(cap, on):
+ assert (g8.rotquant_config(cap, "auto") is not None) is on
+
+
+def test_rotquant_kill_switch_and_force(monkeypatch):
+ assert (
+ g8.rotquant_config((12, 0), "off") is None
+ ) # the fused GEMM's own kill switch also turns it off
+ assert g8.rotquant_config((10, 0), "force") is not None
+ assert g8.rotquant_config((7, 5), "force") is None
+ assert g8.rotquant_config((8, 0), "off") is None
+ monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")
+ assert g8.rotquant_config((12, 0), "auto") is None
+ assert g8.rotquant_config((8, 0), "auto") is None
+ assert g8.rotquant_config((10, 0), "force") is None
+
+
+def forced_rotq(forced):
+ cfg = g8.rotquant_device_config(torch.cuda.current_device())
+ if cfg is None:
+ pytest.skip("fused rotation probe refused this device")
+ return cfg
+
+
+def _act(m, k, seed):
+ g = torch.Generator().manual_seed(seed)
+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
+ x[:, :5] *= 80 # outlier channels, what the rotation is for
+ x[min(1, m - 1)] = 0 # a zero row: the eps clamp
+ return x.to(torch.bfloat16).cuda()
+
+
+@needs_cuda
+ "m, k", [(17, 256), (300, 256 * 3), (1037, 5376), (257, 7168), (129, 14336), (19, 256 * 128)]
+)
[email protected]("kind", ["v1", "v2"])
+def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq, m, k, kind):
+ x = _act(m, k, m + k)
+ q, s = g8._rotq_op()(x, 256, kind == "v2")
+ rq, rs = g8.rotquant_reference(x, 256, kind)
+ # torchao's own activation-scale dtype: bf16, except v2 on torchao >= 0.18 (fp32)
+ want = torch.float32 if kind == "v2" and g8._v2_act_scale_fp32() else torch.bfloat16
+ assert q.dtype == torch.int8 and s.dtype == want and s.shape == (m,)
+ assert torch.equal(q, rq) and torch.equal(s, rs)
+
+
+@needs_cuda
+def test_rotquant_probe_accepts_this_torchao(forced):
+ # with 0.17's bf16 roundings on 0.18, ~4% of codes differed and the probe refused every device
+ assert g8.rotquant_device_config(torch.cuda.current_device()) is not None
+
+
+@needs_cuda
+def test_rotquant_fake_op_matches_the_real_scale_dtype(forced_rotq):
+ from torch._subclasses.fake_tensor import FakeTensorMode
+ x = _act(40, 768, 9)
+ for v2 in (False, True):
+ _, s = g8._rotq_op()(x, 256, v2)
+ with FakeTensorMode() as mode:
+ _, fs = g8._rotq_op()(mode.from_tensor(x), 256, v2)
+ assert fs.dtype == s.dtype
+
+
+@needs_cuda
+def test_rotquant_unsupported_shapes_take_the_stock_math(forced_rotq):
+ x = _act(40, 256 * 129, 3) # more groups than one tile holds
+ assert not g8.rotquant_supported(x, 256, forced_rotq)
+ assert not g8.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)
+ assert not g8.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)
+ q, s = g8._rotq_op()(x, 256, True)
+ rq, rs = g8.rotquant_reference(x, 256, "v2")
+ assert torch.equal(q, rq) and torch.equal(s, rs)
+ xt = _act(1024, 300, 6).t() # non-contiguous input: made contiguous, same result
+ q, s = g8._rotq_op()(xt, 256, True)
+ rq, rs = g8.rotquant_reference(xt.contiguous(), 256, "v2")
+ assert torch.equal(q, rq) and torch.equal(s, rs)
+
+
+@needs_cuda
[email protected]("version", [None, 2])
+def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(
+ forced_rotq, monkeypatch, version
+):
+ """The rotated Linear quantizes its activation with the fused kernel: same output as ConvRotLinear.forward, eager
+ and compiled, one graph, no graph break, no recompile on a second call."""
+ from torch._dynamo.utils import counters
+
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ stock = _convrot(_int8_linear(1024, 768, False, version))
+ fused = _convrot(_int8_linear(1024, 768, False, version))
+ holder = torch.nn.Sequential(fused)
+ assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is True
+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3
+ x[0, 7, :5] *= 200
+ x2 = (
+ x * 0.5
+ ) # made outside inference_mode like x, so a recompile here would be the op's own guards
+ with torch.inference_mode():
+ before = g8.rotquant_call_count()
+ assert torch.equal(fused(x), stock(x))
+ assert g8.rotquant_call_count() == before + 1
+ counters.clear()
+ torch._dynamo.reset()
+ with torch._inductor.config.patch(emulate_precision_casts = True):
+ compiled = torch.compile(fused, fullgraph = True)
+ out = compiled(x)
+ out2 = compiled(x2)
+ assert not counters["graph_break"]
+ assert counters["stats"]["unique_graphs"] == 1
+ assert torch.equal(out, stock(x)) and torch.equal(out2, stock(x2))
+ g8.uninstall(holder)
+
+
+@needs_cuda
+def test_rotquant_kill_switch_keeps_the_stock_rotation(forced, monkeypatch):
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")
+ stock = _convrot(_int8_linear(1024, 768, False, 2))
+ fused = _convrot(_int8_linear(1024, 768, False, 2))
+ holder = torch.nn.Sequential(fused)
+ assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is False
+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ before, gemm = g8.rotquant_call_count(), g8.call_count()
+ assert torch.equal(fused(x), stock(x))
+ assert g8.rotquant_call_count() == before and g8.call_count() == gemm + 1
+ g8.uninstall(holder)
+
+
+@needs_cuda
+def test_rotquant_probe_refusal_keeps_the_stock_rotation(forced, monkeypatch):
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ monkeypatch.setattr(g8, "_rotq_probe", lambda index, cfg: False)
+ fused = _convrot(_int8_linear(1024, 768, False, 2))
+ assert g8.install(torch.nn.Sequential(fused)) == 1 and fused.__dict__[g8._REC][2] is False