Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fuse MiniMax-H3's ConvRot rotation into the int8 activation quant (#12517)
Daniel Han34b87cb6a32a
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 865da207e09..31d338aaf8c 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -23,6 +23,10 @@ Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unmConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.+Rotated Linears on sm120 / sm80 (group 256) run the rotation and the act quant as ONE kernel (``rotq_i8``): the same+rotation GEMM in the same K order, then torchao's per-row quant with every bf16 rounding kept, so codes and scales are+bit-identical (probed per device). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``.+A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so everycall sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``.@@ -39,6 +43,7 @@ from typing import Any, OptionalINT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"+INT8_ROTQUANT_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT"_MIN_TRITON = (3, 2)# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge._MIN_ROWS = 17@@ -46,6 +51,7 @@ _K_ALIGN = 64_N_ALIGN = 16_OP_NAMESPACE = "unsloth_studio"_OP_NAME = "int8_mm_dequant"+_ROTQ_OP_NAME = "convrot_act_quant_int8"_REC = "_unsloth_i8_gemm"_MARK = "_unsloth_i8_gemm_prev"_NO_PREV = object()@@ -70,6 +76,14 @@ _ARCH_CONFIG = {# When the arch tile does not fit this part's shared memory._FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)+# rotq_i8 (BLOCK_M group rows, BLOCK_K, num_warps, num_stages); measured end to end on these archs only.+_ROTQ_CONFIG = {+ (8, 0): (128, 32, 8, 3),+ (12, 0): (128, 32, 8, 4),+}+_ROTQ_FALLBACK = (128, 32, 8, 3)+_ROTQ_GROUPS = (256,)+def int8_gemm_mode() -> str:"""'off', 'force' or 'auto' from the environment."""@@ -91,6 +105,25 @@ def streamed_enabled() -> bool:return raw not in ("0", "off", "false", "no")+def rotquant_enabled() -> bool:+ """ConvRot Linears quantize their activation with the fused rotation kernel unless+ ``UNSLOTH_DIFFUSION_INT8_ROTQUANT=0``."""+ raw = (os.environ.get(INT8_ROTQUANT_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")+++def rotquant_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:+ """The fused rotation tile for this compute capability, or None (stock rotation)."""+ mode = int8_gemm_mode() if mode is None else mode+ if mode == "off" or capability is None or not rotquant_enabled():+ return None+ cap = (int(capability[0]), int(capability[1]))+ cfg = _ROTQ_CONFIG.get(cap)+ if cfg is None and mode == "force" and cap >= (8, 0):+ return _ROTQ_FALLBACK+ return cfg++def _triton_version_ok(version: Optional[str] = None) -> bool:if version is None:try:@@ -202,7 +235,65 @@ def _kernels() -> Optional[types.SimpleNamespace]:mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)- return types.SimpleNamespace(i8mm_dq = i8mm_dq)+ from triton.language.extra import libdevice++ @triton.jit+ def rotq_i8(+ x_ptr,+ h_ptr,+ q_ptr,+ s_ptr,+ M,+ NG,+ G: tl.constexpr,+ BLOCK_M: tl.constexpr,+ BLOCK_K: tl.constexpr,+ ROWS: tl.constexpr,+ ROWS_P2: tl.constexpr,+ QMIN: tl.constexpr,+ QMAX: tl.constexpr,+ DIV: tl.constexpr,+ EPS: tl.constexpr,+ FP32_SCALE: tl.constexpr,+ ):+ # one program = ROWS whole activation rows, so the per-row amax closes inside the tile+ pid = tl.program_id(0)+ t = tl.arange(0, BLOCK_M)+ lr = t // NG+ row = pid.to(tl.int64) * ROWS + lr+ live = (lr < ROWS) & (row < M)+ rg = tl.where(live, row * NG + (t % NG), 0)+ offs_k = tl.arange(0, BLOCK_K)+ offs_n = tl.arange(0, G)+ acc = tl.zeros((BLOCK_M, G), dtype = tl.float32)+ a_ptrs = x_ptr + rg[:, None] * G + offs_k[None, :]+ h_ptrs = h_ptr + offs_k[:, None] * G + offs_n[None, :]+ for _k in range(0, G // BLOCK_K):+ a = tl.load(a_ptrs, mask = live[:, None], other = 0.0)+ acc = tl.dot(a, tl.load(h_ptrs), acc, out_dtype = tl.float32)+ a_ptrs += BLOCK_K+ h_ptrs += BLOCK_K * G+ z = _rbf16(acc)+ # s = bf16(max(bf16(amax / DIV), EPS)), q = clamp(rint(bf16(z * bf16(1 / s)))); FP32_SCALE (torchao >= 0.18+ # Int8Tensor) keeps the reciprocal and the product in fp32.+ j = tl.arange(0, ROWS_P2)+ sel = lr[:, None] == j[None, :]+ amax = tl.max(tl.where(sel, tl.max(tl.abs(z), axis = 1)[:, None], 0.0), axis = 0)+ s = _rbf16(tl.maximum(_rbf16(libdevice.div_rn(amax, DIV)), EPS))+ inv = libdevice.div_rn(tl.full((ROWS_P2,), 1.0, tl.float32), s)+ if not FP32_SCALE:+ inv = _rbf16(inv)+ inv_t = tl.max(tl.where(sel, inv[None, :], 0.0), axis = 1)+ p = z * inv_t[:, None]+ if not FP32_SCALE:+ p = _rbf16(p)+ p = libdevice.rint(p)+ p = tl.minimum(tl.maximum(p, QMIN), QMAX)+ tl.store(q_ptr + rg[:, None] * G + offs_n[None, :], p.to(tl.int8), mask = live[:, None])+ srow = pid.to(tl.int64) * ROWS + j+ tl.store(s_ptr + srow, s.to(s_ptr.dtype.element_ty), mask = (j < ROWS) & (srow < M))++ return types.SimpleNamespace(i8mm_dq = i8mm_dq, rotq_i8 = rotq_i8)# device index -> probed tile (None = stock).@@ -384,6 +475,224 @@ def _probe(index: int, cfg: tuple) -> bool:return False+# (QMIN, QMAX, DIV, EPS): v1 _int8_symm_per_token_reduced_range_quant, v2 Int8Tensor.from_hp(PerRow, SYMMETRIC).+_ROTQ_QPARAMS = {+ "v1": (-127.0, 127.0, 127.0, 1e-5),+ "v2": (-128.0, 127.0, 127.5, 1.1920928955078125e-07),+}+# device index -> probed rotq tile (None = stock).+_ROTQ_DEVICE: dict = {}+_ROTQ_CALLS = [0]+_ROTQ_HANDLE: Any = None+++@lru_cache(maxsize = 1)+def _v2_act_scale_fp32() -> bool:+ """torchao 0.18+ ``Int8Tensor.from_hp`` returns an fp32 activation scale (and quantizes in fp32)."""+ try:+ import torch+ from torchao.quantization.granularity import PerRow+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor++ x = torch.ones(1, 16, dtype = torch.bfloat16)+ return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32+ except Exception: # noqa: BLE001 - unknown: the probe below still has to match bit for bit+ return False+++def _rotq_scale_fp32(kind: str) -> bool:+ return kind == "v2" and _v2_act_scale_fp32()+++def rotquant_reference(x2d: Any, group: int, kind: str) -> tuple:+ """The stock path: ConvRotLinear.forward's rotation (bf16 GEMM), then torchao's activation quant. (int8, scale)."""+ import torch+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation++ xr = rotate_convrot_activation(+ x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group+ )+ if kind == "v1":+ q, scale = _act_quant_v1(xr)+ else:+ from torchao.quantization.granularity import PerRow+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor++ t = Int8Tensor.from_hp(xr, PerRow())+ q, scale = t.qdata, t.scale+ if _rotq_scale_fp32(kind):+ return q, scale.reshape(-1).to(torch.float32)+ return q, scale.reshape(-1).to(torch.bfloat16)+++def _rotq_rows(k: int, group: int, cfg: tuple) -> int:+ """Whole activation rows per program (0: this K does not fit one tile)."""+ return cfg[0] // (k // group)+++def _rotq_launch(x2d: Any, group: int, kind: str, cfg: tuple) -> tuple:+ import torch+ import triton+ from .diffusion_convrot import build_convrot_hadamard++ kern = _kernels()+ m, k = x2d.shape+ bm, bk, warps, stages = cfg+ rows = _rotq_rows(k, group, cfg)+ qmin, qmax, div, eps = _ROTQ_QPARAMS[kind]+ h = build_convrot_hadamard(group, device = x2d.device, dtype = torch.bfloat16)+ q = torch.empty((m, k), device = x2d.device, dtype = torch.int8)+ fp32_scale = _rotq_scale_fp32(kind)+ s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32_scale else torch.bfloat16)+ with torch.cuda.device(x2d.device):+ kern.rotq_i8[(triton.cdiv(m, rows),)](+ x2d,+ h,+ q,+ s,+ m,+ k // group,+ G = group,+ BLOCK_M = bm,+ BLOCK_K = bk,+ ROWS = rows,+ ROWS_P2 = max(triton.next_power_of_2(rows), 2),+ QMIN = qmin,+ QMAX = qmax,+ DIV = div,+ EPS = eps,+ FP32_SCALE = fp32_scale,+ num_warps = warps,+ num_stages = stages,+ enable_fp_fusion = False,+ )+ return q, s+++def rotquant_supported(x2d: Any, group: int, cfg: Optional[tuple]) -> bool:+ """Shape / dtype / device / group gate of the fused kernel (anything else keeps the stock rotation + quant)."""+ import torch++ if (+ cfg is None+ or group not in _ROTQ_GROUPS+ or x2d.dim() != 2+ or x2d.dtype != torch.bfloat16+ or not x2d.is_cuda+ ):+ return False+ k = x2d.shape[1]+ return k % group == 0 and k >= group and _rotq_rows(k, group, cfg) >= 1+++def _rotq_run(x2d: Any, group: int, v2: bool) -> tuple:+ """Op body: the probed tile for this device, the stock rotation + quant if anything does not fit."""+ _ROTQ_CALLS[0] += 1+ kind = "v2" if v2 else "v1"+ x2d = x2d if x2d.is_contiguous() else x2d.contiguous()+ cfg = _ROTQ_DEVICE.get(x2d.device.index)+ if rotquant_supported(x2d, group, cfg):+ try:+ return _rotq_launch(x2d, group, kind, cfg)+ except Exception: # noqa: BLE001 - a failed launch keeps the stock math+ _ROTQ_DEVICE[x2d.device.index] = None+ return rotquant_reference(x2d, group, kind)+++@lru_cache(maxsize = 1)+def _rotq_op() -> Any:+ """``unsloth_studio::convrot_act_quant_int8`` (opaque to Inductor, allocation only), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _ROTQ_OP_NAME):+ return getattr(ns, _ROTQ_OP_NAME)+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return None+ try:++ @custom_op(+ f"{_OP_NAMESPACE}::{_ROTQ_OP_NAME}",+ mutates_args = (),+ schema = "(Tensor x, int group, bool v2) -> (Tensor, Tensor)",+ )+ def _convrot_act_quant_int8(x, group, v2):+ return _rotq_run(x, group, v2)++ @_convrot_act_quant_int8.register_fake+ def _(x, group, v2):+ return (+ x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),+ x.new_empty(+ (x.shape[0],),+ dtype = torch.float32+ if _rotq_scale_fp32("v2" if v2 else "v1")+ else torch.bfloat16,+ ),+ )+ except Exception: # noqa: BLE001+ return None+ return getattr(getattr(torch.ops, _OP_NAMESPACE), _ROTQ_OP_NAME)+++def rotquant_device_config(index: int) -> Optional[tuple]:+ """Once per device: arch gate, then bit-exact codes and scales vs the stock path (v1, v2, outliers, zero row, ragged+ M). None = stock."""+ if index in _ROTQ_DEVICE:+ return _ROTQ_DEVICE[index]+ cfg = None+ try:+ import torch+ if (+ torch.cuda.is_available()+ and not getattr(torch.version, "hip", None)+ and _triton_version_ok()+ and _kernels() is not None+ and _rotq_op() is not None+ ):+ want = rotquant_config(torch.cuda.get_device_capability(index))+ for cand in (want, _ROTQ_FALLBACK) if want is not None else ():+ if _rotq_probe(index, cand):+ cfg = cand+ break+ except Exception: # noqa: BLE001+ cfg = None+ _ROTQ_DEVICE[index] = cfg+ return cfg+++def _rotq_probe(index: int, cfg: tuple) -> bool:+ import torch++ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(2)+ try:+ for m, k in ((257, 256 * 3), (33, 256 * 21), (130, 256 * 56), (17, 256 * 128)):+ if _rotq_rows(k, 256, cfg) < 1:+ continue+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)+ x[:, :3] *= 60+ x[1] = 0+ x[2, 11] = 3e4+ x = x.to(torch.bfloat16).to(dev)+ for kind in ("v1", "v2"):+ q, s = _rotq_launch(x, 256, kind, cfg)+ rq, rs = rotquant_reference(x, 256, kind)+ if not (torch.equal(q, rq) and torch.equal(s, rs)):+ return False+ torch.cuda.synchronize(dev)+ return True+ except Exception: # noqa: BLE001+ return False+++def rotquant_call_count() -> int:+ return _ROTQ_CALLS[0]++def tie_operands(device: Any,rows: int = 32,@@ -526,7 +835,7 @@ def _linear_forward(self: Any, x: Any) -> Any:rec = self.__dict__.get(_REC)if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:return type(self).forward(self, x)- kind, group, _ws, weight = rec+ kind, group, rotq, weight = recif self.weight is not weight: # weight replaced since install (reload / LoRA bake): stockreturn type(self).forward(self, x)# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.@@ -540,16 +849,23 @@ def _linear_forward(self: Any, x: Any) -> Any:lead = x.shape[:-1]if x.numel() < _MIN_ROWS * x.shape[-1]:return type(self).forward(self, x)- if group is not None:- from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation- x = rotate_convrot_activation(- x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group- )- x2d = x.reshape(-1, x.shape[-1])- if kind == "v1":- xq, xs = _act_quant_v1(x2d)+ if (+ rotq+ and _ROTQ_HANDLE is not None+ and rotquant_supported(x.reshape(-1, x.shape[-1]), group, _ROTQ_DEVICE.get(x.device.index))+ ):+ xq, xs = _ROTQ_HANDLE(x.reshape(-1, x.shape[-1]), group, kind == "v2")else:- xq, xs = _act_quant_v2(x2d, weight)+ if group is not None:+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation+ x = rotate_convrot_activation(+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group+ )+ x2d = x.reshape(-1, x.shape[-1])+ if kind == "v1":+ xq, xs = _act_quant_v1(x2d)+ else:+ xq, xs = _act_quant_v2(x2d, weight)y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)return y.reshape(*lead, y.shape[-1])@@ -686,7 +1002,7 @@ def _swap(logger: Any = None,device: Any = None,) -> int:- global _OP_HANDLE+ global _OP_HANDLE, _ROTQ_HANDLEfrom .diffusion_int8_fused import resident_cuda_deviceimport torch@@ -714,29 +1030,29 @@ def _swap(return 0if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):recs = [(m, r) for m, r in recs if r[0] != "v1"]+ rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not Nonecount = 0with _LOCK:_OP_HANDLE = _op()if _OP_HANDLE is None:return 0+ if rotq:+ _ROTQ_HANDLE = _rotq_op()for module, rec in recs:if _MARK in module.__dict__:count += 1continue- module.__dict__[_REC] = (- rec[0],- rec[1],- None,- rec[3],- ) # kind, rotation, the Parameter; no payload alias+ # kind, rotation group, fused rotation + act quant on, the Parameter; no payload alias+ module.__dict__[_REC] = (rec[0], rec[1], bool(rotq and rec[1] is not None), rec[3])module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)count += 1if logger is not None and count:logger.info(- "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d",+ "diffusion.int8_gemm: %d int8 Linear(s) run the fused-dequant GEMM (bf16 out) on sm_%d%d%s",count,*torch.cuda.get_device_capability(index),+ ", ConvRot rotation fused into the act quant" if rotq else "",)return countdiff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex 82e7c45a12a..2100e002b3a 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -16,9 +16,12 @@ torch = pytest.importorskip("torch")@pytest.fixture(autouse = True)def _clean(monkeypatch):monkeypatch.delenv(g8.INT8_GEMM_ENV, raising = False)+ monkeypatch.delenv(g8.INT8_ROTQUANT_ENV, raising = False)g8._DEVICE_CFG.clear()+ g8._ROTQ_DEVICE.clear()yieldg8._DEVICE_CFG.clear()+ g8._ROTQ_DEVICE.clear()@pytest.mark.parametrize(@@ -452,7 +455,11 @@ def test_convrot_linear_swap_is_bit_identical_eager_and_compiled(forced, monkeypwith torch._inductor.config.patch(emulate_precision_casts = True):out = torch.compile(fused, fullgraph = True)(x)assert not counters["graph_break"]- assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))+ if fused.__dict__[g8._REC][2]:+ # the opaque fused op is eager-exact; Inductor's compiled act quant is not on every torch+ assert torch.equal(out, stock(x))+ else:+ assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))g8.uninstall(holder)assert not g8.is_installed(fused)@@ -517,3 +524,153 @@ def test_streamed_kill_switch(forced, monkeypatch):monkeypatch.setenv(g8.INT8_GEMM_STREAMED_ENV, "0")holder = torch.nn.Sequential(_int8_linear(1024, 768, False, None))assert g8.install(holder, device = "cuda") == 0 and not g8.is_installed(holder[0])+++ "cap, on",+ [+ ((8, 0), True), # A100: measured end to end (streamed H3)+ ((8, 6), False),+ ((8, 9), False), # L4: H3 Diffusers does not load there (host RAM floor), never measured+ ((9, 0), False),+ ((10, 0), False),+ ((12, 0), True),+ ],+)+def test_rotquant_arch_gate(cap, on):+ assert (g8.rotquant_config(cap, "auto") is not None) is on+++def test_rotquant_kill_switch_and_force(monkeypatch):+ assert (+ g8.rotquant_config((12, 0), "off") is None+ ) # the fused GEMM's own kill switch also turns it off+ assert g8.rotquant_config((10, 0), "force") is not None+ assert g8.rotquant_config((7, 5), "force") is None+ assert g8.rotquant_config((8, 0), "off") is None+ monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")+ assert g8.rotquant_config((12, 0), "auto") is None+ assert g8.rotquant_config((8, 0), "auto") is None+ assert g8.rotquant_config((10, 0), "force") is None+++def forced_rotq(forced):+ cfg = g8.rotquant_device_config(torch.cuda.current_device())+ if cfg is None:+ pytest.skip("fused rotation probe refused this device")+ return cfg+++def _act(m, k, seed):+ g = torch.Generator().manual_seed(seed)+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)+ x[:, :5] *= 80 # outlier channels, what the rotation is for+ x[min(1, m - 1)] = 0 # a zero row: the eps clamp+ return x.to(torch.bfloat16).cuda()+++@needs_cuda+ "m, k", [(17, 256), (300, 256 * 3), (1037, 5376), (257, 7168), (129, 14336), (19, 256 * 128)]+)[email protected]("kind", ["v1", "v2"])+def test_rotquant_kernel_is_bit_exact_vs_rotation_then_torchao_quant(forced_rotq, m, k, kind):+ x = _act(m, k, m + k)+ q, s = g8._rotq_op()(x, 256, kind == "v2")+ rq, rs = g8.rotquant_reference(x, 256, kind)+ # torchao's own activation-scale dtype: bf16, except v2 on torchao >= 0.18 (fp32)+ want = torch.float32 if kind == "v2" and g8._v2_act_scale_fp32() else torch.bfloat16+ assert q.dtype == torch.int8 and s.dtype == want and s.shape == (m,)+ assert torch.equal(q, rq) and torch.equal(s, rs)+++@needs_cuda+def test_rotquant_probe_accepts_this_torchao(forced):+ # with 0.17's bf16 roundings on 0.18, ~4% of codes differed and the probe refused every device+ assert g8.rotquant_device_config(torch.cuda.current_device()) is not None+++@needs_cuda+def test_rotquant_fake_op_matches_the_real_scale_dtype(forced_rotq):+ from torch._subclasses.fake_tensor import FakeTensorMode+ x = _act(40, 768, 9)+ for v2 in (False, True):+ _, s = g8._rotq_op()(x, 256, v2)+ with FakeTensorMode() as mode:+ _, fs = g8._rotq_op()(mode.from_tensor(x), 256, v2)+ assert fs.dtype == s.dtype+++@needs_cuda+def test_rotquant_unsupported_shapes_take_the_stock_math(forced_rotq):+ x = _act(40, 256 * 129, 3) # more groups than one tile holds+ assert not g8.rotquant_supported(x, 256, forced_rotq)+ assert not g8.rotquant_supported(_act(40, 1024, 4), 64, forced_rotq)+ assert not g8.rotquant_supported(_act(40, 1024, 5).half(), 256, forced_rotq)+ q, s = g8._rotq_op()(x, 256, True)+ rq, rs = g8.rotquant_reference(x, 256, "v2")+ assert torch.equal(q, rq) and torch.equal(s, rs)+ xt = _act(1024, 300, 6).t() # non-contiguous input: made contiguous, same result+ q, s = g8._rotq_op()(xt, 256, True)+ rq, rs = g8.rotquant_reference(xt.contiguous(), 256, "v2")+ assert torch.equal(q, rq) and torch.equal(s, rs)+++@needs_cuda[email protected]("version", [None, 2])+def test_convrot_linear_fuses_rotation_into_act_quant_eager_and_compiled(+ forced_rotq, monkeypatch, version+):+ """The rotated Linear quantizes its activation with the fused kernel: same output as ConvRotLinear.forward, eager+ and compiled, one graph, no graph break, no recompile on a second call."""+ from torch._dynamo.utils import counters++ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ stock = _convrot(_int8_linear(1024, 768, False, version))+ fused = _convrot(_int8_linear(1024, 768, False, version))+ holder = torch.nn.Sequential(fused)+ assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is True+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3+ x[0, 7, :5] *= 200+ x2 = (+ x * 0.5+ ) # made outside inference_mode like x, so a recompile here would be the op's own guards+ with torch.inference_mode():+ before = g8.rotquant_call_count()+ assert torch.equal(fused(x), stock(x))+ assert g8.rotquant_call_count() == before + 1+ counters.clear()+ torch._dynamo.reset()+ with torch._inductor.config.patch(emulate_precision_casts = True):+ compiled = torch.compile(fused, fullgraph = True)+ out = compiled(x)+ out2 = compiled(x2)+ assert not counters["graph_break"]+ assert counters["stats"]["unique_graphs"] == 1+ assert torch.equal(out, stock(x)) and torch.equal(out2, stock(x2))+ g8.uninstall(holder)+++@needs_cuda+def test_rotquant_kill_switch_keeps_the_stock_rotation(forced, monkeypatch):+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ monkeypatch.setenv(g8.INT8_ROTQUANT_ENV, "0")+ stock = _convrot(_int8_linear(1024, 768, False, 2))+ fused = _convrot(_int8_linear(1024, 768, False, 2))+ holder = torch.nn.Sequential(fused)+ assert g8.install(holder) == 1 and fused.__dict__[g8._REC][2] is False+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)+ with torch.inference_mode():+ before, gemm = g8.rotquant_call_count(), g8.call_count()+ assert torch.equal(fused(x), stock(x))+ assert g8.rotquant_call_count() == before and g8.call_count() == gemm + 1+ g8.uninstall(holder)+++@needs_cuda+def test_rotquant_probe_refusal_keeps_the_stock_rotation(forced, monkeypatch):+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ monkeypatch.setattr(g8, "_rotq_probe", lambda index, cfg: False)+ fused = _convrot(_int8_linear(1024, 768, False, 2))+ assert g8.install(torch.nn.Sequential(fused)) == 1 and fused.__dict__[g8._REC][2] is False