Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 measured (every sm8x stock on
Triton >= 3.8, whose int8 dot spills there), other sm80+ behind a bit-exact probe and a latency check
(UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED=0 keeps them stock), sm100 / sm103 stock (ROCm and CPU never reach it);
Linear output bit-identical, compiled block not end to end; sm120 loads its operands through TMA descriptors (UNSLOTH_DIFFUSION_INT8_GEMM_TMA=0 keeps pointer loads);
UNSLOTH_DIFFUSION_INT8_GEMM=0 disables."""
from __future__ import annotations
import os
import threading
import types
from functools import lru_cache
from typing import Any, Optional
from .rocm_bf16 import is_rocm_torch
INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"
INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"
INT8_SWIGLU_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED"
INT8_GEMM_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED"
INT8_GEMM_TMA_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TMA"
INT8_GEMM_RACE_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_RACE"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
_K_ALIGN = 64
_N_ALIGN = 16
_OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_mm_dequant"
_REC = "_unsloth_i8_gemm"
_MARK = "_unsloth_i8_gemm_prev"
# The bound ``_linear_forward`` the swap installed: a traced forward checks the instance ``forward`` by identity with
# it (dynamo resolves ``forward.__func__ is _linear_forward`` to False, so the shared paths never engaged compiled).
_FWD = "_unsloth_i8_gemm_fwd"
# on a SwiGLU MLP whose down projection takes its codes from the gate / value halves
_FF_MARK = "_unsloth_i8_gemm_swiglu_prev"
# fp16 input runs the sm75 prebuilt kernel
_F16 = "_unsloth_i8_gemm_f16"
_OP16_NAME = "int8_mm_dequant_f16"
_SWIGLU_FF_CLASSES = ("QwenImage21SwiGLUFeedForward",)
_NO_PREV = object()
_LOCK = threading.Lock()
_OP_HANDLE: Any = None
# Engagement census; never read inside a traced region.
_CALLS = [0]
_TMA_CALLS = [0]
# (BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, num_warps, num_stages) per (major, minor), tuned on DiT shapes. Absent = off.
_ARCH_CONFIG = {
(8, 0): (
128,
128,
128,
8,
4,
3,
), # A100
(8, 9): (256, 128, 128, 8, 8, 3), # L4
(12, 0): (128, 128, 64, 8, 4, 4), # RTX PRO 6000
}
# When the arch tile does not fit this part's shared memory, and the unlisted archs' tile (64 KB of stages: fits 99 KB).
_FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)
# Measured slower than cuBLAS: not probed unless forced.
_UNLISTED_SKIP = ((10, 0), (10, 3))
# Raced against the fallback on unlisted archs; a tile overflowing shared memory fails the probe and drops out.
_UNLISTED_RACE_TILES = (
(128, 128, 128, 8, 4, 3),
(256, 128, 128, 8, 8, 3),
)
# First match wins, bounds inclusive. No L4 rule on purpose: power-capped clocks lose end to end.
# (major, minor) -> ((N_lo, N_hi, K_lo, K_hi, tile), ...), M >= _SHAPE_MIN_M only.
_SHAPE_TILES: dict = {
(12, 0): (
# MiniMax-H3 fused QKV / FFN in
(16384, 32768, 4096, 6144, (128, 128, 64, 32, 4, 4)),
# deep K: H3 out / FFN down, Qwen-Image-2.1 MLP out, Z-Image w2
(1, 8192, 7168, 16384, (256, 128, 128, 8, 8, 3)),
),
}
_SHAPE_MIN_M = 1024
# Same kernel and epilogue with host TMA descriptors for A / W, one tile for M >= _SHAPE_MIN_M (replaces
# _SHAPE_TILES there); below that the pointer-load kernel is faster.
_TMA_CONFIG = {(12, 0): (128, 128, 64, 8, 4, 4)}
def int8_gemm_mode() -> str:
"""'off', 'force' or 'auto' from the environment."""
raw = (os.environ.get(INT8_GEMM_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no"):
return "off"
if raw in ("1", "on", "true", "yes", "force"):
return "force"
return "auto"
def convrot_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_CONVROT_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def streamed_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_STREAMED_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def swiglu_streamed_enabled() -> bool:
raw = (os.environ.get(INT8_SWIGLU_STREAMED_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def tiles_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_TILES_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def tma_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_TMA_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def tma_config(capability: Optional[tuple]) -> Optional[tuple]:
if capability is None or not tma_enabled():
return None
return _TMA_CONFIG.get((int(capability[0]), int(capability[1])))
def _triton_version_ok(version: Optional[str] = None) -> bool:
if version is None:
try:
import triton
version = str(triton.__version__)
except Exception: # noqa: BLE001
return False
import re
match = re.match(r"(\d+)\.(\d+)", version)
return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON
@lru_cache(maxsize = None)
def _triton_at_least(major: int, minor: int) -> bool:
try:
import triton
except Exception: # noqa: BLE001
return False
import re
match = re.match(r"(\d+)\.(\d+)", str(triton.__version__))
return bool(match) and (int(match.group(1)), int(match.group(2))) >= (major, minor)
def _triton38_sm8x(cap: tuple) -> bool:
"""Triton >= 3.8 spills sm8x int8 tiles (triton-lang/triton#12486); even spill-free ones trail cuBLAS."""
return cap[0] == 8 and _triton_at_least(3, 8)
def unlisted_mode() -> str:
"""'off', 'force' (no latency check) or 'auto' for the sm80+ archs without an ``_ARCH_CONFIG`` tile."""
gemm = int8_gemm_mode()
if gemm != "auto":
return gemm
raw = (os.environ.get(INT8_GEMM_UNLISTED_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no"):
return "off"
if raw in ("1", "on", "true", "yes", "force"):
return "force"
return "auto"
def unlisted_arch(capability: Optional[tuple]) -> bool:
if capability is None:
return False
cap = (int(capability[0]), int(capability[1]))
return cap >= (8, 0) and cap not in _ARCH_CONFIG
def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) -> bool:
mode = unlisted_mode() if mode is None else mode
if mode == "off" or not unlisted_arch(capability):
return False
if mode == "force":
return True
cap = (int(capability[0]), int(capability[1]))
return cap not in _UNLISTED_SKIP and not _triton38_sm8x(cap)
def arch_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:
"""The tile for this compute capability, or None where the lever stays off."""
mode = int8_gemm_mode() if mode is None else mode
if mode == "off" or capability is None:
return None
cap = (int(capability[0]), int(capability[1]))
cfg = _ARCH_CONFIG.get(cap)
if mode == "auto" and _triton38_sm8x(cap):
return None
if cfg is None and mode == "force" and cap >= (8, 0):
return _FALLBACK_CONFIG
return cfg
@lru_cache(maxsize = 1)
def _kernels() -> Optional[types.SimpleNamespace]:
try:
import triton
import triton.language as tl
except Exception: # noqa: BLE001 - no Triton means the stock path
return None
globals().update(triton = triton, tl = tl)
@triton.jit
def _rbf16(x):
return x.to(tl.bfloat16).to(tl.float32)
@triton.jit(do_not_specialize = ["M"])
def i8mm_dq(
a_ptr,
w_ptr,
xs_ptr,
ws_ptr,
b_ptr,
c_ptr,
M,
N,
K,
stride_am,
stride_wn,
stride_cm,
HAS_BIAS: tl.constexpr, # bias present
XS_FP32: tl.constexpr, # activation scale is fp32
WS_FP32: tl.constexpr, # weight scale is fp32
EVEN_K: tl.constexpr, # K divisible by BLOCK_K
BLOCK_M: tl.constexpr, # block size along M (rows)
BLOCK_N: tl.constexpr, # block size along N (out features)
BLOCK_K: tl.constexpr, # block size along K (in features)
GROUP_M: tl.constexpr, # M tiles per swizzle group
):
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
num_pid_n = tl.cdiv(N, BLOCK_N)
num_pid_in_group = GROUP_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_M)
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
# int64 row / column offsets: M * K overflows int32 on large video token grids.
offs_m = pid_m.to(tl.int64) * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n.to(tl.int64) * BLOCK_N + tl.arange(0, BLOCK_N)
rm = tl.max_contiguous(tl.multiple_of(offs_m % M, BLOCK_M), BLOCK_M)
rn = tl.max_contiguous(tl.multiple_of(offs_n % N, BLOCK_N), BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
a_ptrs = a_ptr + rm[:, None] * stride_am + offs_k[None, :]
w_ptrs = w_ptr + rn[None, :] * stride_wn + offs_k[:, None]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.int32)
for k in range(0, tl.cdiv(K, BLOCK_K)):
if EVEN_K:
a = tl.load(a_ptrs)
b = tl.load(w_ptrs)
else:
kmask = offs_k < K - k * BLOCK_K
a = tl.load(a_ptrs, mask = kmask[None, :], other = 0)
b = tl.load(w_ptrs, mask = kmask[:, None], other = 0)
acc = tl.dot(a, b, acc, out_dtype = tl.int32)
a_ptrs += BLOCK_K
w_ptrs += BLOCK_K
# torchao epilogue, every rounding kept: (int32 * xs) -> xs dtype, * ws -> ws dtype, -> bf16, + bias.
xs = tl.load(xs_ptr + rm).to(tl.float32)
ws = tl.load(ws_ptr + rn).to(tl.float32)
if XS_FP32:
y = acc.to(tl.float32)
else:
# int32 -> fp32 -> bf16 rounds twice; Triton folds ``acc.to(fp32).to(bf16)`` into one rounding.
y = _rbf16(tl.extra.cuda.libdevice.int2float_rn(acc))
y = _rbf16(y * xs[:, None])
# *_rn: ptxas fuses packed f32x2 mul + add into an FMA on sm_100 even with fp fusion off.
y = tl.extra.cuda.libdevice.mul_rn(y, ws[None, :])
if not WS_FP32:
y = _rbf16(y)
if HAS_BIAS:
y = tl.extra.cuda.libdevice.add_rn(y, tl.load(b_ptr + rn).to(tl.float32)[None, :])
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :]
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)
@triton.jit(do_not_specialize = ["M"])
def i8mm_dq_tma(
a_desc,
w_desc,
xs_ptr,
ws_ptr,
b_ptr,
c_ptr,
M,
N,
K,
stride_cm,
HAS_BIAS: tl.constexpr, # bias present
XS_FP32: tl.constexpr, # activation scale is fp32
WS_FP32: tl.constexpr, # weight scale is fp32
BLOCK_M: tl.constexpr, # block size along M (rows)
BLOCK_N: tl.constexpr, # block size along N (out features)
BLOCK_K: tl.constexpr, # block size along K (in features)
GROUP_M: tl.constexpr, # M tiles per swizzle group
):
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
num_pid_n = tl.cdiv(N, BLOCK_N)
num_pid_in_group = GROUP_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_M)
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
om = pid_m * BLOCK_M
on = pid_n * BLOCK_N
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.int32)
for k in range(0, tl.cdiv(K, BLOCK_K)):
a = a_desc.load([om, k * BLOCK_K])
b = w_desc.load([on, k * BLOCK_K])
acc = tl.dot(a, b.T, acc, out_dtype = tl.int32)
offs_m = om.to(tl.int64) + tl.arange(0, BLOCK_M)
offs_n = on.to(tl.int64) + tl.arange(0, BLOCK_N)
rm = offs_m % M
rn = offs_n % N
xs = tl.load(xs_ptr + rm).to(tl.float32)
ws = tl.load(ws_ptr + rn).to(tl.float32)
if XS_FP32:
y = acc.to(tl.float32)
else:
y = _rbf16(tl.extra.cuda.libdevice.int2float_rn(acc))
y = _rbf16(y * xs[:, None])
y = tl.extra.cuda.libdevice.mul_rn(y, ws[None, :])
if not WS_FP32:
y = _rbf16(y)
if HAS_BIAS:
y = tl.extra.cuda.libdevice.add_rn(y, tl.load(b_ptr + rn).to(tl.float32)[None, :])
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :]
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, y.to(tl.bfloat16), mask = mask)
return types.SimpleNamespace(i8mm_dq = i8mm_dq, i8mm_dq_tma = i8mm_dq_tma)
# device index -> probed tile (None = stock).
_DEVICE_CFG: dict = {}
_DEVICE_TMA: dict = {}
# device index -> the per-shape rules whose tile passed the probe on that device.
_DEVICE_TILES: dict = {}
def reference(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
"""Eager torchao epilogue, same ops and order as the stock linear."""
import torch
m = a.shape[0]
if m < _MIN_ROWS: # _int_mm's row floor: pad, then drop the pad rows (exact)
a = torch.cat([a, a.new_zeros((_MIN_ROWS - m, a.shape[1]))])
c = torch._int_mm(a, w.t())[:m]
y = (c * xs.reshape(-1, 1)).to(torch.bfloat16)
y = y * ws.reshape(-1)
if bias is not None:
y = y + bias
return y.to(torch.bfloat16)
def _launch(a: Any, w: Any, xs: Any, ws: Any, bias: Any, cfg: tuple) -> Any:
import torch
k = _kernels()
m, kk = a.shape
n = w.shape[0]
bm, bn, bk, gm, warps, stages = cfg
out = torch.empty((m, n), device = a.device, dtype = torch.bfloat16)
grid = (triton.cdiv(m, bm) * triton.cdiv(n, bn),)
with torch.cuda.device(a.device):
k.i8mm_dq[grid](
a,
w,
xs,
ws,
bias if bias is not None else ws,
out,
m,
n,
kk,
a.stride(0),
w.stride(0),
out.stride(0),
HAS_BIAS = bias is not None,
XS_FP32 = xs.dtype == torch.float32,
WS_FP32 = ws.dtype == torch.float32,
EVEN_K = kk % bk == 0,
BLOCK_M = bm,
BLOCK_N = bn,
BLOCK_K = bk,
GROUP_M = gm,
num_warps = warps,
num_stages = stages,
enable_fp_fusion = False,
)
return out
@lru_cache(maxsize = 1)
def _tensor_descriptor() -> Any:
try:
from triton.tools.tensor_descriptor import TensorDescriptor
except Exception: # noqa: BLE001 - Triton before host descriptors
return None
return TensorDescriptor
def _launch_tma(a: Any, w: Any, xs: Any, ws: Any, bias: Any, cfg: tuple) -> Any:
import torch
import triton
k = _kernels()
desc = _tensor_descriptor()
m, kk = a.shape
n = w.shape[0]
bm, bn, bk, gm, warps, stages = cfg
out = torch.empty((m, n), device = a.device, dtype = torch.bfloat16)
grid = (triton.cdiv(m, bm) * triton.cdiv(n, bn),)
with torch.cuda.device(a.device):
k.i8mm_dq_tma[grid](
desc.from_tensor(a, [bm, bk]),
desc.from_tensor(w, [bn, bk]),
xs,
ws,
bias if bias is not None else ws,
out,
m,
n,
kk,
out.stride(0),
HAS_BIAS = bias is not None,
XS_FP32 = xs.dtype == torch.float32,
WS_FP32 = ws.dtype == torch.float32,
BLOCK_M = bm,
BLOCK_N = bn,
BLOCK_K = bk,
GROUP_M = gm,
num_warps = warps,
num_stages = stages,
enable_fp_fusion = False,
)
return out
def _aligned(a: Any, w: Any) -> bool:
"""K, N, both row strides and both int8 base pointers on 16 bytes. Triton specialises on these, and an off-16 variant
spills; the driver then reserves that local memory device-wide for the process, outside PyTorch's allocator."""
return (
a.shape[1] % 16 == 0
and w.shape[0] % 16 == 0
and a.stride(0) % 16 == 0
and w.stride(0) % 16 == 0
and a.data_ptr() % 16 == 0
and w.data_ptr() % 16 == 0
)
def tile_for(index: Any, m: int, n: int, k: int) -> Optional[tuple]:
"""The probed tile this device runs an ``[m, k] x [n, k]`` GEMM with: a per-shape rule, else the arch default."""
cfg = _DEVICE_CFG.get(index)
if cfg is None or m < _SHAPE_MIN_M:
return cfg
for n_lo, n_hi, k_lo, k_hi, tile in _DEVICE_TILES.get(index, ()):
if n_lo <= n <= n_hi and k_lo <= k <= k_hi:
return tile
return cfg
def tma_tile_for(index: Any, m: int) -> Optional[tuple]:
tile = _DEVICE_TMA.get(index)
return None if m < _SHAPE_MIN_M else tile
def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
"""Op body: race-won CUTLASS tile, else probed Triton tile, else the stock epilogue."""
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
index = a.device.index
tile = _cutlass_tile(index, a, w, xs, ws, bias)
if tile is not None:
import torch
from . import diffusion_int8_cutlass as cut
try:
return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.bfloat16)
except Exception: # noqa: BLE001 - drops the prebuilt kernel on this card
_CUT_PLAN.pop(index, None)
cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])
if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):
tma = tma_tile_for(index, a.shape[0])
if tma is not None:
try:
out = _launch_tma(a, w, xs, ws, bias, tma)
_TMA_CALLS[0] += 1
return out
except Exception: # noqa: BLE001 - pointer loads, same math
_DEVICE_TMA[index] = None
try:
return _launch(a, w, xs, ws, bias, cfg)
except Exception: # noqa: BLE001 - a failed launch keeps the stock math
if cfg != _DEVICE_CFG.get(index) and _DEVICE_TILES.get(index):
_DEVICE_TILES[index] = () # drop the shape tiles, keep the probed default
return _run(a, w, xs, ws, bias)
_DEVICE_CFG[index] = None
if index in _UNLISTED_CARDS:
_UNLISTED_PROBE[index] = False
return reference(a, w, xs, ws, bias)
# index -> {(N, K, xs dtype, ws dtype, bias): {race M: tile or None}}; a large-M tile runs ``_cutlass_fused``
_CUT_PLAN: dict = {}
_CUT_FAMILY: dict = {}
_CUT_RACES: dict = {}
# index -> every shape key already planned (winners and losers), so a later load races only new shapes
_CUT_DONE: dict = {}
_CUT_LARGE_M = 1024
_CUT_CACHE_NAME = "diffusion_int8_cutlass_race.json"
_CUT_CACHE_MAX = 512
def _cut_key(n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> tuple:
return (int(n), int(k), str(xs_dtype), str(ws_dtype), bool(bias))
def _cutlass_tile(
index: Any,
a: Any,
w: Any,
xs: Any,
ws: Any,
bias: Any,
out_dtype: Any = None,
) -> Optional[int]:
plan = _CUT_PLAN.get(index)
if not plan:
return None
entry = plan.get(_cut_key(w.shape[0], w.shape[1], xs.dtype, ws.dtype, bias is not None))
if not entry:
return None
from . import diffusion_int8_cutlass as cut
tile = entry.get(max(cut._RACE_M) if a.shape[0] >= _CUT_LARGE_M else min(cut._RACE_M))
if tile is None:
return None
import torch
return tile if cut.eligible(a, w, xs, ws, bias, out_dtype or torch.bfloat16) else None
def _cutlass_fused(index: Any, n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> bool:
from . import diffusion_int8_cutlass as cut
entry = (_CUT_PLAN.get(index) or {}).get(_cut_key(n, k, xs_dtype, ws_dtype, bias))
return bool(entry) and entry.get(max(cut._RACE_M)) is not None
def cutlass_wanted(capability: Optional[tuple]) -> bool:
"""The prebuilt kernel could serve this arch (switch, family, library loads); before any probe."""
try:
from . import diffusion_int8_cutlass as cut
return (
int8_gemm_mode() != "off"
and cut.mode() != "off"
and cut.family(capability) in (75, 80)
and cut.library() is not None
)
except Exception: # noqa: BLE001
return False
def _cut_cache_key(index: int) -> Optional[str]:
"""Card, toolchain, library build and both sources: any change re-races."""
try:
import hashlib
import torch
from . import diffusion_int8_cutlass as cut
h = hashlib.sha256(open(__file__, "rb").read())
h.update(open(cut.__file__, "rb").read())
lib = cut.library_path()
if lib:
h.update(open(lib, "rb").read())
try:
import triton as _triton
tv = _triton.__version__
except Exception: # noqa: BLE001
tv = "none"
cap = torch.cuda.get_device_capability(index)
return "|".join(
(
torch.cuda.get_device_name(index),
f"{cap[0]}.{cap[1]}",
torch.__version__,
str(torch.version.cuda),
tv,
os.environ.get(INT8_GEMM_TILES_ENV, ""),
os.environ.get(INT8_GEMM_UNLISTED_ENV, ""),
h.hexdigest()[:20],
)
)
except Exception: # noqa: BLE001
return None
def _cut_cache_path() -> Optional[Any]:
try:
from pathlib import Path
from utils.paths.storage_roots import cache_root
return Path(cache_root()) / _CUT_CACHE_NAME
except Exception: # noqa: BLE001
return None
def _cut_cache_read() -> dict:
path = _cut_cache_path()
try:
import json
table = (
json.loads(path.read_text(encoding = "utf-8"))
if path is not None and path.is_file()
else {}
)
return table if isinstance(table, dict) else {}
except Exception: # noqa: BLE001
return {}
def _cut_cache_write(table: dict) -> None:
path = _cut_cache_path()
if path is None:
return
try:
import json
table = dict(list(table.items())[-_CUT_CACHE_MAX:])
path.parent.mkdir(parents = True, exist_ok = True)
tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(table), encoding = "utf-8")
os.replace(tmp, path)
except Exception: # noqa: BLE001
pass
def _xs_dtype(
kind: str,
weight: Any,
k: int,
device: Any,
dtype: Any = None,
) -> Any:
"""Activation-scale dtype of this Linear's act quant, read off a one-row quant."""
import torch
dtype = dtype or torch.bfloat16
x = torch.ones((_MIN_ROWS, k), device = device, dtype = dtype)
if dtype == torch.float16:
from .diffusion_int8_fused import _act_quant
return _act_quant(x, weight)[1].dtype
_, xs = _act_quant_v1(x) if kind == "v1" else _act_quant_v2(x, weight)
return xs.dtype
def _cutlass_plan(
index: int,
recs: list,
triton_on: bool,
logger: Any = None,
) -> dict:
"""Race each int8 weight shape vs today's path, publish winners to ``_CUT_PLAN``; outside any capture."""
import torch
from . import diffusion_int8_cutlass as cut
try:
cap = torch.cuda.get_device_capability(index)
if not cutlass_wanted(cap) or torch.cuda.is_current_stream_capturing():
return {}
if torch.compiler.is_compiling():
return {}
except Exception: # noqa: BLE001
return {}
done = _CUT_DONE.setdefault(index, set())
plan: dict = dict(_CUT_PLAN.get(index) or {})
races: dict = dict(_CUT_RACES.get(index) or {})
try:
if not cut.probed_tiles(index):
_CUT_PLAN[index] = {}
return {}
fam = cut.family(cap)
act = cut.out_dtype_for(fam)
ckey = _cut_cache_key(index)
table = _cut_cache_read() if ckey else {}
cached = table.get(ckey) if ckey else None
cached = cached if isinstance(cached, dict) else {}
dev = torch.device("cuda", index)
g = torch.Generator(device = "cpu").manual_seed(11)
seen: dict = {}
for module, rec in recs:
kind, weight = rec[0], rec[3]
wq, ws = _payload(kind, weight)
n, k = int(wq.shape[0]), int(wq.shape[1])
bias = getattr(module, "bias", None)
if fam == 75 and kind != "v2":
continue # fp16 family: torchao Int8Tensor only
xs_dt = seen.get(("xs", kind, k))
if xs_dt is None:
xs_dt = _xs_dtype(kind, weight, k, dev, act)
seen[("xs", kind, k)] = xs_dt
key = _cut_key(n, k, xs_dt, ws.dtype, bias is not None)
if key in plan or key in done:
continue
done.add(key)
skey = repr(key)
hit = cached.get(skey)
if isinstance(hit, dict) and all(
(v is None or v in cut.probed_tiles(index)) for v in hit.values()
):
plan[key] = {int(m): v for m, v in hit.items()}
continue
# random operands: a streamed weight need not be resident
w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
wsr = (torch.rand(n, generator = g) * 1e-3 + 1e-5).to(ws.dtype).to(dev)
br = (torch.randn(n, generator = g) * 0.1).to(act).to(dev) if bias is not None else None
entry = {}
for m in cut._RACE_M:
if triton_on:
today = lambda a, w_, x_, s_, b_: _launch_or_ref(index, a, w_, x_, s_, b_) # noqa: E731
elif m >= _CUT_LARGE_M:
today = None # cuBLAS lower bound
else:
today = lambda a, w_, x_, s_, b_: cut.reference(a, w_, x_, s_, b_, act) # noqa: E731
r = cut.race(index, w, wsr, br, xs_dt, today, m)
if cut.mode() == "force" and r.get("us"):
r["tile"] = min(
(t for t in r["us"] if t.isdigit()), key = lambda t: r["us"][t], default = None
)
r["tile"] = int(r["tile"]) if r["tile"] is not None else None
entry[m] = r["tile"]
races.setdefault(skey, {})[m] = r
plan[key] = entry
cached[skey] = {str(m): v for m, v in entry.items()}
del w, wsr, br
if ckey:
table = _cut_cache_read()
table.pop(ckey, None)
table[ckey] = cached
_cut_cache_write(table)
except Exception as exc: # noqa: BLE001
if logger is not None:
logger.info(
"diffusion.int8_gemm: prebuilt CUTLASS race failed on cuda:%d (%s)", index, exc
)
plan = dict(_CUT_PLAN.get(index) or {})
plan = {k_: v for k_, v in plan.items() if any(t is not None for t in v.values())}
_CUT_PLAN[index] = plan
_CUT_FAMILY[index] = cut.family(torch.cuda.get_device_capability(index))
_CUT_RACES[index] = races
return plan
def _launch_or_ref(index: int, a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])
if cfg is not None and _aligned(a, w):
return _launch(a, w, xs, ws, bias, cfg)
return reference(a, w, xs, ws, bias)
def cutlass_call_count() -> int:
from . import diffusion_int8_cutlass as cut
return cut._CALLS[0]
def _run16(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
"""fp16 op body (sm75 family): the race winner, else torchao's fp16 epilogue after cuBLAS."""
import torch
from . import diffusion_int8_cutlass as cut
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
index = a.device.index
tile = _cutlass_tile(index, a, w, xs, ws, bias, torch.float16)
if tile is not None:
try:
return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.float16)
except Exception: # noqa: BLE001
_CUT_PLAN.pop(index, None)
return cut.reference(a, w, xs, ws, bias, torch.float16)
@lru_cache(maxsize = 1)
def _op16() -> Any:
"""``unsloth_studio::int8_mm_dequant_f16`` (opaque to Inductor), or None."""
try:
import torch
except Exception: # noqa: BLE001
return None
ns = getattr(torch.ops, _OP_NAMESPACE, None)
if ns is not None and hasattr(ns, _OP16_NAME):
return getattr(ns, _OP16_NAME)
custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
if custom_op is None:
return None
try:
@custom_op(
f"{_OP_NAMESPACE}::{_OP16_NAME}",
mutates_args = (),
schema = "(Tensor a, Tensor w, Tensor xs, Tensor ws, Tensor? bias) -> Tensor",
)
def _int8_mm_dequant_f16(a, w, xs, ws, bias):
return _run16(a, w, xs, ws, bias)
@_int8_mm_dequant_f16.register_fake
def _(a, w, xs, ws, bias):
return a.new_empty((a.shape[0], w.shape[0]), dtype = torch.float16)
except Exception: # noqa: BLE001
return None
return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant_f16
def _linear_forward_f16(self: Any, x: Any, rec: tuple) -> Any:
"""sm75 fp16 forward via the prebuilt kernel, bit for bit the stock forward."""
import torch
kind, group, _rotq, weight, _fused = rec
if self.weight is not weight or _OP16_HANDLE is None:
return type(self).forward(self, x)
wq, ws = _payload(kind, weight)
if wq.device != x.device or x.numel() < _MIN_ROWS * x.shape[-1]:
return type(self).forward(self, x)
lead = x.shape[:-1]
if group is not None:
from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
x = rotate_convrot_activation(
x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
)
from .diffusion_int8_fused import _act_quant
xq, xs = _act_quant(x.reshape(-1, x.shape[-1]), weight)
y = _OP16_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
return y.reshape(*lead, y.shape[-1])
_OP16_HANDLE: Any = None
@lru_cache(maxsize = 1)
def _op() -> Any:
"""``unsloth_studio::int8_mm_dequant`` (opaque to Inductor; no host sync, allocation only), or None."""
try:
import torch
except Exception: # noqa: BLE001
return None
ns = getattr(torch.ops, _OP_NAMESPACE, None)
if ns is not None and hasattr(ns, _OP_NAME):
return getattr(ns, _OP_NAME)
custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
if custom_op is None: # torch < 2.4
return None
try:
@custom_op(
f"{_OP_NAMESPACE}::{_OP_NAME}",
mutates_args = (),
schema = "(Tensor a, Tensor w, Tensor xs, Tensor ws, Tensor? bias) -> Tensor",
)
def _int8_mm_dequant(a, w, xs, ws, bias):
return _run(a, w, xs, ws, bias)
@_int8_mm_dequant.register_fake
def _(a, w, xs, ws, bias):
return a.new_empty((a.shape[0], w.shape[0]), dtype = torch.bfloat16)
except Exception: # noqa: BLE001 - a registration failure keeps the stock path
return None
return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant
def shape_tiles(capability: Optional[tuple]) -> tuple:
"""This arch's per-shape rules, or () (``UNSLOTH_DIFFUSION_INT8_GEMM_TILES=0``, unmeasured arch)."""
if capability is None or not tiles_enabled():
return ()
return tuple(_SHAPE_TILES.get((int(capability[0]), int(capability[1])), ()))
def _tma_probe(index: int, capability: tuple) -> Optional[tuple]:
"""This arch's TMA tile when its kernel matches the eager epilogue bit for bit (``_probe``), else None."""
tma = tma_config(capability)
try:
ok = tma is not None and _tensor_descriptor() is not None and _probe(index, tma, tma = True)
except Exception: # noqa: BLE001 - pointer loads stay
ok = False
return tma if ok else None
def device_config(index: int) -> Optional[tuple]:
"""Probed once per device (arch gate, Triton, op registration, bit-exact match with the eager
torchao epilogue); None keeps the stock path, a failing per-shape tile falls back to the
default, and unlisted archs re-read unlisted_config per call."""
if index in _UNLISTED_CARDS:
return unlisted_config(index)
if index in _DEVICE_CFG:
return _DEVICE_CFG[index]
_DEVICE_TMA.pop(index, None)
cfg = None
rules: tuple = ()
try:
import torch
if (
torch.cuda.is_available()
and not is_rocm_torch(torch)
and _triton_version_ok()
and _kernels() is not None
and _op() is not None
):
cap = torch.cuda.get_device_capability(index)
if unlisted_arch(cap):
_UNLISTED_CARDS.add(index)
return unlisted_config(index)
want = arch_config(cap)
for cand in (want, _FALLBACK_CONFIG) if want is not None else ():
if cand is not None and _probe(index, cand):
cfg = cand
break
if cfg is not None:
ok: dict = {}
for rule in shape_tiles(cap):
tile = rule[4]
if tile not in ok:
ok[tile] = tile == cfg or _probe(index, tile)
rules = tuple(rule for rule in shape_tiles(cap) if ok[rule[4]])
tma = _tma_probe(index, cap)
if tma is not None:
_DEVICE_TMA[index] = tma
except Exception: # noqa: BLE001
cfg = None
_DEVICE_TILES[index] = rules if cfg is not None else ()
_DEVICE_CFG[index] = cfg
return cfg
_UNLISTED_CARDS: set = set()
_UNLISTED_PROBE: dict = {}
_UNLISTED_OK: dict = {}
_UNLISTED_TIMING: dict = {}
# (M, N, K): Qwen-Image-2.1 attention projection, MLP in, MLP down
_UNLISTED_LATENCY_SHAPES = ((2048, 4096, 4096), (2048, 12288, 4096), (2048, 4096, 12288))
# Hold per tile (then as long timed): long enough for a power-capped part (L4: 72 W) to settle its clock.
_UNLISTED_RACE_SHAPES = tuple((4096, n, k) for _m, n, k in _UNLISTED_LATENCY_SHAPES)
_RACE_HOLD_S = 0.3
def _unlisted_latency(index: int, cfg: tuple) -> dict:
"""{"fused": us, "stock": us} summed over ``_UNLISTED_LATENCY_SHAPES``, each arm captured in a CUDA graph and
replayed, best of seven interleaved rounds. Stock = cuBLAS ``_int_mm``; its epilogue fuses into the consumer, so
each arm ends in one read of its output (int32 for stock, bf16 here) standing in for that consumer."""
return _graph_latency(
index,
{
"fused": lambda a, w, xs, ws: _launch(a, w, xs, ws, None, cfg).amax(),
"stock": lambda a, w, xs, ws: _int_mm_stock(a, w).amax(),
},
)
def _int_mm_stock(a: Any, w: Any) -> Any:
import torch
return torch._int_mm(a, w.t())
def _unlisted_race(index: int, cfgs: tuple) -> dict:
"""{tile: us}, each held under load: a power-capped part clocks down under a tile that draws more, so a short
burst would rank tiles by boost-clock speed, not render speed."""
fns = {}
for cfg in cfgs:
fns[cfg] = lambda a, w, xs, ws, cfg = cfg: _launch(a, w, xs, ws, None, cfg).amax()
return _graph_latency(index, fns, _UNLISTED_RACE_SHAPES, hold_s = _RACE_HOLD_S)
def _graph_latency(
index: int,
fns: dict,
shapes: tuple = _UNLISTED_LATENCY_SHAPES,
hold_s: float = 0.0,
) -> dict:
"""{name: us} per CUDA-graph arm: best of seven interleaved rounds, or held ``hold_s`` then timed ``hold_s``."""
import torch
from .diffusion_cuda_graph import CAPTURE_MODE, _capturing, synchronize_outside_capture
dev = torch.device("cuda", index)
g = torch.Generator(device = "cpu").manual_seed(5)
ops = []
for m, n, k in shapes:
a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
xs = torch.full((m,), 1e-3, device = dev, dtype = torch.bfloat16)
ws = torch.full((n,), 1e-3, device = dev, dtype = torch.bfloat16)
ops.append((a, w, xs, ws))
reps = 2
graphs = {}
with torch.cuda.device(dev):
side = torch.cuda.Stream(dev)
side.wait_stream(torch.cuda.current_stream(dev))
with torch.cuda.stream(side):
for fn in fns.values():
for op in ops:
fn(*op) # compile / autotune outside the capture
torch.cuda.current_stream(dev).wait_stream(side)
synchronize_outside_capture(dev)
for name, fn in fns.items():
graph = torch.cuda.CUDAGraph()
with _capturing(), torch.cuda.graph(graph, capture_error_mode = CAPTURE_MODE):
for _ in range(reps):
for op in ops:
fn(*op)
graphs[name] = graph
for graph in graphs.values():
graph.replay()
synchronize_outside_capture(dev)
start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
best = {name: float("inf") for name in graphs}
if hold_s > 0:
for name, graph in graphs.items():
best[name] = _held_us(graph, hold_s, start, end) / reps
del graphs
return best
for _ in range(7):
for name, graph in graphs.items():
start.record()
graph.replay()
end.record()
end.synchronize()
best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)
del graphs
return best
def _held_us(graph: Any, hold_s: float, start: Any, end: Any) -> float:
"""us per replay of ``graph`` after ``hold_s`` of back-to-back replays, averaged over ``hold_s`` more."""
import time
t0 = time.perf_counter()
while time.perf_counter() - t0 < hold_s:
graph.replay()
end.record()
end.synchronize()
n = 0
start.record()
t0 = time.perf_counter()
while n == 0 or time.perf_counter() - t0 < hold_s:
graph.replay()
n += 1
if n % 2 == 0:
end.record()
end.synchronize()
end.record()
end.synchronize()
return start.elapsed_time(end) * 1000.0 / n
def unlisted_config(index: int) -> Optional[tuple]:
"""Unlisted sm80+ card: the race winner if the fallback passed the probe and the winner the latency check, else
None. Force skips race and check and runs the fallback. Outside any trace / capture."""
import torch
mode = unlisted_mode()
cfg = None
try:
if unlisted_wanted(torch.cuda.get_device_capability(index), mode):
if index not in _UNLISTED_PROBE:
_UNLISTED_PROBE[index] = _probe(index, _FALLBACK_CONFIG)
if _UNLISTED_PROBE[index]:
tile = _FALLBACK_CONFIG if mode == "force" else _unlisted_tile(index)
key = (index, mode, tile)
if key not in _UNLISTED_OK:
ok = True
if mode != "force":
try:
timing = _unlisted_latency(index, tile)
_UNLISTED_TIMING[index] = {k: round(v, 1) for k, v in timing.items()}
_UNLISTED_TIMING[index].update(_UNLISTED_RACE.get(index) or {})
ok = timing["fused"] <= timing["stock"]
except Exception: # noqa: BLE001 - main's path stays
ok = False
_UNLISTED_OK[key] = ok
cfg = tile if _UNLISTED_OK[key] else None
except Exception: # noqa: BLE001
cfg = None
_DEVICE_TILES[index] = ()
_DEVICE_CFG[index] = cfg
return cfg
_UNLISTED_TILE: dict = {}
_UNLISTED_RACE: dict = {} # index -> {"tile": kept tile, "race": {tile: us}} for the log
_RACE_CACHE_NAME = "diffusion_int8_tile_race.json"
_RACE_CACHE_MAX = 16
def race_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_RACE_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def _race_cache_file() -> Optional[Any]:
try:
from pathlib import Path
from utils.paths.storage_roots import cache_root
return Path(cache_root()) / _RACE_CACHE_NAME
except Exception: # noqa: BLE001 - no resolvable root: race every process
return None
def _race_key(index: int) -> Optional[str]:
"""Card, toolchain and this file's source: any change re-races."""
try:
import hashlib
import torch
import triton as _triton
src = hashlib.sha256(open(__file__, "rb").read()).hexdigest()[:16]
cap = torch.cuda.get_device_capability(index)
return "|".join(
(
torch.cuda.get_device_name(index),
f"{cap[0]}.{cap[1]}",
torch.__version__,
str(torch.version.cuda),
_triton.__version__,
src,
)
)
except Exception: # noqa: BLE001
return None
def _race_cache_load(key: Optional[str]) -> Optional[tuple]:
path = _race_cache_file() if key else None
try:
import json
tile = (
json.loads(path.read_text(encoding = "utf-8")).get(key)
if path is not None and path.is_file()
else None
)
except Exception: # noqa: BLE001 - unreadable or corrupt: a miss
return None
tile = tuple(tile) if isinstance(tile, list) else None
return tile if tile == _FALLBACK_CONFIG or tile in _UNLISTED_RACE_TILES else None
def _race_cache_store(key: Optional[str], tile: tuple) -> None:
path = _race_cache_file() if key else None
if path is None:
return
try:
import json
try:
table = json.loads(path.read_text(encoding = "utf-8")) if path.is_file() else {}
except Exception: # noqa: BLE001
table = {}
table = {k: v for k, v in table.items() if k != key and isinstance(v, list)}
table = dict(list(table.items())[-(_RACE_CACHE_MAX - 1) :])
table[key] = list(tile)
path.parent.mkdir(parents = True, exist_ok = True)
tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(table), encoding = "utf-8")
os.replace(tmp, path)
except Exception: # noqa: BLE001 - persistence is best-effort
pass
def _unlisted_tile(index: int) -> tuple:
"""Once per device: fastest probe-passing tile vs the fallback (ties keep the fallback), persisted per card and
toolchain. Any failure, or UNSLOTH_DIFFUSION_INT8_GEMM_RACE=0, keeps the fallback."""
if (
not race_enabled()
): # read per load, ahead of the kept winner, so the switch acts between loads
return _FALLBACK_CONFIG
if index not in _UNLISTED_TILE:
best = _FALLBACK_CONFIG
try:
key = _race_key(index) if race_enabled() else None
cached = _race_cache_load(key)
if cached is not None and (cached == _FALLBACK_CONFIG or _probe(index, cached)):
best = cached
_UNLISTED_RACE[index] = {"tile": best}
elif race_enabled():
cands = tuple(
t for t in _UNLISTED_RACE_TILES if t != _FALLBACK_CONFIG and _probe(index, t)
)
if cands:
timing = _unlisted_race(index, (_FALLBACK_CONFIG,) + cands)
for tile in cands:
if timing[tile] < timing[best]:
best = tile
_UNLISTED_RACE[index] = {
"tile": best,
"race": {t: round(us, 1) for t, us in timing.items()},
}
_race_cache_store(key, best)
except Exception: # noqa: BLE001 - the fallback stays
best = _FALLBACK_CONFIG
_UNLISTED_TILE[index] = best
return _UNLISTED_TILE[index]
# (M, N, K, bias, fp32 scales). Ragged N / K stay on 16: an off-16 probe compiles the spilling variant (see ``_aligned``).
_PROBE_SHAPES = (
(257, 384, 512, False, False),
(33, 208, 144, True, True),
(300, 528, 1040, True, False),
)
def _probe(
index: int,
cfg: tuple,
tma: bool = False,
) -> bool:
import torch
launch = _launch_tma if tma else _launch
dev = torch.device("cuda", index)
g = torch.Generator(device = "cpu").manual_seed(0)
try:
for m, n, k, bias, xs32 in _PROBE_SHAPES:
a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(torch.bfloat16)
xs = (xs.float() if xs32 else xs).to(dev)
ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(torch.bfloat16).to(dev)
ws = ws.float() if xs32 else ws
b = (torch.randn(n, generator = g) * 0.1).to(torch.bfloat16).to(dev) if bias else None
if not torch.equal(launch(a, w, xs, ws, b, cfg), reference(a, w, xs, ws, b)):
return False
a, w = tie_operands(dev)
xs = torch.ones(a.shape[0], device = dev, dtype = torch.bfloat16)
ws = torch.ones(w.shape[0], device = dev, dtype = torch.bfloat16)
if not torch.equal(launch(a, w, xs, ws, None, cfg), reference(a, w, xs, ws, None)):
return False
from .diffusion_cuda_graph import synchronize_outside_capture
synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001 - out of shared memory, compile failure, ...
return False
def tie_operands(
device: Any,
rows: int = 32,
k: int = 4096,
) -> tuple:
"""int8 (a, w) whose int32 products sit one or two units off a bf16 midpoint in [2^24, 2^26), both signs."""
import torch
k2 = 128
k1 = k - k2
targets = []
for e in (24, 25):
for mult in (0, 3, 50, 100):
mid = (1 << e) + mult * (1 << (e - 7)) + (1 << (e - 8))
for d in (-1, 1, -2, 2):
targets += [mid + d, -(mid + d)]
w = torch.zeros(len(targets), k, dtype = torch.int8)
for j, t in enumerate(targets):
s1, s2 = divmod(abs(t), 127)
full, rem = divmod(s1, 127)
sign = 1 if t >= 0 else -1
w[j, :full] = 127 * sign
if rem:
w[j, full] = rem * sign
w[j, k1] = s2 * sign
a = torch.ones(rows, k, dtype = torch.int8)
a[:, :k1] = 127
return a.to(device), w.to(device)
def _v1_parts(w: Any) -> Optional[tuple]:
"""(int8 [N, K], scale [N]) of a torchao v1 ``LinearActivationQuantizedTensor`` over a plain-layout symmetric
per-channel ``AffineQuantizedTensor`` with the per-token reduced-range activation quant, else None."""
try:
if type(w).__name__ != "LinearActivationQuantizedTensor":
return None
fn = getattr(w, "input_quant_func", None)
if getattr(fn, "__name__", "") != "_int8_symm_per_token_reduced_range_quant":
return None
if getattr(w, "quant_kwargs", None):
return None
aqt = w.original_weight_tensor
if type(aqt).__name__ != "AffineQuantizedTensor":
return None
impl = aqt.tensor_impl
if (
type(impl).__name__ != "PlainAQTTensorImpl"
or type(getattr(impl, "_layout", None)).__name__ != "PlainLayout"
):
return None
data, scale, zp = impl.int_data, impl.scale, getattr(impl, "zero_point", None)
import torch
if data.dtype != torch.int8 or data.dim() != 2 or scale.numel() != data.shape[0]:
return None
if zp is not None and bool((zp != 0).any()):
return None
if tuple(getattr(aqt, "block_size", ())) != (1, data.shape[1]):
return None
return data, scale.reshape(-1)
except Exception: # noqa: BLE001
return None
def _v2_parts(w: Any, check_zero_point: bool = True) -> Optional[tuple]:
"""(int8 [N, K], scale [N]) of a plain dynamic symmetric per-row torchao ``Int8Tensor``, else None.
``check_zero_point=False`` inside a traced forward: the check is a host sync and a graph break."""
try:
from .diffusion_int8_fused import _plain_int8_weight
if not _plain_int8_weight(w):
return None
if (
check_zero_point
and getattr(w, "zero_point", None) is not None
and bool((w.zero_point != 0).any())
):
return None
return w.qdata, w.scale.reshape(-1)
except Exception: # noqa: BLE001
return None
def _act_quant_v1(x2d: Any) -> tuple:
from .diffusion_convrot_quant import act_quant_v1
return act_quant_v1(x2d)
def _act_quant_v2(x2d: Any, weight: Any) -> tuple:
from .diffusion_int8_fused import _mlp_act_quant
return _mlp_act_quant(x2d, weight)
@lru_cache(maxsize = 4)
def _v1_act_quant_matches(index: int) -> bool:
"""The subclass-free v1 activation quant must equal torchao's own, bit for bit, on this torchao."""
try:
import torch
from torchao.quantization.quant_api import _int8_symm_per_token_reduced_range_quant
g = torch.Generator(device = "cpu").manual_seed(1)
x = (torch.randn(67, 256, generator = g) * 3).to(torch.bfloat16)
x[5] = 0
x[9, :7] *= 300
x = x.to(torch.device("cuda", index))
ref = _int8_symm_per_token_reduced_range_quant(x)
q, s = _act_quant_v1(x)
impl = ref.tensor_impl
return bool(
torch.equal(q, impl.int_data) and torch.equal(s.reshape(-1), impl.scale.reshape(-1))
)
except Exception: # noqa: BLE001
return False
def stock_gemm(xq: Any, wq: Any, xs: Any, ws: Any, bias: Any) -> Any:
"""torchao's int8 Linear after its act quant, op for op: cuBLAS ``_int_mm``, ``* x_scale`` in the scale's dtype,
bf16, ``* w_scale``, ``+ bias``, bf16 (``reference`` minus the small-M padding; callers keep M >= 17). torchao 0.17's
zero-point pass subtracts an exact zero for this symmetric quant and is skipped."""
import torch
y = (torch._int_mm(xq, wq.t()) * xs.reshape(-1, 1)).to(torch.bfloat16)
y = y * ws.reshape(-1)
if bias is not None:
y = y + bias
return y.to(torch.bfloat16)
def _payload(kind: str, weight: Any) -> tuple:
"""(int8 [N, K], scale [N]) off the live parameter, never a cached alias (a moved weight must not go stale)."""
if kind == "v1":
impl = weight.original_weight_tensor.tensor_impl
return impl.int_data, impl.scale.reshape(-1)
return weight.qdata, weight.scale.reshape(-1)
def _linear_forward(self: Any, x: Any) -> Any:
"""``nn.Linear.forward`` for a torchao int8 dynamic weight: torchao's activation quant (fused with the ConvRot
rotation where that kernel is on), then the fused-dequant GEMM, or cuBLAS + torchao's epilogue where it is not."""
import torch
rec = self.__dict__.get(_REC)
if rec is None or not x.is_cuda:
return type(self).forward(self, x)
if self.__dict__.get(_F16):
if x.dtype == torch.float16:
return _linear_forward_f16(self, x, rec)
return type(self).forward(self, x)
if x.dtype != torch.bfloat16:
return type(self).forward(self, x)
kind, group, rotq, weight, fused = rec
if self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock
return type(self).forward(self, x)
wq, ws = _payload(kind, weight)
if wq.device != x.device:
return type(self).forward(self, x)
lead = x.shape[:-1]
if x.numel() < _MIN_ROWS * x.shape[-1]:
return type(self).forward(self, x)
q = None
if rotq:
from .diffusion_convrot_quant import convrot_act_quant
q = convrot_act_quant(x.reshape(-1, x.shape[-1]), group, kind == "v2")
if q is not None:
xq, xs = q
elif not fused:
# rotation kernel not covering this call and no fused GEMM: the module's own stock forward
return type(self).forward(self, x)
else:
if group is not None:
from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
x = rotate_convrot_activation(
x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
)
x2d = x.reshape(-1, x.shape[-1])
if kind == "v1":
xq, xs = _act_quant_v1(x2d)
else:
xq, xs = _act_quant_v2(x2d, weight)
if fused:
y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
else:
y = stock_gemm(xq, wq, xs, ws, self.bias)
return y.reshape(*lead, y.shape[-1])
def _hooked(module: Any) -> bool:
"""A forward (pre-)hook, own or global, or an instance ``forward``: a fused path skipping ``module`` would bypass it."""
import torch
d = module.__dict__
if "forward" in d or d.get("_forward_hooks") or d.get("_forward_pre_hooks"):
return True
nn_module = torch.nn.modules.module
return bool(nn_module._global_forward_hooks or nn_module._global_forward_pre_hooks)
def _swapped_rec(module: Any) -> Optional[tuple]:
"""Record of a Linear we swapped with its install-time weight and no offload / forward hook or override, else None:
the shared paths run the GEMM without calling the module, so any of those would be bypassed."""
import torch
d = module.__dict__
rec = d.get(_REC)
if rec is None or _MARK not in d or "_diffusers_hook" in d or "_hf_hook" in d:
return None
fwd = d.get("forward")
if torch.compiler.is_compiling():
if fwd is None or fwd is not d.get(_FWD):
return None
elif getattr(fwd, "__func__", None) is not _linear_forward:
return None
if d.get("_forward_hooks") or d.get("_forward_pre_hooks"):
return None
nn_module = torch.nn.modules.module
if nn_module._global_forward_hooks or nn_module._global_forward_pre_hooks:
return None
if module.weight is not rec[3]:
return None
return rec
def shared_rotq_linears(x: Any, linears: tuple) -> Optional[list]:
"""``[m(x) for m in linears]`` with one shared rotation + act quant; None when any is not covered."""
import torch
if x.dtype != torch.bfloat16 or not x.is_cuda or x.numel() < _MIN_ROWS * x.shape[-1]:
return None
recs = [_swapped_rec(m) for m in linears]
if any(r is None or not r[2] for r in recs):
return None
kind, group = recs[0][0], recs[0][1]
if any(r[0] != kind or r[1] != group for r in recs):
return None
parts = [_payload(r[0], r[3]) for r in recs]
if any(wq.device != x.device for wq, _ in parts):
return None
from .diffusion_convrot_quant import convrot_act_quant
q = convrot_act_quant(x.reshape(-1, x.shape[-1]), group, kind == "v2")
if q is None:
return None
xq, xs = q
lead = x.shape[:-1]
outs = []
for m, r, (wq, ws) in zip(linears, recs, parts):
if r[4]:
y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, m.bias)
else:
y = stock_gemm(xq, wq, xs, ws, m.bias)
outs.append(y.reshape(*lead, y.shape[-1]))
return outs
def _swiglu_down(down: Any, gate: Any, value: Any) -> Optional[Any]:
"""``down(silu(gate) * value)`` via the fused SwiGLU rotation + quant, else None."""
import torch
rec = _swapped_rec(down)
if rec is None:
return None
kind, group, rotq, weight, fused = rec
if (
not rotq
or gate.dtype != torch.bfloat16
or value.dtype != torch.bfloat16
or not gate.is_cuda
or gate.shape != value.shape
or gate.shape[-1] != down.in_features
or gate.numel() < _MIN_ROWS * gate.shape[-1]
):
return None
wq, ws = _payload(kind, weight)
if wq.device != gate.device:
return None
from .diffusion_convrot_quant import convrot_swiglu_act_quant
k = gate.shape[-1]
q = convrot_swiglu_act_quant(gate.reshape(-1, k), value.reshape(-1, k), group, kind == "v2")
if q is None:
return None
xq, xs = q
if fused:
y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, down.bias)
else:
y = stock_gemm(xq, wq, xs, ws, down.bias)
return y.reshape(*gate.shape[:-1], y.shape[-1])
def producer_linear(down: Any, prefix: Any, gate: Any, value: Any) -> Optional[Any]:
"""``down(cat([prefix.flatten(-2), silu(gate) * value], -1))`` (no prefix: ``down(silu(gate) * value)``) with the
product, concat, rotation and act quant in one kernel reading the producers' buffers, else None (caller: stock).
``gate`` / ``value``: ``[..., F]`` halves of one projection output; ``prefix``: the attention output
``[B, S, H, D]`` as the attention backend returned it."""
import torch
rec = _swapped_rec(down)
if rec is None:
return None
kind, group, rotq, weight, fused = rec
lead = gate.shape[:-1]
f = gate.shape[-1]
width = f if prefix is None else f + prefix.shape[-2] * prefix.shape[-1]
if (
not rotq
or group != 256
or gate.dtype != torch.bfloat16
or not gate.is_cuda
or gate.shape != value.shape
or width != down.in_features
or gate.numel() < _MIN_ROWS * f
):
return None
if prefix is not None and (prefix.dim() != 4 or prefix.shape[:2] != lead):
return None
wq, ws = _payload(kind, weight)
if wq.device != gate.device:
return None
from .diffusion_convrot_quant import convrot_producer_act_quant
q = convrot_producer_act_quant(
prefix, gate.reshape(-1, f), value.reshape(-1, f), group, kind == "v2"
)
if q is None:
return None
xq, xs = q
if fused:
y = _OP_HANDLE(xq, wq, xs.reshape(-1), ws, down.bias)
else:
y = stock_gemm(xq, wq, xs, ws, down.bias)
return y.reshape(*lead, y.shape[-1])
def _plain_silu(act: Any) -> bool:
"""``act`` is an exact, unhooked ``nn.SiLU`` (the fused prologue hard-codes SiLU); ``type() is`` avoids a graph break."""
from torch import nn
return type(act) is nn.SiLU and not _hooked(act)
def _swiglu_ff_forward(self: Any, hidden_states: Any) -> Any:
"""``QwenImage21SwiGLUFeedForward.forward`` on the shared and SwiGLU-fused quants."""
pair = shared_rotq_linears(hidden_states, (self.gate_layer, self.proj))
if pair is None:
gate, value = self.gate_layer(hidden_states), self.proj(hidden_states)
else:
gate, value = pair
y = _swiglu_down(self.out, gate, value) if _plain_silu(self.activation_fn) else None
if y is None:
return self.out(self.activation_fn(gate) * value)
return y
def _swiglu_on(index: int) -> bool:
from .diffusion_convrot_quant import _SWIGLU_DEVICE, rotquant_swiglu_enabled
return rotquant_swiglu_enabled() and bool(_SWIGLU_DEVICE.get(index))
def _swiglu_ff_eligible(module: Any) -> bool:
from torch import nn
if type(module).__name__ not in _SWIGLU_FF_CLASSES or "forward" in module.__dict__:
return False
parts = [getattr(module, n, None) for n in ("gate_layer", "proj", "out")]
if any(p is None or _REC not in p.__dict__ for p in parts):
return False
act = getattr(module, "activation_fn", None)
if type(act) is not nn.SiLU or "forward" in act.__dict__:
return False
# only a rotated down projection on the fused rotation kernel gains (its input is the product)
return bool(parts[2].__dict__[_REC][2])
def _install_swiglu_ff(
transformer: Any,
index: int,
streamed: bool = False,
) -> int:
"""Patch the eligible SwiGLU MLPs on card ``index`` (``streamed``: also host-resident ones)."""
count = 0
for module in transformer.modules():
if _FF_MARK in module.__dict__:
count += 1
continue
if not _swiglu_ff_eligible(module):
continue
card = _weight_card(module.out)
if card != index and not (streamed and card is None):
continue
module.__dict__[_FF_MARK] = True
module.forward = types.MethodType(_swiglu_ff_forward, module)
count += 1
return count
def linear_from_q(q: Any, xs: Any, weight: Any, bias: Any) -> Optional[Any]:
"""For ``diffusion_int8_fused``: the down projection on an already-quantized activation, or None (stock)."""
import torch
if _OP_HANDLE is None or int8_gemm_mode() == "off" or q.device.index is None:
return None
# The fused MLP already admitted this weight (symmetric, and its stock epilogue ignores the zero point too).
parts = _v2_parts(weight, check_zero_point = False)
if _DEVICE_CFG.get(q.device.index) is None and not (
parts is not None
and _cutlass_fused(
q.device.index,
parts[0].shape[0],
parts[0].shape[1],
xs.dtype,
parts[1].dtype,
bias is not None,
)
):
return None
if (
parts is None
or q.shape[0] < _MIN_ROWS
or parts[1].dtype not in (torch.bfloat16, torch.float32)
):
return None
if parts[0].shape[1] % _K_ALIGN or parts[0].shape[0] % _N_ALIGN:
return None
if bias is not None and bias.dtype != torch.bfloat16:
return None
return _OP_HANDLE(q, parts[0], xs.reshape(-1), parts[1], bias)
def _rotation_group(module: Any) -> Optional[int]:
"""The ConvRot group the forward above can reproduce, else None."""
try:
from .diffusion_convrot import convrot_linear_class, is_power_of_four
if type(module) is not convrot_linear_class():
return None
group = module.convrot_groupsize
if not is_power_of_four(group) or module.in_features % group:
return None
return int(group)
except Exception: # noqa: BLE001
return None
def _eligible(module: Any) -> Optional[tuple]:
"""(kind, rotation group or None, scale, weight) for a Linear the fused GEMM can run, else None."""
from torch import nn
group = None
if type(module) is not nn.Linear:
# any other subclass transforms its input in a way this forward would skip
group = _rotation_group(module) if convrot_enabled() else None
if group is None:
return None
w = module.weight
bias = module.bias
import torch
# fp16 bias only reaches the sm75 kernel (``_swap_on`` filters per card)
if bias is not None and bias.dtype not in (torch.bfloat16, torch.float16):
return None
# off-grid K runs masked loads, far slower than cuBLAS
out_f, in_f = getattr(module, "out_features", 0), getattr(module, "in_features", 0)
if in_f % _K_ALIGN or out_f % _N_ALIGN:
return None
parts = _v1_parts(w)
if parts is not None and parts[1].dtype == torch.bfloat16:
return ("v1", group, parts[1], w)
parts = _v2_parts(w)
if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32, torch.float16):
return ("v2", group, parts[1], w)
return None
def _card_dtypes_ok(module: Any, rec: tuple, f16_card: bool) -> bool:
import torch
bias = getattr(module, "bias", None)
if f16_card:
return rec[0] == "v2" and (bias is None or bias.dtype == torch.float16)
scale_dtype = getattr(rec[2], "dtype", None)
return (bias is None or bias.dtype == torch.bfloat16) and scale_dtype in (
None,
torch.bfloat16,
torch.float32,
)
def candidates(transformer: Any, rotated_only: bool = False) -> int:
try:
recs = (_eligible(m) for m in transformer.modules())
return sum(1 for r in recs if r is not None and (r[1] is not None or not rotated_only))
except Exception: # noqa: BLE001
return 0
def _gates(capability: Optional[tuple]) -> tuple:
"""(fused GEMM on this arch, rotation kernel on this arch), before any probe."""
from .diffusion_convrot_quant import rotquant_config
gemm = arch_config(capability) is not None
rotq = convrot_enabled() and rotquant_config(capability) is not None
return gemm, rotq
def _unlisted_rotated(index: int) -> bool:
"""Unlisted card: ``rotq_h16`` armed for it (probe + its own latency check). Takes no ``diffusion_int8_fused``
lock: its SwiGLU fusion reaches this through ``rotated_ff`` while holding it."""
if not convrot_enabled():
return False
try:
from . import diffusion_convrot_quant as cq
from .diffusion_int8_fused import _act_quant_device_ok, _act_quant_op
actq = None
if (index, cq.unlisted_mode()) not in cq._UNLISTED_DEVICE:
actq = _act_quant_op() if _act_quant_device_ok(index) else None
return bool(cq.unlisted_device_ok(index, actq))
except Exception: # noqa: BLE001
return False
def rotated_swapped(index: int) -> Optional[bool]:
"""On an unlisted card (outside any trace): True when rotated Linears go through this swap (fused GEMM and the
``rotq_h16`` rotation + act quant both on), so the Qwen-Image-2.1 / Z-Image stock fusions stand down. False keeps
them, exactly as on main. None on a listed arch (``_gates`` decides)."""
import torch
cap = torch.cuda.get_device_capability(index)
if not unlisted_arch(cap) or _gates(cap)[1] or int8_gemm_mode() == "off":
return None
return _unlisted_rotated(index) and device_config(index) is not None
def install(
transformer: Any,
logger: Any = None,
offload_active: bool = False,
device: Any = None,
) -> int:
"""Idempotent, returns the candidate count; must run before the first compiled forward; block-streamed denoisers need device or keep stock path."""
if int8_gemm_mode() == "off" or transformer is None:
return 0
if device is not None:
if not streamed_enabled():
return 0
return _finalize(transformer, logger, device = device)
if offload_active:
return 0
from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
if resident_cuda_device(transformer) is not None:
return _finalize(transformer, logger)
# Weights not on the GPU yet: arch gate now (status never claims a stock arch), probe + swap at the first forward.
try:
import torch
if not torch.cuda.is_available() or is_rocm_torch(torch):
return 0
cap = torch.cuda.get_device_capability(torch.cuda.current_device())
prebuilt = cutlass_wanted(cap)
if not prebuilt and (not _triton_version_ok() or _kernels() is None):
return 0
gemm, rotq = _gates(cap)
gemm = gemm or unlisted_wanted(cap) or prebuilt
if not (gemm or rotq):
return 0
except Exception: # noqa: BLE001
return 0
n = candidates(transformer, rotated_only = not gemm)
if n:
run_on_first_call(transformer, "int8_gemm", lambda t: _finalize(t, logger))
return n
def _finalize(
transformer: Any,
logger: Any = None,
device: Any = None,
) -> int:
count = _swap(transformer, logger, device = device)
try:
transformer._unsloth_int8_gemm = count # the deferred install recorded the candidate count
except Exception: # noqa: BLE001
pass
return count
def _swap(
transformer: Any,
logger: Any = None,
device: Any = None,
) -> int:
from .diffusion_int8_fused import resident_cuda_device
import torch
if device is not None:
try:
dev = torch.device(device)
except Exception: # noqa: BLE001
return 0
if dev.type != "cuda" or is_rocm_torch(torch) or not torch.cuda.is_available():
return 0
else:
dev = resident_cuda_device(transformer)
if dev is None:
cards = _split_cards(transformer)
if cards:
count = sum(_swap_on(transformer, logger, index) for index in cards)
return count
if dev is None:
return 0
index = dev.index if dev.index is not None else torch.cuda.current_device()
return _swap_on(transformer, logger, index, streamed = device is not None)
def _split_cards(transformer: Any) -> tuple:
"""The CUDA ordinals of a layer-split denoiser whose every weight sits on a card (NVIDIA), else ()."""
try:
import torch
from .diffusion_layer_split import is_layer_split
if is_rocm_torch(torch) or not is_layer_split(transformer):
return ()
devices = {p.device for p in transformer.parameters()}
except Exception: # noqa: BLE001
return ()
if not devices or any(d.type != "cuda" or d.index is None for d in devices):
return ()
return tuple(sorted(d.index for d in devices))
def _weight_card(module: Any) -> Optional[int]:
weight = getattr(module, "weight", None)
dev = getattr(weight, "device", None)
return dev.index if dev is not None and dev.type == "cuda" else None
def _swap_on(
transformer: Any,
logger: Any,
index: int,
streamed: bool = False,
) -> int:
"""Swap the eligible int8 Linears of ``transformer`` against card ``index``'s probed kernels. On a layer split
(several cards) only the Linears whose weight sits on ``index``."""
global _OP_HANDLE
import torch
multi = len(_split_cards(transformer)) > 1
recs = [(m, _eligible(m)) for m in transformer.modules()]
recs = [(m, r) for m, r in recs if r is not None and (not multi or _weight_card(m) == index)]
if not recs:
return 0
from .diffusion_convrot_quant import rotquant_device_config
f16_card = tuple(torch.cuda.get_device_capability(index)) == (7, 5)
if f16_card:
recs = [(m, r) for m, r in recs if _card_dtypes_ok(m, r, True)]
if not recs:
return 0
gemm = device_config(index) is not None
triton_gemm = gemm
rotated = any(r[1] is not None for _, r in recs)
rotq = rotated and rotquant_device_config(index) is not None
if gemm and rotated and not rotq and index in _UNLISTED_CARDS:
# unlisted card: rotated Linears only with rotq_h16; else the stock fusions keep them, as on main
rotq = _unlisted_rotated(index)
if not rotq:
recs = [(m, r) for m, r in recs if r[1] is None]
if logger is not None:
logger.info(
"diffusion.int8_gemm: rotated Linears keep the stock path on cuda:%d (rotq_h16 off)",
index,
)
if not gemm and logger is not None and _UNLISTED_TIMING.get(index):
t = _UNLISTED_TIMING[index]
logger.info(
"diffusion.int8_gemm: fused-dequant GEMM kept off on cuda:%d (latency check: %.0f vs %.0f us cuBLAS)",
index,
t["fused"],
t["stock"],
)
if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):
recs = [(m, r) for m, r in recs if r[0] != "v1"]
# bf16 cards: the Triton path takes any bias dtype; only the prebuilt kernel needs bf16 bias
cut_recs = [(m, r) for m, r in recs if f16_card or _card_dtypes_ok(m, r, False)]
plan = _cutlass_plan(index, cut_recs, triton_gemm, logger) if cut_recs else {}
cut_fused: dict = {}
if plan:
import torch as _torch
dev = _torch.device("cuda", index)
xs_seen: dict = {}
act = _torch.float16 if f16_card else _torch.bfloat16
for m, r in cut_recs:
wq, ws_ = _payload(r[0], r[3])
k = int(wq.shape[1])
if (r[0], k) not in xs_seen:
xs_seen[(r[0], k)] = _xs_dtype(r[0], r[3], k, dev, act)
cut_fused[id(m)] = _cutlass_fused(
index,
wq.shape[0],
k,
xs_seen[(r[0], k)],
ws_.dtype,
getattr(m, "bias", None) is not None,
)
gemm = gemm or any(cut_fused.values())
if not triton_gemm:
recs = [(m, r) for m, r in recs if (rotq and r[1] is not None) or cut_fused.get(id(m))]
if not recs:
return 0
count = 0
global _OP16_HANDLE
with _LOCK:
if f16_card:
_OP16_HANDLE = _op16()
if _OP16_HANDLE is None:
return 0
elif gemm:
_OP_HANDLE = _op()
if _OP_HANDLE is None:
return 0
for module, rec in recs:
fused = (triton_gemm or bool(cut_fused.get(id(module)))) and not f16_card
if f16_card:
module.__dict__[_F16] = True
if _MARK in module.__dict__:
if multi:
module.__dict__[_REC] = (
rec[0],
rec[1],
bool(rotq and rec[1] is not None),
rec[3],
fused,
)
count += 1
continue
# kind, rotation group, fused rotation + act quant on, the Parameter (no payload alias), fused GEMM on
module.__dict__[_REC] = (
rec[0],
rec[1],
bool(rotq and rec[1] is not None),
rec[3],
fused,
)
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
module.__dict__[_FWD] = module.forward
count += 1
ff = (
_install_swiglu_ff(
transformer,
index,
streamed = streamed and not multi and swiglu_streamed_enabled(),
)
if rotq
else 0
)
if logger is not None and plan:
n_cut = sum(1 for v in cut_fused.values() if v)
logger.info(
"diffusion.int8_gemm: %d int8 Linear(s) on cuda:%d run the prebuilt CUTLASS GEMM (%d weight shape(s) "
"won the race against %s)",
n_cut,
index,
len(plan),
"the Triton kernel" if triton_gemm else "cuBLAS",
)
if logger is not None and count:
from .diffusion_convrot_quant import _H16_DEVICE
timing = _UNLISTED_TIMING.get(index) if gemm else None
logger.info(
"diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s%s%s",
count,
(
"run the fused-dequant GEMM (bf16 out"
+ (", TMA loads)" if _DEVICE_TMA.get(index) else ")")
)
if gemm
else "keep cuBLAS",
*torch.cuda.get_device_capability(index),
f" (latency check: {timing['fused']:.0f} vs {timing['stock']:.0f} us cuBLAS"
+ (f", tile {timing['tile']}" if timing.get("tile") else "")
+ (
" of "
+ " / ".join(
f"{t[0]}x{t[1]}x{t[2]} {us:.0f} us" for t, us in timing["race"].items()
)
+ " held"
if timing.get("race")
else ""
)
+ ")"
if timing
else "",
(
", ConvRot rotation fused into the act quant ("
+ ("H16 x H16 kernel" if _H16_DEVICE.get(index) else "dense rotation kernel")
+ ")"
)
if rotq
else "",
(
f", one act quant for gate / value on {ff} SwiGLU MLP(s)"
+ (
" + the SiLU product fused into the down projection's act quant"
if _swiglu_on(index)
else ""
)
)
if ff
else "",
)
return count
def after_layer_split(transformer: Any, logger: Any = None) -> dict:
"""Run once a layer split placed ``transformer``'s blocks, before its first forward: re-home the fused int8 / QKV
records built while it sat on one card (``diffusion_int8_fused.relocate_after_layer_split``), and probe each card's
GEMM tiles so the int8 Linears swapped before the split run their own card's kernel; each RoPE patch's probe map
covers every card. Idempotent; a no-op on a denoiser that is not split."""
out: dict = {"cards": (), "gemm": 0}
cards = _split_cards(transformer)
if len(cards) < 2:
return out
out["cards"] = cards
try:
from .diffusion_int8_fused import relocate_after_layer_split
out.update(relocate_after_layer_split(transformer, logger))
except Exception as exc: # noqa: BLE001 - the stock paths stay correct
if logger is not None:
logger.warning(
"diffusion.int8_gemm: fused record re-home after the layer split failed (%s)", exc
)
try:
from .diffusion_qwenimage21_rope import probe_split_cards
out["rope_cards"] = probe_split_cards(cards)
except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE
pass
try:
from .diffusion_zimage_fused import probe_split_cards as zimage_probe
out["zimage_rope_cards"] = zimage_probe(cards)
except Exception: # noqa: BLE001 - an unprobed card keeps the complex RoPE
pass
if int8_gemm_mode() != "off" and any(_MARK in m.__dict__ for m in transformer.modules()):
try:
out["gemm"] = sum(_swap_on(transformer, logger, index) for index in cards)
except Exception as exc: # noqa: BLE001
if logger is not None:
logger.warning(
"diffusion.int8_gemm: per-card swap after the layer split failed (%s)", exc
)
return out
def uninstall(transformer: Any = None) -> None:
if transformer is None:
return
from .diffusion_int8_fused import cancel_first_call
cancel_first_call(transformer, "int8_gemm")
with _LOCK:
for module in transformer.modules():
if module.__dict__.pop(_FF_MARK, None) is not None:
fwd = module.__dict__.get("forward")
if getattr(fwd, "__func__", None) is _swiglu_ff_forward:
module.__dict__.pop("forward", None)
module.__dict__.pop(_F16, None)
prev = module.__dict__.pop(_MARK, None)
if prev is None:
continue
module.__dict__.pop(_REC, None)
module.__dict__.pop(_FWD, None)
if prev is _NO_PREV:
module.__dict__.pop("forward", None)
else:
module.forward = prev
def is_installed(module: Any) -> bool:
d = getattr(module, "__dict__", {})
return _MARK in d or _FF_MARK in d
def call_count() -> int:
return _CALLS[0]
def tma_call_count() -> int:
return _TMA_CALLS[0]