Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Fused int8 weight-only GEMV for small-M Linears (AdaLN modulations, M = batch).
torchao's weight-only linear materialises a bf16 copy of the whole weight per call; this reads the int8 codes and
scales directly with one float32 accumulate. Per-row and per-group symmetric scales only; anything else keeps
torchao's forward. Opaque to Inductor and M is the batch, so no recompiles or CUDA-graph re-captures.
"""
from __future__ import annotations
import contextlib
import os
import threading
from functools import lru_cache
from typing import Any, Optional
INT8_GEMV_ENV = "UNSLOTH_DIFFUSION_INT8_GEMV"
# Above this many rows the stock dequant + GEMM is faster.
MAX_ROWS = 4
MAX_OUTLIERS = 16
OUTLIER_IDX = "outlier_idx"
OUTLIER_WEIGHT = "outlier_weight"
_OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_wo_gemv"
# (BLOCK_N, BLOCK_K, warps, stages), first that launches wins; the first needs ~40 KB smem, the second fits any card.
_CONFIGS = ((128, 128, 8, 3), (64, 64, 4, 2))
# (device index, scale group or 0 per row) -> index into _plans of the plan that launched
_DEVICE_CFG: dict = {}
GROUP_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMV_GROUP_TILES"
_LOCK = threading.Lock()
# Engagement evidence for benches.
COUNTS = {"gemv": 0, "fallback": 0}
# Resolved once by ``install``, outside any trace.
_OP_HANDLE: Any = None
def gemv_enabled() -> bool:
raw = (os.environ.get(INT8_GEMV_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
@lru_cache(maxsize = 1)
def _kernel() -> Any:
try:
import triton
import triton.language as tl
except Exception: # noqa: BLE001 - no Triton: stock path
return None
# M and KO only bound masks
@triton.jit(do_not_specialize = ["M", "KO"])
def i8_wo_gemv(
x_ptr,
q_ptr,
s_ptr,
b_ptr,
oi_ptr,
ow_ptr,
y_ptr,
M,
N,
K,
KO,
stride_xm,
stride_qn,
stride_sn,
stride_on,
stride_ym,
GS,
HAS_BIAS: tl.constexpr, # bias present
HAS_OUT: tl.constexpr, # bf16 outlier columns present
GROUPED: tl.constexpr, # per-group weight scales
BLOCK_M: tl.constexpr, # block size along M (rows)
BLOCK_N: tl.constexpr, # block size along N (out features)
BLOCK_K: tl.constexpr, # block size along K (in features)
):
pid = tl.program_id(0)
rn = pid * BLOCK_N + tl.arange(0, BLOCK_N)
rm = tl.arange(0, BLOCK_M)
n_ok = rn < N
m_ok = rm < M
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.float32)
for k0 in range(0, K, BLOCK_K):
rk = k0 + tl.arange(0, BLOCK_K)
q = tl.load(q_ptr + rn[:, None] * stride_qn + rk[None, :], mask = n_ok[:, None], other = 0)
x = tl.load(
x_ptr + rm[:, None] * stride_xm + rk[None, :], mask = m_ok[:, None], other = 0.0
)
# int8 -> bf16 is exact and bf16 x bf16 products are exact in the float32 accumulator
part = tl.dot(x, tl.trans(q.to(tl.bfloat16)), out_dtype = tl.float32)
if GROUPED:
# GS (the scale group) is a multiple of BLOCK_K
s = tl.load(s_ptr + rn * stride_sn + k0 // GS, mask = n_ok, other = 0.0)
acc += part * s.to(tl.float32)[None, :]
else:
acc += part
if not GROUPED:
s = tl.load(s_ptr + rn * stride_sn, mask = n_ok, other = 0.0)
acc = acc * s.to(tl.float32)[None, :]
if HAS_OUT:
# bf16 outlier columns (their int8 codes are zero): x[:, idx] @ W_out.T, same accumulator
ro = tl.arange(0, 16)
o_ok = ro < KO
idx = tl.load(oi_ptr + ro, mask = o_ok, other = 0)
xo = tl.load(
x_ptr + rm[:, None] * stride_xm + idx[None, :],
mask = m_ok[:, None] & o_ok[None, :],
other = 0.0,
)
wo = tl.load(
ow_ptr + rn[:, None] * stride_on + ro[None, :],
mask = n_ok[:, None] & o_ok[None, :],
other = 0.0,
)
acc += tl.dot(xo, tl.trans(wo), out_dtype = tl.float32)
if HAS_BIAS:
b = tl.load(b_ptr + rn, mask = n_ok, other = 0.0)
acc = acc + b.to(tl.float32)[None, :]
tl.store(
y_ptr + rm[:, None] * stride_ym + rn[None, :],
acc.to(tl.bfloat16),
mask = m_ok[:, None] & n_ok[None, :],
)
return triton, i8_wo_gemv
def reference(
x: Any,
q: Any,
s: Any,
bias: Any,
oidx: Any = None,
ow: Any = None,
) -> Any:
"""Same math in torch (float32 accumulate, one bf16 rounding): the CPU / no-Triton body of the op."""
import torch
n, k = q.shape
groups = s.numel() // n
xf = x.float()
if groups == 1:
y = (xf @ q.float().t()) * s.reshape(1, n).float()
else:
g = k // groups
parts = torch.einsum(
"mgk,ngk->mng", xf.reshape(-1, groups, g), q.float().reshape(n, groups, g)
)
y = (parts * s.reshape(1, n, groups).float()).sum(-1)
if oidx is not None:
y = y + xf.index_select(-1, oidx.long()) @ ow.float().t()
if bias is not None:
y = y + bias.float()
return y.to(torch.bfloat16)
def _group_tiles() -> bool:
return (os.environ.get(GROUP_TILES_ENV) or "").strip().lower() not in (
"0",
"off",
"false",
"no",
)
def _plans(group: int) -> tuple:
"""Tiles to try in order; grouped: one group per K tile first (the old tile), then _CONFIGS' K tiles inside it."""
if not group:
return _CONFIGS
whole = tuple((bn, group, w, st) for bn, _bk, w, st in _CONFIGS)
if not _group_tiles():
return whole
return whole + tuple(
(bn, bk, w, st) for bn, bk, w, st in _CONFIGS if group > bk and group % bk == 0
)
def _launch(x: Any, q: Any, s2: Any, bias: Any, oidx: Any, ow: Any, cfg: tuple) -> Any:
import torch
triton, fn = _kernel()
block_n, block_k, warps, stages = cfg
m, k = x.shape
n, groups = s2.shape
ko = 0 if oidx is None else int(oidx.numel())
y = torch.empty((m, n), device = x.device, dtype = torch.bfloat16)
# Triton launches on the current device, not the tensor's
other_card = x.is_cuda and x.device.index != torch.cuda.current_device()
with torch.cuda.device(x.device) if other_card else contextlib.nullcontext():
fn[(triton.cdiv(n, block_n),)](
x,
q,
s2,
bias if bias is not None else y,
oidx if ko else y,
ow if ko else y,
y,
m,
n,
k,
ko,
x.stride(0),
q.stride(0),
s2.stride(0),
ow.stride(0) if ko else 0,
y.stride(0),
k // groups,
HAS_BIAS = bias is not None,
HAS_OUT = ko > 0,
GROUPED = groups > 1,
# tl.dot wants >= 16 rows: the masked pad rows load zeros and are never stored
BLOCK_M = 16,
BLOCK_N = block_n,
BLOCK_K = block_k,
num_warps = warps,
num_stages = stages,
)
return y
def _run(
x: Any,
q: Any,
s: Any,
bias: Any,
oidx: Any = None,
ow: Any = None,
) -> Any:
m, k = x.shape
n = q.shape[0]
groups = s.numel() // n
block_k = k // groups if groups > 1 else _CONFIGS[-1][1]
ko = 0 if oidx is None else int(oidx.numel())
index = x.device.index if x.is_cuda else None
key = (index, block_k if groups > 1 else 0)
plans = _plans(key[1])
start = _DEVICE_CFG.get(key, 0)
if (
index is None
or _kernel() is None
or start >= len(plans)
or k % block_k
or (groups == 1 and k % plans[start][1])
or q.stride(-1) != 1
or block_k & (block_k - 1)
or block_k < 16
or ko > MAX_OUTLIERS
or (ko and ow.stride(-1) != 1)
):
COUNTS["fallback"] += 1
return reference(x, q, s, bias, oidx, ow)
x = x if x.stride(-1) == 1 else x.contiguous()
s2 = s.reshape(n, groups)
if s2.stride(-1) != 1:
s2 = s2.contiguous()
for pos in range(start, len(plans)):
try:
y = _launch(x, q, s2, bias, oidx, ow, plans[pos])
except Exception: # noqa: BLE001 - a tile this card cannot hold (shared memory): the next, smaller one
continue
_DEVICE_CFG[key] = pos
COUNTS["gemv"] += 1
return y
_DEVICE_CFG[key] = len(plans)
COUNTS["fallback"] += 1
return reference(x, q, s, bias, oidx, ow)
@lru_cache(maxsize = 1)
def _op() -> Any:
"""``unsloth_studio::int8_wo_gemv`` (opaque to Inductor), or None."""
try:
import torch
except Exception: # noqa: BLE001
return None
ns = getattr(torch.ops, _OP_NAMESPACE, None)
if ns is not None and hasattr(ns, _OP_NAME):
return getattr(ns, _OP_NAME)
custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
if custom_op is None:
return None
try:
@custom_op(
f"{_OP_NAMESPACE}::{_OP_NAME}",
mutates_args = (),
schema = "(Tensor x, Tensor q, Tensor s, Tensor? bias, Tensor? oidx, Tensor? ow) -> Tensor",
)
def _int8_wo_gemv(x, q, s, bias, oidx, ow):
return _run(x, q, s, bias, oidx, ow)
@_int8_wo_gemv.register_fake
def _(x, q, s, bias, oidx, ow):
return x.new_empty((x.shape[0], q.shape[0]), dtype = torch.bfloat16)
except Exception: # noqa: BLE001
return None
return getattr(torch.ops, _OP_NAMESPACE).int8_wo_gemv
def weight_only_parts(w: Any) -> Optional[tuple]:
"""(int8 [N, K], scale [N, K / G]) of a symmetric weight-only torchao ``Int8Tensor`` (per row or per group)."""
if type(w).__name__ != "Int8Tensor":
return None
if (
getattr(w, "act_quant_kwargs", None) is not None
or getattr(w, "act_pre_scale", None) is not None
):
return None
q, s = getattr(w, "qdata", None), getattr(w, "scale", None)
block = list(getattr(w, "block_size", ()) or ())
if q is None or s is None or q.dim() != 2 or len(block) != 2 or block[0] != 1:
return None
n, k = q.shape
if k % block[1] or s.numel() != n * (k // block[1]):
return None
return q, s
def _outliers(module: Any) -> tuple:
return getattr(module, OUTLIER_IDX, None), getattr(module, OUTLIER_WEIGHT, None)
def _forward(self: Any, x: Any) -> Any:
import torch
import torch.nn.functional as F
w = self.weight
oidx, ow = _outliers(self)
parts = weight_only_parts(w)
flat_rows = x.numel() // x.shape[-1] if x.shape[-1] else 0
op = _OP_HANDLE
if (
op is None
or parts is None
or x.dtype != torch.bfloat16
or not x.is_cuda
or flat_rows == 0
or flat_rows > MAX_ROWS
or parts[0].device != x.device
or (oidx is not None and (oidx.device != x.device or ow.device != x.device))
):
# stock torchao weight-only forward, plus the dense outlier columns (exact: their codes are zero)
y = F.linear(x, w, self.bias)
if oidx is not None:
y = y + F.linear(
x.index_select(-1, oidx.to(x.device)), ow.to(device = x.device, dtype = x.dtype)
)
return y
lead = x.shape[:-1]
y = op(x.reshape(-1, x.shape[-1]), parts[0], parts[1], self.bias, oidx, ow)
return y.reshape(*lead, y.shape[-1])
@lru_cache(maxsize = None)
def gemv_linear_class() -> Any:
"""Built once: a fresh class per call would make torch.compile retrace every block."""
from torch import nn
class Int8GemvLinear(nn.Linear):
forward = _forward
return Int8GemvLinear
def is_gemv_linear(module: Any) -> bool:
return type(module).__name__ == "Int8GemvLinear"
def split_outliers_(module: Any, channels: Any) -> int:
"""Before quantize_: move the weight columns of ``channels`` into a dense ``outlier_weight`` buffer and zero them
in the weight, so ``x @ W.T == x @ W_rest.T + x[:, idx] @ W_out.T`` exactly while the int8 scales only see the
rest. Returns how many columns moved (0 leaves the module untouched)."""
import torch
if channels is None or not len(channels) or len(channels) > MAX_OUTLIERS:
return 0
w = module.weight.data
idx = torch.as_tensor(sorted(int(c) for c in channels), dtype = torch.int64, device = w.device)
if int(idx.max()) >= w.shape[1]:
return 0
module.register_buffer(OUTLIER_WEIGHT, w.index_select(1, idx).contiguous())
module.register_buffer(OUTLIER_IDX, idx)
rest = w.clone()
rest[:, idx] = 0
module.weight.data = rest
# swapped now, not at install: from here on only this forward adds the columns back (dense or quantized weight)
module.__class__ = gemv_linear_class()
return int(idx.numel())
def prepare_outlier_buffers(transformer: Any, state_dict: Any) -> int:
"""Before ``load_state_dict(strict = True, assign = True)`` of an artifact built with outlier columns: give each
named Linear placeholder buffers of the stored shapes and the GEMV forward, so the strict load assigns them and the
forward adds them back. A key naming no Linear is left alone (the strict load then refuses the artifact)."""
import torch
from torch import nn
suffix = "." + OUTLIER_WEIGHT
count = 0
for key in [k for k in state_dict if isinstance(k, str) and k.endswith(suffix)]:
fqn = key[: -len(suffix)]
idx = state_dict.get(f"{fqn}.{OUTLIER_IDX}")
try:
module = transformer.get_submodule(fqn)
except AttributeError:
continue
if idx is None or not isinstance(module, nn.Linear):
continue
weight = state_dict[key]
module.register_buffer(
OUTLIER_WEIGHT, torch.empty(tuple(weight.shape), dtype = weight.dtype, device = "meta")
)
module.register_buffer(
OUTLIER_IDX, torch.empty(tuple(idx.shape), dtype = idx.dtype, device = "meta")
)
module.__class__ = gemv_linear_class()
count += 1
return count
def timestep_outlier_channels(transformer: Any, count: int) -> tuple:
"""The ``count`` input channels with the largest |SiLU(temb)| over the whole timestep range, from the model's own
``time_text_embed`` (Qwen-Image: the modulations' only input is SiLU of it, a function of the timestep alone, so
this needs no prompts). () when the transformer has no such embedder or anything fails."""
if count <= 0:
return ()
try:
import torch
tte = getattr(transformer, "time_text_embed", None)
proj = getattr(tte, "time_proj", None)
embedder = getattr(tte, "timestep_embedder", None)
if proj is None or embedder is None:
return ()
weight = embedder.linear_1.weight
if getattr(weight, "is_meta", False):
return ()
with torch.no_grad():
sigma = torch.linspace(0.0, 1.0, 1001, device = weight.device)
emb = embedder(proj(sigma).to(weight.dtype))
extra = getattr(tte, "addition_t_embedding", None)
if extra is not None:
rows = extra.weight.to(emb.dtype)
emb = torch.cat([emb + r for r in rows], dim = 0)
score = torch.nn.functional.silu(emb.float()).abs().amax(dim = 0)
if not bool(torch.isfinite(score).all()) or count > score.numel():
return ()
return tuple(sorted(int(i) for i in torch.topk(score, count).indices.tolist()))
except Exception: # noqa: BLE001 - calibration is optional: no outliers, plain int8 codes
return ()
def install(
transformer: Any,
fqns: Any,
*,
logger: Any = None,
) -> int:
"""Swap the named weight-only Linears onto the GEMV forward (class swap, no hook); returns how many. A Linear
carrying outlier columns is swapped even with the kernel off: its forward is what adds them back."""
from torch import nn
global _OP_HANDLE
kernel = gemv_enabled() and _kernel() is not None and _op() is not None
_OP_HANDLE = _op() if kernel else None
modules = dict(transformer.named_modules())
count = 0
with _LOCK:
for fqn in fqns:
module = modules.get(fqn)
if type(module) is not nn.Linear and not is_gemv_linear(module):
continue
if weight_only_parts(module.weight) is None or (
not kernel and _outliers(module)[0] is None
):
continue
module.__class__ = gemv_linear_class()
count += 1
if logger is not None and count:
logger.info(
"diffusion.int8_gemv: %s on %d weight-only Linear(s)",
"fused GEMV" if kernel else "outlier columns (stock GEMM)",
count,
)
return count
def call_count() -> int:
return COUNTS["gemv"]