Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Torchao-free int8/fp8 weight-only quant for ROCm and Windows; plain buffers survive offload
hooks' Module.to(); torch imported lazily"""
from __future__ import annotations
import contextlib
import contextvars
import os
import re
import threading
import time
from functools import lru_cache
from typing import Any, Callable, Optional
from .diffusion_fp8_dequant import dequant_rows, fused_dequant_enabled
NATIVE_INT8 = "int8"
NATIVE_FP8 = "fp8"
_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}
NATIVE_INT8_ACT_ENV = "UNSLOTH_NATIVE_INT8_ACT"
NATIVE_INT8_ROT_ENV = "UNSLOTH_NATIVE_INT8_ROT"
# Apple Silicon int8 weight-only via the Metal ``_weight_int8pack_mm`` kernel. Unset: a load-time probe decides;
# ``0`` keeps the dequantise path; ``1`` takes the kernel without probing.
NATIVE_MPS_INT8PACK_ENV = "UNSLOTH_DIFFUSION_MPS_INT8PACK"
# The Metal kernel asserts N % 32 == 0 and K % 32 == 0.
_INT8PACK_ALIGN = 32
# The kernel runs near fp32 GEMM speed and is not bit-reproducible run to run, so it is taken only where it beats the
# chip's own bf16 GEMM by this much (M1: about 1.3x; M3 parity; M5 4x behind, as its 16-bit GEMM uses matrix units).
_INT8PACK_MIN_SPEEDUP = 1.10
_INT8PACK_PROBE_ROWS = 1024
# (dtype, in_features, out_features) of the probed layer -> verdict for this process (one chip per process). Each
# quantised transformer's layers carry their own verdict, so a later model with other shapes is probed afresh.
_INT8PACK_PROBED: dict = {}
# torch._int_mm needs M > 16 and K, N multiples of 8.
_INT_MM_MIN_ROWS = 17
_INT_MM_CHUNK_ELEMS = 1 << 25
# Compiled linear holds int32 acc + output chunk (6 B/elem); eager's row budget keeps its peak <= eager's (#15183 used 4x).
_INT_MM_FUSED_CHUNK_ELEMS = _INT_MM_CHUNK_ELEMS
# ``0`` keeps eager CUDA W8A8 kernels instead of one compiled function per linear.
NATIVE_INT8_FUSED_ENV = "UNSLOTH_NATIVE_INT8_FUSED"
# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).
NATIVE_FP8_FORMATS = ("float8_e4m3fn", "float8_e5m2")
# ``0`` quantises host-resident linears on the host again instead of one at a time on the card.
NATIVE_QUANT_ON_DEVICE_ENV = "UNSLOTH_NATIVE_QUANT_ON_DEVICE"
_HOST_QUANT_DEVICE: contextvars.ContextVar = contextvars.ContextVar(
"unsloth_native_host_quant_device", default = None
)
def fp8_bits_to_float32(bits: Any, fmt: str = "float8_e4m3fn") -> Any:
"""uint8 fp8 codes to float32 by integer ops, bit-exact; Triton has no e4m3fn convert below sm_89."""
import torch
b = bits.to(torch.int32)
if fmt == "float8_e5m2":
exp_bits, man_bits, bias = 5, 2, 15
else:
exp_bits, man_bits, bias = 4, 3, 7
e = (b >> man_bits) & ((1 << exp_bits) - 1)
m = b & ((1 << man_bits) - 1)
normal = (((e + (127 - bias)) << 23) | (m << (23 - man_bits))).view(torch.float32)
# subnormal: m * 2**(1 - bias - man_bits), exact in float32
sub = m.to(torch.float32) * (2.0 ** (1 - bias - man_bits))
mag = torch.where(e == 0, sub, normal)
if fmt == "float8_e5m2":
top = e == (1 << exp_bits) - 1
mag = torch.where(top, torch.where(m == 0, float("inf"), float("nan")), mag)
else:
mag = torch.where((e == 15) & (m == 7), float("nan"), mag)
return torch.where((b & 0x80) != 0, -mag, mag)
@lru_cache(maxsize = None)
def fp8_lookup_table(fmt: str, device: str) -> Any:
"""float32 value of every fp8 code on ``device``. MPS has no fp8 casts before torch 2.15; indexing this table
with the uint8 codes gives the same bits as the cast on any torch."""
import torch
return torch.arange(256, dtype = torch.uint8).view(getattr(torch, fmt)).float().to(device)
def int8_act_requested() -> bool:
import os
return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "1"
def int8_act_disabled() -> bool:
import os
return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "0"
def mps_int8pack_disabled() -> bool:
import os
return os.environ.get(NATIVE_MPS_INT8PACK_ENV, "").strip().lower() in (
"0",
"off",
"false",
"no",
)
def mps_int8pack_forced() -> bool:
import os
return os.environ.get(NATIVE_MPS_INT8PACK_ENV, "").strip().lower() in ("1", "on", "true", "yes")
def mps_int8pack_selected(dtype: Any, layer: Any = None) -> bool:
"""Metal int8pack use: env override, else the layer's probe verdict; with no probe, dequantise."""
if mps_int8pack_disabled():
return False
if mps_int8pack_forced():
return True
if layer is not None:
return bool(getattr(layer, "_int8pack_verdict", {}).get(dtype, False))
return any(v for (d, _, _), v in _INT8PACK_PROBED.items() if d == dtype)
def _median_call_s(
fn: Callable[[], Any],
sync: Callable[[], None],
reps: int = 5,
) -> float:
import time
times = []
for _ in range(reps):
sync()
t0 = time.perf_counter()
fn()
sync()
times.append(time.perf_counter() - t0)
times.sort()
return times[len(times) // 2]
def _bench_int8pack_vs_dense(layer: Any, dtype: Any, device: Any) -> tuple:
"""(kernel s, dense ``dtype`` F.linear s) per call on ``layer``'s real shape, arms interleaved over 3 rounds."""
import torch
w_q = layer.weight_q.detach().to(device)
scale = layer.weight_scale.view(torch.float32).to(device).to(dtype).contiguous()
w = (w_q.to(dtype) * scale.unsqueeze(1)).contiguous()
x = torch.randn(_INT8PACK_PROBE_ROWS, layer.in_features, dtype = dtype, device = device)
sync = torch.mps.synchronize if device.type == "mps" else (lambda: None)
kernel = lambda: torch._weight_int8pack_mm(x, w_q, scale)
dense = lambda: torch.nn.functional.linear(x, w)
kernel()
dense()
k_s, d_s = [], []
for _ in range(3):
k_s.append(_median_call_s(kernel, sync))
d_s.append(_median_call_s(dense, sync))
k_s.sort()
d_s.sort()
return k_s[1], d_s[1]
def probe_mps_int8pack(
module: Any,
dtype: Any,
*,
device: Any = "mps",
logger: Any = None,
) -> bool:
"""Time the Metal kernel against a dense ``dtype`` GEMM on ``module``'s largest kernel-eligible layer and stamp
the verdict on every eligible layer of ``module``. Verdicts are cached per (dtype, probed shape), so another model
with the same largest layer reuses it and one with a different shape is probed. The env override skips the probe."""
import torch
if mps_int8pack_disabled() or mps_int8pack_forced():
return mps_int8pack_selected(dtype)
cls = native_linear_class()
layers = [
m
for m in module.modules()
if isinstance(m, cls) and m._int8pack_layer_ok(dtype, probe = True)
]
if not layers:
return False
layer = max(layers, key = lambda m: m.in_features * m.out_features)
key = (dtype, layer.in_features, layer.out_features)
def stamp(verdict: bool) -> bool:
for m in layers:
m._int8pack_verdict = {**getattr(m, "_int8pack_verdict", {}), dtype: verdict}
return verdict
if key in _INT8PACK_PROBED:
return stamp(_INT8PACK_PROBED[key])
try:
kernel_s, dense_s = _bench_int8pack_vs_dense(layer, dtype, torch.device(device))
except Exception as exc: # noqa: BLE001 -- a failed probe keeps the dequantise path
if logger is not None:
logger.info("diffusion.native_quant: int8pack probe failed, dequantising: %s", exc)
_INT8PACK_PROBED[key] = False
return stamp(False)
speedup = dense_s / max(kernel_s, 1e-9)
verdict = speedup >= _INT8PACK_MIN_SPEEDUP
_INT8PACK_PROBED[key] = verdict
stamp(verdict)
if logger is not None:
logger.info(
"diffusion.native_quant: int8pack probe %dx%dx%d: kernel %.3f ms, dense %s %.3f ms (%.2fx) -> %s",
_INT8PACK_PROBE_ROWS,
layer.in_features,
layer.out_features,
kernel_s * 1e3,
str(dtype).replace("torch.", ""),
dense_s * 1e3,
speedup,
"Metal int8 kernel" if verdict else "dequantise",
)
return verdict
def snap_scale_up(scale: Any, dtype: Any) -> Any:
"""``scale`` rounded up to a ``dtype``-exact fp32 value: the Metal kernel takes ``dtype`` scales, so a call-time
cast would add per-row gain error (up to 2**-9 in bf16); rounding up never clips a code."""
import torch
# nextafter is not implemented on every backend.
host = scale.detach().to("cpu", torch.float32)
cast = host.to(dtype)
up = torch.nextafter(cast, torch.full_like(cast, float("inf")))
return torch.where(cast.float() < host, up, cast).float().to(scale.device)
def int8_rotation_group() -> int:
import os
if os.environ.get(NATIVE_INT8_ROT_ENV, "").strip() == "0":
return 0
from .diffusion_convrot import DEFAULT_CONVROT_GROUPSIZE
return DEFAULT_CONVROT_GROUPSIZE
@lru_cache(maxsize = 1)
def _row_major_op() -> Any:
"""Row-major copy Inductor cannot re-layout; cuBLASLt int8 GEMMs fail on a column-major activation."""
import torch
ns = getattr(torch.ops, "unsloth_studio", None)
if ns is not None and hasattr(ns, "native_row_major"):
return ns.native_row_major
custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
if custom_op is None:
return None
try:
@custom_op(
"unsloth_studio::native_row_major", mutates_args = (), schema = "(Tensor x) -> Tensor"
)
def _native_row_major(x):
return x.clone(memory_format = torch.contiguous_format)
@_native_row_major.register_fake
def _(x):
return x.new_empty(x.shape)
except Exception: # noqa: BLE001 - a registration failure keeps the plain copy
return None
return torch.ops.unsloth_studio.native_row_major
def int8_fused_disabled() -> bool:
import os
return os.environ.get(NATIVE_INT8_FUSED_ENV, "").strip().lower() in ("0", "false", "no", "off")
def _own_rotation(h: Any, dtype: Any) -> Any:
"""Per-layer copy: at float32 ``h.to(dtype)`` aliases the cached tensor, so group offload moved one shared buffer."""
return h.to(dtype, copy = True)
def _w8a8_rows(x2: Any, rot_h: Any, weight_q: Any, weight_scale: Any, bias: Any) -> Any:
"""Eager ``_rotate`` + ``_int_mm_rows`` maths on tensors only, so offload hooks never invalidate the graph."""
import torch
if rot_h is not None:
g = rot_h.shape[0]
x2 = (x2.reshape(x2.shape[0], -1, g) @ rot_h.to(x2.dtype)).reshape(x2.shape)
x_scale = torch.linalg.vector_norm(
x2, ord = float("inf"), dim = 1, keepdim = True, dtype = torch.float32
).clamp(min = 1e-12)
# fp64 reciprocal: matches eager CUDA bit-for-bit; Triton's fp32 reciprocal is approximate and drifts codes.
x_scale = x_scale * (1.0 / 127.0)
x_recip = (1.0 / x_scale.double()).float()
xq = torch.mul(x2, x_recip).round().clamp(-127, 127).to(torch.int8)
if rot_h is None:
op = _FUSED_STATE.get("row_major")
xq = op(xq) if op is not None else xq.contiguous()
acc = torch._int_mm(xq, weight_q.t())
out = torch.mul(acc, x_scale)
w_scale = weight_scale.view(torch.float32)
if bias is not None:
out = torch.addcmul(bias.float(), out, w_scale)
else:
out = out * w_scale
return out.to(x2.dtype)
def _w8a8_rows_into(
x2: Any, rot_h: Any, weight_q: Any, weight_scale: Any, bias: Any, out: Any
) -> None:
"""``_w8a8_rows`` stored straight into ``out`` (a row slice of the chunked output), no temporary chunk result."""
out.copy_(_w8a8_rows(x2, rot_h, weight_q, weight_scale, bias))
# "devices": per card, None once it may run the compiled linear, else why not (its arch, or its compile failed).
_FUSED_STATE: dict = {"fn": None, "failed": None, "devices": {}}
_UNSEEN = object()
def _device_key(device: Any = None) -> Any:
"""Indexed ``torch.device``; None or an unindexed cuda = the current card (the CPU without one)."""
import torch
if isinstance(device, int):
return torch.device("cuda", device)
if device is not None:
device = torch.device(device)
if device.type != "cuda" or device.index is not None:
return device
try:
if torch.cuda.is_available():
return torch.device("cuda", int(torch.cuda.current_device()))
except Exception: # noqa: BLE001
pass
return torch.device("cpu")
# ROCm archs where the compiled W8A8 linear was measured: RDNA3 / RDNA3.5 / RDNA4 (gfx11xx / gfx12xx). On gfx1151 it
# matches eager exactly and beats bf16 (1.03-1.36x per linear at 4096 rows), where the eager int8 path is 0.93x bf16.
_ROCM_FUSED_W8A8_ARCH = re.compile(r"^gfx1[12][0-9a-f]{2}$")
def _rocm_fused_w8a8_refusal(torch: Any, ordinal: Optional[int] = None) -> Optional[str]:
"""None when the compiled linear may run on ROCm card ``ordinal`` (current when None), else why not."""
from .rocm_bf16 import _device_gfx_arch
arch = _device_gfx_arch(torch, ordinal)
if _ROCM_FUSED_W8A8_ARCH.match(arch):
return None
return f"ROCm {arch or 'unknown arch'}"
def rocm_rdna_w8a8_ready(ordinal: Optional[int] = None) -> bool:
"""ROCm RDNA card ``ordinal`` (current when None) where the compiled W8A8 linear can run and has not failed."""
if _FUSED_STATE["failed"] is not None or int8_fused_disabled():
return False
try:
import torch
from .rocm_bf16 import is_rocm_torch
if not is_rocm_torch(torch) or not torch.cuda.is_available():
return False
key = _device_key(ordinal)
if _FUSED_STATE.get("devices", {}).get(key) is not None:
return False
if _rocm_fused_w8a8_refusal(torch, key.index) is not None:
return False
from .diffusion_speed import torch_compile_runtime_available
return bool(torch_compile_runtime_available())
except Exception: # noqa: BLE001 - unknown device: keep the weight-only default
return False
def _fused_w8a8(device: Any = None) -> Optional[Callable[..., Any]]:
"""Compiled ``_w8a8_rows`` for an input on ``device`` (current card when None), or None if disabled, no inductor
runtime, a ROCm arch other than RDNA3 / RDNA4, or already failed on that card."""
if _FUSED_STATE["failed"] is not None or int8_fused_disabled():
return None
devices = _FUSED_STATE.get("devices")
if devices is None:
devices = _FUSED_STATE.setdefault("devices", {})
key = device if getattr(device, "index", None) is not None else _device_key(device)
seen = devices.get(key, _UNSEEN)
fn = _FUSED_STATE["fn"]
if seen is None and fn is not None:
return fn
if seen is not _UNSEEN and seen is not None:
return None
import torch
from .rocm_bf16 import is_rocm_torch
try:
from .diffusion_speed import torch_compile_runtime_available
runtime = torch_compile_runtime_available()
except Exception: # noqa: BLE001 - unknown runtime: stay eager
runtime = False
try: # Studio's structured logger (the line lands in the server log), plain logging outside Studio
from loggers import get_logger
log = get_logger(__name__)
except Exception: # noqa: BLE001
import logging
log = logging.getLogger(__name__)
if fn is None and not runtime:
_FUSED_STATE["failed"] = "no inductor runtime"
log.info("diffusion.native_quant: int8 W8A8 linear stays eager (no inductor runtime)")
return None
# The current card (no device) or a HIP input's card; a CPU input has no arch to judge.
if is_rocm_torch(torch) and (device is None or key.type == "cuda"):
refusal = _rocm_fused_w8a8_refusal(torch, key.index if key.type == "cuda" else None)
if refusal is not None:
devices[key] = refusal
log.info(
"diffusion.native_quant: int8 W8A8 linear stays eager on %s (%s)",
_key_label(key),
refusal,
)
return None
devices[key] = None
if fn is not None:
return fn
# Registered in eager, before any trace reads the handle.
_FUSED_STATE["row_major"] = _row_major_op()
# Every key exists before the first trace: older dynamo guards on len(_FUSED_STATE), so adding
# "calls" after the first call would recompile each compiled function once.
_FUSED_STATE.setdefault("calls", 0)
fn = torch.compile(_w8a8_rows, dynamic = True, fullgraph = True)
# One graph for every slice offset and chunk height (dynamic storage offset), written in place by inductor.
_FUSED_STATE["into"] = torch.compile(_w8a8_rows_into, dynamic = True, fullgraph = True)
_FUSED_STATE["fn"] = fn
log.info("diffusion.native_quant: int8 W8A8 linear runs as one compiled function")
return fn
def _key_label(key: Any) -> str:
return str(key)
def _fused_w8a8_failed(exc: BaseException, device: Any = None) -> None:
"""The compiled linear failed on ``device``'s card: eager there from here on (other cards keep it)."""
import logging
key = _device_key(device)
_FUSED_STATE.setdefault("devices", {})[key] = repr(exc)
logging.getLogger(__name__).warning(
"diffusion.native_quant: the compiled int8 W8A8 linear failed on %s (%s); eager kernels there from here on "
"(set %s=0 to skip it)",
_key_label(key),
exc,
NATIVE_INT8_FUSED_ENV,
)
def _autocast_on(x: Any) -> bool:
import torch
try:
return bool(torch.is_autocast_enabled(x.device.type))
except TypeError: # torch without the device_type argument: treat any autocast as on
return bool(torch.is_autocast_enabled() or torch.is_autocast_cpu_enabled())
def fused_w8a8_state() -> dict:
"""``failed``: the process-wide reason, else the first card's; ``devices``: each card's (None = runs it)."""
devices = dict(_FUSED_STATE.get("devices", {}))
failed = _FUSED_STATE["failed"]
if failed is None:
failed = next((why for why in devices.values() if why is not None), None)
return {
"calls": _FUSED_STATE.get("calls", 0),
"failed": failed,
"devices": {_key_label(k): why for k, why in devices.items()},
}
# Per-thread fp32 widen buffer: a fresh one per call costs ~4x the widen in page faults.
_CPU_WIDEN = threading.local()
def _cpu_widen_scratch(numel: int) -> Any:
import torch
buf = getattr(_CPU_WIDEN, "buf", None)
if buf is None or buf.numel() < numel:
_CPU_WIDEN.buf = None
# Not an inference tensor: those are read-only outside inference_mode.
with torch.inference_mode(False):
buf = _CPU_WIDEN.buf = torch.empty(numel, dtype = torch.float32)
return buf[:numel]
@lru_cache(maxsize = 1)
def native_linear_class():
"""``NativeWeightOnlyLinear``, defined on first use so this module imports torch-free."""
import torch
from torch import nn
from torch.nn import functional as F
from . import diffusion_native_w8a8_rdna as rdna
class NativeWeightOnlyLinear(nn.Module):
"""A Linear whose weight is stored as int8 / fp8 with a per-output-row scale."""
def __init__(
self,
linear: Any,
scheme: str,
act_int8: bool = False,
rot_group: int = 0,
codes: Any = None,
scale: Any = None,
scale_dtype: Any = None,
):
"""``codes`` / ``scale``: int8 (or, for the fp8 scheme, float8_e4m3fn) codes [out, in] and per-row
scales made elsewhere, stored as given (``linear`` then supplies only the shapes, dtype and bias).
With ``rot_group`` int8 codes are already ConvRot-rotated, so the rotation is installed whether or
not activations go int8. ``scale_dtype``: snap int8 scales exact in that dtype (``snap_scale_up``)."""
super().__init__()
if scheme not in _QMAX:
raise ValueError(f"unsupported native scheme {scheme!r}")
self.in_features = int(linear.in_features)
self.out_features = int(linear.out_features)
self.scheme = scheme
cast = getattr(linear, "_unsloth_compute_dtype", None)
self.compute_dtype = cast if isinstance(cast, torch.dtype) else linear.weight.dtype
self.act_int8 = bool(act_int8) and scheme == NATIVE_INT8
self.rot_group = (
int(rot_group)
if self.act_int8 and rot_group and self.in_features % int(rot_group) == 0
else 0
)
self.fp8_format = "float8_e4m3fn"
if codes is not None:
wanted = torch.int8 if scheme == NATIVE_INT8 else torch.float8_e4m3fn
if (
scheme == NATIVE_FP8
and str(codes.dtype).replace("torch.", "") in NATIVE_FP8_FORMATS
):
wanted = codes.dtype
self.fp8_format = str(codes.dtype).replace("torch.", "")
if codes.dtype != wanted:
raise ValueError(
f"pre-quantized native {scheme} codes must be {str(wanted).replace('torch.', '')}"
)
if rot_group and scheme != NATIVE_INT8:
raise ValueError("a ConvRot rotation needs int8 codes")
self.rot_group = int(rot_group or 0)
if self.rot_group and self.in_features % self.rot_group:
raise ValueError(
f"rotation group {self.rot_group} does not divide in_features {self.in_features}"
)
wq, scale = codes, scale.reshape(-1, 1)
if self.rot_group:
from .diffusion_convrot import build_convrot_hadamard
h = build_convrot_hadamard(
self.rot_group, device = codes.device, dtype = torch.float32
)
self.register_buffer(
"rot_h", _own_rotation(h, self.compute_dtype), persistent = False
)
else:
weight = linear.weight.detach()
with torch.no_grad():
w = weight.float()
if self.rot_group:
from .diffusion_convrot import build_convrot_hadamard
h = build_convrot_hadamard(
self.rot_group, device = w.device, dtype = torch.float32
)
g = self.rot_group
w = (w.reshape(self.out_features, -1, g) @ h.T).reshape(
self.out_features, -1
)
self.register_buffer(
"rot_h", _own_rotation(h, self.compute_dtype), persistent = False
)
scale = w.abs().amax(dim = 1, keepdim = True).clamp(min = 1e-12) / _QMAX[scheme]
if scheme == NATIVE_INT8 and scale_dtype is not None:
scale = snap_scale_up(scale, scale_dtype)
if scheme == NATIVE_INT8:
wq = (w / scale).round_().clamp_(-127, 127).to(torch.int8)
else:
wq = (w / scale).to(torch.float8_e4m3fn)
del w
# Integer views: a module-wide ``.to(dtype)`` casts floating buffers, which would widen fp8 or round the scales.
self.register_buffer("weight_q", wq.view(torch.uint8) if scheme == NATIVE_FP8 else wq)
self.register_buffer(
"weight_scale", scale.squeeze(1).to(torch.float32).contiguous().view(torch.int32)
)
bias = linear.bias
if (
bias is not None
and bias.dtype != self.compute_dtype
and isinstance(cast, torch.dtype)
):
bias = nn.Parameter(bias.detach().to(self.compute_dtype), requires_grad = False)
self.bias = bias
# Registered here, in eager, so a compiled forward only reads the handle (see _row_major_op).
self._row_major = _row_major_op() if self.act_int8 and not self.rot_group else None
self._int8pack_scales: dict = {}
# ROCm RDNA: fused rotate+quant and int8 GEMM kernels, probed per card here, in eager, before any trace.
self._rdna = False
if self.act_int8:
self._rdna = (
rdna.shape_supported(self.in_features, self.out_features, self.rot_group)
and rdna.prepare()
)
def _widen(self, codes: Any) -> Any:
"""Exact float32 values of a row slice of ``weight_q``, as the eager branches below."""
if self.scheme != NATIVE_FP8:
return codes.to(torch.float32)
if codes.device.type == "mps":
lut = fp8_lookup_table(self.fp8_format, str(codes.device))
return lut.index_select(0, codes.reshape(-1).int()).view(codes.shape)
return codes.view(getattr(torch, self.fp8_format)).to(torch.float32)
def _stored_weight(self, dtype: Any) -> Any:
scale = self.weight_scale.view(torch.float32)
if not torch.compiler.is_compiling() and fused_dequant_enabled():
return dequant_rows(
self.weight_q,
scale,
dtype,
self._widen,
self.fp8_format if self.scheme == NATIVE_FP8 else None,
)
if self.scheme != NATIVE_FP8:
wide = self.weight_q.to(torch.float32)
elif torch.compiler.is_compiling():
wide = fp8_bits_to_float32(self.weight_q, self.fp8_format)
elif self.weight_q.device.type == "mps":
lut = fp8_lookup_table(self.fp8_format, str(self.weight_q.device))
wide = lut.index_select(0, self.weight_q.reshape(-1).int()).view(
self.weight_q.shape
)
else:
wide = self.weight_q.view(getattr(torch, self.fp8_format)).to(torch.float32)
return (wide * scale[:, None]).to(dtype)
def _forward_weight(self, x: Any) -> Any:
# Not under autograd: a graph would save the buffer the next layer overwrites.
if (
self.scheme == NATIVE_INT8
and x.dtype is torch.float32
and x.device.type == "cpu"
and self.weight_q.device.type == "cpu"
and not torch.compiler.is_compiling()
and not (torch.is_grad_enabled() and x.requires_grad)
):
out = _cpu_widen_scratch(self.weight_q.numel()).view(self.weight_q.shape)
out.copy_(self.weight_q)
return out.mul_(self.weight_scale.view(torch.float32)[:, None])
return self._stored_weight(x.dtype)
def _rotate(self, x: Any) -> Any:
g = self.rot_group
# Leading dims, not -1: a zero-row input has no inferable -1 on older torch.
grouped = x.reshape(*x.shape[:-1], self.in_features // g, g)
return (grouped @ self.rot_h.to(x.dtype)).reshape(x.shape)
def dequantized_weight(self, dtype: Any) -> Any:
w = self._stored_weight(torch.float32 if self.rot_group else dtype)
if not self.rot_group:
return w
g = self.rot_group
h = self.rot_h.to(torch.float32)
return (
(w.reshape(self.out_features, -1, g) @ h).reshape(self.out_features, -1).to(dtype)
)
@property
def weight(self) -> Any:
"""The dense weight, rebuilt on each read. Code that reads a wrapped Linear's weight directly
(PEFT's DoRA forward reads ``base_layer.weight``) keeps working; nothing holds it."""
return self.dequantized_weight(self.compute_dtype)
def _int_mm_ok(self, rows: int) -> bool:
return (
rows >= _INT_MM_MIN_ROWS
and self.in_features % 8 == 0
and self.out_features % 8 == 0
)
def _int_mm_rows(self, x2: Any) -> Any:
# Must stay fast in eager (Windows ROCm has no Triton): upcasts ride mixed-dtype ops, no .float() copies.
x_scale = torch.linalg.vector_norm(
x2, ord = float("inf"), dim = 1, keepdim = True, dtype = torch.float32
)
x_scale = x_scale.clamp_(min = 1e-12).div_(127.0)
xq = torch.mul(x2, x_scale.reciprocal()).round_().clamp_(-127, 127).to(torch.int8)
# A rotated input is a matmul output, row-major; an un-rotated one follows its producer's layout.
if not self.rot_group and (torch.compiler.is_compiling() or not xq.is_contiguous()):
op = getattr(self, "_row_major", None)
xq = op(xq) if op is not None else xq.contiguous()
w_scale = self.weight_scale.view(torch.float32)
if (
self.bias is None
and x2.dtype == torch.float16
and x2.is_cuda
and not torch.compiler.is_compiling()
):
from . import diffusion_int8_cutlass as _cut # noqa: PLC0415
tile = _cut.native_tile(xq, self.weight_q, x_scale, w_scale)
if tile is not None:
try:
return _cut.native_gemm(xq, self.weight_q, x_scale, w_scale, tile)
except Exception: # noqa: BLE001 - the stock maths below
_cut._NATIVE_PLAN.clear()
acc = torch._int_mm(xq, self.weight_q.t())
out = torch.mul(acc, x_scale)
if self.bias is not None:
out = torch.addcmul(self.bias.float(), out, w_scale)
else:
out = out.mul_(w_scale)
return out
def _forward_rdna(self, x: Any) -> Any:
x2 = x.reshape(-1, self.in_features)
xq, xs = rdna.rotq(x2, self.rot_group)
out = rdna.w8a8_mm(
xq, xs, self.weight_q, self.weight_scale.view(torch.float32), self.bias, x.dtype
)
return out.reshape(*x.shape[:-1], self.out_features)
def _rdna_on(self, x: Any) -> bool:
if not (self._rdna and x.is_cuda and x.dtype in (torch.bfloat16, torch.float16)):
return False
# The custom ops have no backward: a graph that needs input gradients keeps the stock maths.
if x.requires_grad and torch.is_grad_enabled():
return False
index = x.device.index
if not torch.compiler.is_compiling() and index not in rdna._STATE["on"]:
rdna.prepare(x.device)
return rdna.enabled_for(index) and self._int_mm_ok(
x.numel() // max(1, self.in_features)
)
def _forward_int_mm(self, x: Any) -> Any:
x2 = x.reshape(-1, self.in_features)
rows = x2.shape[0]
# Eager materialises fp32 [rows, features] temporaries (inductor fuses them): cap them per chunk.
chunk = _INT_MM_CHUNK_ELEMS // max(self.in_features, self.out_features)
if torch.compiler.is_compiling() or rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:
out = self._int_mm_rows(x2).to(x.dtype)
else:
out = torch.cat(
[
self._int_mm_rows(part).to(x.dtype)
for part in torch.tensor_split(x2, -(-rows // chunk))
]
)
return out.reshape(*x.shape[:-1], self.out_features)
def _forward_fused(self, x: Any) -> Any:
"""Chunked compiled forward; None = use eager kernels (also during CUDA graph capture)."""
in_f = self.in_features
rows = x.numel() // max(1, in_f)
if not self._int_mm_ok(rows) or (
x.is_cuda and torch.cuda.is_current_stream_capturing()
):
return None
from .diffusion_bg_compile import eager_forced # noqa: PLC0415
# Detailer crops and in-flight background compiles must not compile on this thread (#15063).
if eager_forced():
return None
fn = _fused_w8a8(x.device)
if fn is None:
return None
# Detached: guards on a view's base / Parameter shape would recompile per layout / layer.
x2 = x.reshape(-1, in_f).detach()
rot_h = self.rot_h if self.rot_group else None
bias = self.bias.detach() if self.bias is not None else None
chunk = _INT_MM_FUSED_CHUNK_ELEMS // max(in_f, self.out_features)
try:
if rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:
out = fn(x2, rot_h, self.weight_q, self.weight_scale, bias)
else:
# Only the rotation matmul can change the dtype (autocast); without it the output is x2's dtype.
into = _FUSED_STATE.get("into")
if into is not None and (rot_h is None or not _autocast_on(x2)):
out = x2.new_empty(rows, self.out_features)
else:
into = out = None
start = 0
for part in torch.tensor_split(x2, -(-rows // chunk)):
stop = start + part.shape[0]
if into is not None:
into(
part.detach(),
rot_h,
self.weight_q,
self.weight_scale,
bias,
out[start:stop],
)
else:
got = fn(part.detach(), rot_h, self.weight_q, self.weight_scale, bias)
if out is None:
# Allocated from the first chunk to keep the compiled dtype (autocast can differ).
out = got.new_empty(rows, self.out_features)
out[start:stop].copy_(got)
del got
start = stop
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001 - a compile failure falls back to the eager kernels
_fused_w8a8_failed(exc, x.device)
return None
_FUSED_STATE["calls"] = _FUSED_STATE.get("calls", 0) + 1
return out.reshape(*x.shape[:-1], self.out_features)
def _int8pack_scale(self, dtype: Any, device: Any) -> Any:
"""Per-row scale in ``dtype`` if that cast is exact (cached), else None: keep dequantise."""
key = (dtype, str(device))
# Keyed on the buffer's storage and version so load_state_dict / copy_ into it is never served stale.
stamp = (self.weight_scale.data_ptr(), self.weight_scale._version)
hit = self._int8pack_scales.get(key)
if hit is None or hit[0] != stamp:
scale = self.weight_scale.view(torch.float32)
cast = scale.to(dtype)
exact = bool(torch.equal(cast.float(), scale))
hit = (stamp, cast.contiguous() if exact else None)
self._int8pack_scales[key] = hit
return hit[1]
def _int8pack_layer_ok(
self,
dtype: Any,
probe: bool = False,
) -> bool:
"""``probe``: the layer's shape and codes suit the kernel, whatever the probe decided."""
return (
self.scheme == NATIVE_INT8
and not self.act_int8
and not self.rot_group
and dtype in (torch.bfloat16, torch.float16, torch.float32)
and self.in_features % _INT8PACK_ALIGN == 0
and self.out_features % _INT8PACK_ALIGN == 0
and hasattr(torch, "_weight_int8pack_mm")
and (
self._int8pack_scale(dtype, self.weight_q.device) is not None
if probe
else mps_int8pack_selected(dtype, self)
)
)
def _int8pack_ok(self, x: Any) -> bool:
return x.device.type == "mps" and x.numel() > 0 and self._int8pack_layer_ok(x.dtype)
def int8pack_engaged(self, dtype: Any) -> bool:
"""Whether a ``dtype`` forward on the stored device takes the Metal int8 kernel."""
device = self.weight_q.device
return (
device.type == "mps"
and self._int8pack_layer_ok(dtype)
and self._int8pack_scale(dtype, device) is not None
)
def _forward_int8pack(self, x: Any) -> Any:
"""None = take the dequantise path."""
scale = self._int8pack_scale(x.dtype, x.device)
if scale is None:
return None
x2 = x.reshape(-1, self.in_features)
if not x2.is_contiguous():
x2 = x2.contiguous()
out = torch._weight_int8pack_mm(x2, self.weight_q, scale)
if self.bias is not None:
out = out + self.bias.to(out.dtype)
return out.reshape(*x.shape[:-1], self.out_features)
def forward(self, x: Any) -> Any:
if (
self.act_int8
and self.bias is None
and x.is_cuda
and x.dtype == torch.float16
and not torch.compiler.is_compiling()
):
# race once per card and shape, before any graph capture
from . import diffusion_int8_cutlass as _cut # noqa: PLC0415
_cut.native_prepare(self.weight_q, x.dtype, False)
if self._int8pack_ok(x):
out = self._forward_int8pack(x)
if out is not None:
return out
if self.act_int8 and self._rdna_on(x):
return self._forward_rdna(x)
if self.act_int8 and x.is_cuda and not torch.compiler.is_compiling():
out = self._forward_fused(x)
if out is not None:
return out
if self.rot_group:
x = self._rotate(x)
if (
self.act_int8
and x.is_cuda
and self._int_mm_ok(x.numel() // max(1, self.in_features))
):
return self._forward_int_mm(x)
return F.linear(x, self._forward_weight(x), self.bias)
def extra_repr(self) -> str:
return (
f"in_features={self.in_features}, out_features={self.out_features}, "
f"scheme={self.scheme}, act_int8={self.act_int8}, rot_group={self.rot_group}, bias={self.bias is not None}"
+ (f", codes={self.fp8_format}" if self.scheme == NATIVE_FP8 else "")
)
return NativeWeightOnlyLinear
def is_native_linear(module: Any) -> bool:
return type(module).__name__ == "NativeWeightOnlyLinear" and hasattr(module, "weight_q")
def is_native_quantised(module: Any) -> bool:
"""Whether any layer of ``module`` has been swapped for a native weight-only Linear."""
try:
return any(is_native_linear(sub) for sub in module.modules())
except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes
return False
def apply_native_weight_quant(
transformer: Any,
scheme: str,
*,
filter_fn: Callable[[Any, str], bool],
act_int8: bool,
logger: Any = None,
scale_dtype: Any = None,
) -> int:
"""Swap every Linear ``filter_fn`` keeps for a native weight-only twin; returns layers swapped. One layer
at a time; a failure part-way leaves a partial conversion the caller must refuse."""
cls = native_linear_class()
act_int8 = scheme == NATIVE_INT8 and bool(act_int8)
rot_group = int8_rotation_group() if act_int8 else 0
# Names only: holding the old modules would keep every bf16 weight alive until the pass ends.
names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]
def build(linear: Any) -> Any:
return cls(
linear,
scheme,
act_int8 = act_int8,
rot_group = rot_group,
scale_dtype = scale_dtype if scheme == NATIVE_INT8 and not act_int8 else None,
)
device = _HOST_QUANT_DEVICE.get()
mover = _HostLinearMover(device) if device is not None else None
for name in names:
parent_name, _, leaf = name.rpartition(".")
parent = transformer.get_submodule(parent_name) if parent_name else transformer
linear = getattr(parent, leaf)
setattr(parent, leaf, mover.build(linear, build) if mover is not None else build(linear))
if mover is not None:
mover.close(logger)
if logger is not None:
logger.info(
"diffusion.transformer_quant: %s %s (torchao-free) on %d linears",
scheme,
("W8A8 torch._int_mm" + (f" ConvRot g{rot_group}" if rot_group else ""))
if act_int8
else "weight-only",
len(names),
)
return len(names)
def native_quant_on_device_enabled() -> bool:
return os.environ.get(NATIVE_QUANT_ON_DEVICE_ENV, "").strip().lower() not in (
"0",
"false",
"no",
"off",
)
@contextlib.contextmanager
def quantise_host_linears_on(device: Any):
"""Inside, ``apply_native_weight_quant`` builds each host-resident Linear on ``device`` and returns it to the
host. Same kernels as a layer staged on the card, so the codes match a fully staged build bit for bit."""
token = _HOST_QUANT_DEVICE.set(device)
try:
yield
finally:
_HOST_QUANT_DEVICE.reset(token)
class _HostLinearMover:
"""A card failure builds that layer and the rest on the host, as without this."""
def __init__(self, device: Any) -> None:
import torch
self.device = torch.device(device)
self._pinned: dict = {}
self.layers = 0
self.up_bytes = 0
self.started = time.perf_counter()
self.failed: Optional[str] = None
def _staging(self, slot: str, nbytes: int) -> Any:
import torch
from .host_pinning import host_pinning_allowed
if not host_pinning_allowed(self.device):
return None
buf = self._pinned.get(slot)
if buf is None or buf.numel() < nbytes:
self._pinned[slot] = None
try:
buf = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)
except Exception: # noqa: BLE001 -- no pinnable memory: pageable copies, same bytes
buf = torch.empty(0, dtype = torch.uint8)
self._pinned[slot] = buf
return buf if buf.numel() >= nbytes else None
def _to_device(self, tensor: Any) -> Any:
import torch
nbytes = tensor.numel() * tensor.element_size()
buf = self._staging("up", nbytes) if tensor.is_contiguous() else None
if buf is None:
return tensor.to(self.device)
host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)
host.copy_(tensor)
out = torch.empty(tensor.shape, dtype = tensor.dtype, device = self.device)
out.copy_(host, non_blocking = True)
return out
def _to_host(self, tensor: Any) -> Any:
import torch
nbytes = tensor.numel() * tensor.element_size()
buf = self._staging("down", nbytes) if tensor.is_contiguous() else None
if buf is None:
return tensor.to("cpu")
host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)
host.copy_(tensor)
return torch.empty(tensor.shape, dtype = tensor.dtype).copy_(host)
def build(self, linear: Any, make: Callable[[Any], Any]) -> Any:
import torch
weight = getattr(linear, "weight", None)
if (
self.failed is not None
or not isinstance(weight, torch.nn.Parameter)
or weight.device.type != "cpu"
):
return make(linear)
cuda = self.device.type == "cuda"
try:
linear.weight = torch.nn.Parameter(
self._to_device(weight.detach()), requires_grad = False
)
built = make(linear)
for key, buf in list(built._buffers.items()):
if buf is not None and buf.device.type != "cpu":
built._buffers[key] = self._to_host(buf)
# The next upload reuses the pinned buffer.
if cuda:
torch.cuda.synchronize(self.device)
except Exception as exc: # noqa: BLE001 -- out of memory or a device fault: the host build, as before
self.failed = f"{type(exc).__name__}: {exc}".splitlines()[0][:200]
linear.weight = weight
if cuda:
torch.cuda.empty_cache()
return make(linear)
finally:
if linear.weight is not weight:
linear.weight = weight
self.layers += 1
self.up_bytes += weight.numel() * weight.element_size()
return built
def close(self, logger: Any = None) -> None:
self._pinned.clear()
if logger is None or not (self.layers or self.failed):
return
logger.info(
"diffusion.transformer_quant: built %d host linear(s) on %s one at a time (%.1f GB in %.1f s)%s",
self.layers,
self.device,
self.up_bytes / 1e9,
time.perf_counter() - self.started,
f"; the rest on the host ({self.failed})" if self.failed else "",
)
def _first_native_linear(module: Any) -> Any:
try:
return next((sub for sub in module.modules() if is_native_linear(sub)), None)
except Exception: # noqa: BLE001 -- a probe must never raise
return None
def _rotation_group(module: Any, layer: Any) -> int:
# Every layer: rotation skips widths the group does not divide, so the first layer alone can read unrotated.
try:
return max(
int(getattr(sub, "rot_group", 0) or 0)
for sub in module.modules()
if is_native_linear(sub)
)
except Exception: # noqa: BLE001 -- a probe must never raise
return int(getattr(layer, "rot_group", 0) or 0)
def native_quant_signature(module: Any) -> Optional[str]:
layer = _first_native_linear(module)
if layer is None:
return None
if not getattr(layer, "act_int8", False):
return f"{layer.scheme}-wo"
rot = _rotation_group(module, layer)
return f"{layer.scheme}-w8a8" + (f"-rot{rot}" if rot else "")
def _int8pack_reported(layer: Any) -> bool:
engaged = getattr(layer, "int8pack_engaged", None)
if engaged is None or getattr(getattr(layer, "weight_q", None), "device", None) is None:
return False
import torch
bias = getattr(layer, "bias", None)
try:
return bool(engaged(bias.dtype if bias is not None else torch.bfloat16))
except Exception: # noqa: BLE001 -- a status line must not fail the load
return False
def native_quant_reason(module: Any, scheme: str) -> str:
layer = _first_native_linear(module)
if layer is not None and getattr(layer, "act_int8", False):
rot = _rotation_group(module, layer)
return (
f"W8A8: {scheme} weights and activations through torch._int_mm"
+ (f" with a g{rot} ConvRot rotation" if rot else "")
+ " (torchao-free; plain buffers, so the offload hooks can move it)"
)
if layer is not None and _int8pack_reported(layer):
return (
f"weight-only: {scheme} weights through the Apple Silicon int8 kernel, which measured faster than "
"bf16 here (torchao-free; half the weight memory; not bit-reproducible run to run, "
f"{NATIVE_MPS_INT8PACK_ENV}=0 for repeatable images)"
)
return (
f"weight-only: {scheme} weights, bf16 compute "
"(torchao-free, a memory saving rather than a speed-up)"
)
def native_weight_error(linear: Any, scheme: str) -> Optional[float]:
"""Relative Frobenius error of one Linear's native reconstruction; for tests and probes."""
try:
import torch
layer = native_linear_class()(linear, scheme)
ref = linear.weight.detach().float()
rec = layer.dequantized_weight(torch.float32)
return float((rec - ref).norm() / ref.norm().clamp(min = 1e-12))
except Exception: # noqa: BLE001
return None