PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import torch
import torch.nn as nn
try:
import auto_round_kernel
ARK_INSTALLED = True
ark = auto_round_kernel
except:
ARK_INSTALLED = False
BITS_DTYPE_MAPPING = {
2: "int2",
4: "int4",
8: "int8",
}
FP8_DTYPE_MAPPING = {
"fp8_e4m3": "fp8_e4m3",
"fp8_e5m2": "fp8_e5m2",
}
AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
shifts = torch.arange(0, 32, bits, device=qzeros.device)
# unpacking columnwise
iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
torch.int8 # smallest dtype available
)
iweights = iweights.view(iweights.shape[0], -1)
# unpacking columnwise
if qzeros is not None:
izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
torch.int8 # smallest dtype available
)
izeros = izeros.view(izeros.shape[0], -1)
else:
izeros = qzeros
return iweights, izeros
def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):
reverse_order_tensor = torch.arange(
iweights.shape[-1],
dtype=torch.int32,
device=izeros.device,
)
reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
reverse_order_tensor = reverse_order_tensor.view(-1)
if izeros is not None:
izeros = izeros[:, reverse_order_tensor]
iweights = iweights[:, reverse_order_tensor]
return iweights, izeros
def convert_dtype_torch2str(dtype):
if dtype == torch.int8:
return "int8"
elif dtype == torch.float:
return "fp32"
elif dtype == torch.float16:
return "fp16"
elif dtype == torch.bfloat16:
return "bf16"
elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
return dtype
else:
assert False, f"Unsupported pytorch dtype {dtype} to str dtype"
class QuantLinear(nn.Module):
QUANT_TYPE = "ark_gptq_nozp"
ZP_BIAS = 0
def __init__(
self,
bits,
group_size,
sym,
in_features,
out_features,
bias,
weight_dtype=torch.bfloat16,
**kwargs,
):
super().__init__()
if bits not in [2, 4, 8]:
raise NotImplementedError("Only 2, 4, 8 bits are supported for ARK.")
if not ARK_INSTALLED:
raise ModuleNotFoundError(
"The 'auto_round_kernel' module is required but not installed. "
"Please install the 'auto-round-lib' package, for example:\n"
" pip install auto-round-lib"
)
self.infeatures = in_features
self.outfeatures = out_features
self.bits = bits
self.group_size = group_size if group_size != -1 else self.infeatures
self.maxq = 2**self.bits - 1
self.qbias = 2 ** (self.bits - 1)
self.pack_num = 32 // self.bits
self.weight_dtype = weight_dtype
self.asym = not sym
if not self.infeatures % self.group_size == 0:
raise NotImplementedError("in_features must be divisible by group_size")
if "awq" in self.QUANT_TYPE:
self.register_buffer(
"qweight",
torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32),
)
else:
self.register_buffer(
"qweight",
torch.zeros(
(self.infeatures // 32 * self.bits, self.outfeatures),
dtype=torch.int32,
),
)
self.register_buffer(
"qzeros",
torch.zeros(
(
math.ceil(self.infeatures / self.group_size),
self.outfeatures // 32 * self.bits,
),
dtype=torch.int32,
),
)
self.register_buffer(
"scales",
torch.zeros(
(math.ceil(self.infeatures / self.group_size), self.outfeatures),
dtype=weight_dtype,
),
)
if bias:
self.register_buffer("bias", torch.zeros((self.outfeatures), dtype=torch.float))
else:
self.bias = None
def extra_repr(self) -> str:
return (
f"in_features={self.infeatures}, out_features={self.outfeatures}, bias={self.bias is not None}, "
f"w_bit={self.bits}, group_size={self.group_size}"
)
def post_init(self):
if self.qweight.device.type not in ["cpu", "xpu"]:
raise NotImplementedError(
f"Device type {self.qweight.device.type} is not supported. Only CPU and XPU devices are supported."
)
if self.qweight.device.type != "cpu" and self.asym:
raise NotImplementedError("Asymmetric quantization is only supported on CPU devices")
if "awq" in self.QUANT_TYPE:
intweight, zeros = unpack_awq(
self.qweight, self.qzeros, self.bits
) # weight: k x n zeros: k / group_size x n
intweight, zeros = reverse_awq_order(intweight, zeros, self.bits) # weight: k x n zeros: k / group_size x n
intweight = torch.bitwise_and(intweight, self.maxq)
zeros = torch.bitwise_and(zeros, self.maxq)
intweight = intweight - self.qbias # from uint8_t to int8_t
zeros = zeros - self.qbias # from uint8_t to int8_t
else:
# intweight: k x n, zeros: k / group_size x n
intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS, self.asym)
if zeros is None:
zeros = torch.empty(0, dtype=torch.int8)
else:
# change it to int8 with offset 128
zeros = (zeros.to(torch.int32) - self.qbias).to(torch.int8)
if self.asym:
intweight = (intweight.to(torch.int32) - self.qbias).to(torch.int8)
else:
intweight -= self.qbias
intweight = intweight.to(torch.int8)
if self.qweight.device.type == "xpu":
self.sdt = "fp16"
self.cdt = "int8"
self.torch_dt = torch.float16
else:
self.sdt = "fp32"
self.cdt = "auto"
self.torch_dt = torch.float32
self.wdt = BITS_DTYPE_MAPPING[self.bits]
scales = self.scales.to(self.torch_dt).contiguous()
self.qweight = ark.repack_quantized_weight(
intweight.contiguous(),
scales,
zeros.contiguous(),
self.group_size,
# compute_dtype
self.cdt,
# weight_dtype
self.wdt,
# scale_dtype
self.sdt,
self.asym,
)
# free mem
self.qzeros = torch.empty(0)
self.scales = torch.empty(0)
if self.bias is not None:
self.bias = self.bias.to(self.torch_dt)
else:
self.bias = torch.empty(0)
def forward(self, x: torch.Tensor):
raw_input_dtype = x.dtype
x = x.to(self.torch_dt)
out_shape = x.shape[:-1] + (self.outfeatures,)
x = x.view(-1, x.shape[-1])
self.bias = self.bias.to(self.torch_dt)
outputs = ark.woqgemm(
x, # convert xd to 2d,
self.qweight,
self.bias,
self.outfeatures,
self.infeatures,
self.group_size,
self.cdt, # compute_dtype
self.wdt, # weight_dtype
self.sdt, # scale_dtype
self.asym,
)
return outputs.to(raw_input_dtype).view(out_shape)
class QuantLinearGPTQ(QuantLinear):
QUANT_TYPE = "ark_gptq"
ZP_BIAS = 1
class QuantLinearAWQ(QuantLinear):
QUANT_TYPE = "ark_awq"
@torch.no_grad()
def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias, asym):
wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32, device=qweight.device).unsqueeze(0)
zeros = None
if asym:
zp_shape = list(qzeros.shape)
zp_shape[1] = zp_shape[1] * (32 // bits)
zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(
torch.int16 if bits == 8 else torch.int8
)
torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)
if bits == 8:
zeros = zeros.to(torch.uint8)
zeros += gptq_bias
try:
zeros = zeros.reshape(zp_shape)
except:
# zeros and scales have different iteam numbers.
# remove 1 (due to 0 + 1 in line 252)
zeros = zeros[zeros != 1]
zeros = zeros.reshape(zp_shape)
weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(
torch.int16 if bits == 8 else torch.int8
)
weight.bitwise_and_((2**bits) - 1)
weight = weight.view(-1, weight.shape[-1])
return weight, zeros
# Copied from qlinear_marlin.py
@torch.no_grad()
def dequantize_weight(qweight, qzeros, scales, bits):
unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)
group_size = unpacked_qweight.shape[0] // scales.shape[0]
scales = scales.repeat_interleave(group_size, dim=0)
if unpacked_qzeros is not None:
unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)
else:
unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32, device=qweight.device)
unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales
return unpacked_qweight, unpacked_qzeros
def ark_post_init(model):
for _, submodule in model.named_modules():
if isinstance(submodule, (QuantLinear, QuantLinearFP8)):
submodule.post_init()
return model
class QuantLinearFP8(nn.Module):
"""Quantized Linear layer using FP8 format with ARK kernel."""
QUANT_TYPE = "ark_fp8"
def __init__(
self,
bits,
group_size,
infeatures,
outfeatures,
bias,
kernel_switch_threshold=128,
trainable=False,
weight_dtype=torch.float16,
data_type="fp8_e4m3",
**kwargs,
):
super().__init__()
data_type = str(data_type)
self.original_data_type = data_type
self.is_mx = "mx_fp" in data_type.lower()
# Normalize to the underlying FP8 weight format. MXFP8 stores weights in
# FP8 (E4M3/E5M2) but uses E8M0 exponent-only scales.
if self.is_mx:
if "e5m2" in data_type.lower():
data_type = "fp8_e5m2"
else:
data_type = "fp8_e4m3"
else:
# Support both fp8_e4m3 and fp8_e5m2 format names
if data_type not in ["fp8_e4m3", "fp8_e5m2", "fp8"]:
data_type = "fp8_e4m3" # Default to E4M3
if data_type == "fp8":
data_type = "fp8_e4m3"
assert ARK_INSTALLED, "Please install auto_round_kernel."
self.infeatures = infeatures
self.outfeatures = outfeatures
self.bits = 8 # FP8 is always 8-bit
self.group_size = group_size if group_size != -1 else infeatures
self.weight_dtype = weight_dtype
self.data_type = data_type
if hasattr(ark, "set_threads"):
ark.set_threads(torch.get_num_threads())
# FP8 weights - load from safetensors with correct FP8 dtype
# Shape is [outfeatures, infeatures] to match standard Linear layer
# IMPORTANT: Must use float8 dtype to match safetensors, not uint8
fp8_dtype = torch.float8_e5m2 if data_type == "fp8_e5m2" else torch.float8_e4m3fn
self.register_buffer(
"weight",
torch.zeros((outfeatures, infeatures), dtype=fp8_dtype),
)
# Scales for dequantization - load from safetensors as "weight_scale"
# Shape is [outfeatures, num_groups] to match per-channel quantization
# where num_groups = ceil(infeatures / group_size)
scale_storage_dtype = torch.uint8 if self.is_mx else weight_dtype
self.register_buffer(
"weight_scale",
torch.zeros(
(outfeatures, math.ceil(infeatures / self.group_size)),
dtype=scale_storage_dtype,
),
)
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))
else:
self.bias = None
self.kernel_switch_threshold = kernel_switch_threshold
self.trainable = trainable
def _load_from_state_dict(
self,
state_dict,
prefix,
local_metadata,
strict,
missing_keys,
unexpected_keys,
error_msgs,
):
"""Override to handle weight_scale shape mismatch between checkpoint and model."""
weight_scale_key = prefix + "weight_scale"
if weight_scale_key in state_dict:
loaded_scale = state_dict[weight_scale_key]
expected_shape = self.weight_scale.shape # [outfeatures, num_groups]
out_features, num_groups = expected_shape
expected_numel = out_features * num_groups
if loaded_scale.ndim == 2 and loaded_scale.shape == (
num_groups,
out_features,
):
state_dict[weight_scale_key] = loaded_scale.t().contiguous()
loaded_scale = state_dict[weight_scale_key]
# If loaded scale is 1D but we expect 2D, reshape or broadcast it.
if loaded_scale.ndim == 1 and len(expected_shape) == 2:
numel = loaded_scale.numel()
if numel == expected_numel:
state_dict[weight_scale_key] = loaded_scale.view(expected_shape)
elif numel == out_features:
expanded = loaded_scale.view(out_features, 1).expand(-1, num_groups)
state_dict[weight_scale_key] = expanded.to(self.weight_scale.dtype)
elif numel == num_groups:
expanded = loaded_scale.view(1, num_groups).expand(out_features, -1)
state_dict[weight_scale_key] = expanded.to(self.weight_scale.dtype)
elif numel % out_features == 0:
inferred_num_groups = numel // out_features
inferred_shape = (out_features, inferred_num_groups)
state_dict[weight_scale_key] = loaded_scale.view(inferred_shape)
self.weight_scale = torch.zeros(
inferred_shape,
dtype=self.weight_scale.dtype,
device=self.weight_scale.device,
)
else:
print(
f"Cannot reshape scale! loaded_scale.numel()={numel}, expected={expected_numel}, out_features={out_features}"
)
super()._load_from_state_dict(
state_dict,
prefix,
local_metadata,
strict,
missing_keys,
unexpected_keys,
error_msgs,
)
def post_init(self):
target_fp8_dtype = torch.float8_e5m2 if self.data_type == "fp8_e5m2" else torch.float8_e4m3fn
weight_tensor = self.weight
converted_via_cpu = False
if weight_tensor.dtype != target_fp8_dtype:
weight_cpu = weight_tensor.detach().to("cpu")
fp8_cpu = weight_cpu.to(target_fp8_dtype)
converted_via_cpu = True
else:
fp8_cpu = weight_tensor.detach()
if weight_tensor.device.type != "cpu":
fp8_cpu = fp8_cpu.to("cpu")
converted_via_cpu = True
fp8_cpu = fp8_cpu.contiguous().view(self.outfeatures, self.infeatures)
fp8_uint8_cpu = fp8_cpu.view(torch.uint8)
if converted_via_cpu and self.weight.device.type != "cpu":
fp8_uint8 = fp8_uint8_cpu.to(self.weight.device)
else:
fp8_uint8 = fp8_uint8_cpu
fp8_weight = fp8_uint8.view(self.outfeatures, self.infeatures).t().contiguous()
zeros = torch.empty(0, dtype=torch.uint8, device=fp8_weight.device) # No zero points for FP8
# Compute type selection (independent of how scales are stored).
if self.weight.device.type == "xpu":
self.cdt = "fp16"
else:
self.cdt = "fp32"
if self.is_mx:
# MXFP8 stores per-group scales as E8M0 exponents in uint8 with bias 127.
# ARK expects fp8_e8m0 scales as float32 exponents (int-valued), which will
# be packed to bestla::utils::f8 (int8 exponent) internally.
self.sdt = "fp8_e8m0"
if self.weight_scale.dtype == torch.uint8:
exponents = self.weight_scale.to(torch.int16) - 127
else:
# Fallback: allow feeding exponents directly.
exponents = self.weight_scale.to(torch.int16)
exponents = exponents.clamp(-127, 127)
scales = exponents.t().to(torch.float32).contiguous() # [num_groups,out]
else:
if self.weight.device.type == "xpu":
self.sdt = "fp16"
scales = self.weight_scale.t().to(torch.float16).contiguous() # [num_groups,out]
else:
self.sdt = "fp32"
scales = self.weight_scale.t().float().contiguous() # [num_groups,out]
self.wdt = FP8_DTYPE_MAPPING[self.data_type]
# Repack weights using ARK kernel
packed_weight = ark.repack_quantized_weight(
fp8_weight,
scales,
zeros.contiguous(),
torch.empty(0, dtype=torch.int32, device=fp8_weight.device),
self.cdt, # compute_dtype
self.wdt, # weight_dtype (fp8_e4m3 or fp8_e5m2)
self.sdt, # scale_dtype
False, # asym = False for FP8
self.group_size,
)
# Replace weight buffer with packed weight
self.weight = packed_weight
# Free original weight_scale
self.weight_scale = torch.empty(0)
if self.bias is not None:
if self.bias.device.type == "cpu":
self.bias = self.bias.to(torch.float32)
else:
self.bias = self.bias.to(torch.float16)
def forward(self, x: torch.Tensor):
raw_input_dtype = x.dtype
if x.device.type == "cpu":
odt = torch.float32
if raw_input_dtype != torch.float32:
x = x.to(torch.float32)
else:
odt = x.dtype
out_shape = x.shape[:-1] + (self.outfeatures,)
x = x.view(-1, x.shape[-1]) # convert xd to 2d
out_2d_shape = x.shape[:-1] + (self.outfeatures,)
outputs = torch.empty(out_2d_shape, device=x.device, dtype=odt)
if self.bias is None:
bias = torch.empty(0, device=x.device, dtype=odt)
else:
bias = self.bias
if bias.device != x.device:
bias = bias.to(x.device)
if bias.dtype != odt:
bias = bias.to(odt)
ark.woq_linear(
x,
self.weight,
bias,
outputs,
self.cdt, # compute_dtype
self.wdt, # weight_dtype
self.sdt, # scale_dtype
False, # asym
self.group_size,
)
return outputs.to(raw_input_dtype).view(out_shape)
def extra_repr(self) -> str:
return (
f"in_features={self.infeatures}, out_features={self.outfeatures}, bias={self.bias is not None}, bits={self.bits}, "
f"group_size={self.group_size}, data_type={self.data_type}"
)
__all__ = ["QuantLinear", "QuantLinearAWQ", "QuantLinearFP8", "QuantLinearGPTQ"]