PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Residual low-rank decomposition inspired by SVDQuant.
Reference: Li et al., "SVDQuant: Absorbing Outliers by Low-Rank Components for
4-Bit Diffusion Models", ICLR 2025, https://arxiv.org/abs/2411.05007.
This module is an AutoRound implementation and does not copy source code from
the reference implementation.
"""
import math
from dataclasses import dataclass
import torch
from auto_round.data_type.utils import get_quant_func
_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})
_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})
def _validate_scheme_values(scheme):
values = {}
for field in ("data_type", "bits", "group_size", "sym"):
try:
values[field] = getattr(scheme, field)
except AttributeError as exc:
raise ValueError(f"Residual quantization scheme is missing required value {field!r}.") from exc
if not isinstance(values["data_type"], str) or not values["data_type"].strip():
raise ValueError("Residual quantization scheme data_type must be a non-empty string.")
if not isinstance(values["bits"], int) or isinstance(values["bits"], bool) or values["bits"] <= 0:
raise ValueError("Residual quantization scheme bits must be a positive integer.")
if values["data_type"] in _FIXED_MXFP4_DTYPES and values["bits"] != 4:
raise ValueError(
f"Residual quantization scheme data_type={values['data_type']!r} requires bits=4; "
f"got bits={values['bits']}."
)
group_size = values["group_size"]
scalar_group_size = isinstance(group_size, int) and not isinstance(group_size, bool) and group_size >= -1
block_group_size = (
isinstance(group_size, tuple)
and len(group_size) == 2
and all(isinstance(size, int) and not isinstance(size, bool) and size > 0 for size in group_size)
)
if not scalar_group_size and not block_group_size:
raise ValueError(
"Residual quantization scheme group_size must be -1, 0, a positive integer, or a pair of positive integers."
)
if not isinstance(values["sym"], bool):
raise ValueError("Residual quantization scheme sym must be a boolean.")
return values
@dataclass(frozen=True)
class ResidualQuantScheme:
"""Weight quantization settings for stateless residual QDQ."""
data_type: str | None = None
bits: int | None = None
group_size: int | tuple[int, int] | None = None
sym: bool | None = None
def __post_init__(self) -> None:
_validate_scheme_values(self)
@dataclass(frozen=True)
class ActivationQuantScheme:
"""Activation quantization settings for stateless calibration QDQ."""
data_type: str | None = None
bits: int | None = None
group_size: int | tuple[int, int] | None = None
sym: bool | None = None
def __post_init__(self) -> None:
_validate_scheme_values(self)
@dataclass(frozen=True)
class ResidualDecomposition:
"""Best deployment-materialized candidate from residual outer iteration."""
residual: torch.Tensor
down: torch.Tensor
up: torch.Tensor
selected_iteration: int
error: float
def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.Tensor:
values = _validate_scheme_values(scheme)
requested_dtype = values["data_type"]
if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:
requested_dtype = f"{requested_dtype}_rceil"
quant_func, resolved_dtype = get_quant_func(
dtype=requested_dtype,
bits=values["bits"],
sym=values["sym"],
disable_opt_rtn=True,
group_size=values["group_size"],
iters=0,
)
logical_dtype = resolved_dtype.removeprefix("rtn_")
resolved_base_dtype = logical_dtype.removesuffix("_rceil")
if (
resolved_base_dtype in _MXFP4_ALIASES
and values["bits"] == 4
and (
not isinstance(values["group_size"], int)
or isinstance(values["group_size"], bool)
or values["group_size"] != 32
)
):
raise ValueError(
f"Deployable MXFP4 {tensor_name} QDQ requires scalar group_size=32; "
f"got group_size={values['group_size']!r}."
)
qdq, _, _ = quant_func(
tensor=tensor,
bits=values["bits"],
group_size=values["group_size"],
data_type=logical_dtype,
)
if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:
raise ValueError(
f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "
f"got shape={tuple(qdq.shape)}, dtype={qdq.dtype}."
)
if qdq.device != tensor.device:
raise ValueError(
f"{tensor_name.capitalize()} RTN QDQ must preserve the input device; "
f"got input device={tensor.device}, output device={qdq.device}."
)
if not torch.isfinite(qdq).all():
raise ValueError(f"{tensor_name.capitalize()} RTN QDQ produced non-finite values.")
return qdq
@torch.inference_mode()
def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch.Tensor:
"""Apply the registered RTN quantize-dequantize function to a residual."""
return _rtn_qdq_tensor(weight, scheme, tensor_name="residual")
@torch.inference_mode()
def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:
"""Apply deployment-compatible dynamic activation quantize-dequantize."""
return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")
@torch.inference_mode()
def compute_svd_factors(
weight: torch.Tensor, rank: int, *, driver: str | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return shared down/up factors without materializing a dense reconstruction."""
if weight.ndim != 2:
raise ValueError(f"SVDQuant expects a two-dimensional weight matrix, got shape={tuple(weight.shape)}.")
max_rank = min(weight.shape)
if not isinstance(rank, int) or isinstance(rank, bool) or not 0 <= rank <= max_rank:
raise ValueError(f"SVDQuant rank must be an integer in [0, {max_rank}], got {rank!r}.")
out_features, in_features = weight.shape
if rank == 0:
down_weight = torch.empty((0, in_features), dtype=weight.dtype, device=weight.device)
up_weight = torch.empty((out_features, 0), dtype=weight.dtype, device=weight.device)
return down_weight, up_weight
u, s, vh = torch.linalg.svd(weight, full_matrices=False, driver=driver if weight.is_cuda else None)
down_weight = vh[:rank, :]
up_weight = u[:, :rank] * s[:rank].reshape(1, -1)
return down_weight, up_weight
@torch.inference_mode()
def truncated_svd(
weight: torch.Tensor, rank: int, *, driver: str | None = None
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Return a rank-limited reconstruction and its shared down/up factors."""
down_weight, up_weight = compute_svd_factors(weight, rank, driver=driver)
low_rank = torch.zeros_like(weight) if rank == 0 else up_weight @ down_weight
return low_rank, down_weight, up_weight
@torch.inference_mode()
def iterate_residual_decomposition(
weight: torch.Tensor,
*,
rank: int,
scheme: ResidualQuantScheme,
iterations: int,
early_stop: bool,
residual_dtype: torch.dtype,
low_rank_dtype: torch.dtype,
driver: str | None = None,
) -> ResidualDecomposition:
"""Select the lowest weight-MSE residual/low-rank candidate after deployment casting."""
if type(iterations) is not int or iterations < 1:
raise ValueError(f"SVDQuant residual iterations must be a positive integer, got {iterations!r}.")
quantized_residual = torch.zeros_like(weight)
best_down = None
best_up = None
best_error = float("inf")
best_iteration = None
for iteration in range(1, iterations + 1):
low_rank, down, up = truncated_svd(weight - quantized_residual, rank, driver=driver)
if not all(torch.isfinite(tensor).all() for tensor in (low_rank, down, up)):
break
residual = (weight - low_rank).to(residual_dtype)
quantized_residual = rtn_qdq_residual(residual, scheme).to(weight.dtype)
deployed_down = down.to(low_rank_dtype)
deployed_up = up.to(low_rank_dtype)
deployed_low_rank = deployed_up.to(weight.dtype) @ deployed_down.to(weight.dtype)
error = torch.sum((weight - (quantized_residual + deployed_low_rank)).square()).item()
accepted = math.isfinite(error) and error < best_error
if accepted:
best_down = deployed_down.clone()
best_up = deployed_up.clone()
best_error = error
best_iteration = iteration
elif early_stop and best_iteration is not None and (not math.isfinite(error) or error > best_error):
break
if best_down is None or best_up is None or best_iteration is None:
raise ValueError("SVDQuant residual iteration did not produce a finite candidate.")
deployed_low_rank = best_up.to(weight.dtype) @ best_down.to(weight.dtype)
residual = weight - deployed_low_rank
if not torch.isfinite(residual).all():
raise ValueError("SVDQuant residual iteration produced a non-finite residual.")
return ResidualDecomposition(residual, best_down, best_up, best_iteration, best_error)