PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Extensible Module Weight Type Conversion Framework.
This module provides a registry-based system for detecting and converting quantized
module weights to high precision.
Table of Contents:
1. CORE FRAMEWORK
- ModuleWeightType: Enum of supported weight types
- WeightTypeHandler: Abstract base class for handlers
- Registry functions: register_weight_type_handler, get_handler, etc.
2. PUBLIC API
- detect_weight_type(): Detect weight type of a layer or model
- check_and_mark_quantized_module(): Check and mark quantized layers
- is_quantized_input_module(): Check if model has quantized weights
- convert_module_to_hp_if_necessary(): Main conversion function
3. HANDLER IMPLEMENTATIONS
- FP8Handler: Fully implemented for FP8 block-wise quantization
- MXFP8Handler: CompressedLinear with MXFP8PackedCompressor
- MXFP4Handler: CompressedLinear with MXFP4PackedCompressor
- NVFP4Handler: CompressedLinear with NVFP4PackedCompressor
- WOQHandler: CompressedLinear with weight-only quantization
Quick Start Guide:
Usage - Detect and Convert:
>>> from auto_round.utils.weight_handler import (
... check_and_mark_quantized_module,
... convert_module_to_hp_if_necessary,
... )
>>> check_and_mark_quantized_module(model)
>>> model = convert_module_to_hp_if_necessary(model)
Adding a New Weight Type Handler:
1. Add new type to ModuleWeightType enum
2. Create handler class inheriting from WeightTypeHandler
3. Register with @register_weight_type_handler decorator
Example:
@register_weight_type_handler(ModuleWeightType.MY_NEW_TYPE)
class MyNewTypeHandler(WeightTypeHandler):
def detect_layer(self, module): ...
def convert_layer(self, layer, dtype, device, to_cpu): ...
"""
import os
from abc import ABC, abstractmethod
from collections.abc import Callable
from contextlib import ContextDecorator
from dataclasses import fields
from enum import Enum, auto
import psutil
import torch
from auto_round import envs
from auto_round.logger import logger
# ============================================================================
# Section 0: UTILITY HELPERS
# ============================================================================
def _pad_weight(weight: torch.Tensor, block_size: list) -> tuple[torch.Tensor, int, int]:
"""Pads a matrix to make its dimensions multiples of block_size."""
M, N = weight.shape[-2:]
block_size_m, block_size_n = block_size
pad_M = (block_size_m - M % block_size_m) % block_size_m
pad_N = (block_size_n - N % block_size_n) % block_size_n
if pad_M == 0 and pad_N == 0:
return weight, M, N # No padding needed
padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
return padded_weight, M, N # Return original dimensions for unpadding
def _unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:
"""Removes padding from the matrix to restore its original shape."""
if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):
return weight
if keep_first_dim:
return weight[:, :original_M, :original_N]
else:
return weight[:original_M, :original_N]
class with_thread_limits(ContextDecorator):
"""
Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.
Inspired by vLLM's thread limit decorator.
https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115
"""
def __init__(self, div=1):
self.div = div
self.old_omp = None
self.old_torch = None
def __enter__(self):
# Save original settings
self.old_omp = envs.AR_OMP_NUM_THREADS
self.old_torch = torch.get_num_threads()
try:
if psutil is not None:
num_cores = len(psutil.Process().cpu_affinity() or [])
if num_cores == 0:
num_cores = os.cpu_count() or 1
else:
num_cores = os.cpu_count() or 1
# Set new limits
new_threads = max(1, num_cores // self.div)
os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)
torch.set_num_threads(new_threads)
except Exception:
pass
return self
def __exit__(self, exc_type, exc_val, exc_tb):
# Restore original settings
if self.old_omp is None:
os.environ.pop("AR_OMP_NUM_THREADS", None)
else:
os.environ["AR_OMP_NUM_THREADS"] = self.old_omp
torch.set_num_threads(self.old_torch)
# ============================================================================
# Section 1: CORE FRAMEWORK - Types, Base Classes, and Registry
# ============================================================================
class ModuleWeightType(Enum):
"""Enumeration of supported quantized module weight types.
When adding a new weight type:
1. Add the new type here
2. Create a handler class (see Section 3)
3. Register with @register_weight_type_handler decorator
"""
FP8 = auto() # FP8 with block-wise scaling (FP8Linear/CompressedLinear from transformers)
MXFP8 = auto() # MX FP8 (CompressedLinear with MXFP8PackedCompressor)
MXFP4 = auto() # MX FP4 (CompressedLinear with MXFP4PackedCompressor)
NVFP4 = auto() # NV FP4 (CompressedLinear with NVFP4PackedCompressor)
WOQ = auto() # Weight-Only Quantization (CompressedLinear with weight-only quantization)
class WeightTypeHandler(ABC):
"""Abstract base class for weight type detection and conversion handlers.
Subclasses must implement:
- detect_layer(): Check if a single layer is of this weight type
- convert_layer(): Convert a single layer to high precision
"""
@abstractmethod
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a single layer is of this weight type.
Args:
module: The module to check.
Returns:
True if the module is of this weight type, False otherwise.
"""
def attach_weight_shape(self, module: torch.nn.Module):
"""Optional helper to attach weight shape information to the module for detection."""
if not hasattr(module, "weight") or module.weight is None:
module.weight = torch.empty(
module.out_features,
module.in_features,
device="meta",
)
@abstractmethod
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert a single layer to high precision.
Args:
layer: The quantized layer to convert.
dtype: Target dtype for the converted layer.
device: Target device for the conversion.
to_cpu: If True, move converted layer to CPU.
Returns:
A new high-precision layer with dequantized weights.
"""
# --- Handler Registry ---
_WEIGHT_TYPE_HANDLERS: dict[ModuleWeightType, WeightTypeHandler] = {}
def register_weight_type_handler(weight_type: ModuleWeightType):
"""Decorator to register a weight type handler.
Args:
weight_type: The ModuleWeightType this handler supports.
Returns:
Decorator function that registers the handler class.
Example:
@register_weight_type_handler(ModuleWeightType.MXFP4)
class MXFP4Handler(WeightTypeHandler):
...
"""
def decorator(handler_cls: type[WeightTypeHandler]):
if not issubclass(handler_cls, WeightTypeHandler):
raise TypeError(f"Handler {handler_cls.__name__} must be a subclass of WeightTypeHandler")
_WEIGHT_TYPE_HANDLERS[weight_type] = handler_cls()
return handler_cls
return decorator
def get_handler(weight_type: ModuleWeightType) -> WeightTypeHandler | None:
"""Get the registered handler for a weight type.
Args:
weight_type: The ModuleWeightType to get the handler for.
Returns:
The registered handler, or None if not registered.
"""
return _WEIGHT_TYPE_HANDLERS.get(weight_type)
def get_all_handlers() -> dict[ModuleWeightType, WeightTypeHandler]:
"""Get all registered weight type handlers.
Returns:
Dictionary mapping weight types to their handlers.
"""
return _WEIGHT_TYPE_HANDLERS.copy()
# ============================================================================
# Section 2: PUBLIC API - Detection and Conversion Functions
# ============================================================================
def detect_weight_type(module: torch.nn.Module) -> ModuleWeightType | None:
"""Detect the weight type of a module or model.
First checks if the module itself has a quantized_weight_type attribute.
If not found, scans all submodules to find any with quantized_weight_type attribute.
Args:
module: The module or model to check.
Returns:
The detected ModuleWeightType, or None if no match found.
"""
# Check if module itself is marked
if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:
return module.quantized_weight_type
# Scan submodules for quantized_weight_type attribute
for submodule in module.modules():
if hasattr(submodule, "quantized_weight_type") and submodule.quantized_weight_type is not None:
return submodule.quantized_weight_type
return None
# --- Model Marking Functions ---
def check_and_mark_quantized_module(model: torch.nn.Module) -> set[ModuleWeightType]:
"""Check if model contains quantized layers and mark them accordingly.
This function scans the model (including the model itself) for quantized layers using
handlers' detect_layer method (which checks actual characteristics). It marks detected
layers with `quantized_weight_type`.
Args:
model: The model to check and mark.
Returns:
A set of detected ModuleWeightType values. Empty set if no quantized layers found.
"""
detected_types: set[ModuleWeightType] = set()
for weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():
# Check model itself first
if handler.detect_layer(model):
handler.attach_weight_shape(model)
model._is_quantized_input_module = True
model.quantized_weight_type = weight_type
detected_types.add(weight_type)
# Then check all submodules
for n, m in model.named_modules():
# Use handler to detect based on actual characteristics
if handler.detect_layer(m):
handler.attach_weight_shape(m)
# Mark the layer itself
m.quantized_weight_type = weight_type
# for gguf format, gguf format need to mark the quantized input module
# in order to skip weight_scale_inv, etc. (extra tensor from file)
if not getattr(model, "_is_quantized_input_module", False):
model._is_quantized_input_module = True
# Record detected types
detected_types.add(weight_type)
# remove decompress_hook for CT models
if hasattr(model, "ct_decompress_hook"):
model.ct_decompress_hook.remove()
return detected_types
def is_quantized_input_module(model: torch.nn.Module) -> ModuleWeightType | None:
"""Check if a model has quantized input weights and return the weight type.
This traverses all submodules to check for the `quantized_weight_type` attribute
set by `check_and_mark_quantized_module`.
Args:
model: The model to check.
Returns:
The ModuleWeightType if the model has quantized weights, None otherwise.
"""
# Check model itself first
if hasattr(model, "quantized_weight_type") and model.quantized_weight_type is not None:
return model.quantized_weight_type
# Traverse all submodules to find quantized layers
for module in model.modules():
if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:
return module.quantized_weight_type
return None
def remove_existed_quantization_config(model: torch.nn.Module):
"""Removes the existing quantization configuration from the model's config if it exists.
This is necessary to prevent conflicts during conversion, especially for models that have a
`quantization_config` attribute in their config or sub-configs. It checks the model and its
config for any `quantization_config` attributes and deletes them if found.
"""
if hasattr(model, "config") and model.config is not None:
if hasattr(model.config, "quantization_config"):
delattr(model.config, "quantization_config")
for attr in dir(model.config): # for text_config, vision_config, etc.
if "config" not in attr:
continue
config_attr = getattr(model.config, attr)
if hasattr(config_attr, "quantization_config"):
delattr(config_attr, "quantization_config")
# --- Main Conversion Function ---
def convert_module_to_hp_if_necessary(
model_or_layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str | torch.device = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert quantized layer(s) to high-precision Linear layer(s) if necessary.
This function automatically detects the weight type and uses the appropriate
handler for conversion. If the input is not quantized, it returns unchanged.
Args:
model_or_layer: Either a single layer, a block, or an entire model.
dtype: Target dtype for the converted layer(s). Defaults to torch.bfloat16.
device: Target device for the conversion. Defaults to "cpu".
to_cpu: If True, move converted layers to CPU. Defaults to False.
Returns:
If input is a quantized layer: A new high-precision layer with dequantized weights.
If input is a model containing quantized layers: The model with layers converted.
If input has no quantized layers: Returns the input unchanged.
"""
from auto_round.utils.device import clear_memory
from auto_round.utils.model import set_module
def _sync_serialization_attrs(src_module: torch.nn.Module, dst_module: torch.nn.Module) -> None:
"""Copy serialization-related attributes from source to destination module."""
from auto_round.compressors.base import SERIALIZATION_KEYS
orig_module_keys = list(SERIALIZATION_KEYS) + ["global_name"]
for key in orig_module_keys:
if hasattr(src_module, key):
setattr(dst_module, key, getattr(src_module, key))
remove_existed_quantization_config(model_or_layer)
# Check if it's a single quantized layer (has the attribute directly)
if hasattr(model_or_layer, "quantized_weight_type") and model_or_layer.quantized_weight_type is not None:
handler = get_handler(model_or_layer.quantized_weight_type)
logger.info(
f"Converting layer {model_or_layer.__class__.__name__} from "
+ f"{model_or_layer.quantized_weight_type.name} to {dtype}"
)
new_module = handler.convert_layer(model_or_layer, dtype, device, to_cpu)
_sync_serialization_attrs(model_or_layer, new_module)
return new_module
# Otherwise, traverse model and convert all quantized layers
# Get handler for each layer to support mixed quantization types
cnt = 0
type_counts: dict = {}
for n, m in model_or_layer.named_modules():
if hasattr(m, "quantized_weight_type") and m.quantized_weight_type is not None:
handler = get_handler(m.quantized_weight_type)
type_counts[m.quantized_weight_type.name] = type_counts.get(m.quantized_weight_type.name, 0) + 1
new_module = handler.convert_layer(m, dtype, device, to_cpu)
_sync_serialization_attrs(m, new_module)
new_module.quantized_weight_type = None # Clear quantized type after conversion
set_module(model_or_layer, n, new_module)
cnt += 1
if cnt % 10 == 0:
clear_memory()
if type_counts:
summary = ", ".join(f"{k}: {v} layers" for k, v in type_counts.items())
logger.info(f"Converted quantized layers to {dtype}: {summary}")
return model_or_layer
# ============================================================================
# Section 3: FP8 BLOCK DEQUANTIZATION HELPERS
# ============================================================================
def _pad_block_fp8_weight_naive(
weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list
) -> tuple[torch.Tensor, int, int]:
assert len(block_size) == 2
block_size_m, block_size_n = block_size
weight_scale_m, weight_scale_n = weight_scale.shape[-2:]
weight, orig_M, orig_N = _pad_weight(weight, block_size)
M, N = weight.shape[-2:]
# Check if scale tensor is large enough to cover the weight dimensions
# Some models may use over-provisioned scale tensors (e.g., for memory alignment),
# but undersized scales are invalid and should be rejected early
expected_scale_m = M // block_size_m
expected_scale_n = N // block_size_n
if weight_scale_m < expected_scale_m or weight_scale_n < expected_scale_n:
raise ValueError(
"FP8 weight scale shape is smaller than required for the padded weight: "
f"weight_shape={tuple(weight.shape)}, scale_shape={tuple(weight_scale.shape)}, block_size={block_size}"
)
# Handle over-provisioned scale tensors: if the scale tensor covers more blocks
# than the weight dimensions require, pad the weight to match the scale coverage.
target_M = max(M, weight_scale_m * block_size_m)
target_N = max(N, weight_scale_n * block_size_n)
if target_M != M or target_N != N:
pad_M = target_M - M
pad_N = target_N - N
weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
return weight, orig_M, orig_N
@with_thread_limits()
def _dequant_fp8_linear_weight(
weight: torch.Tensor,
weight_scale: torch.Tensor,
block_size: list | None = None,
data_type: str | None = None,
output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
"""Core dequantization logic for block-wise FP8 weights.
``data_type`` is the source quantization data type; ``output_dtype`` is the torch dtype of the result.
"""
if weight_scale is None:
return weight
# If weight is stored as uint8, view it as float8_e4m3fn
if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:
weight = weight.view(torch.float8_e4m3fn)
if block_size is None:
if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:
if weight_scale.numel() == weight.shape[0]:
weight_scale = weight_scale.view(-1, 1)
elif weight_scale.numel() == weight.shape[-1]:
weight_scale = weight_scale.view(1, -1)
return weight.to(output_dtype) * weight_scale.to(output_dtype)
assert len(block_size) == 2
weight, orig_M, orig_N = _pad_block_fp8_weight_naive(weight, weight_scale, block_size)
weight_shape_len = len(weight.shape)
block_size_m, block_size_n = block_size
# mul scale
if weight_shape_len == 2:
weight_scale_m, weight_scale_n = weight_scale.shape
weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)
weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)
dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
keep_first_dim = False
elif weight_shape_len == 3:
fd, weight_scale_m, weight_scale_n = weight_scale.shape
weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)
dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
keep_first_dim = True
else:
raise ValueError("Only support original weight shape is either 2 or 3")
dequant_weight = _unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)
return dequant_weight
# ============================================================================
# Section 4: HANDLER IMPLEMENTATIONS
# ============================================================================
# ----------------------------------------------------------------------------
# FP8 Block Handler - Full Implementation (Reference Example)
# ----------------------------------------------------------------------------
@register_weight_type_handler(ModuleWeightType.FP8)
class FP8Handler(WeightTypeHandler):
"""Handler for FP8 block-wise quantized layers.
This handler supports:
- FP8Linear layers with block-wise scaling
- CompressedLinear layers with compressor
- torch.nn.Linear layers with FP8 dtype weights
"""
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a module is an FP8 linear layer based on actual characteristics."""
# Check for CompressedLinear layer type (exclude MXFP4, MXFP8, NVFP4)
if module.__class__.__name__ == "CompressedLinear":
if hasattr(module, "compressor") and module.compressor is not None:
compressor_name = module.compressor.__class__.__name__
return "Float" in compressor_name
return True
if hasattr(module, "quantization_scheme"):
from compressed_tensors.quantization.utils import is_module_quantized # pylint: disable=E0401
if is_module_quantized(module) and module.quantization_status.value == "compressed":
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits == 8
and q_scheme.weights.type == "float"
and q_scheme.input_activations.num_bits == 8
and q_scheme.input_activations.type == "float"
):
return True
# Check for FP8Linear layer type
if module.__class__.__name__ == "FP8Linear":
return True
# Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)
if type(module) == torch.nn.Linear and getattr(module, "weight", None) is not None:
if str(module.weight.dtype).startswith("torch.float8"):
return True
return False
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert a single FP8/CompressedLinear layer to a standard Linear layer."""
if hasattr(layer, "quantization_scheme") and layer.__class__.__name__ == "Linear":
from compressed_tensors.compressors.base import decompress_module # pylint: disable=E0401
decompress_module(layer)
return layer
from auto_round.schemes import QuantizationScheme
from auto_round.utils.device import is_gaudi2
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
# Copy quantization scheme attributes
scheme_keys = (f.name for f in fields(QuantizationScheme))
for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
setattr(new_layer, key, getattr(layer, key, None))
# Handle Gaudi2 device compatibility
if is_gaudi2():
device = "cpu"
layer = layer.to(device)
# Check if layer has compressor.decompress_module method
if hasattr(layer, "compressor") and hasattr(layer.compressor, "decompress_module"):
dq_weight = layer.compressor.decompress_module(layer)
else:
# Use FP8 block-based dequantization
weight_scale = getattr(layer, "weight_scale", None)
block_size = getattr(layer, "block_size", None)
data_type = getattr(layer, "data_type", None)
if weight_scale is None:
weight_scale = getattr(layer, "weight_scale_inv", None)
if weight_scale is None:
raise AttributeError(
"FP8 layer is missing both 'weight_scale' and 'weight_scale_inv' "
"attributes required for dequantization."
)
dq_weight = _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
# Free intermediate CUDA tensors to avoid memory buildup
del dq_weight
layer.to("meta")
if to_cpu:
new_layer = new_layer.to("cpu")
return new_layer
# ----------------------------------------------------------------------------
# MXFP4 Handler - CompressedLinear with MXFP4PackedCompressor
# ----------------------------------------------------------------------------
@register_weight_type_handler(ModuleWeightType.MXFP4)
class MXFP4Handler(WeightTypeHandler):
"""Handler for MXFP4 quantized layers (CompressedLinear with MXFP4PackedCompressor)."""
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a module is an MXFP4 CompressedLinear layer."""
if module.__class__.__name__ == "CompressedLinear":
if hasattr(module, "compressor") and module.compressor is not None:
compressor_name = module.compressor.__class__.__name__
return "MXFP4" in compressor_name
if hasattr(module, "quantization_scheme"):
from compressed_tensors.quantization.utils import is_module_quantized # pylint: disable=E0401
if is_module_quantized(module) and module.quantization_status.value == "compressed":
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits == 4
and q_scheme.weights.type == "float"
and q_scheme.weights.group_size == 32
and q_scheme.input_activations.num_bits == 4
and q_scheme.input_activations.type == "float"
and q_scheme.input_activations.group_size == 32
):
return True
return False
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert an MXFP4 CompressedLinear layer to a standard Linear layer."""
from auto_round.schemes import QuantizationScheme
from auto_round.utils.device import is_gaudi2
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
# Copy quantization scheme attributes
scheme_keys = (f.name for f in fields(QuantizationScheme))
for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
setattr(new_layer, key, getattr(layer, key, None))
# Handle Gaudi2 device compatibility
if is_gaudi2():
device = "cpu"
layer = layer.to(device)
from auto_round.utils.model_free_utils import dequant_mxfp4
# Get packed weight and scale from layer
# MXFP4 weights are stored as packed uint8 in weight_packed
weight_packed = getattr(layer, "weight_packed", None)
if weight_packed is None:
weight_packed = getattr(layer, "weight", None)
weight_scale = getattr(layer, "weight_scale", None)
weight_scale = weight_scale.to(torch.uint8)
if weight_packed is None or weight_scale is None:
raise ValueError("MXFP4 layer must have weight_packed and weight_scale attributes")
# Dequantize using to_dtype function
dq_weight = dequant_mxfp4(
data_lp=weight_packed,
scale_e8m0=weight_scale,
elem_dtype="fp4_e2m1",
block_size=32,
target_dtype=dtype,
)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
# Free intermediate CUDA tensors to avoid memory buildup
del dq_weight
layer.to("meta")
if to_cpu:
new_layer = new_layer.to("cpu")
return new_layer
# ----------------------------------------------------------------------------
# MXFP8 Handler - CompressedLinear with MXFP8PackedCompressor
# ----------------------------------------------------------------------------
@register_weight_type_handler(ModuleWeightType.MXFP8)
class MXFP8Handler(WeightTypeHandler):
"""Handler for MXFP8 quantized layers (CompressedLinear with MXFP8PackedCompressor)."""
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a module is an MXFP8 CompressedLinear layer."""
if module.__class__.__name__ == "CompressedLinear":
if hasattr(module, "compressor") and module.compressor is not None:
compressor_name = module.compressor.__class__.__name__
return "MXFP8" in compressor_name
if hasattr(module, "quantization_scheme"):
from compressed_tensors.quantization.utils import is_module_quantized # pylint: disable=E0401
if is_module_quantized(module) and module.quantization_status.value == "compressed":
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits == 8
and q_scheme.weights.type == "float"
and q_scheme.weights.group_size == 32
and q_scheme.input_activations.num_bits == 8
and q_scheme.input_activations.type == "float"
and q_scheme.input_activations.group_size == 32
):
return True
return False
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert an MXFP8 CompressedLinear layer to a standard Linear layer."""
from auto_round.schemes import QuantizationScheme
from auto_round.utils.device import is_gaudi2
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
# Copy quantization scheme attributes
scheme_keys = (f.name for f in fields(QuantizationScheme))
for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
setattr(new_layer, key, getattr(layer, key, None))
# Handle Gaudi2 device compatibility
if is_gaudi2():
device = "cpu"
layer = layer.to(device)
from auto_round.utils.model_free_utils import dequant_mx_fp8
# Get weight and scale from layer
weight = layer.weight
weight_scale = getattr(layer, "weight_scale", None)
weight_scale = weight_scale.to(torch.uint8)
if weight is None or weight_scale is None:
raise ValueError("MXFP8 layer must have weight and weight_scale attributes")
# Dequantize using dequant_mx_fp8 function
dq_weight = dequant_mx_fp8(
weight_fp8=weight,
scale_e8m0=weight_scale,
block_size=32,
target_dtype=dtype,
)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
# Free intermediate CUDA tensors to avoid memory buildup
del dq_weight
layer.to("meta")
if to_cpu:
new_layer = new_layer.to("cpu")
return new_layer
# ----------------------------------------------------------------------------
# NVFP4 Handler - CompressedLinear with NVFP4PackedCompressor
# ----------------------------------------------------------------------------
@register_weight_type_handler(ModuleWeightType.NVFP4)
class NVFP4Handler(WeightTypeHandler):
"""Handler for NVFP4 quantized layers (CompressedLinear with NVFP4PackedCompressor)."""
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a module is an NVFP4 CompressedLinear layer."""
if module.__class__.__name__ == "CompressedLinear":
if hasattr(module, "compressor") and module.compressor is not None:
compressor_name = module.compressor.__class__.__name__
return "NVFP4" in compressor_name
if hasattr(module, "quantization_scheme"):
from compressed_tensors.quantization.utils import is_module_quantized # pylint: disable=E0401
if is_module_quantized(module) and module.quantization_status.value == "compressed":
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits == 4
and q_scheme.weights.type == "float"
and q_scheme.weights.group_size == 16
and q_scheme.input_activations.num_bits == 4
and q_scheme.input_activations.type == "float"
and q_scheme.input_activations.group_size == 16
):
return True
return False
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert an NVFP4 CompressedLinear layer to a standard Linear layer."""
if hasattr(layer, "quantization_scheme") and layer.__class__.__name__ == "Linear":
from compressed_tensors.compressors.base import decompress_module # pylint: disable=E0401
decompress_module(layer)
return layer
from auto_round.schemes import QuantizationScheme
from auto_round.utils.device import is_gaudi2
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
# Copy quantization scheme attributes
scheme_keys = (f.name for f in fields(QuantizationScheme))
for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
setattr(new_layer, key, getattr(layer, key, None))
# Handle Gaudi2 device compatibility
if is_gaudi2():
device = "cpu"
layer = layer.to(device)
# Use compressor.decompress_module for dequantization
dq_weight = layer.compressor.decompress_module(layer)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
# Free intermediate CUDA tensors to avoid memory buildup
del dq_weight
layer.to("meta")
if to_cpu:
new_layer = new_layer.to("cpu")
return new_layer
# ----------------------------------------------------------------------------
# WOQ Handler - CompressedLinear with weight-only quantization
# ----------------------------------------------------------------------------
@register_weight_type_handler(ModuleWeightType.WOQ)
class WOQHandler(WeightTypeHandler):
"""Handler for integer 4-bit weight-only quantized layers (Compressed Tensor only)."""
def detect_layer(self, module: torch.nn.Module) -> bool:
"""Check if a module is a CompressedLinear layer."""
if module.__class__.__name__ == "CompressedLinear":
if hasattr(module, "compressor") and module.compressor is not None:
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits in [2, 4, 8]
and q_scheme.weights.type == "int"
and q_scheme.input_activations is None
):
return True
if hasattr(module, "quantization_scheme"):
from compressed_tensors.quantization.utils import is_module_quantized # pylint: disable=E0401
if is_module_quantized(module) and module.quantization_status.value == "compressed":
q_scheme = module.quantization_scheme
if (
q_scheme.weights.num_bits in [2, 4, 8]
and q_scheme.weights.type == "int"
and q_scheme.input_activations is None
):
return True
return False
def convert_layer(
self,
layer: torch.nn.Module,
dtype: torch.dtype = torch.bfloat16,
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
"""Convert an integer weight-only quantized layer to a standard Linear layer."""
if hasattr(layer, "quantization_scheme") and layer.__class__.__name__ == "Linear":
from compressed_tensors.compressors.base import decompress_module # pylint: disable=E0401
decompress_module(layer)
return layer
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
# Use compressor.decompress_module for dequantization
dq_weight = layer.compressor.decompress_module(layer)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
# Free intermediate CUDA tensors to avoid memory buildup
del dq_weight
layer.to("meta")
if to_cpu:
new_layer = new_layer.to("cpu")
return new_layer