Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Extensible Module Weight Type Conversion Framework. (#1375)

Xin He53f20cf12d76
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 57e771478..97702db3e 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -73,14 +73,13 @@ from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
TORCH_VERSION_AT_LEAST_2_6,
CpuInfo,
- check_and_mark_fp8_model,
+ check_and_mark_quantized_module,
check_seqlen_compatible,
check_to_quantized,
clear_memory,
compile_func,
convert_dtype_str2torch,
- convert_fp8_layer_to_linear,
- convert_fp8_module_to_16b,
+ convert_module_to_hp_if_necessary,
detect_device,
find_matching_blocks,
flatten_list,
@@ -92,11 +91,9 @@ from auto_round.utils import (
htcore,
is_auto_device_mapping,
is_debug_mode,
- is_fp8_linear,
- is_fp8_model,
is_hpex_available,
is_moe_model,
- is_moe_model_via_config,
+ is_quantized_input_module,
llm_load_model,
memory_monitor,
mv_module_from_gpu,
@@ -296,7 +293,7 @@ class BaseCompressor(object):
"AutoRound does not support parameters on meta device. "
"Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
)
- check_and_mark_fp8_model(model)
+ check_and_mark_quantized_module(model)
self.model = model.eval()
self.tokenizer = tokenizer
self.shared_cache_keys = get_shared_keys(self.model)
@@ -514,8 +511,8 @@ class BaseCompressor(object):
logger.info("AutoScheme is not yet supported for multimodal LLMs.")
sys.exit(-1)
- if getattr(model, "is_fp8", False):
- logger.info("AutoScheme does not currently support FP8 models.")
+ if is_quantized_input_module(model):
+ logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")
sys.exit(-1)
all_dtypes = []
@@ -1145,9 +1142,8 @@ class BaseCompressor(object):
if dtype is not None:
m = m.to(dtype)
- if is_fp8_linear(m):
- m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)
- set_module(self.model, name, m)
+ m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)
+ set_module(self.model, name, m)
tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device
# Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic
if self.is_immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:
@@ -1254,9 +1250,8 @@ class BaseCompressor(object):
for name in pbar:
pbar.set_description(f"Calculate weight global scale: {name}")
m = get_module(self.model, name)
- if is_fp8_linear(m):
- m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)
- set_module(self.model, name, m)
+ m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)
+ set_module(self.model, name, m)
weight_global_scale = calculate_gparam(m.weight, self.group_size)
setattr(m, "weight_global_scale", weight_global_scale)
@@ -1383,8 +1378,7 @@ class BaseCompressor(object):
m.to("meta")
# Convert remaining fp8
- if is_fp8_model(self.model):
- convert_fp8_module_to_16b(self.model, self.amp_dtype, self.device)
+ convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)
if self.is_immediate_saving:
shard_writer(self, is_finalize=True)
@@ -1455,8 +1449,7 @@ class BaseCompressor(object):
block = get_module(self.model, block_name)
materialize_model_(block)
block.to("cpu")
- if is_fp8_model(self.model):
- convert_fp8_module_to_16b(block, dtype=self.amp_dtype, device=self.device)
+ convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)
if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:
set_auto_device_map_for_block_with_tuning(
@@ -1751,11 +1744,7 @@ class BaseCompressor(object):
pbar.close()
self._quantize_layers(layer_names, all_inputs)
- if is_fp8_model(self.model):
- for n, m in self.model.named_modules():
- if is_fp8_linear(m):
- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to("cpu")
- set_module(self.model, n, new_layer)
+ convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device, to_cpu=True)
if self.is_immediate_saving:
shard_writer(self, is_finalize=True)
@@ -1819,10 +1808,8 @@ class BaseCompressor(object):
layer = get_module(self.model, layer_name)
layer = layer.to(self.device)
- if is_fp8_linear(layer):
- new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype, self.device).to(self.device)
- set_module(self.model, layer_name, new_layer)
- layer = new_layer
+ layer = convert_module_to_hp_if_necessary(layer, self.amp_dtype, self.device)
+ set_module(self.model, layer_name, layer)
wrapper_layer = WrapperLinear(
layer,
@@ -2129,7 +2116,7 @@ class BaseCompressor(object):
Raises:
Exception: If caching on GPU fails, switches to CPU and caches there.
"""
- if is_fp8_model(self.model):
+ if is_quantized_input_module(self.model):
layer_names = []
if layer_names is None:
layer_names = []
@@ -2780,11 +2767,7 @@ class BaseCompressor(object):
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
materialize_model_(block)
- if is_fp8_model(self.model):
- for n, m in block.named_modules():
- if is_fp8_linear(m):
- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to(device)
- set_module(block, n, new_layer)
+ convert_module_to_hp_if_necessary(block, self.amp_dtype, device)
if auto_offload:
# card_0_in_high_risk indicates that card_0 memory is already in high usage (90%) w/o any weights
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
index 730f8e205..b5cfd8598 100644
--- a/auto_round/compressors/mllm/compressor.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -41,7 +41,6 @@ from auto_round.utils import (
find_matching_blocks,
get_block_names,
get_max_vram,
- is_fp8_model,
mllm_load_model,
mv_module_from_gpu,
to_device,
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 500ddc115..8d890df8f 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -26,7 +26,7 @@ from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
-from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_model
+from auto_round.utils import check_to_quantized
class BackendDataType(str, Enum):
@@ -120,19 +120,6 @@ def block_forward(
return output
-def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:
- if is_fp8_model(model):
- return True
- for n, m in model.named_modules():
- if is_fp8_linear(m):
- m.is_fp8_linear = True
- if not hasattr(model, "is_fp8"):
- model.is_fp8 = True
- if hasattr(model, "is_fp8"):
- return True
- return False
-
-
def check_skippable_keywords(key):
"""
Prints a reminder if a key is not stored during quantization fine-tuning.
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 5b9252fe2..6a22a13e2 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -56,7 +56,7 @@ from auto_round.utils import (
clear_memory,
get_module,
get_packing_device,
- is_fp8_model,
+ is_quantized_input_module,
is_separate_tensor,
logger,
)
@@ -159,7 +159,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
yield name, tensor
def is_extra_tensor(tensor_name):
- if is_fp8_model(cls.model) and "scale" in tensor_name.split(".")[-1]:
+ if is_quantized_input_module(cls.model) and "scale" in tensor_name.split(".")[-1]:
return False
if tensor_name not in cls.model.tensor_name_list:
return True
diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.py
index d005b7d35..32d976702 100644
--- a/auto_round/utils/__init__.py
+++ b/auto_round/utils/__init__.py
@@ -15,6 +15,10 @@
from auto_round.utils.device import *
from auto_round.utils.common import *
from auto_round.utils.model import *
+from auto_round.utils.weight_handler import (
+ detect_weight_type,
+ is_quantized_input_module,
+)
import transformers
from packaging.version import Version
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 66af26c9e..0371caf2f 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -16,9 +16,6 @@ import json
import os
import re
from collections import UserDict
-from contextlib import ContextDecorator, contextmanager
-from dataclasses import asdict, fields
-from functools import wraps
from pathlib import Path
from typing import Union
@@ -31,58 +28,12 @@ from auto_round import envs
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
-
-# ============================================================================
-# FP8 Dequantization Registry
-# ============================================================================
-# Registry mapping FP8 layer class names to their dequantization handlers.
-# This ensures detection and dequantization logic stay in sync.
-FP8_DEQUANT_REGISTRY = {}
-
-
-def register_fp8_layer(layer_name: str):
- """Register a dequantization handler for an FP8 layer type.
-
- Args:
- layer_name: The class name of the FP8 layer type (e.g., "FP8Linear", "CompressedLinear")
-
- Returns:
- Decorator function that registers the handler.
-
- Example:
- @register_fp8_layer("FP8Linear")
- def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
- # Dequantization logic
- return dequantized_weight
- """
-
- def decorator(fn):
- FP8_DEQUANT_REGISTRY[layer_name] = fn
- return fn
-
- return decorator
-
-
-def dequant_fp8_layer(layer, dtype=torch.bfloat16, device: str = "cpu"):
- """Dequantize an FP8 layer using the registry.
-
- Args:
- layer: The FP8 layer to dequantize.
- dtype: Target dtype for dequantized weights.
- device: Target device for dequantization.
-
- Returns:
- Dequantized weight tensor.
-
- Raises:
- NotImplementedError: If the layer type is not registered.
- """
- name = layer.__class__.__name__
- if name not in FP8_DEQUANT_REGISTRY:
- raise NotImplementedError(
- f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(FP8_DEQUANT_REGISTRY.keys())}"
- )
- return FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)
+from auto_round.utils.weight_handler import (
+ _dequant_fp8_linear_weight,
+ check_and_mark_quantized_module,
+ convert_module_to_hp_if_necessary,
+ is_quantized_input_module,
+)
def clean_module_parameter(submodule: torch.nn.Module, param_name: str) -> None:
@@ -192,19 +143,6 @@ def convert_dtype_torch2str_hf(dtype):
return str_dtype
-def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:
- if is_fp8_model(model):
- return True
- for n, m in model.named_modules():
- if is_fp8_linear(m):
- m.is_fp8_linear = True
- if not hasattr(model, "is_fp8"):
- model.is_fp8 = True
- if hasattr(model, "is_fp8"):
- return True
- return False
-
-
def check_diffusers_installed(): # pragma: no cover
try:
import diffusers # noqa: F401
@@ -390,7 +328,7 @@ def llm_load_model(
)
model = model.eval()
- check_and_mark_fp8_model(model)
+ check_and_mark_quantized_module(model)
model = _to_model_dtype(model, model_dtype)
return model, tokenizer
@@ -540,7 +478,7 @@ def mllm_load_model(
pass
model = model.eval()
- check_and_mark_fp8_model(model)
+ check_and_mark_quantized_module(model)
model = _to_model_dtype(model, model_dtype)
return model, processor, tokenizer, image_processor
@@ -673,45 +611,6 @@ def is_moe_layer(module: torch.nn.Module) -> bool:
)
-def is_fp8_model(model: torch.nn.Module) -> bool:
- if not hasattr(model, "is_fp8"):
- return False
- else:
- return model.is_fp8
-
-
-def is_fp8_linear(module: torch.nn.Module) -> bool:
- """Check if a module is an FP8 linear layer.
-
- Detection follows this priority:
- 1. Explicit `is_fp8_linear` attribute (highest priority)
- 2. Registry lookup (for registered FP8 layer types)
- 3. Fallback dtype check (for backward compatibility with unregistered FP8 layers)
-
- Args:
- module: The module to check.
-
- Returns:
- True if the module is an FP8 linear layer, False otherwise.
- """
- # First check if explicitly marked
- if hasattr(module, "is_fp8_linear"):
- return module.is_fp8_linear
-
- # Check registry for supported FP8 layer types
- layer_name = module.__class__.__name__
- if layer_name in FP8_DEQUANT_REGISTRY:
- return True
-
- # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)
- # This maintains backward compatibility for layers not yet in registry
- if type(module) == torch.nn.Linear and module.weight is not None:
- if str(module.weight.dtype).startswith("torch.float8"):
- return True
-
- return False
-
-
def get_block_names(model, quant_vision=False):
"""Get the block names for transformers-like networks.
@@ -948,167 +847,6 @@ def set_nested_attr(module, attr_name: str, value):
setattr(module, attrs[-1], value)
-def _pad_weight(weight: torch.Tensor, block_size: list) -> tuple[torch.Tensor, int, int]:
- """Pads a matrix to make its dimensions multiples of block_size."""
- M, N = weight.shape[-2:]
- block_size_m, block_size_n = block_size
- pad_M = (block_size_m - M % block_size_m) % block_size_m
- pad_N = (block_size_n - N % block_size_n) % block_size_n
-
- if pad_M == 0 and pad_N == 0:
- return weight, M, N # No padding needed
- padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
- return padded_weight, M, N # Return original dimensions for unpadding
-
-
-def _unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:
- """Removes padding from the matrix to restore its original shape."""
- if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):
- return weight
- if keep_first_dim:
- return weight[:, :original_M, :original_N]
- else:
- return weight[:original_M, :original_N]
-
-
-def pad_block_fp8_weight_naive(
- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list
-) -> tuple[torch.Tensor, int, int]:
- assert len(block_size) == 2
-
- block_size_m, block_size_n = block_size
- weight_scale_m, weight_scale_n = weight_scale.shape[-2:]
-
- weight, orig_M, orig_N = _pad_weight(weight, block_size)
- M, N = weight.shape[-2:]
-
- assert weight_scale_m == M // block_size_m
- assert weight_scale_n == N // block_size_n
-
- return weight, orig_M, orig_N
-
-
-class with_thread_limits(ContextDecorator):
- """
- Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.
- Inspired by vLLM's thread limit decorator.
- https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115
- """
-
- def __init__(self, div=1):
- self.div = div
- self.old_omp = None
- self.old_torch = None
-
- def __enter__(self):
- # Save original settings
- self.old_omp = envs.AR_OMP_NUM_THREADS
- self.old_torch = torch.get_num_threads()
-
- try:
- if psutil is not None:
- num_cores = len(psutil.Process().cpu_affinity() or [])
- if num_cores == 0:
- num_cores = os.cpu_count() or 1
- else:
- num_cores = os.cpu_count() or 1
-
- # Set new limits
- new_threads = max(1, num_cores // self.div)
- os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)
- torch.set_num_threads(new_threads)
- except Exception:
- pass
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- # Restore original settings
- if self.old_omp is None:
- os.environ.pop("AR_OMP_NUM_THREADS", None)
- else:
- os.environ["AR_OMP_NUM_THREADS"] = self.old_omp
- torch.set_num_threads(self.old_torch)
-
-
-@with_thread_limits()
-def _dequant_fp8_linear_weight(
- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None
-) -> torch.Tensor:
- """Core dequantization logic for block-wise FP8 weights."""
- dtype = torch.bfloat16
- if weight_scale is None:
- return weight
-
- # If weight is stored as uint8, view it as float8_e4m3fn
- if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:
- weight = weight.view(torch.float8_e4m3fn)
-
- if block_size is None:
- if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:
- if weight_scale.numel() == weight.shape[0]:
- weight_scale = weight_scale.view(-1, 1)
- elif weight_scale.numel() == weight.shape[-1]:
- weight_scale = weight_scale.view(1, -1)
- return weight.to(dtype) * weight_scale.to(dtype)
-
- assert len(block_size) == 2
-
- weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)
-
- weight_shape_len = len(weight.shape)
- block_size_m, block_size_n = block_size
-
- # mul scale
- if weight_shape_len == 2:
- weight_scale_m, weight_scale_n = weight_scale.shape
- weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)
- weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
- dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
- keep_first_dim = False
- elif weight_shape_len == 3:
- fd, weight_scale_m, weight_scale_n = weight_scale.shape
- weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
- weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
- dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
- keep_first_dim = True
- else:
- raise ValueError("Only support original weight shape is either 2 or 3")
-
- dequant_weight = _unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)
-
- return dequant_weight
-
-
-# Register FP8 layer dequantization handlers
-# Note: Handlers are registered here (after _dequant_fp8_linear_weight is defined)
-# to ensure all dependencies are available.
-@register_fp8_layer("CompressedLinear")
-def dequant_compressed_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
- """Dequantize CompressedLinear layer using compressor."""
- layer = layer.to(device)
- return layer.compressor.decompress_module(layer)
-
-
-@register_fp8_layer("FP8Linear")
-def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
- """Dequantize FP8Linear layer using block-based dequantization."""
- layer = layer.to(device)
- weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv
- block_size = getattr(layer, "block_size", None)
- data_type = getattr(layer, "data_type", None)
- # Pass data_type if _dequant_fp8_linear_weight supports it
- # Check if function accepts data_type parameter
- import inspect
-
- sig = inspect.signature(_dequant_fp8_linear_weight)
- if "data_type" in sig.parameters:
- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
- else:
- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size)
-
-
def check_to_quantized(config):
"""Checks if the configuration is valid for quantization.
@@ -1166,76 +904,6 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):
)
-def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
- """Convert an FP8 layer to a standard Linear layer.
-
- Uses the FP8 dequantization registry to handle different FP8 layer types.
- Preserves quantization scheme attributes and other metadata.
-
- Args:
- layer: The FP8 layer to convert.
- dtype: Target dtype for the converted layer.
- device: Target device for the conversion.
-
- Returns:
- A new torch.nn.Linear layer with dequantized weights.
-
- Raises:
- NotImplementedError: If the layer type is not registered in the FP8 registry.
- """
- from auto_round.schemes import QuantizationScheme
-
- new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
- if layer.bias is not None:
- new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
-
- # Copy quantization scheme attributes
- scheme_keys = (f.name for f in fields(QuantizationScheme))
- keys = tuple(scheme_keys) + ("global_name", "scale_dtype")
- for key in keys:
- setattr(new_layer, key, getattr(layer, key, None))
-
- # Handle Gaudi2 device compatibility
- from auto_round.utils.device import is_gaudi2
-
- if is_gaudi2():
- device = "cpu"
- # Use registry-based dequantization
- dq_weight = dequant_fp8_layer(layer, dtype=dtype, device=device)
- new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
-
- return new_layer
-
-
-def convert_fp8_module_to_16b(model, dtype=torch.bfloat16, device: str = "cpu"):
- """
- Convert a model with FP8 quantized layers to a model with 16-bit linear layers.
- This is useful for compatibility with other frameworks or for further processing.
-
- Uses `is_fp8_linear` for detection, which supports all registered FP8 layer types
- via the FP8 dequantization registry.
-
- Args:
- model: The model with FP8 layers to convert.
- dtype: Target dtype for converted layers.
- device: Target device for conversion.
-
- Returns:
- The model with FP8 layers converted to 16-bit Linear layers.
- """
- from auto_round.utils.device import clear_memory
-
- cnt = 0
- for n, m in model.named_modules():
- if is_fp8_linear(m): # Use registry-based detection
- new_module = convert_fp8_layer_to_linear(m, dtype=dtype, device=device)
- set_module(model, n, new_module)
- cnt += 1
- if cnt % 10 == 0: # Tricky setting
- clear_memory()
- return model
-
-
def _to_model_dtype(model, model_dtype):
if model_dtype is not None:
try:
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
new file mode 100644
index 000000000..fb4e40298
--- /dev/null
+++ b/auto_round/utils/weight_handler.py
@@ -0,0 +1,643 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Extensible Module Weight Type Conversion Framework.
+
+This module provides a registry-based system for detecting and converting quantized
+module weights to high precision.
+
+Table of Contents:
+ 1. CORE FRAMEWORK
+ - ModuleWeightType: Enum of supported weight types
+ - WeightTypeHandler: Abstract base class for handlers
+ - Registry functions: register_weight_type_handler, get_handler, etc.
+
+ 2. PUBLIC API
+ - detect_weight_type(): Detect weight type of a layer or model
+ - check_and_mark_quantized_module(): Check and mark quantized layers
+ - is_quantized_input_module(): Check if model has quantized weights
+ - convert_module_to_hp_if_necessary(): Main conversion function
+
+ 3. HANDLER IMPLEMENTATIONS
+ - FP8BlockHandler: Fully implemented for FP8 block-wise quantization
+ - MXFP8Handler: Placeholder (TODO)
+ - MXFP4Handler: Placeholder (TODO)
+ - NVFP4Handler: Placeholder (TODO)
+
+Quick Start Guide:
+ Usage - Detect and Convert:
+ >>> from auto_round.utils.weight_handler import (
+ ... check_and_mark_quantized_module,
+ ... convert_module_to_hp_if_necessary,
+ ... )
+ >>> check_and_mark_quantized_module(model)
+ >>> model = convert_module_to_hp_if_necessary(model)
+
+ Adding a New Weight Type Handler:
+ 1. Add new type to ModuleWeightType enum
+ 2. Create handler class inheriting from WeightTypeHandler
+ 3. Register with @register_weight_type_handler decorator
+
+ Example:
+ @register_weight_type_handler(ModuleWeightType.MY_NEW_TYPE)
+ class MyNewTypeHandler(WeightTypeHandler):
+ def detect_layer(self, module): ...
+ def convert_layer(self, layer, dtype, device, to_cpu): ...
+"""
+
+import os
+from abc import ABC, abstractmethod
+from contextlib import ContextDecorator
+from dataclasses import fields
+from enum import Enum, auto
+from typing import Callable, Dict, Optional, Set, Type
+
+import psutil
+import torch
+
+from auto_round import envs
+from auto_round.logger import logger
+
+# ============================================================================
+# Section 0: UTILITY HELPERS
+# ============================================================================
+
+
+def _pad_weight(weight: torch.Tensor, block_size: list) -> tuple[torch.Tensor, int, int]:
+ """Pads a matrix to make its dimensions multiples of block_size."""
+ M, N = weight.shape[-2:]
+ block_size_m, block_size_n = block_size
+ pad_M = (block_size_m - M % block_size_m) % block_size_m
+ pad_N = (block_size_n - N % block_size_n) % block_size_n
+
+ if pad_M == 0 and pad_N == 0:
+ return weight, M, N # No padding needed
+ padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
+ return padded_weight, M, N # Return original dimensions for unpadding
+
+
+def _unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:
+ """Removes padding from the matrix to restore its original shape."""
+ if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):
+ return weight
+ if keep_first_dim:
+ return weight[:, :original_M, :original_N]
+ else:
+ return weight[:original_M, :original_N]
+
+
+class with_thread_limits(ContextDecorator):
+ """
+ Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.
+ Inspired by vLLM's thread limit decorator.
+ https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115
+ """
+
+ def __init__(self, div=1):
+ self.div = div
+ self.old_omp = None
+ self.old_torch = None
+
+ def __enter__(self):
+ # Save original settings
+ self.old_omp = envs.AR_OMP_NUM_THREADS
+ self.old_torch = torch.get_num_threads()
+
+ try:
+ if psutil is not None:
+ num_cores = len(psutil.Process().cpu_affinity() or [])
+ if num_cores == 0:
+ num_cores = os.cpu_count() or 1
+ else:
+ num_cores = os.cpu_count() or 1
+
+ # Set new limits
+ new_threads = max(1, num_cores // self.div)
+ os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)
+ torch.set_num_threads(new_threads)
+ except Exception:
+ pass
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Restore original settings
+ if self.old_omp is None:
+ os.environ.pop("AR_OMP_NUM_THREADS", None)
+ else:
+ os.environ["AR_OMP_NUM_THREADS"] = self.old_omp
+ torch.set_num_threads(self.old_torch)
+
+
+# ============================================================================
+# Section 1: CORE FRAMEWORK - Types, Base Classes, and Registry
+# ============================================================================
+
+
+class ModuleWeightType(Enum):
+ """Enumeration of supported quantized module weight types.
+
+ When adding a new weight type:
+ 1. Add the new type here
+ 2. Create a handler class (see Section 3)
+ 3. Register with @register_weight_type_handler decorator
+ """
+
+ FP8_BLOCK = auto() # FP8 with block-wise scaling (fully implemented)
+ MXFP8 = auto() # MX FP8 (placeholder)
+ MXFP4 = auto() # MX FP4 (placeholder)
+ NVFP4 = auto() # NV FP4 (placeholder)
+
+
+class WeightTypeHandler(ABC):
+ """Abstract base class for weight type detection and conversion handlers.
+
+ Subclasses must implement:
+ - detect_layer(): Check if a single layer is of this weight type
+ - convert_layer(): Convert a single layer to high precision
+ """
+
+ @abstractmethod
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a single layer is of this weight type.
+
+ Args:
+ module: The module to check.
+
+ Returns:
+ True if the module is of this weight type, False otherwise.
+ """
+ pass
+
+ @abstractmethod
+ def convert_layer(
+ self,
+ layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+ ) -> torch.nn.Module:
+ """Convert a single layer to high precision.
+
+ Args:
+ layer: The quantized layer to convert.
+ dtype: Target dtype for the converted layer.
+ device: Target device for the conversion.
+ to_cpu: If True, move converted layer to CPU.
+
+ Returns:
+ A new high-precision layer with dequantized weights.
+ """
+ pass
+
+
+# --- Handler Registry ---
+
+_WEIGHT_TYPE_HANDLERS: Dict[ModuleWeightType, WeightTypeHandler] = {}
+
+
+def register_weight_type_handler(weight_type: ModuleWeightType):
+ """Decorator to register a weight type handler.
+
+ Args:
+ weight_type: The ModuleWeightType this handler supports.
+
+ Returns:
+ Decorator function that registers the handler class.
+
+ Example:
+ @register_weight_type_handler(ModuleWeightType.MXFP4)
+ class MXFP4Handler(WeightTypeHandler):
+ ...
+ """
+
+ def decorator(handler_cls: Type[WeightTypeHandler]):
+ if not issubclass(handler_cls, WeightTypeHandler):
+ raise TypeError(f"Handler {handler_cls.__name__} must be a subclass of WeightTypeHandler")
+ _WEIGHT_TYPE_HANDLERS[weight_type] = handler_cls()
+ return handler_cls
+
+ return decorator
+
+
+def get_handler(weight_type: ModuleWeightType) -> Optional[WeightTypeHandler]:
+ """Get the registered handler for a weight type.
+
+ Args:
+ weight_type: The ModuleWeightType to get the handler for.
+
+ Returns:
+ The registered handler, or None if not registered.
+ """
+ return _WEIGHT_TYPE_HANDLERS.get(weight_type)
+
+
+def get_all_handlers() -> Dict[ModuleWeightType, WeightTypeHandler]:
+ """Get all registered weight type handlers.
+
+ Returns:
+ Dictionary mapping weight types to their handlers.
+ """
+ return _WEIGHT_TYPE_HANDLERS.copy()
+
+
+# ============================================================================
+# Section 2: PUBLIC API - Detection and Conversion Functions
+# ============================================================================
+def detect_weight_type(module: torch.nn.Module) -> Optional[ModuleWeightType]:
+ """Detect the weight type of a module or model.
+
+ First checks if the module itself has a quantized_weight_type attribute.
+ If not found, scans all submodules to find any with quantized_weight_type attribute.
+
+ Args:
+ module: The module or model to check.
+
+ Returns:
+ The detected ModuleWeightType, or None if no match found.
+ """
+ # Check if module itself is marked
+ if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:
+ return module.quantized_weight_type
+
+ # Scan submodules for quantized_weight_type attribute
+ for submodule in module.modules():
+ if hasattr(submodule, "quantized_weight_type") and submodule.quantized_weight_type is not None:
+ return submodule.quantized_weight_type
+
+ return None
+
+
+# --- Model Marking Functions ---
+def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightType]:
+ """Check if model contains quantized layers and mark them accordingly.
+
+ This function scans the model (including the model itself) for quantized layers using
+ handlers' detect_layer method (which checks actual characteristics). It marks detected
+ layers with `quantized_weight_type`.
+
+ Args:
+ model: The model to check and mark.
+
+ Returns:
+ A set of detected ModuleWeightType values. Empty set if no quantized layers found.
+ """
+ detected_types: Set[ModuleWeightType] = set()
+ for weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():
+ # Check model itself first
+ if handler.detect_layer(model):
+ model.quantized_weight_type = weight_type
+ detected_types.add(weight_type)
+
+ # Then check all submodules
+ for n, m in model.named_modules():
+ # Use handler to detect based on actual characteristics
+ if handler.detect_layer(m):
+ # Mark the layer itself
+ m.quantized_weight_type = weight_type
+ # Record detected types
+ detected_types.add(weight_type)
+
+ return detected_types
+
+
+def is_quantized_input_module(model: torch.nn.Module) -> Optional[ModuleWeightType]:
+ """Check if a model has quantized input weights and return the weight type.
+
+ This traverses all submodules to check for the `quantized_weight_type` attribute
+ set by `check_and_mark_quantized_module`.
+
+ Args:
+ model: The model to check.
+
+ Returns:
+ The ModuleWeightType if the model has quantized weights, None otherwise.
+ """
+ # Check model itself first
+ if hasattr(model, "quantized_weight_type") and model.quantized_weight_type is not None:
+ return model.quantized_weight_type
+
+ # Traverse all submodules to find quantized layers
+ for module in model.modules():
+ if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:
+ return module.quantized_weight_type
+
+ return None
+
+
+# --- Main Conversion Function ---
+def convert_module_to_hp_if_necessary(
+ model_or_layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+) -> torch.nn.Module:
+ """Convert quantized layer(s) to high-precision Linear layer(s) if necessary.
+
+ This function automatically detects the weight type and uses the appropriate
+ handler for conversion. If the input is not quantized, it returns unchanged.
+
+ Args:
+ model_or_layer: Either a single layer, a block, or an entire model.
+ dtype: Target dtype for the converted layer(s). Defaults to torch.bfloat16.
+ device: Target device for the conversion. Defaults to "cpu".
+ to_cpu: If True, move converted layers to CPU. Defaults to False.
+
+ Returns:
+ If input is a quantized layer: A new high-precision layer with dequantized weights.
+ If input is a model containing quantized layers: The model with layers converted.
+ If input has no quantized layers: Returns the input unchanged.
+ """
+ from auto_round.utils.device import clear_memory
+ from auto_round.utils.model import set_module
+
+ # Check if it's a single quantized layer (has the attribute directly)
+ if hasattr(model_or_layer, "quantized_weight_type") and model_or_layer.quantized_weight_type is not None:
+ handler = get_handler(model_or_layer.quantized_weight_type)
+ return handler.convert_layer(model_or_layer, dtype, device, to_cpu)
+
+ # Otherwise, traverse model and convert all quantized layers
+ # Get handler for each layer to support mixed quantization types
+ cnt = 0
+ for n, m in model_or_layer.named_modules():
+ if hasattr(m, "quantized_weight_type") and m.quantized_weight_type is not None:
+ handler = get_handler(m.quantized_weight_type)
+ new_module = handler.convert_layer(m, dtype, device, to_cpu)
+ set_module(model_or_layer, n, new_module)
+ cnt += 1
+ if cnt % 10 == 0:
+ clear_memory()
+
+ return model_or_layer
+
+
+# ============================================================================
+# Section 3: FP8 BLOCK DEQUANTIZATION HELPERS
+# ============================================================================
+
+
+def _pad_block_fp8_weight_naive(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list
+) -> tuple[torch.Tensor, int, int]:
+ assert len(block_size) == 2
+
+ block_size_m, block_size_n = block_size
+ weight_scale_m, weight_scale_n = weight_scale.shape[-2:]
+
+ weight, orig_M, orig_N = _pad_weight(weight, block_size)
+ M, N = weight.shape[-2:]
+
+ assert weight_scale_m == M // block_size_m
+ assert weight_scale_n == N // block_size_n
+
+ return weight, orig_M, orig_N
+
+
+@with_thread_limits()
+def _dequant_fp8_linear_weight(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None
+) -> torch.Tensor:
+ """Core dequantization logic for block-wise FP8 weights."""
+ dtype = torch.bfloat16
+ if weight_scale is None:
+ return weight
+
+ # If weight is stored as uint8, view it as float8_e4m3fn
+ if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:
+ weight = weight.view(torch.float8_e4m3fn)
+
+ if block_size is None:
+ if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:
+ if weight_scale.numel() == weight.shape[0]:
+ weight_scale = weight_scale.view(-1, 1)
+ elif weight_scale.numel() == weight.shape[-1]:
+ weight_scale = weight_scale.view(1, -1)
+ return weight.to(dtype) * weight_scale.to(dtype)
+
+ assert len(block_size) == 2
+
+ weight, orig_M, orig_N = _pad_block_fp8_weight_naive(weight, weight_scale, block_size)
+
+ weight_shape_len = len(weight.shape)
+ block_size_m, block_size_n = block_size
+
+ # mul scale
+ if weight_shape_len == 2:
+ weight_scale_m, weight_scale_n = weight_scale.shape
+ weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)
+ weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ keep_first_dim = False
+ elif weight_shape_len == 3:
+ fd, weight_scale_m, weight_scale_n = weight_scale.shape
+ weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
+ weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ keep_first_dim = True
+ else:
+ raise ValueError("Only support original weight shape is either 2 or 3")
+
+ dequant_weight = _unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)
+
+ return dequant_weight
+
+
+# ============================================================================
+# Section 4: HANDLER IMPLEMENTATIONS
+# ============================================================================
+
+
+# ----------------------------------------------------------------------------
+# FP8 Block Handler - Full Implementation (Reference Example)
+# ----------------------------------------------------------------------------
+@register_weight_type_handler(ModuleWeightType.FP8_BLOCK)
+class FP8BlockHandler(WeightTypeHandler):
+ """Handler for FP8 block-wise quantized layers.
+
+ This handler supports:
+ - FP8Linear layers with block-wise scaling
+ - CompressedLinear layers with FP8 compression
+ - torch.nn.Linear layers with FP8 dtype weights
+ """
+
+ # Registry mapping FP8 layer class names to their dequantization handlers
+ FP8_DEQUANT_REGISTRY: Dict[str, Callable] = {}
+
+ @classmethod
+ def register_fp8_layer(cls, layer_name: str):
+ """Register a dequantization handler for an FP8 layer type.
+
+ Args:
+ layer_name: The class name of the FP8 layer type.
+
+ Returns:
+ Decorator function that registers the handler.
+ """
+
+ def decorator(fn: Callable):
+ cls.FP8_DEQUANT_REGISTRY[layer_name] = fn
+ return fn
+
+ return decorator
+
+ def _dequant_layer(self, layer: torch.nn.Module, dtype: torch.dtype, device: str) -> torch.Tensor:
+ """Dequantize an FP8 layer using the internal registry.
+
+ Args:
+ layer: The FP8 layer to dequantize.
+ dtype: Target dtype for dequantized weights.
+ device: Target device for dequantization.
+
+ Returns:
+ Dequantized weight tensor.
+
+ Raises:
+ NotImplementedError: If the layer type is not registered.
+ """
+ name = layer.__class__.__name__
+ if name not in self.FP8_DEQUANT_REGISTRY:
+ raise NotImplementedError(
+ f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(self.FP8_DEQUANT_REGISTRY.keys())}"
+ )
+ return self.FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)
+
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a module is an FP8 linear layer based on actual characteristics."""
+ # Check registry for supported FP8 layer types
+ layer_name = module.__class__.__name__
+ if layer_name in self.FP8_DEQUANT_REGISTRY:
+ return True
+
+ # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)
+ if type(module) == torch.nn.Linear and module.weight is not None:
+ if str(module.weight.dtype).startswith("torch.float8"):
+ return True
+
+ return False
+
+ def convert_layer(
+ self,
+ layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+ ) -> torch.nn.Module:
+ """Convert a single FP8 layer to a standard Linear layer."""
+ from auto_round.schemes import QuantizationScheme
+ from auto_round.utils.device import is_gaudi2
+ from auto_round.utils.model import set_module
+
+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
+ if layer.bias is not None:
+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
+
+ # Copy quantization scheme attributes
+ scheme_keys = (f.name for f in fields(QuantizationScheme))
+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
+ setattr(new_layer, key, getattr(layer, key, None))
+
+ # Handle Gaudi2 device compatibility
+ if is_gaudi2():
+ device = "cpu"
+
+ # Use registry-based dequantization
+ dq_weight = self._dequant_layer(layer, dtype=dtype, device=device)
+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+
+ if to_cpu:
+ new_layer = new_layer.to("cpu")
+
+ return new_layer
+
+
+# --- FP8 Layer Dequantization Handlers ---
+# Register specific dequantization logic for different FP8 layer types
+
+
[email protected]_fp8_layer("CompressedLinear")
+def _dequant_compressed_linear(
+ layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"
+) -> torch.Tensor:
+ """Dequantize CompressedLinear layer using compressor.
+
+ Args:
+ layer: The CompressedLinear layer to dequantize.
+ dtype: Target dtype for dequantized weights.
+ device: Target device for dequantization.
+
+ Returns:
+ Dequantized weight tensor.
+ """
+ layer = layer.to(device)
+ return layer.compressor.decompress_module(layer)
+
+
[email protected]_fp8_layer("FP8Linear")
+def _dequant_fp8_linear(
+ layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"
+) -> torch.Tensor:
+ """Dequantize FP8Linear layer using block-based dequantization.
+
+ Args:
+ layer: The FP8Linear layer to dequantize.
+ dtype: Target dtype for dequantized weights.
+ device: Target device for dequantization.
+
+ Returns:
+ Dequantized weight tensor.
+ """
+ layer = layer.to(device)
+ weight_scale = getattr(layer, "weight_scale", None)
+ block_size = getattr(layer, "block_size", None)
+ data_type = getattr(layer, "data_type", None)
+ if weight_scale is None:
+ weight_scale = getattr(layer, "weight_scale_inv", None)
+ if weight_scale is None:
+ raise AttributeError(
+ "FP8Linear layer is missing both 'weight_scale' and 'weight_scale_inv' "
+ "attributes required for dequantization."
+ )
+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
+
+
+# ----------------------------------------------------------------------------
+# Placeholder Handlers for Unimplemented Types
+# ----------------------------------------------------------------------------
+
+
+def _create_placeholder_handler(weight_type_name: str) -> Type[WeightTypeHandler]:
+ """Factory function to create placeholder handlers for unimplemented weight types."""
+
+ class PlaceholderHandler(WeightTypeHandler):
+ """Placeholder handler - detection always returns False."""
+
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ return False
+
+ def convert_layer(self, layer, dtype=torch.bfloat16, device="cpu", to_cpu=False):
+ raise NotImplementedError(f"{weight_type_name} layer conversion not yet implemented")
+
+ PlaceholderHandler.__name__ = f"{weight_type_name}Handler"
+ PlaceholderHandler.__doc__ = (
+ f"Placeholder handler for {weight_type_name}. TODO: Implement detection and conversion."
+ )
+ return PlaceholderHandler
+
+
+# Register placeholder handlers for unimplemented types
+register_weight_type_handler(ModuleWeightType.MXFP8)(_create_placeholder_handler("MXFP8"))
+register_weight_type_handler(ModuleWeightType.MXFP4)(_create_placeholder_handler("MXFP4"))
+register_weight_type_handler(ModuleWeightType.NVFP4)(_create_placeholder_handler("NVFP4"))
diff --git a/test/fixtures.py b/test/fixtures.py
index cf9ad806e..482b74598 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -123,6 +123,15 @@ def tiny_qwen_2_5_vl_model_path():
shutil.rmtree(tiny_model_path, ignore_errors=True)
+# Mock torch.cuda.get_device_capability to always return (9, 0) like H100
[email protected](autouse=True, scope="session")
+def mock_cuda_capability():
+ from unittest.mock import patch
+
+ with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
+ yield
+
+
@pytest.fixture(autouse=True, scope="session")
def clean_tmp_model_folder():
yield
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 42a7dc83f..d2f6cabcd 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -660,10 +660,14 @@ class TestAutoRound:
assert dequant_weight.shape.numel() == 32 * 5760 * 1440
def test_dequant_g2_fp8_weight(self):
- """Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""
+ """Test Gaudi2-specific dequantization redirection logic in convert_module_to_hp_if_necessary."""
from unittest.mock import MagicMock, patch
- from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear
+ from auto_round.utils.model import (
+ _dequant_fp8_linear_weight,
+ check_and_mark_quantized_module,
+ convert_module_to_hp_if_necessary,
+ )
# Test 1: Verify Standard e4m3fn native dequantization result
weight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)
@@ -674,7 +678,7 @@ class TestAutoRound:
assert dq_weight.dtype == torch.bfloat16
assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3
- # Test 2: Verify Gaudi2 redirection in convert_fp8_layer_to_linear
+ # Test 2: Verify Gaudi2 redirection in convert_module_to_hp_if_necessary
mock_layer = MagicMock()
mock_layer.in_features = 2
mock_layer.out_features = 1
@@ -689,14 +693,16 @@ class TestAutoRound:
mock_layer.to.return_value = mock_layer
with patch("auto_round.utils.device.is_gaudi2", return_value=True):
- convert_fp8_layer_to_linear(mock_layer, device="hpu")
+ check_and_mark_quantized_module(mock_layer)
+ convert_module_to_hp_if_necessary(mock_layer, device="hpu")
# Verify it was moved to CPU
mock_layer.to.assert_called_with("cpu")
with patch("auto_round.utils.device.is_gaudi2", return_value=False):
# Reset mock
mock_layer.to.reset_mock()
- convert_fp8_layer_to_linear(mock_layer, device="hpu")
+ check_and_mark_quantized_module(mock_layer)
+ convert_module_to_hp_if_necessary(mock_layer, device="hpu")
# Verify it was moved to HPU (as requested in device arg)
mock_layer.to.assert_called_with("hpu")
diff --git a/test/test_cpu/core/test_fp8_re_quant.py b/test/test_cpu/utils/test_fp8_re_quant.py
similarity index 93%
rename from test/test_cpu/core/test_fp8_re_quant.py
rename to test/test_cpu/utils/test_fp8_re_quant.py
index 59e42a543..21596f2de 100644
--- a/test/test_cpu/core/test_fp8_re_quant.py
+++ b/test/test_cpu/utils/test_fp8_re_quant.py
@@ -4,7 +4,11 @@ import torch
import torch.nn as nn
from transformers import AutoConfig
-from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear
+from auto_round.utils.model import (
+ _dequant_fp8_linear_weight,
+ check_and_mark_quantized_module,
+ convert_module_to_hp_if_necessary,
+)
class MockFP8Layer:
@@ -85,7 +89,8 @@ class TestFP8ReQuant(unittest.TestCase):
mock_layer = FP8Linear(in_features, out_features)
# This should now pass without AttributeError for 'block_size'
- new_layer = convert_fp8_layer_to_linear(mock_layer, dtype=torch.bfloat16)
+ check_and_mark_quantized_module(mock_layer)
+ new_layer = convert_module_to_hp_if_necessary(mock_layer, dtype=torch.bfloat16)
self.assertIsInstance(new_layer, nn.Linear)
self.assertEqual(new_layer.in_features, in_features)
diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.py
index e2ca5b359..a1dfe7023 100644
--- a/test/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py
@@ -1,6 +1,5 @@
import copy
import shutil
-from unittest.mock import patch
import pytest
import torch
@@ -13,13 +12,6 @@ from auto_round.testing_utils import require_awq, require_optimum
from ...helpers import get_model_path, save_tiny_model
-# Mock torch.cuda.get_device_capability to always return (9, 0) like H100
-def mock_cuda_capability():
- with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
- yield
-
-
class TestAutoRound:
save_dir = "./saved"