PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Extensible Module Weight Type Conversion Framework. (#1375)
Xin He53f20cf12d76
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 57e771478..97702db3e 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -73,14 +73,13 @@ from auto_round.utils import (SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,CpuInfo,- check_and_mark_fp8_model,+ check_and_mark_quantized_module,check_seqlen_compatible,check_to_quantized,clear_memory,compile_func,convert_dtype_str2torch,- convert_fp8_layer_to_linear,- convert_fp8_module_to_16b,+ convert_module_to_hp_if_necessary,detect_device,find_matching_blocks,flatten_list,@@ -92,11 +91,9 @@ from auto_round.utils import (htcore,is_auto_device_mapping,is_debug_mode,- is_fp8_linear,- is_fp8_model,is_hpex_available,is_moe_model,- is_moe_model_via_config,+ is_quantized_input_module,llm_load_model,memory_monitor,mv_module_from_gpu,@@ -296,7 +293,7 @@ class BaseCompressor(object):"AutoRound does not support parameters on meta device. ""Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU.")- check_and_mark_fp8_model(model)+ check_and_mark_quantized_module(model)self.model = model.eval()self.tokenizer = tokenizerself.shared_cache_keys = get_shared_keys(self.model)@@ -514,8 +511,8 @@ class BaseCompressor(object):logger.info("AutoScheme is not yet supported for multimodal LLMs.")sys.exit(-1)- if getattr(model, "is_fp8", False):- logger.info("AutoScheme does not currently support FP8 models.")+ if is_quantized_input_module(model):+ logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")sys.exit(-1)all_dtypes = []@@ -1145,9 +1142,8 @@ class BaseCompressor(object):if dtype is not None:m = m.to(dtype)- if is_fp8_linear(m):- m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)- set_module(self.model, name, m)+ m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)+ set_module(self.model, name, m)tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device# Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logicif self.is_immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:@@ -1254,9 +1250,8 @@ class BaseCompressor(object):for name in pbar:pbar.set_description(f"Calculate weight global scale: {name}")m = get_module(self.model, name)- if is_fp8_linear(m):- m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)- set_module(self.model, name, m)+ m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)+ set_module(self.model, name, m)weight_global_scale = calculate_gparam(m.weight, self.group_size)setattr(m, "weight_global_scale", weight_global_scale)@@ -1383,8 +1378,7 @@ class BaseCompressor(object):m.to("meta")# Convert remaining fp8- if is_fp8_model(self.model):- convert_fp8_module_to_16b(self.model, self.amp_dtype, self.device)+ convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)if self.is_immediate_saving:shard_writer(self, is_finalize=True)@@ -1455,8 +1449,7 @@ class BaseCompressor(object):block = get_module(self.model, block_name)materialize_model_(block)block.to("cpu")- if is_fp8_model(self.model):- convert_fp8_module_to_16b(block, dtype=self.amp_dtype, device=self.device)+ convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:set_auto_device_map_for_block_with_tuning(@@ -1751,11 +1744,7 @@ class BaseCompressor(object):pbar.close()self._quantize_layers(layer_names, all_inputs)- if is_fp8_model(self.model):- for n, m in self.model.named_modules():- if is_fp8_linear(m):- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to("cpu")- set_module(self.model, n, new_layer)+ convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device, to_cpu=True)if self.is_immediate_saving:shard_writer(self, is_finalize=True)@@ -1819,10 +1808,8 @@ class BaseCompressor(object):layer = get_module(self.model, layer_name)layer = layer.to(self.device)- if is_fp8_linear(layer):- new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype, self.device).to(self.device)- set_module(self.model, layer_name, new_layer)- layer = new_layer+ layer = convert_module_to_hp_if_necessary(layer, self.amp_dtype, self.device)+ set_module(self.model, layer_name, layer)wrapper_layer = WrapperLinear(layer,@@ -2129,7 +2116,7 @@ class BaseCompressor(object):Raises:Exception: If caching on GPU fails, switches to CPU and caches there."""- if is_fp8_model(self.model):+ if is_quantized_input_module(self.model):layer_names = []if layer_names is None:layer_names = []@@ -2780,11 +2767,7 @@ class BaseCompressor(object):Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)"""materialize_model_(block)- if is_fp8_model(self.model):- for n, m in block.named_modules():- if is_fp8_linear(m):- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to(device)- set_module(block, n, new_layer)+ convert_module_to_hp_if_necessary(block, self.amp_dtype, device)if auto_offload:# card_0_in_high_risk indicates that card_0 memory is already in high usage (90%) w/o any weightsdiff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 730f8e205..b5cfd8598 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -41,7 +41,6 @@ from auto_round.utils import (find_matching_blocks,get_block_names,get_max_vram,- is_fp8_model,mllm_load_model,mv_module_from_gpu,to_device,diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 500ddc115..8d890df8f 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -26,7 +26,7 @@ from torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme-from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_model+from auto_round.utils import check_to_quantizedclass BackendDataType(str, Enum):@@ -120,19 +120,6 @@ def block_forward(return output-def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:- if is_fp8_model(model):- return True- for n, m in model.named_modules():- if is_fp8_linear(m):- m.is_fp8_linear = True- if not hasattr(model, "is_fp8"):- model.is_fp8 = True- if hasattr(model, "is_fp8"):- return True- return False--def check_skippable_keywords(key):"""Prints a reminder if a key is not stored during quantization fine-tuning.diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 5b9252fe2..6a22a13e2 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -56,7 +56,7 @@ from auto_round.utils import (clear_memory,get_module,get_packing_device,- is_fp8_model,+ is_quantized_input_module,is_separate_tensor,logger,)@@ -159,7 +159,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:yield name, tensordef is_extra_tensor(tensor_name):- if is_fp8_model(cls.model) and "scale" in tensor_name.split(".")[-1]:+ if is_quantized_input_module(cls.model) and "scale" in tensor_name.split(".")[-1]:return Falseif tensor_name not in cls.model.tensor_name_list:return Truediff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.pyindex d005b7d35..32d976702 100644--- a/auto_round/utils/__init__.py+++ b/auto_round/utils/__init__.py@@ -15,6 +15,10 @@from auto_round.utils.device import *from auto_round.utils.common import *from auto_round.utils.model import *+from auto_round.utils.weight_handler import (+ detect_weight_type,+ is_quantized_input_module,+)import transformersfrom packaging.version import Versiondiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 66af26c9e..0371caf2f 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -16,9 +16,6 @@ import jsonimport osimport refrom collections import UserDict-from contextlib import ContextDecorator, contextmanager-from dataclasses import asdict, fields-from functools import wrapsfrom pathlib import Pathfrom typing import Union@@ -31,58 +28,12 @@ from auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme--# ============================================================================-# FP8 Dequantization Registry-# ============================================================================-# Registry mapping FP8 layer class names to their dequantization handlers.-# This ensures detection and dequantization logic stay in sync.-FP8_DEQUANT_REGISTRY = {}---def register_fp8_layer(layer_name: str):- """Register a dequantization handler for an FP8 layer type.-- Args:- layer_name: The class name of the FP8 layer type (e.g., "FP8Linear", "CompressedLinear")-- Returns:- Decorator function that registers the handler.-- Example:- @register_fp8_layer("FP8Linear")- def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):- # Dequantization logic- return dequantized_weight- """-- def decorator(fn):- FP8_DEQUANT_REGISTRY[layer_name] = fn- return fn-- return decorator---def dequant_fp8_layer(layer, dtype=torch.bfloat16, device: str = "cpu"):- """Dequantize an FP8 layer using the registry.-- Args:- layer: The FP8 layer to dequantize.- dtype: Target dtype for dequantized weights.- device: Target device for dequantization.-- Returns:- Dequantized weight tensor.-- Raises:- NotImplementedError: If the layer type is not registered.- """- name = layer.__class__.__name__- if name not in FP8_DEQUANT_REGISTRY:- raise NotImplementedError(- f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(FP8_DEQUANT_REGISTRY.keys())}"- )- return FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)+from auto_round.utils.weight_handler import (+ _dequant_fp8_linear_weight,+ check_and_mark_quantized_module,+ convert_module_to_hp_if_necessary,+ is_quantized_input_module,+)def clean_module_parameter(submodule: torch.nn.Module, param_name: str) -> None:@@ -192,19 +143,6 @@ def convert_dtype_torch2str_hf(dtype):return str_dtype-def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:- if is_fp8_model(model):- return True- for n, m in model.named_modules():- if is_fp8_linear(m):- m.is_fp8_linear = True- if not hasattr(model, "is_fp8"):- model.is_fp8 = True- if hasattr(model, "is_fp8"):- return True- return False--def check_diffusers_installed(): # pragma: no covertry:import diffusers # noqa: F401@@ -390,7 +328,7 @@ def llm_load_model()model = model.eval()- check_and_mark_fp8_model(model)+ check_and_mark_quantized_module(model)model = _to_model_dtype(model, model_dtype)return model, tokenizer@@ -540,7 +478,7 @@ def mllm_load_model(passmodel = model.eval()- check_and_mark_fp8_model(model)+ check_and_mark_quantized_module(model)model = _to_model_dtype(model, model_dtype)return model, processor, tokenizer, image_processor@@ -673,45 +611,6 @@ def is_moe_layer(module: torch.nn.Module) -> bool:)-def is_fp8_model(model: torch.nn.Module) -> bool:- if not hasattr(model, "is_fp8"):- return False- else:- return model.is_fp8---def is_fp8_linear(module: torch.nn.Module) -> bool:- """Check if a module is an FP8 linear layer.-- Detection follows this priority:- 1. Explicit `is_fp8_linear` attribute (highest priority)- 2. Registry lookup (for registered FP8 layer types)- 3. Fallback dtype check (for backward compatibility with unregistered FP8 layers)-- Args:- module: The module to check.-- Returns:- True if the module is an FP8 linear layer, False otherwise.- """- # First check if explicitly marked- if hasattr(module, "is_fp8_linear"):- return module.is_fp8_linear-- # Check registry for supported FP8 layer types- layer_name = module.__class__.__name__- if layer_name in FP8_DEQUANT_REGISTRY:- return True-- # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)- # This maintains backward compatibility for layers not yet in registry- if type(module) == torch.nn.Linear and module.weight is not None:- if str(module.weight.dtype).startswith("torch.float8"):- return True-- return False--def get_block_names(model, quant_vision=False):"""Get the block names for transformers-like networks.@@ -948,167 +847,6 @@ def set_nested_attr(module, attr_name: str, value):setattr(module, attrs[-1], value)-def _pad_weight(weight: torch.Tensor, block_size: list) -> tuple[torch.Tensor, int, int]:- """Pads a matrix to make its dimensions multiples of block_size."""- M, N = weight.shape[-2:]- block_size_m, block_size_n = block_size- pad_M = (block_size_m - M % block_size_m) % block_size_m- pad_N = (block_size_n - N % block_size_n) % block_size_n-- if pad_M == 0 and pad_N == 0:- return weight, M, N # No padding needed- padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)- return padded_weight, M, N # Return original dimensions for unpadding---def _unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:- """Removes padding from the matrix to restore its original shape."""- if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):- return weight- if keep_first_dim:- return weight[:, :original_M, :original_N]- else:- return weight[:original_M, :original_N]---def pad_block_fp8_weight_naive(- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list-) -> tuple[torch.Tensor, int, int]:- assert len(block_size) == 2-- block_size_m, block_size_n = block_size- weight_scale_m, weight_scale_n = weight_scale.shape[-2:]-- weight, orig_M, orig_N = _pad_weight(weight, block_size)- M, N = weight.shape[-2:]-- assert weight_scale_m == M // block_size_m- assert weight_scale_n == N // block_size_n-- return weight, orig_M, orig_N---class with_thread_limits(ContextDecorator):- """- Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.- Inspired by vLLM's thread limit decorator.- https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115- """-- def __init__(self, div=1):- self.div = div- self.old_omp = None- self.old_torch = None-- def __enter__(self):- # Save original settings- self.old_omp = envs.AR_OMP_NUM_THREADS- self.old_torch = torch.get_num_threads()-- try:- if psutil is not None:- num_cores = len(psutil.Process().cpu_affinity() or [])- if num_cores == 0:- num_cores = os.cpu_count() or 1- else:- num_cores = os.cpu_count() or 1-- # Set new limits- new_threads = max(1, num_cores // self.div)- os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)- torch.set_num_threads(new_threads)- except Exception:- pass- return self-- def __exit__(self, exc_type, exc_val, exc_tb):- # Restore original settings- if self.old_omp is None:- os.environ.pop("AR_OMP_NUM_THREADS", None)- else:- os.environ["AR_OMP_NUM_THREADS"] = self.old_omp- torch.set_num_threads(self.old_torch)---@with_thread_limits()-def _dequant_fp8_linear_weight(- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None-) -> torch.Tensor:- """Core dequantization logic for block-wise FP8 weights."""- dtype = torch.bfloat16- if weight_scale is None:- return weight-- # If weight is stored as uint8, view it as float8_e4m3fn- if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:- weight = weight.view(torch.float8_e4m3fn)-- if block_size is None:- if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:- if weight_scale.numel() == weight.shape[0]:- weight_scale = weight_scale.view(-1, 1)- elif weight_scale.numel() == weight.shape[-1]:- weight_scale = weight_scale.view(1, -1)- return weight.to(dtype) * weight_scale.to(dtype)-- assert len(block_size) == 2-- weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)-- weight_shape_len = len(weight.shape)- block_size_m, block_size_n = block_size-- # mul scale- if weight_shape_len == 2:- weight_scale_m, weight_scale_n = weight_scale.shape- weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)- weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)- dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)- keep_first_dim = False- elif weight_shape_len == 3:- fd, weight_scale_m, weight_scale_n = weight_scale.shape- weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)- weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)- dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)- keep_first_dim = True- else:- raise ValueError("Only support original weight shape is either 2 or 3")-- dequant_weight = _unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)-- return dequant_weight---# Register FP8 layer dequantization handlers-# Note: Handlers are registered here (after _dequant_fp8_linear_weight is defined)-# to ensure all dependencies are available.-@register_fp8_layer("CompressedLinear")-def dequant_compressed_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):- """Dequantize CompressedLinear layer using compressor."""- layer = layer.to(device)- return layer.compressor.decompress_module(layer)---@register_fp8_layer("FP8Linear")-def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):- """Dequantize FP8Linear layer using block-based dequantization."""- layer = layer.to(device)- weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv- block_size = getattr(layer, "block_size", None)- data_type = getattr(layer, "data_type", None)- # Pass data_type if _dequant_fp8_linear_weight supports it- # Check if function accepts data_type parameter- import inspect-- sig = inspect.signature(_dequant_fp8_linear_weight)- if "data_type" in sig.parameters:- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)- else:- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size)--def check_to_quantized(config):"""Checks if the configuration is valid for quantization.@@ -1166,76 +904,6 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):)-def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):- """Convert an FP8 layer to a standard Linear layer.-- Uses the FP8 dequantization registry to handle different FP8 layer types.- Preserves quantization scheme attributes and other metadata.-- Args:- layer: The FP8 layer to convert.- dtype: Target dtype for the converted layer.- device: Target device for the conversion.-- Returns:- A new torch.nn.Linear layer with dequantized weights.-- Raises:- NotImplementedError: If the layer type is not registered in the FP8 registry.- """- from auto_round.schemes import QuantizationScheme-- new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)- if layer.bias is not None:- new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))-- # Copy quantization scheme attributes- scheme_keys = (f.name for f in fields(QuantizationScheme))- keys = tuple(scheme_keys) + ("global_name", "scale_dtype")- for key in keys:- setattr(new_layer, key, getattr(layer, key, None))-- # Handle Gaudi2 device compatibility- from auto_round.utils.device import is_gaudi2-- if is_gaudi2():- device = "cpu"- # Use registry-based dequantization- dq_weight = dequant_fp8_layer(layer, dtype=dtype, device=device)- new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))-- return new_layer---def convert_fp8_module_to_16b(model, dtype=torch.bfloat16, device: str = "cpu"):- """- Convert a model with FP8 quantized layers to a model with 16-bit linear layers.- This is useful for compatibility with other frameworks or for further processing.-- Uses `is_fp8_linear` for detection, which supports all registered FP8 layer types- via the FP8 dequantization registry.-- Args:- model: The model with FP8 layers to convert.- dtype: Target dtype for converted layers.- device: Target device for conversion.-- Returns:- The model with FP8 layers converted to 16-bit Linear layers.- """- from auto_round.utils.device import clear_memory-- cnt = 0- for n, m in model.named_modules():- if is_fp8_linear(m): # Use registry-based detection- new_module = convert_fp8_layer_to_linear(m, dtype=dtype, device=device)- set_module(model, n, new_module)- cnt += 1- if cnt % 10 == 0: # Tricky setting- clear_memory()- return model--def _to_model_dtype(model, model_dtype):if model_dtype is not None:try:diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pynew file mode 100644index 000000000..fb4e40298--- /dev/null+++ b/auto_round/utils/weight_handler.py@@ -0,0 +1,643 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Extensible Module Weight Type Conversion Framework.++This module provides a registry-based system for detecting and converting quantized+module weights to high precision.++Table of Contents:+ 1. CORE FRAMEWORK+ - ModuleWeightType: Enum of supported weight types+ - WeightTypeHandler: Abstract base class for handlers+ - Registry functions: register_weight_type_handler, get_handler, etc.++ 2. PUBLIC API+ - detect_weight_type(): Detect weight type of a layer or model+ - check_and_mark_quantized_module(): Check and mark quantized layers+ - is_quantized_input_module(): Check if model has quantized weights+ - convert_module_to_hp_if_necessary(): Main conversion function++ 3. HANDLER IMPLEMENTATIONS+ - FP8BlockHandler: Fully implemented for FP8 block-wise quantization+ - MXFP8Handler: Placeholder (TODO)+ - MXFP4Handler: Placeholder (TODO)+ - NVFP4Handler: Placeholder (TODO)++Quick Start Guide:+ Usage - Detect and Convert:+ >>> from auto_round.utils.weight_handler import (+ ... check_and_mark_quantized_module,+ ... convert_module_to_hp_if_necessary,+ ... )+ >>> check_and_mark_quantized_module(model)+ >>> model = convert_module_to_hp_if_necessary(model)++ Adding a New Weight Type Handler:+ 1. Add new type to ModuleWeightType enum+ 2. Create handler class inheriting from WeightTypeHandler+ 3. Register with @register_weight_type_handler decorator++ Example:+ @register_weight_type_handler(ModuleWeightType.MY_NEW_TYPE)+ class MyNewTypeHandler(WeightTypeHandler):+ def detect_layer(self, module): ...+ def convert_layer(self, layer, dtype, device, to_cpu): ...+"""++import os+from abc import ABC, abstractmethod+from contextlib import ContextDecorator+from dataclasses import fields+from enum import Enum, auto+from typing import Callable, Dict, Optional, Set, Type++import psutil+import torch++from auto_round import envs+from auto_round.logger import logger++# ============================================================================+# Section 0: UTILITY HELPERS+# ============================================================================+++def _pad_weight(weight: torch.Tensor, block_size: list) -> tuple[torch.Tensor, int, int]:+ """Pads a matrix to make its dimensions multiples of block_size."""+ M, N = weight.shape[-2:]+ block_size_m, block_size_n = block_size+ pad_M = (block_size_m - M % block_size_m) % block_size_m+ pad_N = (block_size_n - N % block_size_n) % block_size_n++ if pad_M == 0 and pad_N == 0:+ return weight, M, N # No padding needed+ padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)+ return padded_weight, M, N # Return original dimensions for unpadding+++def _unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:+ """Removes padding from the matrix to restore its original shape."""+ if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):+ return weight+ if keep_first_dim:+ return weight[:, :original_M, :original_N]+ else:+ return weight[:original_M, :original_N]+++class with_thread_limits(ContextDecorator):+ """+ Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.+ Inspired by vLLM's thread limit decorator.+ https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115+ """++ def __init__(self, div=1):+ self.div = div+ self.old_omp = None+ self.old_torch = None++ def __enter__(self):+ # Save original settings+ self.old_omp = envs.AR_OMP_NUM_THREADS+ self.old_torch = torch.get_num_threads()++ try:+ if psutil is not None:+ num_cores = len(psutil.Process().cpu_affinity() or [])+ if num_cores == 0:+ num_cores = os.cpu_count() or 1+ else:+ num_cores = os.cpu_count() or 1++ # Set new limits+ new_threads = max(1, num_cores // self.div)+ os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)+ torch.set_num_threads(new_threads)+ except Exception:+ pass+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ # Restore original settings+ if self.old_omp is None:+ os.environ.pop("AR_OMP_NUM_THREADS", None)+ else:+ os.environ["AR_OMP_NUM_THREADS"] = self.old_omp+ torch.set_num_threads(self.old_torch)+++# ============================================================================+# Section 1: CORE FRAMEWORK - Types, Base Classes, and Registry+# ============================================================================+++class ModuleWeightType(Enum):+ """Enumeration of supported quantized module weight types.++ When adding a new weight type:+ 1. Add the new type here+ 2. Create a handler class (see Section 3)+ 3. Register with @register_weight_type_handler decorator+ """++ FP8_BLOCK = auto() # FP8 with block-wise scaling (fully implemented)+ MXFP8 = auto() # MX FP8 (placeholder)+ MXFP4 = auto() # MX FP4 (placeholder)+ NVFP4 = auto() # NV FP4 (placeholder)+++class WeightTypeHandler(ABC):+ """Abstract base class for weight type detection and conversion handlers.++ Subclasses must implement:+ - detect_layer(): Check if a single layer is of this weight type+ - convert_layer(): Convert a single layer to high precision+ """++ @abstractmethod+ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a single layer is of this weight type.++ Args:+ module: The module to check.++ Returns:+ True if the module is of this weight type, False otherwise.+ """+ pass++ @abstractmethod+ def convert_layer(+ self,+ layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+ ) -> torch.nn.Module:+ """Convert a single layer to high precision.++ Args:+ layer: The quantized layer to convert.+ dtype: Target dtype for the converted layer.+ device: Target device for the conversion.+ to_cpu: If True, move converted layer to CPU.++ Returns:+ A new high-precision layer with dequantized weights.+ """+ pass+++# --- Handler Registry ---++_WEIGHT_TYPE_HANDLERS: Dict[ModuleWeightType, WeightTypeHandler] = {}+++def register_weight_type_handler(weight_type: ModuleWeightType):+ """Decorator to register a weight type handler.++ Args:+ weight_type: The ModuleWeightType this handler supports.++ Returns:+ Decorator function that registers the handler class.++ Example:+ @register_weight_type_handler(ModuleWeightType.MXFP4)+ class MXFP4Handler(WeightTypeHandler):+ ...+ """++ def decorator(handler_cls: Type[WeightTypeHandler]):+ if not issubclass(handler_cls, WeightTypeHandler):+ raise TypeError(f"Handler {handler_cls.__name__} must be a subclass of WeightTypeHandler")+ _WEIGHT_TYPE_HANDLERS[weight_type] = handler_cls()+ return handler_cls++ return decorator+++def get_handler(weight_type: ModuleWeightType) -> Optional[WeightTypeHandler]:+ """Get the registered handler for a weight type.++ Args:+ weight_type: The ModuleWeightType to get the handler for.++ Returns:+ The registered handler, or None if not registered.+ """+ return _WEIGHT_TYPE_HANDLERS.get(weight_type)+++def get_all_handlers() -> Dict[ModuleWeightType, WeightTypeHandler]:+ """Get all registered weight type handlers.++ Returns:+ Dictionary mapping weight types to their handlers.+ """+ return _WEIGHT_TYPE_HANDLERS.copy()+++# ============================================================================+# Section 2: PUBLIC API - Detection and Conversion Functions+# ============================================================================+def detect_weight_type(module: torch.nn.Module) -> Optional[ModuleWeightType]:+ """Detect the weight type of a module or model.++ First checks if the module itself has a quantized_weight_type attribute.+ If not found, scans all submodules to find any with quantized_weight_type attribute.++ Args:+ module: The module or model to check.++ Returns:+ The detected ModuleWeightType, or None if no match found.+ """+ # Check if module itself is marked+ if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:+ return module.quantized_weight_type++ # Scan submodules for quantized_weight_type attribute+ for submodule in module.modules():+ if hasattr(submodule, "quantized_weight_type") and submodule.quantized_weight_type is not None:+ return submodule.quantized_weight_type++ return None+++# --- Model Marking Functions ---+def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightType]:+ """Check if model contains quantized layers and mark them accordingly.++ This function scans the model (including the model itself) for quantized layers using+ handlers' detect_layer method (which checks actual characteristics). It marks detected+ layers with `quantized_weight_type`.++ Args:+ model: The model to check and mark.++ Returns:+ A set of detected ModuleWeightType values. Empty set if no quantized layers found.+ """+ detected_types: Set[ModuleWeightType] = set()+ for weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():+ # Check model itself first+ if handler.detect_layer(model):+ model.quantized_weight_type = weight_type+ detected_types.add(weight_type)++ # Then check all submodules+ for n, m in model.named_modules():+ # Use handler to detect based on actual characteristics+ if handler.detect_layer(m):+ # Mark the layer itself+ m.quantized_weight_type = weight_type+ # Record detected types+ detected_types.add(weight_type)++ return detected_types+++def is_quantized_input_module(model: torch.nn.Module) -> Optional[ModuleWeightType]:+ """Check if a model has quantized input weights and return the weight type.++ This traverses all submodules to check for the `quantized_weight_type` attribute+ set by `check_and_mark_quantized_module`.++ Args:+ model: The model to check.++ Returns:+ The ModuleWeightType if the model has quantized weights, None otherwise.+ """+ # Check model itself first+ if hasattr(model, "quantized_weight_type") and model.quantized_weight_type is not None:+ return model.quantized_weight_type++ # Traverse all submodules to find quantized layers+ for module in model.modules():+ if hasattr(module, "quantized_weight_type") and module.quantized_weight_type is not None:+ return module.quantized_weight_type++ return None+++# --- Main Conversion Function ---+def convert_module_to_hp_if_necessary(+ model_or_layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+) -> torch.nn.Module:+ """Convert quantized layer(s) to high-precision Linear layer(s) if necessary.++ This function automatically detects the weight type and uses the appropriate+ handler for conversion. If the input is not quantized, it returns unchanged.++ Args:+ model_or_layer: Either a single layer, a block, or an entire model.+ dtype: Target dtype for the converted layer(s). Defaults to torch.bfloat16.+ device: Target device for the conversion. Defaults to "cpu".+ to_cpu: If True, move converted layers to CPU. Defaults to False.++ Returns:+ If input is a quantized layer: A new high-precision layer with dequantized weights.+ If input is a model containing quantized layers: The model with layers converted.+ If input has no quantized layers: Returns the input unchanged.+ """+ from auto_round.utils.device import clear_memory+ from auto_round.utils.model import set_module++ # Check if it's a single quantized layer (has the attribute directly)+ if hasattr(model_or_layer, "quantized_weight_type") and model_or_layer.quantized_weight_type is not None:+ handler = get_handler(model_or_layer.quantized_weight_type)+ return handler.convert_layer(model_or_layer, dtype, device, to_cpu)++ # Otherwise, traverse model and convert all quantized layers+ # Get handler for each layer to support mixed quantization types+ cnt = 0+ for n, m in model_or_layer.named_modules():+ if hasattr(m, "quantized_weight_type") and m.quantized_weight_type is not None:+ handler = get_handler(m.quantized_weight_type)+ new_module = handler.convert_layer(m, dtype, device, to_cpu)+ set_module(model_or_layer, n, new_module)+ cnt += 1+ if cnt % 10 == 0:+ clear_memory()++ return model_or_layer+++# ============================================================================+# Section 3: FP8 BLOCK DEQUANTIZATION HELPERS+# ============================================================================+++def _pad_block_fp8_weight_naive(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list+) -> tuple[torch.Tensor, int, int]:+ assert len(block_size) == 2++ block_size_m, block_size_n = block_size+ weight_scale_m, weight_scale_n = weight_scale.shape[-2:]++ weight, orig_M, orig_N = _pad_weight(weight, block_size)+ M, N = weight.shape[-2:]++ assert weight_scale_m == M // block_size_m+ assert weight_scale_n == N // block_size_n++ return weight, orig_M, orig_N+++@with_thread_limits()+def _dequant_fp8_linear_weight(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None+) -> torch.Tensor:+ """Core dequantization logic for block-wise FP8 weights."""+ dtype = torch.bfloat16+ if weight_scale is None:+ return weight++ # If weight is stored as uint8, view it as float8_e4m3fn+ if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:+ weight = weight.view(torch.float8_e4m3fn)++ if block_size is None:+ if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:+ if weight_scale.numel() == weight.shape[0]:+ weight_scale = weight_scale.view(-1, 1)+ elif weight_scale.numel() == weight.shape[-1]:+ weight_scale = weight_scale.view(1, -1)+ return weight.to(dtype) * weight_scale.to(dtype)++ assert len(block_size) == 2++ weight, orig_M, orig_N = _pad_block_fp8_weight_naive(weight, weight_scale, block_size)++ weight_shape_len = len(weight.shape)+ block_size_m, block_size_n = block_size++ # mul scale+ if weight_shape_len == 2:+ weight_scale_m, weight_scale_n = weight_scale.shape+ weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)+ weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ keep_first_dim = False+ elif weight_shape_len == 3:+ fd, weight_scale_m, weight_scale_n = weight_scale.shape+ weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)+ weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ keep_first_dim = True+ else:+ raise ValueError("Only support original weight shape is either 2 or 3")++ dequant_weight = _unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)++ return dequant_weight+++# ============================================================================+# Section 4: HANDLER IMPLEMENTATIONS+# ============================================================================+++# ----------------------------------------------------------------------------+# FP8 Block Handler - Full Implementation (Reference Example)+# ----------------------------------------------------------------------------+@register_weight_type_handler(ModuleWeightType.FP8_BLOCK)+class FP8BlockHandler(WeightTypeHandler):+ """Handler for FP8 block-wise quantized layers.++ This handler supports:+ - FP8Linear layers with block-wise scaling+ - CompressedLinear layers with FP8 compression+ - torch.nn.Linear layers with FP8 dtype weights+ """++ # Registry mapping FP8 layer class names to their dequantization handlers+ FP8_DEQUANT_REGISTRY: Dict[str, Callable] = {}++ @classmethod+ def register_fp8_layer(cls, layer_name: str):+ """Register a dequantization handler for an FP8 layer type.++ Args:+ layer_name: The class name of the FP8 layer type.++ Returns:+ Decorator function that registers the handler.+ """++ def decorator(fn: Callable):+ cls.FP8_DEQUANT_REGISTRY[layer_name] = fn+ return fn++ return decorator++ def _dequant_layer(self, layer: torch.nn.Module, dtype: torch.dtype, device: str) -> torch.Tensor:+ """Dequantize an FP8 layer using the internal registry.++ Args:+ layer: The FP8 layer to dequantize.+ dtype: Target dtype for dequantized weights.+ device: Target device for dequantization.++ Returns:+ Dequantized weight tensor.++ Raises:+ NotImplementedError: If the layer type is not registered.+ """+ name = layer.__class__.__name__+ if name not in self.FP8_DEQUANT_REGISTRY:+ raise NotImplementedError(+ f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(self.FP8_DEQUANT_REGISTRY.keys())}"+ )+ return self.FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)++ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a module is an FP8 linear layer based on actual characteristics."""+ # Check registry for supported FP8 layer types+ layer_name = module.__class__.__name__+ if layer_name in self.FP8_DEQUANT_REGISTRY:+ return True++ # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)+ if type(module) == torch.nn.Linear and module.weight is not None:+ if str(module.weight.dtype).startswith("torch.float8"):+ return True++ return False++ def convert_layer(+ self,+ layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+ ) -> torch.nn.Module:+ """Convert a single FP8 layer to a standard Linear layer."""+ from auto_round.schemes import QuantizationScheme+ from auto_round.utils.device import is_gaudi2+ from auto_round.utils.model import set_module++ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)+ if layer.bias is not None:+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))++ # Copy quantization scheme attributes+ scheme_keys = (f.name for f in fields(QuantizationScheme))+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):+ setattr(new_layer, key, getattr(layer, key, None))++ # Handle Gaudi2 device compatibility+ if is_gaudi2():+ device = "cpu"++ # Use registry-based dequantization+ dq_weight = self._dequant_layer(layer, dtype=dtype, device=device)+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))++ if to_cpu:+ new_layer = new_layer.to("cpu")++ return new_layer+++# --- FP8 Layer Dequantization Handlers ---+# Register specific dequantization logic for different FP8 layer types++[email protected]_fp8_layer("CompressedLinear")+def _dequant_compressed_linear(+ layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"+) -> torch.Tensor:+ """Dequantize CompressedLinear layer using compressor.++ Args:+ layer: The CompressedLinear layer to dequantize.+ dtype: Target dtype for dequantized weights.+ device: Target device for dequantization.++ Returns:+ Dequantized weight tensor.+ """+ layer = layer.to(device)+ return layer.compressor.decompress_module(layer)++[email protected]_fp8_layer("FP8Linear")+def _dequant_fp8_linear(+ layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"+) -> torch.Tensor:+ """Dequantize FP8Linear layer using block-based dequantization.++ Args:+ layer: The FP8Linear layer to dequantize.+ dtype: Target dtype for dequantized weights.+ device: Target device for dequantization.++ Returns:+ Dequantized weight tensor.+ """+ layer = layer.to(device)+ weight_scale = getattr(layer, "weight_scale", None)+ block_size = getattr(layer, "block_size", None)+ data_type = getattr(layer, "data_type", None)+ if weight_scale is None:+ weight_scale = getattr(layer, "weight_scale_inv", None)+ if weight_scale is None:+ raise AttributeError(+ "FP8Linear layer is missing both 'weight_scale' and 'weight_scale_inv' "+ "attributes required for dequantization."+ )+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)+++# ----------------------------------------------------------------------------+# Placeholder Handlers for Unimplemented Types+# ----------------------------------------------------------------------------+++def _create_placeholder_handler(weight_type_name: str) -> Type[WeightTypeHandler]:+ """Factory function to create placeholder handlers for unimplemented weight types."""++ class PlaceholderHandler(WeightTypeHandler):+ """Placeholder handler - detection always returns False."""++ def detect_layer(self, module: torch.nn.Module) -> bool:+ return False++ def convert_layer(self, layer, dtype=torch.bfloat16, device="cpu", to_cpu=False):+ raise NotImplementedError(f"{weight_type_name} layer conversion not yet implemented")++ PlaceholderHandler.__name__ = f"{weight_type_name}Handler"+ PlaceholderHandler.__doc__ = (+ f"Placeholder handler for {weight_type_name}. TODO: Implement detection and conversion."+ )+ return PlaceholderHandler+++# Register placeholder handlers for unimplemented types+register_weight_type_handler(ModuleWeightType.MXFP8)(_create_placeholder_handler("MXFP8"))+register_weight_type_handler(ModuleWeightType.MXFP4)(_create_placeholder_handler("MXFP4"))+register_weight_type_handler(ModuleWeightType.NVFP4)(_create_placeholder_handler("NVFP4"))diff --git a/test/fixtures.py b/test/fixtures.pyindex cf9ad806e..482b74598 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -123,6 +123,15 @@ def tiny_qwen_2_5_vl_model_path():shutil.rmtree(tiny_model_path, ignore_errors=True)+# Mock torch.cuda.get_device_capability to always return (9, 0) like H100[email protected](autouse=True, scope="session")+def mock_cuda_capability():+ from unittest.mock import patch++ with patch("torch.cuda.get_device_capability", return_value=(9, 0)):+ yield++@pytest.fixture(autouse=True, scope="session")def clean_tmp_model_folder():yielddiff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 42a7dc83f..d2f6cabcd 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -660,10 +660,14 @@ class TestAutoRound:assert dequant_weight.shape.numel() == 32 * 5760 * 1440def test_dequant_g2_fp8_weight(self):- """Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""+ """Test Gaudi2-specific dequantization redirection logic in convert_module_to_hp_if_necessary."""from unittest.mock import MagicMock, patch- from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear+ from auto_round.utils.model import (+ _dequant_fp8_linear_weight,+ check_and_mark_quantized_module,+ convert_module_to_hp_if_necessary,+ )# Test 1: Verify Standard e4m3fn native dequantization resultweight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)@@ -674,7 +678,7 @@ class TestAutoRound:assert dq_weight.dtype == torch.bfloat16assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3- # Test 2: Verify Gaudi2 redirection in convert_fp8_layer_to_linear+ # Test 2: Verify Gaudi2 redirection in convert_module_to_hp_if_necessarymock_layer = MagicMock()mock_layer.in_features = 2mock_layer.out_features = 1@@ -689,14 +693,16 @@ class TestAutoRound:mock_layer.to.return_value = mock_layerwith patch("auto_round.utils.device.is_gaudi2", return_value=True):- convert_fp8_layer_to_linear(mock_layer, device="hpu")+ check_and_mark_quantized_module(mock_layer)+ convert_module_to_hp_if_necessary(mock_layer, device="hpu")# Verify it was moved to CPUmock_layer.to.assert_called_with("cpu")with patch("auto_round.utils.device.is_gaudi2", return_value=False):# Reset mockmock_layer.to.reset_mock()- convert_fp8_layer_to_linear(mock_layer, device="hpu")+ check_and_mark_quantized_module(mock_layer)+ convert_module_to_hp_if_necessary(mock_layer, device="hpu")# Verify it was moved to HPU (as requested in device arg)mock_layer.to.assert_called_with("hpu")diff --git a/test/test_cpu/core/test_fp8_re_quant.py b/test/test_cpu/utils/test_fp8_re_quant.pysimilarity index 93%rename from test/test_cpu/core/test_fp8_re_quant.pyrename to test/test_cpu/utils/test_fp8_re_quant.pyindex 59e42a543..21596f2de 100644--- a/test/test_cpu/core/test_fp8_re_quant.py+++ b/test/test_cpu/utils/test_fp8_re_quant.py@@ -4,7 +4,11 @@ import torchimport torch.nn as nnfrom transformers import AutoConfig-from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear+from auto_round.utils.model import (+ _dequant_fp8_linear_weight,+ check_and_mark_quantized_module,+ convert_module_to_hp_if_necessary,+)class MockFP8Layer:@@ -85,7 +89,8 @@ class TestFP8ReQuant(unittest.TestCase):mock_layer = FP8Linear(in_features, out_features)# This should now pass without AttributeError for 'block_size'- new_layer = convert_fp8_layer_to_linear(mock_layer, dtype=torch.bfloat16)+ check_and_mark_quantized_module(mock_layer)+ new_layer = convert_module_to_hp_if_necessary(mock_layer, dtype=torch.bfloat16)self.assertIsInstance(new_layer, nn.Linear)self.assertEqual(new_layer.in_features, in_features)diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex e2ca5b359..a1dfe7023 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -1,6 +1,5 @@import copyimport shutil-from unittest.mock import patchimport pytestimport torch@@ -13,13 +12,6 @@ from auto_round.testing_utils import require_awq, require_optimumfrom ...helpers import get_model_path, save_tiny_model-# Mock torch.cuda.get_device_capability to always return (9, 0) like H100-def mock_cuda_capability():- with patch("torch.cuda.get_device_capability", return_value=(9, 0)):- yield--class TestAutoRound:save_dir = "./saved"