PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Implement MXFP4Handler MXFP8Handler NVFP4Handler (#1385)
Xin Hef7db867a9d4f
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex fb4e40298..ccdf44c3d 100644--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -31,10 +31,10 @@ Table of Contents:- convert_module_to_hp_if_necessary(): Main conversion function3. HANDLER IMPLEMENTATIONS- - FP8BlockHandler: Fully implemented for FP8 block-wise quantization- - MXFP8Handler: Placeholder (TODO)- - MXFP4Handler: Placeholder (TODO)- - NVFP4Handler: Placeholder (TODO)+ - FP8Handler: Fully implemented for FP8 block-wise quantization+ - MXFP8Handler: CompressedLinear with MXFP8PackedCompressor+ - MXFP4Handler: CompressedLinear with MXFP4PackedCompressor+ - NVFP4Handler: CompressedLinear with NVFP4PackedCompressorQuick Start Guide:Usage - Detect and Convert:@@ -154,10 +154,10 @@ class ModuleWeightType(Enum):3. Register with @register_weight_type_handler decorator"""- FP8_BLOCK = auto() # FP8 with block-wise scaling (fully implemented)- MXFP8 = auto() # MX FP8 (placeholder)- MXFP4 = auto() # MX FP4 (placeholder)- NVFP4 = auto() # NV FP4 (placeholder)+ FP8 = auto() # FP8 with block-wise scaling (FP8Linear/CompressedLinear from transformers)+ MXFP8 = auto() # MX FP8 (CompressedLinear with MXFP8PackedCompressor)+ MXFP4 = auto() # MX FP4 (CompressedLinear with MXFP4PackedCompressor)+ NVFP4 = auto() # NV FP4 (CompressedLinear with NVFP4PackedCompressor)class WeightTypeHandler(ABC):@@ -463,69 +463,101 @@ def _dequant_fp8_linear_weight(# ----------------------------------------------------------------------------# FP8 Block Handler - Full Implementation (Reference Example)# -----------------------------------------------------------------------------@register_weight_type_handler(ModuleWeightType.FP8_BLOCK)-class FP8BlockHandler(WeightTypeHandler):+@register_weight_type_handler(ModuleWeightType.FP8)+class FP8Handler(WeightTypeHandler):"""Handler for FP8 block-wise quantized layers.This handler supports:- FP8Linear layers with block-wise scaling- - CompressedLinear layers with FP8 compression+ - CompressedLinear layers with compressor- torch.nn.Linear layers with FP8 dtype weights"""- # Registry mapping FP8 layer class names to their dequantization handlers- FP8_DEQUANT_REGISTRY: Dict[str, Callable] = {}+ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a module is an FP8 linear layer based on actual characteristics."""+ # Check for CompressedLinear layer type (exclude MXFP4, MXFP8, NVFP4)+ if module.__class__.__name__ == "CompressedLinear":+ if hasattr(module, "compressor") and module.compressor is not None:+ compressor_name = module.compressor.__class__.__name__+ return "Float" in compressor_name+ return True- @classmethod- def register_fp8_layer(cls, layer_name: str):- """Register a dequantization handler for an FP8 layer type.+ # Check for FP8Linear layer type+ if module.__class__.__name__ == "FP8Linear":+ return True- Args:- layer_name: The class name of the FP8 layer type.+ # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)+ if type(module) == torch.nn.Linear and module.weight is not None:+ if str(module.weight.dtype).startswith("torch.float8"):+ return True- Returns:- Decorator function that registers the handler.- """+ return False- def decorator(fn: Callable):- cls.FP8_DEQUANT_REGISTRY[layer_name] = fn- return fn+ def convert_layer(+ self,+ layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+ ) -> torch.nn.Module:+ """Convert a single FP8/CompressedLinear layer to a standard Linear layer."""+ from auto_round.schemes import QuantizationScheme+ from auto_round.utils.device import is_gaudi2- return decorator+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)+ if layer.bias is not None:+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))- def _dequant_layer(self, layer: torch.nn.Module, dtype: torch.dtype, device: str) -> torch.Tensor:- """Dequantize an FP8 layer using the internal registry.+ # Copy quantization scheme attributes+ scheme_keys = (f.name for f in fields(QuantizationScheme))+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):+ setattr(new_layer, key, getattr(layer, key, None))- Args:- layer: The FP8 layer to dequantize.- dtype: Target dtype for dequantized weights.- device: Target device for dequantization.+ # Handle Gaudi2 device compatibility+ if is_gaudi2():+ device = "cpu"- Returns:- Dequantized weight tensor.+ layer = layer.to(device)- Raises:- NotImplementedError: If the layer type is not registered.- """- name = layer.__class__.__name__- if name not in self.FP8_DEQUANT_REGISTRY:- raise NotImplementedError(- f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(self.FP8_DEQUANT_REGISTRY.keys())}"- )- return self.FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)+ # Check if layer has compressor.decompress_module method+ if hasattr(layer, "compressor") and hasattr(layer.compressor, "decompress_module"):+ dq_weight = layer.compressor.decompress_module(layer)+ else:+ # Use FP8 block-based dequantization+ weight_scale = getattr(layer, "weight_scale", None)+ block_size = getattr(layer, "block_size", None)+ data_type = getattr(layer, "data_type", None)+ if weight_scale is None:+ weight_scale = getattr(layer, "weight_scale_inv", None)+ if weight_scale is None:+ raise AttributeError(+ "FP8 layer is missing both 'weight_scale' and 'weight_scale_inv' "+ "attributes required for dequantization."+ )+ dq_weight = _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)- def detect_layer(self, module: torch.nn.Module) -> bool:- """Check if a module is an FP8 linear layer based on actual characteristics."""- # Check registry for supported FP8 layer types- layer_name = module.__class__.__name__- if layer_name in self.FP8_DEQUANT_REGISTRY:- return True+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))- # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)- if type(module) == torch.nn.Linear and module.weight is not None:- if str(module.weight.dtype).startswith("torch.float8"):- return True+ if to_cpu:+ new_layer = new_layer.to("cpu")++ return new_layer++# ----------------------------------------------------------------------------+# MXFP4 Handler - CompressedLinear with MXFP4PackedCompressor+# ----------------------------------------------------------------------------+@register_weight_type_handler(ModuleWeightType.MXFP4)+class MXFP4Handler(WeightTypeHandler):+ """Handler for MXFP4 quantized layers (CompressedLinear with MXFP4PackedCompressor)."""++ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a module is an MXFP4 CompressedLinear layer."""+ if module.__class__.__name__ != "CompressedLinear":+ return False+ if hasattr(module, "compressor") and module.compressor is not None:+ compressor_name = module.compressor.__class__.__name__+ return "MXFP4" in compressor_namereturn Falsedef convert_layer(@@ -535,10 +567,9 @@ class FP8BlockHandler(WeightTypeHandler):device: str = "cpu",to_cpu: bool = False,) -> torch.nn.Module:- """Convert a single FP8 layer to a standard Linear layer."""+ """Convert an MXFP4 CompressedLinear layer to a standard Linear layer."""from auto_round.schemes import QuantizationSchemefrom auto_round.utils.device import is_gaudi2- from auto_round.utils.model import set_modulenew_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)if layer.bias is not None:@@ -553,8 +584,30 @@ class FP8BlockHandler(WeightTypeHandler):if is_gaudi2():device = "cpu"- # Use registry-based dequantization- dq_weight = self._dequant_layer(layer, dtype=dtype, device=device)+ layer = layer.to(device)++ # MXFP4 dequantization using to_dtype from mxfp4_qdq_utils+ from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype++ # Get packed weight and scale from layer+ # MXFP4 weights are stored as packed uint8 in weight_packed+ weight_packed = getattr(layer, "weight_packed", None)+ if weight_packed is None:+ weight_packed = getattr(layer, "weight", None)+ weight_scale = getattr(layer, "weight_scale", None)+ weight_scale = weight_scale.to(torch.uint8)++ if weight_packed is None or weight_scale is None:+ raise ValueError("MXFP4 layer must have weight_packed and weight_scale attributes")++ # Dequantize using to_dtype function+ dq_weight = to_dtype(+ data_lp=weight_packed,+ scale_e8m0=weight_scale,+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=dtype,+ )new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))if to_cpu:@@ -563,81 +616,121 @@ class FP8BlockHandler(WeightTypeHandler):return new_layer-# --- FP8 Layer Dequantization Handlers ----# Register specific dequantization logic for different FP8 layer types+# ----------------------------------------------------------------------------+# MXFP8 Handler - CompressedLinear with MXFP8PackedCompressor+# ----------------------------------------------------------------------------+@register_weight_type_handler(ModuleWeightType.MXFP8)+class MXFP8Handler(WeightTypeHandler):+ """Handler for MXFP8 quantized layers (CompressedLinear with MXFP8PackedCompressor)."""+ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a module is an MXFP8 CompressedLinear layer."""+ if module.__class__.__name__ != "CompressedLinear":+ return False+ if hasattr(module, "compressor") and module.compressor is not None:+ compressor_name = module.compressor.__class__.__name__+ return "MXFP8" in compressor_name+ return False[email protected]_fp8_layer("CompressedLinear")-def _dequant_compressed_linear(- layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"-) -> torch.Tensor:- """Dequantize CompressedLinear layer using compressor.+ def convert_layer(+ self,+ layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+ ) -> torch.nn.Module:+ """Convert an MXFP8 CompressedLinear layer to a standard Linear layer."""+ from auto_round.schemes import QuantizationScheme+ from auto_round.utils.device import is_gaudi2- Args:- layer: The CompressedLinear layer to dequantize.- dtype: Target dtype for dequantized weights.- device: Target device for dequantization.+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)+ if layer.bias is not None:+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))- Returns:- Dequantized weight tensor.- """- layer = layer.to(device)- return layer.compressor.decompress_module(layer)+ # Copy quantization scheme attributes+ scheme_keys = (f.name for f in fields(QuantizationScheme))+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):+ setattr(new_layer, key, getattr(layer, key, None))+ # Handle Gaudi2 device compatibility+ if is_gaudi2():+ device = "cpu"[email protected]_fp8_layer("FP8Linear")-def _dequant_fp8_linear(- layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"-) -> torch.Tensor:- """Dequantize FP8Linear layer using block-based dequantization.+ layer = layer.to(device)- Args:- layer: The FP8Linear layer to dequantize.- dtype: Target dtype for dequantized weights.- device: Target device for dequantization.+ # MXFP8 dequantization using dequant_mx_fp8 from mxfp8_qdq_utils+ from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8- Returns:- Dequantized weight tensor.- """- layer = layer.to(device)- weight_scale = getattr(layer, "weight_scale", None)- block_size = getattr(layer, "block_size", None)- data_type = getattr(layer, "data_type", None)- if weight_scale is None:- weight_scale = getattr(layer, "weight_scale_inv", None)- if weight_scale is None:- raise AttributeError(- "FP8Linear layer is missing both 'weight_scale' and 'weight_scale_inv' "- "attributes required for dequantization."+ # Get weight and scale from layer+ weight = layer.weight+ weight_scale = getattr(layer, "weight_scale", None)+ weight_scale = weight_scale.to(torch.uint8)++ if weight is None or weight_scale is None:+ raise ValueError("MXFP8 layer must have weight and weight_scale attributes")++ # Dequantize using dequant_mx_fp8 function+ dq_weight = dequant_mx_fp8(+ weight_fp8=weight,+ scale_e8m0=weight_scale,+ block_size=32,+ target_dtype=dtype,)- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))++ if to_cpu:+ new_layer = new_layer.to("cpu")++ return new_layer# -----------------------------------------------------------------------------# Placeholder Handlers for Unimplemented Types+# NVFP4 Handler - CompressedLinear with NVFP4PackedCompressor# ----------------------------------------------------------------------------+@register_weight_type_handler(ModuleWeightType.NVFP4)+class NVFP4Handler(WeightTypeHandler):+ """Handler for NVFP4 quantized layers (CompressedLinear with NVFP4PackedCompressor)."""+ def detect_layer(self, module: torch.nn.Module) -> bool:+ """Check if a module is an NVFP4 CompressedLinear layer."""+ if module.__class__.__name__ != "CompressedLinear":+ return False+ if hasattr(module, "compressor") and module.compressor is not None:+ compressor_name = module.compressor.__class__.__name__+ return "NVFP4" in compressor_name+ return False-def _create_placeholder_handler(weight_type_name: str) -> Type[WeightTypeHandler]:- """Factory function to create placeholder handlers for unimplemented weight types."""+ def convert_layer(+ self,+ layer: torch.nn.Module,+ dtype: torch.dtype = torch.bfloat16,+ device: str = "cpu",+ to_cpu: bool = False,+ ) -> torch.nn.Module:+ """Convert an NVFP4 CompressedLinear layer to a standard Linear layer."""+ from auto_round.schemes import QuantizationScheme+ from auto_round.utils.device import is_gaudi2- class PlaceholderHandler(WeightTypeHandler):- """Placeholder handler - detection always returns False."""+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)+ if layer.bias is not None:+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))- def detect_layer(self, module: torch.nn.Module) -> bool:- return False+ # Copy quantization scheme attributes+ scheme_keys = (f.name for f in fields(QuantizationScheme))+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):+ setattr(new_layer, key, getattr(layer, key, None))- def convert_layer(self, layer, dtype=torch.bfloat16, device="cpu", to_cpu=False):- raise NotImplementedError(f"{weight_type_name} layer conversion not yet implemented")+ # Handle Gaudi2 device compatibility+ if is_gaudi2():+ device = "cpu"- PlaceholderHandler.__name__ = f"{weight_type_name}Handler"- PlaceholderHandler.__doc__ = (- f"Placeholder handler for {weight_type_name}. TODO: Implement detection and conversion."- )- return PlaceholderHandler+ layer = layer.to(device)+ # Use compressor.decompress_module for dequantization+ dq_weight = layer.compressor.decompress_module(layer)+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))-# Register placeholder handlers for unimplemented types-register_weight_type_handler(ModuleWeightType.MXFP8)(_create_placeholder_handler("MXFP8"))-register_weight_type_handler(ModuleWeightType.MXFP4)(_create_placeholder_handler("MXFP4"))-register_weight_type_handler(ModuleWeightType.NVFP4)(_create_placeholder_handler("NVFP4"))+ if to_cpu:+ new_layer = new_layer.to("cpu")++ return new_layerdiff --git a/test/fixtures.py b/test/fixtures.pyindex 482b74598..4ec157b1f 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -124,8 +124,8 @@ def tiny_qwen_2_5_vl_model_path():# Mock torch.cuda.get_device_capability to always return (9, 0) like H100[email protected](autouse=True, scope="session")-def mock_cuda_capability():+def mock_fp8_capable_device():from unittest.mock import patchwith patch("torch.cuda.get_device_capability", return_value=(9, 0)):diff --git a/test/test_cpu/advanced/test_low_precision_input_model.py b/test/test_cpu/advanced/test_low_precision_input_model.pynew file mode 100644index 000000000..48aa59b24--- /dev/null+++ b/test/test_cpu/advanced/test_low_precision_input_model.py@@ -0,0 +1,51 @@+import torch++from auto_round.utils.weight_handler import (+ ModuleWeightType,+ check_and_mark_quantized_module,+ convert_module_to_hp_if_necessary,+)++from ...helpers import get_model_path, get_tiny_model+++class TestCompressedTensor:+ nvfp4_model_path = "kaitchup/Qwen3-0.6B-NVFP4"+ mxfp4_model_path = "QuixiAI/Llama-3.2-1B-MXFP4"+ fp8_block_model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"++ def test_fp8_block(self):+ model = get_tiny_model(get_model_path(self.fp8_block_model_path))+ assert (+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"+ ), "Model does not contain CompressedLinear layers"+ detected_types = check_and_mark_quantized_module(model)+ assert ModuleWeightType.FP8 in detected_types+ model = convert_module_to_hp_if_necessary(model)+ assert (+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear+ ), "CompressedLinear layer was not converted to Linear"++ def test_nvfp4(self):+ model = get_tiny_model(get_model_path(self.nvfp4_model_path))+ assert (+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"+ ), "Model does not contain CompressedLinear layers"+ detected_types = check_and_mark_quantized_module(model)+ assert ModuleWeightType.NVFP4 in detected_types+ model = convert_module_to_hp_if_necessary(model)+ assert (+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear+ ), "CompressedLinear layer was not converted to Linear"++ def test_mxfp4(self):+ model = get_tiny_model(get_model_path(self.mxfp4_model_path))+ assert (+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"+ ), "Model does not contain CompressedLinear layers"+ detected_types = check_and_mark_quantized_module(model)+ assert ModuleWeightType.MXFP4 in detected_types+ model = convert_module_to_hp_if_necessary(model)+ assert (+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear+ ), "CompressedLinear layer was not converted to Linear"diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex d2f6cabcd..780582be0 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -688,6 +688,8 @@ class TestAutoRound:mock_layer.block_size = block_sizemock_layer.data_type = "fp8_e4m3fn"mock_layer.__class__.__name__ = "FP8Linear"+ # Explicitly set compressor to None to prevent MagicMock auto-creating it+ mock_layer.compressor = None# mock_layer.to should return a mock layer that we can checkmock_layer.to.return_value = mock_layerdiff --git a/test/test_cuda/advanced/test_fp8_input.py b/test/test_cuda/advanced/test_fp8_input.pyindex de4dfa453..1874b2ccd 100644--- a/test/test_cuda/advanced/test_fp8_input.py+++ b/test/test_cuda/advanced/test_fp8_input.py@@ -3,17 +3,37 @@ import shutilimport pytestimport torch-import transformersfrom packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluatefrom auto_round.utils import llm_load_model+from auto_round.utils.weight_handler import (+ ModuleWeightType,+ check_and_mark_quantized_module,+ convert_module_to_hp_if_necessary,+)from ...helpers import get_model_path, get_tiny_model, transformers_version+def generate_prompt(model, tokenizer):+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+++def evaluate_accuracy(save_dir, task="lambada_openai", threshold=0.25):+ """Helper function to evaluate model accuracy on a given task."""+ model_args = f"pretrained={save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks=task, batch_size="auto")+ acc = result["results"][task]["acc,none"]+ print(f"{task} accuracy: {acc}")+ assert acc > threshold, f"Accuracy {acc} is below threshold {threshold}"+ return acc++class TestAutoRound:save_dir = "./saved"@@ -36,15 +56,13 @@ class TestAutoRound:shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- def test_small_model_rtn_generation(self):+ def test_small_model_rtn_generation(self, mock_fp8_capable_device):model, tokenizer = self.tiny_fp8_model()ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)ar.quantize_and_save(output_dir=self.save_dir)model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.save_dir)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ generate_prompt(model, tokenizer)shutil.rmtree(self.save_dir, ignore_errors=True)@pytest.mark.skipif(@@ -52,7 +70,7 @@ class TestAutoRound:reason="GGUF format saving and loading failed in transformers v5, \https://github.com/huggingface/transformers/issues/43482",)- def test_gguf_imatrix(self):+ def test_gguf_imatrix(self, mock_fp8_capable_device):model, tokenizer = self.tiny_fp8_model()ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)@@ -69,49 +87,33 @@ class TestAutoRound:# inputs = tokenizer(text, return_tensors="pt").to(model.device)# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- def test_small_model_rtn(self):+ def test_small_model_rtn(self, mock_fp8_capable_device):model_name = get_model_path("qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=0)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- model_args = f"pretrained={self.save_dir}"- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")- print(result["results"]["lambada_openai"]["acc,none"])- assert result["results"]["lambada_openai"]["acc,none"] > 0.25-+ evaluate_accuracy(self.save_dir, threshold=0.25)shutil.rmtree(self.save_dir, ignore_errors=True)- def test_small_model_iters1(self):+ def test_small_model_iters1(self, mock_fp8_capable_device):model_name = get_model_path("qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=1)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- model_args = f"pretrained={self.save_dir}"- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")- print(result["results"]["lambada_openai"]["acc,none"])- assert result["results"]["lambada_openai"]["acc,none"] > 0.25-+ evaluate_accuracy(self.save_dir, threshold=0.25)shutil.rmtree(self.save_dir, ignore_errors=True)- def test_medium_model_rtn(self):+ def test_medium_model_rtn(self, mock_fp8_capable_device):model_name = get_model_path("qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=0)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- model_args = f"pretrained={self.save_dir}"- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")- print(result["results"]["lambada_openai"]["acc,none"])- assert result["results"]["lambada_openai"]["acc,none"] > 0.33-+ evaluate_accuracy(self.save_dir, threshold=0.33)shutil.rmtree(self.save_dir, ignore_errors=True)- def test_medium_model_rtn_with_lm_head(self):+ def test_medium_model_rtn_with_lm_head(self, mock_fp8_capable_device):model_name = get_model_path("qwen/Qwen3-0.6B-FP8")layer_config = {"lm_head": {"bits": 4}}ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- model_args = f"pretrained={self.save_dir}"- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")- print(result["results"]["lambada_openai"]["acc,none"])- assert result["results"]["lambada_openai"]["acc,none"] > 0.33-+ evaluate_accuracy(self.save_dir, threshold=0.33)shutil.rmtree(self.save_dir, ignore_errors=True)@pytest.mark.skipif(@@ -119,7 +121,7 @@ class TestAutoRound:reason="GGUF format saving and loading failed in transformers v5, \https://github.com/huggingface/transformers/issues/43482",)- def test_fp8_model_gguf(self):+ def test_fp8_model_gguf(self, mock_fp8_capable_device):from llama_cpp import Llamamodel, tokenizer = self.tiny_fp8_model()@@ -149,7 +151,7 @@ class TestAutoRound:reason="We need this patch for fp8 model loading without dequantization.""https://github.com/intel/auto-round/blob/72e1cecb4a984db101e26700618266115029b9ac/test/test_cuda/quantization/test_mxfp_nvfp.py#L19C5-L19C25",)- def test_diff_datatype(self):+ def test_diff_datatype(self, mock_fp8_capable_device):for scheme in ["NVFP4", "MXFP4"]:model_name = get_model_path("qwen/Qwen3-0.6B-FP8")for iters in [0, 1]:@@ -157,3 +159,16 @@ class TestAutoRound:ar = AutoRound(model_name, iters=iters, scheme=scheme)ar.quantize_and_save(output_dir=self.save_dir)shutil.rmtree(self.save_dir, ignore_errors=True)+++# requires GPU to load FP8Linear+class TestFP8Linear:+ def test_fp8_input(self, mock_fp8_capable_device):+ model = get_tiny_model(get_model_path("qwen/Qwen3-0.6B-FP8"))+ assert (+ type(model.model.layers[0].mlp.up_proj).__name__ == "FP8Linear"+ ), "Model does not contain FP8Linear layers"+ detected_types = check_and_mark_quantized_module(model)+ assert ModuleWeightType.FP8 in detected_types+ model = convert_module_to_hp_if_necessary(model)+ assert type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear, "FP8Linear layer was not converted to Linear"diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex a1dfe7023..11843249d 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -28,8 +28,7 @@ class TestAutoRound:shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, mock_cuda_capability):- # mock_cuda_capability is not used but needed to enable fp8 support in torch+ def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, mock_fp8_capable_device):model_name = get_model_path("qwen/Qwen3-0.6B-FP8")tiny_model_path = "./tmp/tiny_qwen3_fp8"save_tiny_model(model_name, tiny_model_path)