Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Implement MXFP4Handler MXFP8Handler NVFP4Handler (#1385)

Xin Hef7db867a9d4f
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index fb4e40298..ccdf44c3d 100644
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -31,10 +31,10 @@ Table of Contents:
- convert_module_to_hp_if_necessary(): Main conversion function
3. HANDLER IMPLEMENTATIONS
- - FP8BlockHandler: Fully implemented for FP8 block-wise quantization
- - MXFP8Handler: Placeholder (TODO)
- - MXFP4Handler: Placeholder (TODO)
- - NVFP4Handler: Placeholder (TODO)
+ - FP8Handler: Fully implemented for FP8 block-wise quantization
+ - MXFP8Handler: CompressedLinear with MXFP8PackedCompressor
+ - MXFP4Handler: CompressedLinear with MXFP4PackedCompressor
+ - NVFP4Handler: CompressedLinear with NVFP4PackedCompressor
Quick Start Guide:
Usage - Detect and Convert:
@@ -154,10 +154,10 @@ class ModuleWeightType(Enum):
3. Register with @register_weight_type_handler decorator
"""
- FP8_BLOCK = auto() # FP8 with block-wise scaling (fully implemented)
- MXFP8 = auto() # MX FP8 (placeholder)
- MXFP4 = auto() # MX FP4 (placeholder)
- NVFP4 = auto() # NV FP4 (placeholder)
+ FP8 = auto() # FP8 with block-wise scaling (FP8Linear/CompressedLinear from transformers)
+ MXFP8 = auto() # MX FP8 (CompressedLinear with MXFP8PackedCompressor)
+ MXFP4 = auto() # MX FP4 (CompressedLinear with MXFP4PackedCompressor)
+ NVFP4 = auto() # NV FP4 (CompressedLinear with NVFP4PackedCompressor)
class WeightTypeHandler(ABC):
@@ -463,69 +463,101 @@ def _dequant_fp8_linear_weight(
# ----------------------------------------------------------------------------
# FP8 Block Handler - Full Implementation (Reference Example)
# ----------------------------------------------------------------------------
-@register_weight_type_handler(ModuleWeightType.FP8_BLOCK)
-class FP8BlockHandler(WeightTypeHandler):
+@register_weight_type_handler(ModuleWeightType.FP8)
+class FP8Handler(WeightTypeHandler):
"""Handler for FP8 block-wise quantized layers.
This handler supports:
- FP8Linear layers with block-wise scaling
- - CompressedLinear layers with FP8 compression
+ - CompressedLinear layers with compressor
- torch.nn.Linear layers with FP8 dtype weights
"""
- # Registry mapping FP8 layer class names to their dequantization handlers
- FP8_DEQUANT_REGISTRY: Dict[str, Callable] = {}
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a module is an FP8 linear layer based on actual characteristics."""
+ # Check for CompressedLinear layer type (exclude MXFP4, MXFP8, NVFP4)
+ if module.__class__.__name__ == "CompressedLinear":
+ if hasattr(module, "compressor") and module.compressor is not None:
+ compressor_name = module.compressor.__class__.__name__
+ return "Float" in compressor_name
+ return True
- @classmethod
- def register_fp8_layer(cls, layer_name: str):
- """Register a dequantization handler for an FP8 layer type.
+ # Check for FP8Linear layer type
+ if module.__class__.__name__ == "FP8Linear":
+ return True
- Args:
- layer_name: The class name of the FP8 layer type.
+ # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)
+ if type(module) == torch.nn.Linear and module.weight is not None:
+ if str(module.weight.dtype).startswith("torch.float8"):
+ return True
- Returns:
- Decorator function that registers the handler.
- """
+ return False
- def decorator(fn: Callable):
- cls.FP8_DEQUANT_REGISTRY[layer_name] = fn
- return fn
+ def convert_layer(
+ self,
+ layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+ ) -> torch.nn.Module:
+ """Convert a single FP8/CompressedLinear layer to a standard Linear layer."""
+ from auto_round.schemes import QuantizationScheme
+ from auto_round.utils.device import is_gaudi2
- return decorator
+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
+ if layer.bias is not None:
+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
- def _dequant_layer(self, layer: torch.nn.Module, dtype: torch.dtype, device: str) -> torch.Tensor:
- """Dequantize an FP8 layer using the internal registry.
+ # Copy quantization scheme attributes
+ scheme_keys = (f.name for f in fields(QuantizationScheme))
+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
+ setattr(new_layer, key, getattr(layer, key, None))
- Args:
- layer: The FP8 layer to dequantize.
- dtype: Target dtype for dequantized weights.
- device: Target device for dequantization.
+ # Handle Gaudi2 device compatibility
+ if is_gaudi2():
+ device = "cpu"
- Returns:
- Dequantized weight tensor.
+ layer = layer.to(device)
- Raises:
- NotImplementedError: If the layer type is not registered.
- """
- name = layer.__class__.__name__
- if name not in self.FP8_DEQUANT_REGISTRY:
- raise NotImplementedError(
- f"Unsupported FP8 layer type: {name}. " f"Supported types: {list(self.FP8_DEQUANT_REGISTRY.keys())}"
- )
- return self.FP8_DEQUANT_REGISTRY[name](layer, dtype=dtype, device=device)
+ # Check if layer has compressor.decompress_module method
+ if hasattr(layer, "compressor") and hasattr(layer.compressor, "decompress_module"):
+ dq_weight = layer.compressor.decompress_module(layer)
+ else:
+ # Use FP8 block-based dequantization
+ weight_scale = getattr(layer, "weight_scale", None)
+ block_size = getattr(layer, "block_size", None)
+ data_type = getattr(layer, "data_type", None)
+ if weight_scale is None:
+ weight_scale = getattr(layer, "weight_scale_inv", None)
+ if weight_scale is None:
+ raise AttributeError(
+ "FP8 layer is missing both 'weight_scale' and 'weight_scale_inv' "
+ "attributes required for dequantization."
+ )
+ dq_weight = _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
- def detect_layer(self, module: torch.nn.Module) -> bool:
- """Check if a module is an FP8 linear layer based on actual characteristics."""
- # Check registry for supported FP8 layer types
- layer_name = module.__class__.__name__
- if layer_name in self.FP8_DEQUANT_REGISTRY:
- return True
+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
- # Fallback: Check for FP8 dtype (for torch.nn.Linear with FP8 weights)
- if type(module) == torch.nn.Linear and module.weight is not None:
- if str(module.weight.dtype).startswith("torch.float8"):
- return True
+ if to_cpu:
+ new_layer = new_layer.to("cpu")
+
+ return new_layer
+
+# ----------------------------------------------------------------------------
+# MXFP4 Handler - CompressedLinear with MXFP4PackedCompressor
+# ----------------------------------------------------------------------------
+@register_weight_type_handler(ModuleWeightType.MXFP4)
+class MXFP4Handler(WeightTypeHandler):
+ """Handler for MXFP4 quantized layers (CompressedLinear with MXFP4PackedCompressor)."""
+
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a module is an MXFP4 CompressedLinear layer."""
+ if module.__class__.__name__ != "CompressedLinear":
+ return False
+ if hasattr(module, "compressor") and module.compressor is not None:
+ compressor_name = module.compressor.__class__.__name__
+ return "MXFP4" in compressor_name
return False
def convert_layer(
@@ -535,10 +567,9 @@ class FP8BlockHandler(WeightTypeHandler):
device: str = "cpu",
to_cpu: bool = False,
) -> torch.nn.Module:
- """Convert a single FP8 layer to a standard Linear layer."""
+ """Convert an MXFP4 CompressedLinear layer to a standard Linear layer."""
from auto_round.schemes import QuantizationScheme
from auto_round.utils.device import is_gaudi2
- from auto_round.utils.model import set_module
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
if layer.bias is not None:
@@ -553,8 +584,30 @@ class FP8BlockHandler(WeightTypeHandler):
if is_gaudi2():
device = "cpu"
- # Use registry-based dequantization
- dq_weight = self._dequant_layer(layer, dtype=dtype, device=device)
+ layer = layer.to(device)
+
+ # MXFP4 dequantization using to_dtype from mxfp4_qdq_utils
+ from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype
+
+ # Get packed weight and scale from layer
+ # MXFP4 weights are stored as packed uint8 in weight_packed
+ weight_packed = getattr(layer, "weight_packed", None)
+ if weight_packed is None:
+ weight_packed = getattr(layer, "weight", None)
+ weight_scale = getattr(layer, "weight_scale", None)
+ weight_scale = weight_scale.to(torch.uint8)
+
+ if weight_packed is None or weight_scale is None:
+ raise ValueError("MXFP4 layer must have weight_packed and weight_scale attributes")
+
+ # Dequantize using to_dtype function
+ dq_weight = to_dtype(
+ data_lp=weight_packed,
+ scale_e8m0=weight_scale,
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=dtype,
+ )
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
if to_cpu:
@@ -563,81 +616,121 @@ class FP8BlockHandler(WeightTypeHandler):
return new_layer
-# --- FP8 Layer Dequantization Handlers ---
-# Register specific dequantization logic for different FP8 layer types
+# ----------------------------------------------------------------------------
+# MXFP8 Handler - CompressedLinear with MXFP8PackedCompressor
+# ----------------------------------------------------------------------------
+@register_weight_type_handler(ModuleWeightType.MXFP8)
+class MXFP8Handler(WeightTypeHandler):
+ """Handler for MXFP8 quantized layers (CompressedLinear with MXFP8PackedCompressor)."""
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a module is an MXFP8 CompressedLinear layer."""
+ if module.__class__.__name__ != "CompressedLinear":
+ return False
+ if hasattr(module, "compressor") and module.compressor is not None:
+ compressor_name = module.compressor.__class__.__name__
+ return "MXFP8" in compressor_name
+ return False
[email protected]_fp8_layer("CompressedLinear")
-def _dequant_compressed_linear(
- layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"
-) -> torch.Tensor:
- """Dequantize CompressedLinear layer using compressor.
+ def convert_layer(
+ self,
+ layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+ ) -> torch.nn.Module:
+ """Convert an MXFP8 CompressedLinear layer to a standard Linear layer."""
+ from auto_round.schemes import QuantizationScheme
+ from auto_round.utils.device import is_gaudi2
- Args:
- layer: The CompressedLinear layer to dequantize.
- dtype: Target dtype for dequantized weights.
- device: Target device for dequantization.
+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
+ if layer.bias is not None:
+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
- Returns:
- Dequantized weight tensor.
- """
- layer = layer.to(device)
- return layer.compressor.decompress_module(layer)
+ # Copy quantization scheme attributes
+ scheme_keys = (f.name for f in fields(QuantizationScheme))
+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
+ setattr(new_layer, key, getattr(layer, key, None))
+ # Handle Gaudi2 device compatibility
+ if is_gaudi2():
+ device = "cpu"
[email protected]_fp8_layer("FP8Linear")
-def _dequant_fp8_linear(
- layer: torch.nn.Module, dtype: torch.dtype = torch.bfloat16, device: str = "cpu"
-) -> torch.Tensor:
- """Dequantize FP8Linear layer using block-based dequantization.
+ layer = layer.to(device)
- Args:
- layer: The FP8Linear layer to dequantize.
- dtype: Target dtype for dequantized weights.
- device: Target device for dequantization.
+ # MXFP8 dequantization using dequant_mx_fp8 from mxfp8_qdq_utils
+ from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8
- Returns:
- Dequantized weight tensor.
- """
- layer = layer.to(device)
- weight_scale = getattr(layer, "weight_scale", None)
- block_size = getattr(layer, "block_size", None)
- data_type = getattr(layer, "data_type", None)
- if weight_scale is None:
- weight_scale = getattr(layer, "weight_scale_inv", None)
- if weight_scale is None:
- raise AttributeError(
- "FP8Linear layer is missing both 'weight_scale' and 'weight_scale_inv' "
- "attributes required for dequantization."
+ # Get weight and scale from layer
+ weight = layer.weight
+ weight_scale = getattr(layer, "weight_scale", None)
+ weight_scale = weight_scale.to(torch.uint8)
+
+ if weight is None or weight_scale is None:
+ raise ValueError("MXFP8 layer must have weight and weight_scale attributes")
+
+ # Dequantize using dequant_mx_fp8 function
+ dq_weight = dequant_mx_fp8(
+ weight_fp8=weight,
+ scale_e8m0=weight_scale,
+ block_size=32,
+ target_dtype=dtype,
)
- return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+
+ if to_cpu:
+ new_layer = new_layer.to("cpu")
+
+ return new_layer
# ----------------------------------------------------------------------------
-# Placeholder Handlers for Unimplemented Types
+# NVFP4 Handler - CompressedLinear with NVFP4PackedCompressor
# ----------------------------------------------------------------------------
+@register_weight_type_handler(ModuleWeightType.NVFP4)
+class NVFP4Handler(WeightTypeHandler):
+ """Handler for NVFP4 quantized layers (CompressedLinear with NVFP4PackedCompressor)."""
+ def detect_layer(self, module: torch.nn.Module) -> bool:
+ """Check if a module is an NVFP4 CompressedLinear layer."""
+ if module.__class__.__name__ != "CompressedLinear":
+ return False
+ if hasattr(module, "compressor") and module.compressor is not None:
+ compressor_name = module.compressor.__class__.__name__
+ return "NVFP4" in compressor_name
+ return False
-def _create_placeholder_handler(weight_type_name: str) -> Type[WeightTypeHandler]:
- """Factory function to create placeholder handlers for unimplemented weight types."""
+ def convert_layer(
+ self,
+ layer: torch.nn.Module,
+ dtype: torch.dtype = torch.bfloat16,
+ device: str = "cpu",
+ to_cpu: bool = False,
+ ) -> torch.nn.Module:
+ """Convert an NVFP4 CompressedLinear layer to a standard Linear layer."""
+ from auto_round.schemes import QuantizationScheme
+ from auto_round.utils.device import is_gaudi2
- class PlaceholderHandler(WeightTypeHandler):
- """Placeholder handler - detection always returns False."""
+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
+ if layer.bias is not None:
+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
- def detect_layer(self, module: torch.nn.Module) -> bool:
- return False
+ # Copy quantization scheme attributes
+ scheme_keys = (f.name for f in fields(QuantizationScheme))
+ for key in tuple(scheme_keys) + ("global_name", "scale_dtype"):
+ setattr(new_layer, key, getattr(layer, key, None))
- def convert_layer(self, layer, dtype=torch.bfloat16, device="cpu", to_cpu=False):
- raise NotImplementedError(f"{weight_type_name} layer conversion not yet implemented")
+ # Handle Gaudi2 device compatibility
+ if is_gaudi2():
+ device = "cpu"
- PlaceholderHandler.__name__ = f"{weight_type_name}Handler"
- PlaceholderHandler.__doc__ = (
- f"Placeholder handler for {weight_type_name}. TODO: Implement detection and conversion."
- )
- return PlaceholderHandler
+ layer = layer.to(device)
+ # Use compressor.decompress_module for dequantization
+ dq_weight = layer.compressor.decompress_module(layer)
+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
-# Register placeholder handlers for unimplemented types
-register_weight_type_handler(ModuleWeightType.MXFP8)(_create_placeholder_handler("MXFP8"))
-register_weight_type_handler(ModuleWeightType.MXFP4)(_create_placeholder_handler("MXFP4"))
-register_weight_type_handler(ModuleWeightType.NVFP4)(_create_placeholder_handler("NVFP4"))
+ if to_cpu:
+ new_layer = new_layer.to("cpu")
+
+ return new_layer
diff --git a/test/fixtures.py b/test/fixtures.py
index 482b74598..4ec157b1f 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -124,8 +124,8 @@ def tiny_qwen_2_5_vl_model_path():
# Mock torch.cuda.get_device_capability to always return (9, 0) like H100
[email protected](autouse=True, scope="session")
-def mock_cuda_capability():
+def mock_fp8_capable_device():
from unittest.mock import patch
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
diff --git a/test/test_cpu/advanced/test_low_precision_input_model.py b/test/test_cpu/advanced/test_low_precision_input_model.py
new file mode 100644
index 000000000..48aa59b24
--- /dev/null
+++ b/test/test_cpu/advanced/test_low_precision_input_model.py
@@ -0,0 +1,51 @@
+import torch
+
+from auto_round.utils.weight_handler import (
+ ModuleWeightType,
+ check_and_mark_quantized_module,
+ convert_module_to_hp_if_necessary,
+)
+
+from ...helpers import get_model_path, get_tiny_model
+
+
+class TestCompressedTensor:
+ nvfp4_model_path = "kaitchup/Qwen3-0.6B-NVFP4"
+ mxfp4_model_path = "QuixiAI/Llama-3.2-1B-MXFP4"
+ fp8_block_model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"
+
+ def test_fp8_block(self):
+ model = get_tiny_model(get_model_path(self.fp8_block_model_path))
+ assert (
+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"
+ ), "Model does not contain CompressedLinear layers"
+ detected_types = check_and_mark_quantized_module(model)
+ assert ModuleWeightType.FP8 in detected_types
+ model = convert_module_to_hp_if_necessary(model)
+ assert (
+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear
+ ), "CompressedLinear layer was not converted to Linear"
+
+ def test_nvfp4(self):
+ model = get_tiny_model(get_model_path(self.nvfp4_model_path))
+ assert (
+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"
+ ), "Model does not contain CompressedLinear layers"
+ detected_types = check_and_mark_quantized_module(model)
+ assert ModuleWeightType.NVFP4 in detected_types
+ model = convert_module_to_hp_if_necessary(model)
+ assert (
+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear
+ ), "CompressedLinear layer was not converted to Linear"
+
+ def test_mxfp4(self):
+ model = get_tiny_model(get_model_path(self.mxfp4_model_path))
+ assert (
+ type(model.model.layers[0].mlp.up_proj).__name__ == "CompressedLinear"
+ ), "Model does not contain CompressedLinear layers"
+ detected_types = check_and_mark_quantized_module(model)
+ assert ModuleWeightType.MXFP4 in detected_types
+ model = convert_module_to_hp_if_necessary(model)
+ assert (
+ type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear
+ ), "CompressedLinear layer was not converted to Linear"
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index d2f6cabcd..780582be0 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -688,6 +688,8 @@ class TestAutoRound:
mock_layer.block_size = block_size
mock_layer.data_type = "fp8_e4m3fn"
mock_layer.__class__.__name__ = "FP8Linear"
+ # Explicitly set compressor to None to prevent MagicMock auto-creating it
+ mock_layer.compressor = None
# mock_layer.to should return a mock layer that we can check
mock_layer.to.return_value = mock_layer
diff --git a/test/test_cuda/advanced/test_fp8_input.py b/test/test_cuda/advanced/test_fp8_input.py
index de4dfa453..1874b2ccd 100644
--- a/test/test_cuda/advanced/test_fp8_input.py
+++ b/test/test_cuda/advanced/test_fp8_input.py
@@ -3,17 +3,37 @@ import shutil
import pytest
import torch
-import transformers
from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
from auto_round.eval.evaluation import simple_evaluate
from auto_round.utils import llm_load_model
+from auto_round.utils.weight_handler import (
+ ModuleWeightType,
+ check_and_mark_quantized_module,
+ convert_module_to_hp_if_necessary,
+)
from ...helpers import get_model_path, get_tiny_model, transformers_version
+def generate_prompt(model, tokenizer):
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+
+
+def evaluate_accuracy(save_dir, task="lambada_openai", threshold=0.25):
+ """Helper function to evaluate model accuracy on a given task."""
+ model_args = f"pretrained={save_dir}"
+ result = simple_evaluate(model="hf", model_args=model_args, tasks=task, batch_size="auto")
+ acc = result["results"][task]["acc,none"]
+ print(f"{task} accuracy: {acc}")
+ assert acc > threshold, f"Accuracy {acc} is below threshold {threshold}"
+ return acc
+
+
class TestAutoRound:
save_dir = "./saved"
@@ -36,15 +56,13 @@ class TestAutoRound:
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- def test_small_model_rtn_generation(self):
+ def test_small_model_rtn_generation(self, mock_fp8_capable_device):
model, tokenizer = self.tiny_fp8_model()
ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)
ar.quantize_and_save(output_dir=self.save_dir)
model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ generate_prompt(model, tokenizer)
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.mark.skipif(
@@ -52,7 +70,7 @@ class TestAutoRound:
reason="GGUF format saving and loading failed in transformers v5, \
https://github.com/huggingface/transformers/issues/43482",
)
- def test_gguf_imatrix(self):
+ def test_gguf_imatrix(self, mock_fp8_capable_device):
model, tokenizer = self.tiny_fp8_model()
ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)
ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)
@@ -69,49 +87,33 @@ class TestAutoRound:
# inputs = tokenizer(text, return_tensors="pt").to(model.device)
# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- def test_small_model_rtn(self):
+ def test_small_model_rtn(self, mock_fp8_capable_device):
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=0)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- model_args = f"pretrained={self.save_dir}"
- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
- print(result["results"]["lambada_openai"]["acc,none"])
- assert result["results"]["lambada_openai"]["acc,none"] > 0.25
-
+ evaluate_accuracy(self.save_dir, threshold=0.25)
shutil.rmtree(self.save_dir, ignore_errors=True)
- def test_small_model_iters1(self):
+ def test_small_model_iters1(self, mock_fp8_capable_device):
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=1)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- model_args = f"pretrained={self.save_dir}"
- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
- print(result["results"]["lambada_openai"]["acc,none"])
- assert result["results"]["lambada_openai"]["acc,none"] > 0.25
-
+ evaluate_accuracy(self.save_dir, threshold=0.25)
shutil.rmtree(self.save_dir, ignore_errors=True)
- def test_medium_model_rtn(self):
+ def test_medium_model_rtn(self, mock_fp8_capable_device):
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=0)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- model_args = f"pretrained={self.save_dir}"
- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
- print(result["results"]["lambada_openai"]["acc,none"])
- assert result["results"]["lambada_openai"]["acc,none"] > 0.33
-
+ evaluate_accuracy(self.save_dir, threshold=0.33)
shutil.rmtree(self.save_dir, ignore_errors=True)
- def test_medium_model_rtn_with_lm_head(self):
+ def test_medium_model_rtn_with_lm_head(self, mock_fp8_capable_device):
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
layer_config = {"lm_head": {"bits": 4}}
ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- model_args = f"pretrained={self.save_dir}"
- result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
- print(result["results"]["lambada_openai"]["acc,none"])
- assert result["results"]["lambada_openai"]["acc,none"] > 0.33
-
+ evaluate_accuracy(self.save_dir, threshold=0.33)
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.mark.skipif(
@@ -119,7 +121,7 @@ class TestAutoRound:
reason="GGUF format saving and loading failed in transformers v5, \
https://github.com/huggingface/transformers/issues/43482",
)
- def test_fp8_model_gguf(self):
+ def test_fp8_model_gguf(self, mock_fp8_capable_device):
from llama_cpp import Llama
model, tokenizer = self.tiny_fp8_model()
@@ -149,7 +151,7 @@ class TestAutoRound:
reason="We need this patch for fp8 model loading without dequantization."
"https://github.com/intel/auto-round/blob/72e1cecb4a984db101e26700618266115029b9ac/test/test_cuda/quantization/test_mxfp_nvfp.py#L19C5-L19C25",
)
- def test_diff_datatype(self):
+ def test_diff_datatype(self, mock_fp8_capable_device):
for scheme in ["NVFP4", "MXFP4"]:
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
for iters in [0, 1]:
@@ -157,3 +159,16 @@ class TestAutoRound:
ar = AutoRound(model_name, iters=iters, scheme=scheme)
ar.quantize_and_save(output_dir=self.save_dir)
shutil.rmtree(self.save_dir, ignore_errors=True)
+
+
+# requires GPU to load FP8Linear
+class TestFP8Linear:
+ def test_fp8_input(self, mock_fp8_capable_device):
+ model = get_tiny_model(get_model_path("qwen/Qwen3-0.6B-FP8"))
+ assert (
+ type(model.model.layers[0].mlp.up_proj).__name__ == "FP8Linear"
+ ), "Model does not contain FP8Linear layers"
+ detected_types = check_and_mark_quantized_module(model)
+ assert ModuleWeightType.FP8 in detected_types
+ model = convert_module_to_hp_if_necessary(model)
+ assert type(model.model.layers[0].mlp.up_proj) is torch.nn.Linear, "FP8Linear layer was not converted to Linear"
diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.py
index a1dfe7023..11843249d 100644
--- a/test/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py
@@ -28,8 +28,7 @@ class TestAutoRound:
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, mock_cuda_capability):
- # mock_cuda_capability is not used but needed to enable fp8 support in torch
+ def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, mock_fp8_capable_device):
model_name = get_model_path("qwen/Qwen3-0.6B-FP8")
tiny_model_path = "./tmp/tiny_qwen3_fp8"
save_tiny_model(model_name, tiny_model_path)