PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Loosen the packing restrictions for mxfp&nvfp (#911)
Weiweidcfacefaa552
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 34cbb6f94..f7979e269 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -38,7 +38,7 @@ import transformersfrom auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMALfrom auto_round.data_type.nvfp import cast_to_fp4, get_reciprocalfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad-from auto_round.utils import _get_packing_device, is_mx_fp, is_nv_fp+from auto_round.utils import BackendDataType, _get_packing_device, is_mx_fp, is_nv_fp# from auto_round.utils import get_weight_compress_dtypelogger = getLogger(__name__)@@ -72,14 +72,22 @@ class QuantLinear(nn.Module):super().__init__()if bits not in [4, 8]:raise NotImplementedError("Only 4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")self.is_mx = is_mx_fp(data_type)self.is_nv = is_nv_fp(data_type)- if self.is_mx and group_size != 32:- raise NotImplementedError("Only group_size 32 are supported for mxfp.")- if self.is_nv and group_size not in [16, 32]:- raise NotImplementedError("Only group_size 16 are supported for nvfp.")+ if self.is_mx:+ if group_size != 32:+ raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")+ if infeatures % group_size != 0:+ raise NotImplementedError(+ f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type."+ )+ if self.is_nv:+ if group_size % 16 != 0:+ raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")+ if infeatures % group_size != 0:+ raise NotImplementedError(+ f"in_feature must be divisible by {group_size} for {BackendDataType.NV_FP} data type."+ )self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bitsdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex aa74f37e1..4d26eb143 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -127,12 +127,19 @@ def feature_multiply_checker_group_size()+def in_feature_checker_group_size(in_feature, out_feature, config):+ group_size = config["group_size"]+ return in_feature % group_size == 0++feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)feature_multiply_checker_16 = functools.partial(feature_multiply_checker, in_feature_multiplier=16)in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32)-+in_feature_multiply_checker_32 = functools.partial(+ feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=None+)exllamav2_feature_checker = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32)@@ -141,6 +148,8 @@ gptqmodel_marlin_feature_checker = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=1, out_feature_multiplier=64)+mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)+def fp8_static_scheme_checker(in_feature: int,@@ -239,7 +248,7 @@ BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(act_data_type=["mx_fp_rceil"],act_dynamic=[True],priority=0,- checkers=[feature_multiply_checker_32],+ checkers=[mxfp_nvfp_feature_checker],alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)@@ -259,7 +268,7 @@ BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(act_data_type=["mx_fp_rceil"],act_dynamic=[True],priority=0,- checkers=[feature_multiply_checker_32],+ checkers=[mxfp_nvfp_feature_checker],alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)@@ -280,7 +289,7 @@ BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(act_data_type=["nv_fp4_with_static_gs"],act_dynamic=[True],priority=0,- checkers=[feature_multiply_checker_16],+ checkers=[mxfp_nvfp_feature_checker],alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 8c8c9acc5..6742011fa 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -2963,6 +2963,18 @@ def set_layer_config(layer_config.setdefault(n, copy.deepcopy(default_dict))layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")+ # enforce shape divisibility for mxfp/nvfp+ if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[1] % default_dict["group_size"]:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update(+ {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}+ )+ logger.warning_once(+ f"{n} skipped quantization (shape not divisible by {default_dict['group_size']})."+ )# 9. block layers: mark as in_blocks=Truefor name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 180fd8f2f..ea484316b 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -302,275 +302,6 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)shutil.rmtree(quantized_model_path, ignore_errors=True)- def test_mxfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "MXFP4"- layer_config = {}- fp_layers_str = "k_proj"- from auto_round.utils import get_fp_layer_names-- not_quantize_layer_names = get_fp_layer_names(model, fp_layers_str)- for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float"}- autoround = AutoRound(- model,- self.tokenizer,- scheme=scheme,- iters=2,- seqlen=2,- layer_config=layer_config,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- autoround.quantize()- compressed_model = autoround.save_quantized(- output_dir=quantized_model_path, inplace=True, format="llm_compressor"- )- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- skip_layer = compressed_model.model.decoder.layers[3].self_attn.k_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_packed")- and tmp_layer.weight_scale.dtype is torch.uint8- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal MXFP4 packing name or data_type or shape"- assert not hasattr(skip_layer, "weight_scale") and not hasattr( ## check skipped layers- skip_layer, "weight_packed"- ), "Illegal MXFP4 quantization for fp_layers"- quantization_config = AutoConfig.from_pretrained(- quantized_model_path, trust_remote_code=True- ).quantization_config- assert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True- and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4- ), f"Invalid MXFP4 quantization configuration: {quantization_config}"-- shutil.rmtree("./saved", ignore_errors=True)-- def test_rtn_mxfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "MXFP4"- layer_config = {}- fp_layers_str = "k_proj"- from auto_round.utils import get_fp_layer_names-- not_quantize_layer_names = get_fp_layer_names(model, fp_layers_str)- for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float"}- autoround = AutoRound(- model,- self.tokenizer,- scheme=scheme,- iters=0,- seqlen=2,- layer_config=layer_config,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- autoround.quantize()- compressed_model = autoround.save_quantized(- output_dir=quantized_model_path, inplace=True, format="llm_compressor"- )- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- skip_layer = compressed_model.model.decoder.layers[3].self_attn.k_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_packed")- and tmp_layer.weight_scale.dtype is torch.uint8- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal MXFP4 packing name or data_type or shape"- assert not hasattr(skip_layer, "weight_scale") and not hasattr( ## check skipped layers- skip_layer, "weight_packed"- ), "Illegal MXFP4 quantization for fp_layers"- quantization_config = AutoConfig.from_pretrained(- quantized_model_path, trust_remote_code=True- ).quantization_config- assert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True- and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4- ), f"Invalid MXFP4 quantization configuration: {quantization_config}"- shutil.rmtree("./saved", ignore_errors=True)-- def test_mxfp8_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "MXFP8"- autoround = AutoRound(- model,- self.tokenizer,- scheme=scheme,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight")- and tmp_layer.weight.dtype is torch.float8_e4m3fn- and tmp_layer.weight_scale.dtype is torch.uint8- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal MXFP8 packing name or data_type or shape"- quantization_config = AutoConfig.from_pretrained(- quantized_model_path, trust_remote_code=True- ).quantization_config- assert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True- and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8- ), f"Invalid MXFP8 quantization configuration: {quantization_config}"- folder_size_gb = _get_folder_size(quantized_model_path)- # Original opt-125m is < 0.5GB -> quantized mxfp8 model should be smaller but not empty- assert (- 0.15 < folder_size_gb < 0.2- ), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.2 GB)"- shutil.rmtree("./saved", ignore_errors=True)-- def test_nvfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "NVFP4"- autoround = AutoRound(- model,- self.tokenizer,- scheme=scheme,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_global_scale")- and hasattr(tmp_layer, "input_global_scale")- and tmp_layer.weight_packed.dtype is torch.uint8- and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal NVFP4 packing name or data_type or shape"- quantization_config = AutoConfig.from_pretrained(- quantized_model_path, trust_remote_code=True- ).quantization_config- assert (- quantization_config["format"] == "nvfp4-pack-quantized"- and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4- ), f"Invalid NVFP4 quantization configuration: {quantization_config}"- folder_size_gb = _get_folder_size(quantized_model_path)- # Original opt-125m is < 0.5GB -> quantized nvfp4 model should be smaller but not empty- assert (- 0.1 < folder_size_gb < 0.15- ), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.15 GB)"- shutil.rmtree("./saved", ignore_errors=True)-- def test_nvfp4_autoround_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "NVFP4"- autoround = AutoRound(- model,- self.tokenizer,- scheme="NVFP4",- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_global_scale")- and hasattr(tmp_layer, "input_global_scale")- and tmp_layer.weight_packed.dtype is torch.uint8- and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal NVFP4 packing name or data_type or shape"- shutil.rmtree("./saved", ignore_errors=True)-- def test_nvfp4_autoround_save_quantized(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- from transformers import AutoConfig-- scheme = "NVFP4"- autoround = AutoRound(- model,- self.tokenizer,- scheme="NVFP4",- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- autoround.quantize()- compressed_model = autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_global_scale")- and hasattr(tmp_layer, "input_global_scale")- and tmp_layer.weight_packed.dtype is torch.uint8- and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal NVFP4 packing name or data_type or shape"- shutil.rmtree("./saved", ignore_errors=True)-- def test_nvfp4_moe_actmax_rtn(self):- model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"- layer_config = {- "self_attn": {"bits": 16, "act_bits": 16},- "mlp.shared_experts": {"bits": 16, "act_bits": 16},- }- scheme = "nvfp4"- autoround = AutoRound(- model_name,- scheme=scheme,- iters=0,- seqlen=2,- nsamples=2,- dataset=self.llm_dataloader,- layer_config=layer_config,- )- compressed_model, _ = autoround.quantize()- assert hasattr(compressed_model.model.layers[1].mlp.experts[0].gate_proj.orig_layer, "act_max")-- def test_nvfp4_moe_actmax_ar(self):- model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"- layer_config = {- "q_proj": {"bits": 16, "act_bits": 16},- "mlp.shared_experts": {"bits": 16, "act_bits": 16},- "experts.*2": {"bits": 16, "act_bits": 16},- "experts.*5": {"bits": 16, "act_bits": 16},- }- scheme = "nvfp4"- autoround = AutoRound(- model_name,- scheme=scheme,- iters=1,- seqlen=2,- nsamples=2,- dataset=self.llm_dataloader,- layer_config=layer_config,- )- autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")-if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_mxfp_nvfp.py b/test/test_cpu/test_mxfp_nvfp.pynew file mode 100644index 000000000..4fcd25135--- /dev/null+++ b/test/test_cpu/test_mxfp_nvfp.py@@ -0,0 +1,322 @@+import os+import shutil+import sys+import unittest++from parameterized import parameterized++sys.path.insert(0, "../..")+import torch+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer++from auto_round import AutoRound+++def _get_folder_size(path: str) -> float:+ """Return folder size in GB."""+ total_size = 0+ for dirpath, _, filenames in os.walk(path):+ for f in filenames:+ fp = os.path.join(dirpath, f)+ if os.path.isfile(fp):+ total_size += os.path.getsize(fp)+ return total_size / (1024**3) # convert to GB+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundFP(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ model_name = "facebook/opt-125m" # /tf_dataset/auto_round/models/+ self.save_dir = "./saved"+ self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_nvfp4_moe_actmax_rtn(self):+ model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"+ layer_config = {+ "self_attn": {"bits": 16, "act_bits": 16},+ "mlp.shared_experts": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=0,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ compressed_model, _ = autoround.quantize()+ assert hasattr(compressed_model.model.layers[1].mlp.experts[0].gate_proj.orig_layer, "act_max")++ def test_nvfp4_moe_actmax_ar(self):+ model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"+ layer_config = {+ "q_proj": {"bits": 16, "act_bits": 16},+ "mlp.shared_experts": {"bits": 16, "act_bits": 16},+ "experts.*2": {"bits": 16, "act_bits": 16},+ "experts.*5": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")++ def test_mxfp4_llmcompressor_format(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "MXFP4"+ layer_config = {"k_proj": {"bits": 16, "act_bits": 16, "data_type": "float"}}+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize()+ compressed_model = autoround.save_quantized(+ output_dir=quantized_model_path, inplace=True, format="llm_compressor"+ )+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ skip_layer = compressed_model.model.decoder.layers[3].self_attn.k_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_packed")+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP4 packing name or data_type or shape"+ assert not hasattr(skip_layer, "weight_scale") and not hasattr( ## check skipped layers+ skip_layer, "weight_packed"+ ), "Illegal MXFP4 quantization for fp_layers"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4+ ), f"Invalid MXFP4 quantization configuration: {quantization_config}"++ shutil.rmtree("./saved", ignore_errors=True)++ def test_rtn_mxfp4_llmcompressor_format(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "MXFP4"+ layer_config = {"k_proj": {"bits": 16, "act_bits": 16, "data_type": "float"}}+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=0,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize()+ compressed_model = autoround.save_quantized(+ output_dir=quantized_model_path, inplace=True, format="llm_compressor"+ )+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ skip_layer = compressed_model.model.decoder.layers[3].self_attn.k_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_packed")+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP4 packing name or data_type or shape"+ assert not hasattr(skip_layer, "weight_scale") and not hasattr( ## check skipped layers+ skip_layer, "weight_packed"+ ), "Illegal MXFP4 quantization for fp_layers"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4+ ), f"Invalid MXFP4 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_mxfp8_llmcompressor_format(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "MXFP8"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight")+ and tmp_layer.weight.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP8 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8+ ), f"Invalid MXFP8 quantization configuration: {quantization_config}"+ folder_size_gb = _get_folder_size(quantized_model_path)+ # Original opt-125m is < 0.5GB -> quantized mxfp8 model should be smaller but not empty+ assert (+ 0.15 < folder_size_gb < 0.2+ ), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.2 GB)"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_nvfp4_llmcompressor_format(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "NVFP4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "nvfp4-pack-quantized"+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"+ folder_size_gb = _get_folder_size(quantized_model_path)+ # Original opt-125m is < 0.5GB -> quantized nvfp4 model should be smaller but not empty+ assert (+ 0.1 < folder_size_gb < 0.15+ ), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.15 GB)"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_nvfp4_autoround_format(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "NVFP4"+ autoround = AutoRound(+ model_name,+ scheme="NVFP4",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_nvfp4_autoround_save_quantized(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ from transformers import AutoConfig++ scheme = "NVFP4"+ autoround = AutoRound(+ model_name,+ scheme="NVFP4",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize()+ compressed_model = autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_qwen_moe_quant_infer(self):+ model_name = "/tf_dataset/auto_round/models/Qwen/Qwen1.5-MoE-A2.7B"+ layer_config = {+ "layers\.(?:[3-9]|1[0-9]|2[0-3])": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="cpu")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa", limit=10)+ print(result["results"]["piqa"]["acc,none"])+ self.assertGreater(result["results"]["piqa"]["acc,none"], 0.60)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.pyindex 0ab05134f..d6d6c1f93 100644--- a/test/test_cuda/test_export.py+++ b/test/test_cuda/test_export.py@@ -322,116 +322,6 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)- def test_fp8input_mxfp4_llmcompressor_format(self):- model_name = "/models/Qwen3-0.6B-FP8"- scheme = "mxfp4"- ar = AutoRound(- model=model_name,- iters=2,- seqlen=2,- scheme=scheme,- dataset=self.llm_dataloader,- )- compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")- tmp_layer = compressed_model.model.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_packed")- and tmp_layer.weight_scale.dtype is torch.uint8- and tmp_layer.weight_scale.shape[0] == 2048- ), "Illegal MXFP4 packing name or data_type or shape"- quantization_config = AutoConfig.from_pretrained(self.save_dir, trust_remote_code=True).quantization_config- assert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True- and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4- ), f"Invalid MXFP4 quantization configuration: {quantization_config}"- shutil.rmtree(self.save_dir, ignore_errors=True)-- def test_nvfp4_llmcompressor_format(self):- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- scheme = "nvfp4"- autoround = AutoRound(- model,- tokenizer,- scheme=scheme,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")- tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj- assert (- hasattr(tmp_layer, "weight_scale")- and hasattr(tmp_layer, "weight_global_scale")- and hasattr(tmp_layer, "input_global_scale")- and tmp_layer.weight_packed.dtype is torch.uint8- and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn- and tmp_layer.weight_scale.shape[0] == 768- ), "Illegal NVFP4 packing name or data_type or shape"- quantization_config = AutoConfig.from_pretrained(- quantized_model_path, trust_remote_code=True- ).quantization_config- assert (- quantization_config["format"] == "nvfp4-pack-quantized"- and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4- ), f"Invalid NVFP4 quantization configuration: {quantization_config}"- shutil.rmtree("./saved", ignore_errors=True)- # from vllm import LLM, SamplingParams- # prompts = [- # "The capital of France is",- # "The future of AI is",- # ]- ## Create a sampling params object.- # sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)- # QUANTIZATION = "compressed-tensors"- # llm = LLM(model=quantized_model_path,- # # quantization=QUANTIZATION,- # trust_remote_code=True,- # tensor_parallel_size=1,- # enforce_eager=True,- # gpu_memory_utilization=0.7,- # )- # outputs = llm.generate(prompts, sampling_params)- # # Print the outputs.- # for output in outputs:- # prompt = output.prompt- # generated_text = output.outputs[0].text- # if "France" in prompt:- # assert "Paris" in generated_text-- def test_nvfp4_moe_actmax_rtn(self):- model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"- scheme = "nvfp4"- autoround = AutoRound(- model_name,- scheme=scheme,- iters=0,- seqlen=2,- nsamples=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")-- def test_nvfp4_moe_actmax_ar(self):- model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"- scheme = "nvfp4"- autoround = AutoRound(- model_name,- scheme=scheme,- iters=1,- seqlen=2,- nsamples=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")-if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_mxfp_nvfp.py b/test/test_cuda/test_mxfp_nvfp.pynew file mode 100644index 000000000..48dd27d9b--- /dev/null+++ b/test/test_cuda/test_mxfp_nvfp.py@@ -0,0 +1,171 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "../..")+import torch+import transformers+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round.testing_utils import require_awq, require_optimum+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "facebook/opt-125m"+ self.save_dir = "./saved"+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_fp8input_mxfp4_llmcompressor_format(self):+ model_name = "/models/Qwen3-0.6B-FP8"+ scheme = "mxfp4"+ ar = AutoRound(+ model=model_name,+ iters=2,+ seqlen=2,+ scheme=scheme,+ dataset=self.llm_dataloader,+ )+ compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")+ tmp_layer = compressed_model.model.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_packed")+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 2048+ ), "Illegal MXFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(self.save_dir, trust_remote_code=True).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4+ ), f"Invalid MXFP4 quantization configuration: {quantization_config}"+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_nvfp4_llmcompressor_format(self):+ scheme = "nvfp4"+ autoround = AutoRound(+ self.model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "nvfp4-pack-quantized"+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)+ # from vllm import LLM, SamplingParams+ # prompts = [+ # "The capital of France is",+ # "The future of AI is",+ # ]+ ## Create a sampling params object.+ # sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)+ # QUANTIZATION = "compressed-tensors"+ # llm = LLM(model=quantized_model_path,+ # # quantization=QUANTIZATION,+ # trust_remote_code=True,+ # tensor_parallel_size=1,+ # enforce_eager=True,+ # gpu_memory_utilization=0.7,+ # )+ # outputs = llm.generate(prompts, sampling_params)+ # # Print the outputs.+ # for output in outputs:+ # prompt = output.prompt+ # generated_text = output.outputs[0].text+ # if "France" in prompt:+ # assert "Paris" in generated_text++ def test_nvfp4_moe_actmax_rtn(self):+ model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=0,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")++ def test_nvfp4_moe_actmax_ar(self):+ model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")++ def test_qwen_moe_quant_infer(self):+ model_name = "/models/Qwen1.5-MoE-A2.7B"+ layer_config = {+ "layers\.(?:[3-9]|1[0-9]|2[0-3])": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa")+ print(result["results"]["piqa"]["acc,none"])+ self.assertGreater(result["results"]["piqa"]["acc,none"], 0.7)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()