PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix nvfp act quantization bug (#891)
Weiweifbc41c8d9449
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex cd5b09bee..3a7c1f823 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1651,9 +1651,7 @@ class BaseCompressor(object):if self.device_map is not None:accelerate.hooks.remove_hook_from_submodules(block)- if (- is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)- ) or is_static_wfp8afp8(self):+ if is_nv_fp(self.act_data_type) or is_static_wfp8afp8(self):# enable moe experts act_max automatic generation for Linearset_amax_for_all_moe_layers(block, attr_name="act_max")# Normalize imatrix and quantize layers@@ -2911,11 +2909,7 @@ class BaseCompressor(object):with torch.no_grad():unwrapper_block(block, best_params)- if (- is_nv_fp(self.act_data_type)- and hasattr(self, "formats")- and any("nv_fp" in format_ for format_ in self.formats)- ):+ if is_nv_fp(self.act_data_type):# enable moe experts act_max automatic generation for WrapperWALayerset_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 82b426186..0b819e381 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -2481,7 +2481,7 @@ def set_nested_attr(module, attr_name: str, value):attrs = attr_name.split(".")for attr in attrs[:-1]:if not hasattr(module, attr):- raise AttributeError(f"{module} has no attribute '{attr}'")+ return None # No need to set act_max for fp layersmodule = getattr(module, attr)setattr(module, attrs[-1], value)@@ -2546,7 +2546,7 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na# For other MoE models (like Mixtral) with iterable expertstry:set_amax_for_uncalibrated_experts(- [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name+ [getattr(expert, linear_name, None) for expert in sub_module.experts], attr_name=attr_name)except AttributeError as e:# Provide more helpful debugging informationdiff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 382f5651b..180fd8f2f 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -532,6 +532,45 @@ class TestAutoRound(unittest.TestCase):), "Illegal NVFP4 packing name or data_type or shape"shutil.rmtree("./saved", ignore_errors=True)+ def test_nvfp4_moe_actmax_rtn(self):+ model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"+ layer_config = {+ "self_attn": {"bits": 16, "act_bits": 16},+ "mlp.shared_experts": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=0,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ compressed_model, _ = autoround.quantize()+ assert hasattr(compressed_model.model.layers[1].mlp.experts[0].gate_proj.orig_layer, "act_max")++ def test_nvfp4_moe_actmax_ar(self):+ model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"+ layer_config = {+ "q_proj": {"bits": 16, "act_bits": 16},+ "mlp.shared_experts": {"bits": 16, "act_bits": 16},+ "experts.*2": {"bits": 16, "act_bits": 16},+ "experts.*5": {"bits": 16, "act_bits": 16},+ }+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")+if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.pyindex e9699da52..0ab05134f 100644--- a/test/test_cuda/test_export.py+++ b/test/test_cuda/test_export.py@@ -402,6 +402,36 @@ class TestAutoRound(unittest.TestCase):# if "France" in prompt:# assert "Paris" in generated_text+ def test_nvfp4_moe_actmax_rtn(self):+ model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=0,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")++ def test_nvfp4_moe_actmax_ar(self):+ model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"+ scheme = "nvfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")+if __name__ == "__main__":unittest.main()