PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support mxfp nvfp lmhead quant (#1051)
Weiweic4a14799a50d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 524dd90b9..91048eb80 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -509,7 +509,7 @@ class BaseCompressor(object):if len(tmp_devices) > 1:logger.warning(f"there are multiple device types in the device_map, "- f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "+ f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device.")self.device = tmp_devices[0]@@ -526,7 +526,7 @@ class BaseCompressor(object):if "bits" not in kwargs:data_type = kwargs["data_type"]raise KeyError(- f"please set bits when setting data_type={data_type}, or using scheme as an alternative.."+ f"please set bits when setting data_type={data_type}, or using scheme as an alternative.")bits = kwargs["bits"]scheme = f"gguf:q{bits}_k" if bits == 6 else f"gguf:q{bits}_k_s"@@ -1469,8 +1469,8 @@ class BaseCompressor(object):raise ValueError("Could not find any blocks. Check the model or quant_block_list.")all_first_block_names = [block[0] for block in all_blocks]- if self.act_bits < 16 and not self.act_dynamic:- layer_names = self._get_quantized_layer_names_outside_blocks()+ layer_names = self._get_quantized_layer_names_outside_blocks()+ if self.act_bits < 16 and (not self.act_dynamic or len(layer_names) > 0):if len(layer_names) > 0:logger.warning("quantize layers outside blocks for static activation quantizaiton"@@ -1783,6 +1783,21 @@ class BaseCompressor(object):for layer_name in copy.deepcopy(layer_names):if layer_name not in layer_inputs:+ if self.act_bits < 16 and not self.act_dynamic:+ # Activation quantization requires collected inputs+ msg_prefix = (+ f"Activation max hook for layer '{layer_name}' is unavailable due to "+ f"insufficient collected inputs. "+ )+ if "fp8_e5m2" in self.act_data_type:+ logger.warning(msg_prefix + "Please notes that unit scale is used for this layer.")+ else:+ logger.warning(+ msg_prefix + "Static activation quantization is not supported or ineffective, "+ "Skipping quantization for this layer."+ )+ layer_names.remove(layer_name)+ continuelogger.info(f"using rtn to quantize {layer_name}")from auto_round.data_type import QUANT_FUNC_WITH_DTYPE@@ -1813,6 +1828,7 @@ class BaseCompressor(object):q_layer_inputs = Noneenable_quanted_input = self.enable_quanted_inputhas_gguf = False+if hasattr(self, "formats"):has_gguf = any("gguf" in format_ for format_ in self.formats)if has_gguf and self.immediate_packing:@@ -2334,6 +2350,64 @@ class BaseCompressor(object):hook_handle = m.register_forward_hook(hook_func)self.hook_handles.append(hook_handle)+ def _register_act_max_hook(self, model):+ def get_act_max_hook(module, input, output):+ if isinstance(input, (tuple, list)):+ input = input[0]+ if input.numel() == 0:+ return # as no needs for act_max update+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ else:+ act_max = act_max.to(module.act_max.device)+ if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage+ module.act_max = torch.max(+ torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)+ )+ else:+ module.act_max = torch.max(act_max, module.act_max)++ hook_handles = []+ # for single layers out of blocks, like lm_head+ if isinstance(model, SUPPORTED_LAYER_TYPES):+ m = model+ if (+ hasattr(m, "act_dynamic")+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)+ and check_to_quantized(m)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ return hook_handles++ for n, m in model.named_modules():+ if (+ hasattr(m, "act_dynamic")+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)+ and check_to_quantized(m)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue++ # for whole model, RTN+ if n in self.layer_config:+ config = self.layer_config[n]+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_bits", 16)+ if (+ config["bits"] <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and check_to_quantized(config)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue+ return hook_handles+def _quantize_layer(self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"):@@ -2359,6 +2433,19 @@ class BaseCompressor(object):if q_inputs is not None:q_inputs[i] = q_inputs[i].to(layer.weight.dtype)+ if q_inputs is None:+ hook_handles = self._register_act_max_hook(layer)+ with torch.no_grad():+ layer(torch.cat(inputs, dim=0))+ for handle in hook_handles:+ handle.remove()+ else:+ hook_handles = self._register_act_max_hook(layer)+ if hook_handles:+ layer(torch.cat(q_inputs, dim=0))+ for handle in hook_handles:+ handle.remove()+wrapper_linear = WrapperLinear(layer,enable_minmax_tuning=self.enable_minmax_tuning,@@ -2495,54 +2582,6 @@ class BaseCompressor(object):dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"logger.info(dump_info)- def _register_act_max_hook(self, model):-- def get_act_max_hook(module, input, output):- if isinstance(input, (tuple, list)):- input = input[0]- if input.numel() == 0:- return # as no needs for act_max update- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- else:- act_max = act_max.to(module.act_max.device)- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage- module.act_max = torch.max(- torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)- )- else:- module.act_max = torch.max(act_max, module.act_max)-- hook_handles = []-- for n, m in model.named_modules():- if (- hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)- and check_to_quantized(m)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue-- # for whole model, RTN- if n in self.layer_config:- config = self.layer_config[n]- act_dynamic = config.get("act_dynamic", True)- act_data_type = config.get("act_data_type", None)- act_bits = config.get("act_bits", 16)- if (- config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)- and check_to_quantized(config)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue- return hook_handles-def _get_current_output(self, output: list[torch.Tensor], indices: list[int]) -> torch.Tensor:current_output = [output[x] for x in indices]current_output = torch.cat(current_output, dim=self.batch_dim)diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 99c86fd67..533143609 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -130,8 +130,6 @@ def convert_from_autogptq_dynamic(dynamic_config: dict) -> dict:def pack_layer(name, model, backend, device=None):- if name == "lm_head": ##dese not support lm-head- returnlayer = get_module(model, name)if type(layer) not in SUPPORTED_LAYER_TYPES: # already packeddiff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex 819b9d99c..7cbb3266a 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -52,8 +52,6 @@ __all__ = [def pack_layer(name, model, backend, device=None):- if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature- returnlayer = get_module(model, name)if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packedreturn@@ -82,8 +80,6 @@ def pack_layer(name, model, backend, device=None):setattr(layer, "input_global_scale", input_global_scale)delattr(layer, "act_max")- # QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)-if type(layer) == nn.Linear:in_features = layer.in_featuresout_features = layer.out_featuresdiff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex f821c2d56..b9282af27 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -46,8 +46,6 @@ from auto_round.utils import (def pack_layer(name, model, backend, device=None):- if name == "lm_head": ##dese not support lm-head- returnlayer = get_module(model, name)if type(layer) not in SUPPORTED_LAYER_TYPES: ##already packeddiff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex af9702484..69600c8af 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -51,8 +51,6 @@ __all__ = [def pack_layer(name, model, backend, device=None):- if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature- returnlayer = get_module(model, name)if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packedreturndiff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 4e41b9a96..7b646127d 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -35,10 +35,10 @@ class LLMDataLoader:class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ self.model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"self.save_dir = "./saved"- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@classmethod@@ -49,7 +49,7 @@ class TestAutoRound(unittest.TestCase):def test_autogptq_format(self):for group_size in [-1, 32, 128]:bits, sym = 4, False- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_nameautoround = AutoRound(model=model_name,bits=bits,@@ -79,7 +79,7 @@ class TestAutoRound(unittest.TestCase):def test_autoround_format(self):for group_size in [-1, 32, 128]:bits, sym = 4, True- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_nameautoround = AutoRound(model=model_name,bits=bits,@@ -105,7 +105,7 @@ class TestAutoRound(unittest.TestCase):def test_autoround_awq_format(self):for group_size in [-1, 32, 128]:bits, sym = 4, False- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_nameautoround = AutoRound(model=model_name,bits=bits,@@ -217,7 +217,7 @@ class TestAutoRound(unittest.TestCase):from safetensors import safe_open- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -307,7 +307,7 @@ class TestAutoRound(unittest.TestCase):from safetensors import safe_open- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -334,6 +334,69 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(quantized_model_path, ignore_errors=True)+ def test_awq_lmhead_export(self):+ bits, sym, group_size = 4, False, 128+ model_name = "/tf_dataset/auto_round/models/microsoft/phi-2"+ layer_config = {+ "lm_head": {"bits": 4}, # set lm_head quant+ }+ autoround = AutoRound(+ model=model_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ lm_head = compressed_model.lm_head+ from auto_round.export.export_to_awq.utils import WQLinear_GEMM++ assert isinstance(lm_head, WQLinear_GEMM), "Illegal GPTQ quantization for lm_head layer"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_gptq_lmhead_export(self):+ bits, sym, group_size = 4, True, 128+ model_name = "/tf_dataset/auto_round/models/microsoft/phi-2"+ layer_config = {+ "lm_head": {"bits": 4}, # set lm_head quant+ }+ autoround = AutoRound(+ model=model_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ lm_head = compressed_model.lm_head+ assert hasattr(lm_head, "bits") and lm_head.bits == 4, "Illegal GPTQ quantization for lm_head layer"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_mxfp_nvfp.py b/test/test_cpu/test_mxfp_nvfp.pyindex 4fcd25135..ce61a9435 100644--- a/test/test_cpu/test_mxfp_nvfp.py+++ b/test/test_cpu/test_mxfp_nvfp.py@@ -35,10 +35,10 @@ class LLMDataLoader:class TestAutoRoundFP(unittest.TestCase):@classmethoddef setUpClass(self):- model_name = "facebook/opt-125m" # /tf_dataset/auto_round/models/+ self.model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"self.save_dir = "./saved"- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")- self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto")+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@classmethod@@ -51,6 +51,9 @@ class TestAutoRoundFP(unittest.TestCase):layer_config = {"self_attn": {"bits": 16, "act_bits": 16},"mlp.shared_experts": {"bits": 16, "act_bits": 16},+ "experts.*2": {"bits": 16, "act_bits": 16},+ "experts.*5": {"bits": 16, "act_bits": 16},+ "lm_head": {"bits": 4, "act_bits": 4}, ## test lm_head quantization and exporting}scheme = "nvfp4"autoround = AutoRound(@@ -64,6 +67,11 @@ class TestAutoRoundFP(unittest.TestCase):)compressed_model, _ = autoround.quantize()assert hasattr(compressed_model.model.layers[1].mlp.experts[0].gate_proj.orig_layer, "act_max")+ lm_head = compressed_model.lm_head+ assert hasattr(lm_head, "orig_layer") and hasattr(+ lm_head.orig_layer, "act_max"+ ), "Illegal NVFP4 quantization for lm_head layer"+ shutil.rmtree(self.save_dir, ignore_errors=True)def test_nvfp4_moe_actmax_ar(self):model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"@@ -72,6 +80,7 @@ class TestAutoRoundFP(unittest.TestCase):"mlp.shared_experts": {"bits": 16, "act_bits": 16},"experts.*2": {"bits": 16, "act_bits": 16},"experts.*5": {"bits": 16, "act_bits": 16},+ "lm_head": {"bits": 4, "act_bits": 4}, ## test lm_head quantization and exporting}scheme = "nvfp4"autoround = AutoRound(@@ -83,10 +92,55 @@ class TestAutoRoundFP(unittest.TestCase):dataset=self.llm_dataloader,layer_config=layer_config,)- autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")+ compressed_model, _ = autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")+ lm_head = compressed_model.lm_head+ assert (+ hasattr(lm_head, "weight_scale")+ and hasattr(lm_head, "weight_global_scale")+ and hasattr(lm_head, "input_global_scale")+ and lm_head.weight_packed.dtype is torch.uint8+ and lm_head.weight_scale.dtype is torch.float8_e4m3fn+ ), "Illegal NVFP4 packing for lm_head layer"+ quantized_model_path = self.save_dir+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ result = simple_evaluate_user_model(model, tokenizer, batch_size=4, tasks="piqa", limit=4)+ print(result["results"]["piqa"]["acc,none"])+ self.assertGreater(result["results"]["piqa"]["acc,none"], 0.7)+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_mxfp4_moe_ar(self):+ model_name = "/tf_dataset/auto_round/models/deepseek-ai/DeepSeek-V2-Lite"+ layer_config = {+ "q_proj": {"bits": 16, "act_bits": 16, "data_type": "float"},+ "mlp.shared_experts": {"bits": 16, "act_bits": 16, "data_type": "float"},+ "experts.*2": {"bits": 16, "act_bits": 16, "data_type": "float"},+ "experts.*5": {"bits": 16, "act_bits": 16, "data_type": "float"},+ "lm_head": {"bits": 4, "act_bits": 4}, ## test lm_head quantization and exporting+ }+ scheme = "mxfp4"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=1,+ seqlen=2,+ nsamples=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ compressed_model, _ = autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")+ lm_head = compressed_model.lm_head+ assert (+ hasattr(lm_head, "weight_scale")+ and hasattr(lm_head, "weight_packed")+ and lm_head.weight_scale.dtype is torch.uint8+ ), "Illegal MXFP4 packing for lm_head layer"+ shutil.rmtree(self.save_dir, ignore_errors=True)def test_mxfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "MXFP4"@@ -123,11 +177,10 @@ class TestAutoRoundFP(unittest.TestCase):and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is Trueand quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"-- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_rtn_mxfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "MXFP4"@@ -164,10 +217,10 @@ class TestAutoRoundFP(unittest.TestCase):and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is Trueand quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_mxfp8_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "MXFP8"@@ -201,10 +254,10 @@ class TestAutoRoundFP(unittest.TestCase):assert (0.15 < folder_size_gb < 0.2), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.2 GB)"- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_nvfp4_llmcompressor_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "NVFP4"@@ -238,16 +291,16 @@ class TestAutoRoundFP(unittest.TestCase):assert (0.1 < folder_size_gb < 0.15), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.1~0.15 GB)"- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_nvfp4_autoround_format(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "NVFP4"autoround = AutoRound(model_name,- scheme="NVFP4",+ scheme=scheme,iters=2,seqlen=2,dataset=self.llm_dataloader,@@ -263,16 +316,16 @@ class TestAutoRoundFP(unittest.TestCase):and tmp_layer.weight_scale.dtype is torch.float8_e4m3fnand tmp_layer.weight_scale.shape[0] == 768), "Illegal NVFP4 packing name or data_type or shape"- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_nvfp4_autoround_save_quantized(self):- model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model_name = self.model_namefrom transformers import AutoConfigscheme = "NVFP4"autoround = AutoRound(model_name,- scheme="NVFP4",+ scheme=scheme,iters=2,seqlen=2,dataset=self.llm_dataloader,@@ -289,7 +342,7 @@ class TestAutoRoundFP(unittest.TestCase):and tmp_layer.weight_scale.dtype is torch.float8_e4m3fnand tmp_layer.weight_scale.shape[0] == 768), "Illegal NVFP4 packing name or data_type or shape"- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_qwen_moe_quant_infer(self):model_name = "/tf_dataset/auto_round/models/Qwen/Qwen1.5-MoE-A2.7B"diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.pyindex d6d6c1f93..6bb2612e8 100644--- a/test/test_cuda/test_export.py+++ b/test/test_cuda/test_export.py@@ -6,7 +6,7 @@ import unittestsys.path.insert(0, "../..")import torchimport transformers-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer+from transformers import AutoConfig, AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_awq, require_optimum, require_package_version_ut@@ -322,6 +322,69 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)+ def test_awq_lmhead_export(self):+ bits, sym, group_size = 4, False, 128+ model_name = "/models/phi-2"+ layer_config = {+ "lm_head": {"bits": 4}, # set lm_head quant+ }+ autoround = AutoRound(+ model=model_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ lm_head = compressed_model.lm_head+ from auto_round.export.export_to_awq.utils import WQLinear_GEMM++ assert isinstance(lm_head, WQLinear_GEMM), "Illegal GPTQ quantization for lm_head layer"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_gptq_lmhead_export(self):+ bits, sym, group_size = 4, True, 128+ model_name = "/models/phi-2"+ layer_config = {+ "lm_head": {"bits": 4}, # set lm_head quant+ }+ autoround = AutoRound(+ model=model_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ lm_head = compressed_model.lm_head+ assert hasattr(lm_head, "bits") and lm_head.bits == 4, "Illegal GPTQ quantization for lm_head layer"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()