PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix norm_bias_tuning (#639)
Wenhua Cheng28b01fe35b59
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 9d265072d..bc439ede5 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -431,8 +431,8 @@ class AutoRound(object):if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")- if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size != 16):- logger.warning("mx_fp should only support group_size of 16/32 in real deployment")+ if "nv_fp" in self.data_type and (self.group_size != 16):+ logger.warning("nv_fp should only support group_size of 16 in real deployment")if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:@@ -445,8 +445,11 @@ class AutoRound(object):f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps}"f" as nsamples must equal or greater"f" than gradient_accumulate_steps * batch_size")- self._dq_check()+ if self.enable_norm_bias_tuning:+ logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")++ self._dq_check()def _check_compatibility(self):@@ -2286,10 +2289,6 @@ class AutoRound(object):if pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))- for n, m in self.model.named_modules():- if isinstance(m, tuple(self.supported_types)):- m.name = n-for i in range(0, len(block_names), nblocks):if i != 0:pbar.update(1)@@ -2325,7 +2324,7 @@ class AutoRound(object):pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,self.tokenizer)else:- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])pbar.set_description(f"Quantizing done")pbar.update(1)pbar.close()diff --git a/auto_round/version.py b/auto_round/version.pyindex d9f21f781..3a8c9f1ae 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -15,3 +15,4 @@supporting popular model weight only compression based on signround."""__version__ = "0.6.0"+diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 54f3a4565..c43ce3462 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -20,7 +20,7 @@ from .utils import (check_to_quantized,get_scale_shape,set_module,- logger+ logger, SUPPORTED_LAYER_TYPES)@@ -254,7 +254,7 @@ class WrapperLinear(torch.nn.Module):if isinstance(scale, dict):_set_dict_attr(scale, "scale")- elif scale.numel()>1:+ elif scale.numel() > 1:self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")else:self.orig_layer.scale = scale.view(-1).to("cpu")@@ -264,7 +264,7 @@ class WrapperLinear(torch.nn.Module):_set_dict_attr(zp, "zp")elif zp is None:self.orig_layer.zp = None- elif zp.numel()>1 :+ elif zp.numel() > 1:zp = zp.reshape(shape[0], -1)self.orig_layer.zp = zp.to("cpu")else:@@ -475,12 +475,13 @@ class WrapperLlamaNorm(torch.nn.Module):return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)-norm_mapping = {}-norm_mapping["LayerNorm"] = WrapperLayerNorm-norm_mapping["LlamaRMSNorm"] = WrapperLlamaNorm-norm_mapping["Qwen2RMSNorm"] = WrapperLlamaNorm-norm_mapping["Phi3RMSNorm"] = WrapperLlamaNorm-norm_mapping["MistralRMSNorm"] = WrapperLlamaNorm+NORM_MAPPING = {}+NORM_MAPPING["LayerNorm"] = WrapperLayerNorm+NORM_MAPPING["LlamaRMSNorm"] = WrapperLlamaNorm+NORM_MAPPING["Qwen2RMSNorm"] = WrapperLlamaNorm+NORM_MAPPING["Phi3RMSNorm"] = WrapperLlamaNorm+NORM_MAPPING["MistralRMSNorm"] = WrapperLlamaNorm+NORM_MAPPING["Qwen3RMSNorm"] = WrapperLlamaNormclass WrapperMultiblock(torch.nn.Module):@@ -517,7 +518,7 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='quantized_layers = []unquantized_layers = []for n, m in block.named_modules():- if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):+ if isinstance(m, SUPPORTED_LAYER_TYPES):if not check_to_quantized(m):unquantized_layers.append(n)continue@@ -531,18 +532,18 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='set_module(block, n, new_m)quantized_layers.append(n)- if enable_norm_bias_tuning:+ elif enable_norm_bias_tuning:if "norm" in m.__class__.__name__.lower():- if m.__class__.__name__ in norm_mapping.keys():- wrapper_layer_class = norm_mapping[m.__class__.__name__]+ if m.__class__.__name__ in NORM_MAPPING.keys():+ wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]new_m = wrapper_layer_class(m, device=device)- setattr(block, n, new_m)+ set_module(block, n, new_m)elif "RMSNorm" in m.__class__.__name__:logger.warning_once(f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself")- wrapper_layer_class = norm_mapping["LlamaRMSNorm"]+ wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]new_m = wrapper_layer_class(m, device=device)- setattr(block, n, new_m)+ set_module(block, n, new_m)else:logger.warning_once(f"{m.__class__.__name__} is not supported")@@ -584,4 +585,3 @@ def unwrapper_block(block, best_params):best_param = Noneorig_layer = m.unwrapper(best_param)set_module(block, n, orig_layer)-diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 261471f20..25b5afe14 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -224,6 +224,25 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()++ def test_enable_norm_bias_tuning_qwen3(self):+ bits, group_size, sym = 4, 128, True+ model_name = "Qwen/Qwen3-0.6B"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=10,+ enable_norm_bias_tuning=True,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+def test_disable_minmax_tuning(self):bits, group_size, sym = 4, -1, Trueautoround = AutoRound(