Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix norm_bias_tuning (#639)

Wenhua Cheng28b01fe35b59
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 9d265072d..bc439ede5 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -431,8 +431,8 @@ class AutoRound(object):
if "mx_fp" in self.data_type and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
- if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size != 16):
- logger.warning("mx_fp should only support group_size of 16/32 in real deployment")
+ if "nv_fp" in self.data_type and (self.group_size != 16):
+ logger.warning("nv_fp should only support group_size of 16 in real deployment")
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
@@ -445,8 +445,11 @@ class AutoRound(object):
f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps}"
f" as nsamples must equal or greater"
f" than gradient_accumulate_steps * batch_size")
- self._dq_check()
+ if self.enable_norm_bias_tuning:
+ logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")
+
+ self._dq_check()
def _check_compatibility(self):
@@ -2286,10 +2289,6 @@ class AutoRound(object):
if pbar is None:
pbar = tqdm(range(0, len(block_names), nblocks))
- for n, m in self.model.named_modules():
- if isinstance(m, tuple(self.supported_types)):
- m.name = n
-
for i in range(0, len(block_names), nblocks):
if i != 0:
pbar.update(1)
@@ -2325,7 +2324,7 @@ class AutoRound(object):
pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,
self.tokenizer)
else:
- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])
+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
pbar.set_description(f"Quantizing done")
pbar.update(1)
pbar.close()
diff --git a/auto_round/version.py b/auto_round/version.py
index d9f21f781..3a8c9f1ae 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -15,3 +15,4 @@
supporting popular model weight only compression based on signround."""
__version__ = "0.6.0"
+
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 54f3a4565..c43ce3462 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -20,7 +20,7 @@ from .utils import (
check_to_quantized,
get_scale_shape,
set_module,
- logger
+ logger, SUPPORTED_LAYER_TYPES
)
@@ -254,7 +254,7 @@ class WrapperLinear(torch.nn.Module):
if isinstance(scale, dict):
_set_dict_attr(scale, "scale")
- elif scale.numel()>1:
+ elif scale.numel() > 1:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
else:
self.orig_layer.scale = scale.view(-1).to("cpu")
@@ -264,7 +264,7 @@ class WrapperLinear(torch.nn.Module):
_set_dict_attr(zp, "zp")
elif zp is None:
self.orig_layer.zp = None
- elif zp.numel()>1 :
+ elif zp.numel() > 1:
zp = zp.reshape(shape[0], -1)
self.orig_layer.zp = zp.to("cpu")
else:
@@ -475,12 +475,13 @@ class WrapperLlamaNorm(torch.nn.Module):
return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)
-norm_mapping = {}
-norm_mapping["LayerNorm"] = WrapperLayerNorm
-norm_mapping["LlamaRMSNorm"] = WrapperLlamaNorm
-norm_mapping["Qwen2RMSNorm"] = WrapperLlamaNorm
-norm_mapping["Phi3RMSNorm"] = WrapperLlamaNorm
-norm_mapping["MistralRMSNorm"] = WrapperLlamaNorm
+NORM_MAPPING = {}
+NORM_MAPPING["LayerNorm"] = WrapperLayerNorm
+NORM_MAPPING["LlamaRMSNorm"] = WrapperLlamaNorm
+NORM_MAPPING["Qwen2RMSNorm"] = WrapperLlamaNorm
+NORM_MAPPING["Phi3RMSNorm"] = WrapperLlamaNorm
+NORM_MAPPING["MistralRMSNorm"] = WrapperLlamaNorm
+NORM_MAPPING["Qwen3RMSNorm"] = WrapperLlamaNorm
class WrapperMultiblock(torch.nn.Module):
@@ -517,7 +518,7 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='
quantized_layers = []
unquantized_layers = []
for n, m in block.named_modules():
- if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):
+ if isinstance(m, SUPPORTED_LAYER_TYPES):
if not check_to_quantized(m):
unquantized_layers.append(n)
continue
@@ -531,18 +532,18 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='
set_module(block, n, new_m)
quantized_layers.append(n)
- if enable_norm_bias_tuning:
+ elif enable_norm_bias_tuning:
if "norm" in m.__class__.__name__.lower():
- if m.__class__.__name__ in norm_mapping.keys():
- wrapper_layer_class = norm_mapping[m.__class__.__name__]
+ if m.__class__.__name__ in NORM_MAPPING.keys():
+ wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]
new_m = wrapper_layer_class(m, device=device)
- setattr(block, n, new_m)
+ set_module(block, n, new_m)
elif "RMSNorm" in m.__class__.__name__:
logger.warning_once(
f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself")
- wrapper_layer_class = norm_mapping["LlamaRMSNorm"]
+ wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]
new_m = wrapper_layer_class(m, device=device)
- setattr(block, n, new_m)
+ set_module(block, n, new_m)
else:
logger.warning_once(f"{m.__class__.__name__} is not supported")
@@ -584,4 +585,3 @@ def unwrapper_block(block, best_params):
best_param = None
orig_layer = m.unwrapper(best_param)
set_module(block, n, orig_layer)
-
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 261471f20..25b5afe14 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -224,6 +224,25 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+
+ def test_enable_norm_bias_tuning_qwen3(self):
+ bits, group_size, sym = 4, 128, True
+ model_name = "Qwen/Qwen3-0.6B"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=10,
+ enable_norm_bias_tuning=True,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
def test_disable_minmax_tuning(self):
bits, group_size, sym = 4, -1, True
autoround = AutoRound(