PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support for static activation quantization calibration with group_size (#693)
Heng Guo6afe07cbc0fd
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex a96978b91..5f2e57955 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -26,6 +26,7 @@ from tqdm import tqdmfrom transformers import set_seedfrom auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_blockfrom auto_round.utils import (@@ -242,7 +243,7 @@ class AutoRound(object):self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device##activation- self.act_group_size = act_group_size if act_group_size is not None else self.group_size+ self.act_group_size = act_group_size if act_group_size is not None else group_sizeself.act_bits = act_bits if act_bits is not None else self.bitsself.act_sym = act_sym if act_sym is not None else self.symself.act_dynamic = act_dynamic@@ -648,7 +649,11 @@ class AutoRound(object):" change format to auto_round")format = "auto_round"-+ if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":+ logger.warning(+ f"Please note that quantize activation with act_group_size={self.act_group_size}"+ " may result in failure to export or import normally."+ )if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):logger.error(f"datatype<{self.data_type}> not support to export {format} format."@@ -848,7 +853,7 @@ class AutoRound(object):def register_act_hook(model):"""Registers hooks to accumulate activation squared norms into `imatrix`."""- def get_act_max_hook(module, input, output):+ def get_imatrix_hook(module, input, output):input = input[0] if isinstance(input, (tuple, list)) else inputflattened = input.reshape(-1, input.shape[-1]).to(torch.float32)squared = torch.sum(flattened**2, dim=0).to(torch.float32)@@ -863,7 +868,7 @@ class AutoRound(object):hook_handles = []for name, module in model.named_modules():if isinstance(module, self.supported_types) and check_to_quantized(module):- hook = module.register_forward_hook(get_act_max_hook)+ hook = module.register_forward_hook(get_imatrix_hook)hook_handles.append(hook)return hook_handles@@ -2151,10 +2156,12 @@ class AutoRound(object):def get_act_max_hook(module, input, output):if isinstance(input, (tuple, list)):input = input[0]+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).valuesif not hasattr(module, "act_max"):- module.act_max = torch.abs(input).max().item()+ module.act_max = act_maxelse:- module.act_max = max(torch.abs(input).max().item(), module.act_max)+ module.act_max = torch.max(act_max.to(module.act_max.device), module.act_max)hook_handles = []diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex e175341e6..b5d4f937b 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -79,7 +79,7 @@ class FP8WOQLinear(torch.nn.Module):if weight_zp:self.register_buffer("weight_zp", weight_zp.to(dtype))- if act_scale:+ if act_scale is not None:self.register_buffer("act_scale", act_scale.to(dtype))diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 827ffff2a..12b267d13 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -177,6 +177,8 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--act_bits", default=16, type=int, help="activation bits")+ self.add_argument("--act_group_size", default=0, type=int, help="activation group size")+self.add_argument("--fp_layers", default="", type=str, help="list of Layer names to maintain original data type")@@ -513,6 +515,7 @@ def tune(args):layer_config=layer_config,enable_minmax_tuning=not args.disable_minmax_tuning,act_bits=args.act_bits,+ act_group_size=args.act_group_size,low_cpu_mem_usage=low_cpu_mem_usage,data_type=args.data_type,enable_norm_bias_tuning=args.enable_norm_bias_tuning,diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 0614ebd0f..c817a34ff 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -309,10 +309,12 @@ class WrapperLinear(torch.nn.Module):if not self.orig_layer.act_dynamic:act_max_scale = best_params.get("act_max_scale", torch.tensor(1.0)).to(self.device)act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None- tmp_shape = 1- if self.orig_layer.act_group_size > 1:- tmp_shape = (1, self.orig_layer.act_group_size)if act_max is not None:+ tmp_shape = 1+ if self.orig_layer.act_group_size > 1:+ tmp_shape = (act_max.shape[0], self.orig_layer.act_group_size)+ elif self.orig_layer.act_group_size == -1:+ tmp_shape = (act_max.shape[0], 1)_, act_scale, _ = self._qdq_act(torch.zeros(tmp_shape).to(self.device), act_max_scale=self.act_max_scale, act_max=act_max)@@ -398,7 +400,7 @@ class WrapperWALayer(torch.nn.Module):x, _, _ = self.orig_layer.act_quant_func(x,bits=self.orig_layer.act_bits,- group_size=self.orig_layer.group_size,+ group_size=self.orig_layer.act_group_size,scale_dtype=self.orig_layer.scale_dtype,q_scale_thresh=self.orig_layer.q_scale_thresh,data_type=self.orig_layer.act_data_type,diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.pyindex 2ff5c8501..fd9ed2406 100644--- a/test/test_cpu/test_act_quantization.py+++ b/test/test_cpu/test_act_quantization.py@@ -88,6 +88,57 @@ class TestAutoRoundAct(unittest.TestCase):)autoround.quantize()+ def test_wfp8afp8_static(self):+ model_name = "facebook/opt-125m"+ from auto_round.wrapper import WrapperWALayer++ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ autoround = AutoRound(+ model,+ tokenizer,+ group_size=128,+ act_group_size=-1,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ data_type="fp8",+ act_dynamic=False,+ act_data_type="fp8",+ )+ autoround.quantize()++ self.assertTrue(isinstance(autoround.model.model.decoder.layers[2].self_attn.k_proj, WrapperWALayer))+ self.assertEqual(autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_scale.shape[0], 30)+ self.assertEqual(autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_max.shape[0], 30)++ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ autoround = AutoRound(+ model,+ tokenizer,+ group_size=128,+ act_group_size=128,+ iters=0,+ seqlen=2,+ dataset=self.llm_dataloader,+ data_type="fp8",+ act_dynamic=False,+ act_data_type="fp8",+ )+ autoround.quantize()+ self.assertTrue(isinstance(autoround.model.model.decoder.layers[2].self_attn.k_proj, WrapperWALayer))++ self.assertEqual(+ autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_scale.shape[0],+ int(3 * 10 * 768 / 128),+ )+ self.assertEqual(+ autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_max.shape[0],+ int(3 * 10 * 768 / 128),+ )+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 9d363e921..ae780cc44 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -217,6 +217,7 @@ class TestAutoRound(unittest.TestCase):data_type="fp8",act_data_type="fp8",act_dynamic=False,+ act_group_size=0,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")@@ -239,6 +240,7 @@ class TestAutoRound(unittest.TestCase):data_type="fp8",act_data_type="fp8",act_dynamic=False,+ act_group_size=0,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")