Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support for static activation quantization calibration with group_size (#693)

Heng Guo6afe07cbc0fd
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index a96978b91..5f2e57955 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -26,6 +26,7 @@ from tqdm import tqdm
from transformers import set_seed
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
from auto_round.utils import (
@@ -242,7 +243,7 @@ class AutoRound(object):
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
##activation
- self.act_group_size = act_group_size if act_group_size is not None else self.group_size
+ self.act_group_size = act_group_size if act_group_size is not None else group_size
self.act_bits = act_bits if act_bits is not None else self.bits
self.act_sym = act_sym if act_sym is not None else self.sym
self.act_dynamic = act_dynamic
@@ -648,7 +649,11 @@ class AutoRound(object):
" change format to auto_round"
)
format = "auto_round"
-
+ if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":
+ logger.warning(
+ f"Please note that quantize activation with act_group_size={self.act_group_size}"
+ " may result in failure to export or import normally."
+ )
if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
logger.error(
f"datatype<{self.data_type}> not support to export {format} format."
@@ -848,7 +853,7 @@ class AutoRound(object):
def register_act_hook(model):
"""Registers hooks to accumulate activation squared norms into `imatrix`."""
- def get_act_max_hook(module, input, output):
+ def get_imatrix_hook(module, input, output):
input = input[0] if isinstance(input, (tuple, list)) else input
flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
squared = torch.sum(flattened**2, dim=0).to(torch.float32)
@@ -863,7 +868,7 @@ class AutoRound(object):
hook_handles = []
for name, module in model.named_modules():
if isinstance(module, self.supported_types) and check_to_quantized(module):
- hook = module.register_forward_hook(get_act_max_hook)
+ hook = module.register_forward_hook(get_imatrix_hook)
hook_handles.append(hook)
return hook_handles
@@ -2151,10 +2156,12 @@ class AutoRound(object):
def get_act_max_hook(module, input, output):
if isinstance(input, (tuple, list)):
input = input[0]
+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
+ act_max = torch.max(torch.abs(input), dim=-1).values
if not hasattr(module, "act_max"):
- module.act_max = torch.abs(input).max().item()
+ module.act_max = act_max
else:
- module.act_max = max(torch.abs(input).max().item(), module.act_max)
+ module.act_max = torch.max(act_max.to(module.act_max.device), module.act_max)
hook_handles = []
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
index e175341e6..b5d4f937b 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -79,7 +79,7 @@ class FP8WOQLinear(torch.nn.Module):
if weight_zp:
self.register_buffer("weight_zp", weight_zp.to(dtype))
- if act_scale:
+ if act_scale is not None:
self.register_buffer("act_scale", act_scale.to(dtype))
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 827ffff2a..12b267d13 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -177,6 +177,8 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--act_bits", default=16, type=int, help="activation bits")
+ self.add_argument("--act_group_size", default=0, type=int, help="activation group size")
+
self.add_argument(
"--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"
)
@@ -513,6 +515,7 @@ def tune(args):
layer_config=layer_config,
enable_minmax_tuning=not args.disable_minmax_tuning,
act_bits=args.act_bits,
+ act_group_size=args.act_group_size,
low_cpu_mem_usage=low_cpu_mem_usage,
data_type=args.data_type,
enable_norm_bias_tuning=args.enable_norm_bias_tuning,
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 0614ebd0f..c817a34ff 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -309,10 +309,12 @@ class WrapperLinear(torch.nn.Module):
if not self.orig_layer.act_dynamic:
act_max_scale = best_params.get("act_max_scale", torch.tensor(1.0)).to(self.device)
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
- tmp_shape = 1
- if self.orig_layer.act_group_size > 1:
- tmp_shape = (1, self.orig_layer.act_group_size)
if act_max is not None:
+ tmp_shape = 1
+ if self.orig_layer.act_group_size > 1:
+ tmp_shape = (act_max.shape[0], self.orig_layer.act_group_size)
+ elif self.orig_layer.act_group_size == -1:
+ tmp_shape = (act_max.shape[0], 1)
_, act_scale, _ = self._qdq_act(
torch.zeros(tmp_shape).to(self.device), act_max_scale=self.act_max_scale, act_max=act_max
)
@@ -398,7 +400,7 @@ class WrapperWALayer(torch.nn.Module):
x, _, _ = self.orig_layer.act_quant_func(
x,
bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.group_size,
+ group_size=self.orig_layer.act_group_size,
scale_dtype=self.orig_layer.scale_dtype,
q_scale_thresh=self.orig_layer.q_scale_thresh,
data_type=self.orig_layer.act_data_type,
diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.py
index 2ff5c8501..fd9ed2406 100644
--- a/test/test_cpu/test_act_quantization.py
+++ b/test/test_cpu/test_act_quantization.py
@@ -88,6 +88,57 @@ class TestAutoRoundAct(unittest.TestCase):
)
autoround.quantize()
+ def test_wfp8afp8_static(self):
+ model_name = "facebook/opt-125m"
+ from auto_round.wrapper import WrapperWALayer
+
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ group_size=128,
+ act_group_size=-1,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ data_type="fp8",
+ act_dynamic=False,
+ act_data_type="fp8",
+ )
+ autoround.quantize()
+
+ self.assertTrue(isinstance(autoround.model.model.decoder.layers[2].self_attn.k_proj, WrapperWALayer))
+ self.assertEqual(autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_scale.shape[0], 30)
+ self.assertEqual(autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_max.shape[0], 30)
+
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ group_size=128,
+ act_group_size=128,
+ iters=0,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ data_type="fp8",
+ act_dynamic=False,
+ act_data_type="fp8",
+ )
+ autoround.quantize()
+ self.assertTrue(isinstance(autoround.model.model.decoder.layers[2].self_attn.k_proj, WrapperWALayer))
+
+ self.assertEqual(
+ autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_scale.shape[0],
+ int(3 * 10 * 768 / 128),
+ )
+ self.assertEqual(
+ autoround.model.model.decoder.layers[2].self_attn.k_proj.orig_layer.act_max.shape[0],
+ int(3 * 10 * 768 / 128),
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index 9d363e921..ae780cc44 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -217,6 +217,7 @@ class TestAutoRound(unittest.TestCase):
data_type="fp8",
act_data_type="fp8",
act_dynamic=False,
+ act_group_size=0,
)
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
@@ -239,6 +240,7 @@ class TestAutoRound(unittest.TestCase):
data_type="fp8",
act_data_type="fp8",
act_dynamic=False,
+ act_group_size=0,
)
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")