Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support quant lm_head for rtn w8afp8 static quant (#754)

Heng Guode7ecc1b8698
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c30d2280a..fed33df34 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -73,6 +73,7 @@ from auto_round.utils import (
is_nv_fp,
is_optimum_habana_available,
is_standard_fp,
+ is_static_afp8,
llm_load_model,
logger,
mv_module_from_gpu,
@@ -1328,7 +1329,16 @@ class AutoRound(object):
raise ValueError("Could not find any blocks. Check the model or quant_block_list.")
all_first_block_names = [block[0] for block in all_blocks]
- all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)
+ if self.act_bits < 16 and not self.act_dynamic:
+ layer_names = self.get_quantized_layer_names_outside_blocks()
+ if len(layer_names) > 0:
+ logger.warning(
+ "quantize layers outside blocks for static activation quantizaiton"
+ " will significantly increase calibration time"
+ )
+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)
+ else:
+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)
# Clear hooks for multi-GPU setups
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
@@ -1394,7 +1404,9 @@ class AutoRound(object):
if self.device_map is not None:
accelerate.hooks.remove_hook_from_submodules(block)
- if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):
+ if (
+ is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)
+ ) or is_static_afp8(self):
from auto_round.utils import set_amax_for_all_moe_layers
# enable moe experts act_max automatic generation for linears
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
index cca0626f8..b8a32896f 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -22,6 +22,7 @@ import torch
import transformers
from tqdm import tqdm
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -101,22 +102,24 @@ def pack_layer(layer_name, model, data_type, packing_device=None):
return
device = layer.weight.device
- scale = layer.scale
+ scale = layer.scale.view(-1)
zp = layer.zp
weight = layer.weight
- act_scale = layer.act_scale if hasattr(layer, "act_scale") else None
+ weight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)
+ act_scale = layer.act_scale.view(-1) if hasattr(layer, "act_scale") else None
torch_dtype = torch.float8_e4m3fn
if "fp8_e5m2" in data_type:
torch_dtype = torch.float8_e5m2
info = torch.finfo(torch_dtype)
if zp is not None:
q_weight = (
- weight.to(packing_device) / scale.to(packing_device) + zp.to(packing_device)
+ weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp.to(packing_device)
if isinstance(zp, torch.Tensor)
else zp
)
else:
- q_weight = weight.to(packing_device) / scale.to(packing_device)
+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
+ q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)
q_weight = torch.clamp(q_weight, info.min, info.max)
q_weight = q_weight.to(torch_dtype)
if isinstance(layer, torch.nn.Linear):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index fec2177e2..74999c624 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -87,6 +87,14 @@ if deepspeed_exists:
def infer_bits_by_data_type(data_type: str):
+ """Infer bits by data_type
+
+ Args:
+ data_type (str): data_type
+
+ Returns:
+ int: bits inferred by data_type, None means cannot infer correct bits by data_type
+ """
if data_type is None:
return 16
for supported_dtype in SUPPORTED_DTYPES:
@@ -2504,3 +2512,7 @@ def is_mx_fp(backend):
def is_nv_fp(backend):
return BackendDataType.NV_FP in backend
+
+
+def is_static_afp8(ar):
+ return not ar.act_dynamic and "fp8" in ar.act_data_type
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 8942a2c0d..74efad368 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -365,7 +365,7 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.act_max = self.orig_layer.act_max * act_max_scale.item()
self.orig_layer.act_max = self.orig_layer.act_max.to("cpu")
else:
- act_scale = torch.ones_like(act_max_scale, dtype=self.orig_layer.scale_dtype)
+ act_scale = torch.ones(1, dtype=self.orig_layer.scale_dtype)
self.orig_layer.act_scale = act_scale.to("cpu")
self.orig_layer.q_scale_thresh = self.q_scale_thresh
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index dc0939541..68942ac77 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -228,7 +228,7 @@ class TestAutoRound(unittest.TestCase):
f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")
self.assertIn("model.decoder.layers.8.self_attn.k_proj.input_scale", f.keys())
self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())
- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1, 1]))
+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))
self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)
if static_kv_dtype == "fp8":
@@ -259,7 +259,7 @@ class TestAutoRound(unittest.TestCase):
f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")
self.assertIn("model.decoder.layers.8.self_attn.k_proj.input_scale", f.keys())
self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())
- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1, 1]))
+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))
self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)
shutil.rmtree(quantized_model_path, ignore_errors=True)