PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support quant lm_head for rtn w8afp8 static quant (#754)
Heng Guode7ecc1b8698
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c30d2280a..fed33df34 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -73,6 +73,7 @@ from auto_round.utils import (is_nv_fp,is_optimum_habana_available,is_standard_fp,+ is_static_afp8,llm_load_model,logger,mv_module_from_gpu,@@ -1328,7 +1329,16 @@ class AutoRound(object):raise ValueError("Could not find any blocks. Check the model or quant_block_list.")all_first_block_names = [block[0] for block in all_blocks]- all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)+ if self.act_bits < 16 and not self.act_dynamic:+ layer_names = self.get_quantized_layer_names_outside_blocks()+ if len(layer_names) > 0:+ logger.warning(+ "quantize layers outside blocks for static activation quantizaiton"+ " will significantly increase calibration time"+ )+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)+ else:+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)# Clear hooks for multi-GPU setupsif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:@@ -1394,7 +1404,9 @@ class AutoRound(object):if self.device_map is not None:accelerate.hooks.remove_hook_from_submodules(block)- if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):+ if (+ is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)+ ) or is_static_afp8(self):from auto_round.utils import set_amax_for_all_moe_layers# enable moe experts act_max automatic generation for linearsdiff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex cca0626f8..b8a32896f 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -22,6 +22,7 @@ import torchimport transformersfrom tqdm import tqdm+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -101,22 +102,24 @@ def pack_layer(layer_name, model, data_type, packing_device=None):returndevice = layer.weight.device- scale = layer.scale+ scale = layer.scale.view(-1)zp = layer.zpweight = layer.weight- act_scale = layer.act_scale if hasattr(layer, "act_scale") else None+ weight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)+ act_scale = layer.act_scale.view(-1) if hasattr(layer, "act_scale") else Nonetorch_dtype = torch.float8_e4m3fnif "fp8_e5m2" in data_type:torch_dtype = torch.float8_e5m2info = torch.finfo(torch_dtype)if zp is not None:q_weight = (- weight.to(packing_device) / scale.to(packing_device) + zp.to(packing_device)+ weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp.to(packing_device)if isinstance(zp, torch.Tensor)else zp)else:- q_weight = weight.to(packing_device) / scale.to(packing_device)+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)+ q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)q_weight = torch.clamp(q_weight, info.min, info.max)q_weight = q_weight.to(torch_dtype)if isinstance(layer, torch.nn.Linear):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex fec2177e2..74999c624 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -87,6 +87,14 @@ if deepspeed_exists:def infer_bits_by_data_type(data_type: str):+ """Infer bits by data_type++ Args:+ data_type (str): data_type++ Returns:+ int: bits inferred by data_type, None means cannot infer correct bits by data_type+ """if data_type is None:return 16for supported_dtype in SUPPORTED_DTYPES:@@ -2504,3 +2512,7 @@ def is_mx_fp(backend):def is_nv_fp(backend):return BackendDataType.NV_FP in backend+++def is_static_afp8(ar):+ return not ar.act_dynamic and "fp8" in ar.act_data_typediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 8942a2c0d..74efad368 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -365,7 +365,7 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.act_max = self.orig_layer.act_max * act_max_scale.item()self.orig_layer.act_max = self.orig_layer.act_max.to("cpu")else:- act_scale = torch.ones_like(act_max_scale, dtype=self.orig_layer.scale_dtype)+ act_scale = torch.ones(1, dtype=self.orig_layer.scale_dtype)self.orig_layer.act_scale = act_scale.to("cpu")self.orig_layer.q_scale_thresh = self.q_scale_threshdiff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex dc0939541..68942ac77 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -228,7 +228,7 @@ class TestAutoRound(unittest.TestCase):f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")self.assertIn("model.decoder.layers.8.self_attn.k_proj.input_scale", f.keys())self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1, 1]))+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)if static_kv_dtype == "fp8":@@ -259,7 +259,7 @@ class TestAutoRound(unittest.TestCase):f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")self.assertIn("model.decoder.layers.8.self_attn.k_proj.input_scale", f.keys())self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1, 1]))+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)shutil.rmtree(quantized_model_path, ignore_errors=True)