PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine model config file for mixed precision quantization (#300)
wenhuach21fccae4bc3e1f
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 513bde7c5..579248876 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -74,7 +74,7 @@ class AutoRound(object):'group_size': 128,'sym': True'act_data_type': None,- 'act_bits': 32,+ 'act_bits': 16,'act_group_size': None,'act_sym': None,@@ -105,7 +105,7 @@ class AutoRound(object):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.- act_bits (int): Number of bits for activation quantization. Default is 32.+ act_bits (int): Number of bits for activation quantization. Default is 16.act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.@@ -144,7 +144,7 @@ class AutoRound(object):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",- act_bits: int = 32,+ act_bits: int = 16,act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,@@ -354,7 +354,7 @@ class AutoRound(object):self.layer_config[n]["data_type"] = "float"if self.amp_dtype == torch.bfloat16:self.layer_config[n]["data_type"] = "bfloat"- self.layer_config[n]["bits"] = 32+ self.layer_config[n]["bits"] = 16self.layer_config[n]["group_size"] = Noneself.layer_config[n]["sym"] = None@@ -429,8 +429,8 @@ class AutoRound(object):layer_config[n] = {}for key in keys:layer_config[n][key] = getattr(self, key)- layer_config[n]["bits"] = 32- layer_config[n]["act_bits"] = 32+ layer_config[n]["bits"] = 16+ layer_config[n]["act_bits"] = 16for key in keys:setattr(m, key, layer_config[n][key])@@ -1375,7 +1375,7 @@ class AutoRoundOPT(AutoRound):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.- act_bits (int): Number of bits for activation quantization. Default is 32.+ act_bits (int): Number of bits for activation quantization. Default is 16.act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.@@ -1416,7 +1416,7 @@ class AutoRoundOPT(AutoRound):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",- act_bits: int = 32,+ act_bits: int = 16,act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,@@ -1541,7 +1541,7 @@ class AutoRoundAdam(AutoRoundOPT):optimizer: string or objectscale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.- act_bits (int): Number of bits for activation quantization. Default is 32.+ act_bits (int): Number of bits for activation quantization. Default is 16.act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.@@ -1580,7 +1580,7 @@ class AutoRoundAdam(AutoRoundOPT):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",- act_bits: int = 32,+ act_bits: int = 16,act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 0ca355c38..aa0ce85a3 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -221,7 +221,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif len(neq_keys) > 0:extra_config[layer_name] = {}for key in neq_keys:- extra_config[layer_name][key] = layer_config[layer_name][key]+ if layer_config[layer_name][key] is not None:+ extra_config[layer_name][key] = layer_config[layer_name][key]if len(extra_config) > 0:quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 2623592d2..871e0e410 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -145,7 +145,7 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--model_dtype", default=None, type=str,help="force to convert the dtype, some backends supports fp16 dtype better")- self.add_argument("--act_bits", default=32, type=int,+ self.add_argument("--act_bits", default=16, type=int,help="activation bits")self.add_argument("--fp_layers_list", default="", type=str,@@ -322,7 +322,7 @@ def tune(args):for n, m in model.named_modules():if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- layer_config[n] = {"bits": 32}+ layer_config[n] = {"bits": 16}logger.info(f"{n} will not be quantized due to its shape not being divisible by 32,"" resulting in an exporting issue to autogptq")@@ -332,7 +332,7 @@ def tune(args):if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):name = n.split('.')[-1]if n in fp_layers_list or name in fp_layers_list:- layer_config[n] = {"bits": 32}+ layer_config[n] = {"bits": 16}logger.info(f"{n} will not be quantized.")lm_head_layer_name = "lm_head"@@ -376,7 +376,11 @@ def tune(args):if "cpu" not in device_str:torch.cuda.empty_cache()- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"+ if model_name.split('/')[-1] == ".":+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")+ else:+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")+format_list = args.format.replace(' ', '').split(',')inplace = False if len(format_list) > 1 else Truediff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 3631d867e..3ec7404db 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -282,7 +282,10 @@ def tune(args):if args.device != "cpu":torch.cuda.empty_cache()- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"+ if model_name.split('/')[-1] == ".":+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")+ else:+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")format_list = args.format.replace(' ', '').split(',')inplace = False if len(format_list) > 1 else Truediff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex d209f04e8..eab5069a1 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -150,7 +150,7 @@ if __name__ == '__main__':parser.add_argument("--model_dtype", default=None, type=str,help="force to convert the dtype, some backends supports fp16 dtype better")- parser.add_argument("--act_bits", default=32, type=int,+ parser.add_argument("--act_bits", default=16, type=int,help="activation bits")parser.add_argument("--fp_layers", default="", type=str,@@ -288,7 +288,7 @@ if __name__ == '__main__':for n, m in model.named_modules():if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- layer_config[n] = {"bits": 32}+ layer_config[n] = {"bits": 16}print(f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")fp_layers = args.fp_layers.split(",")@@ -297,7 +297,7 @@ if __name__ == '__main__':if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):name = n.split('.')[-1]if n in fp_layers or name in fp_layers:- layer_config[n] = {"bits": 32}+ layer_config[n] = {"bits": 16}print(f"{n} will not be quantized.")lm_head_layer_name = "lm_head"@@ -344,8 +344,10 @@ if __name__ == '__main__':if "cpu" not in device_str:torch.cuda.empty_cache()- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}-qdq"+ if model_name.split('/')[-1] == ".":+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")+ else:+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")eval_folder = Noneif args.format:@@ -382,6 +384,7 @@ if __name__ == '__main__':if "cpu" in deployment_device:autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)if "fake" in deployment_device:+ output_dir = f'{export_dir}-qdq'model = model.to("cpu")model.save_pretrained(output_dir)tokenizer.save_pretrained(output_dir)@@ -416,29 +419,6 @@ if __name__ == '__main__':if isinstance(tasks, str):tasks = tasks.split(',')- if lm_eval_version < Version("0.4.2"):- if args.eval_bs is None:- args.eval_bs = 1- if use_eval_legacy:- if "mmlu" in tasks:- tmp_tasks = tasks- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:- tmp_tasks = tasks- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]- seen = set()- tmp_tasks = tasks- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]-- excel_name = f"{output_dir}_result.xlsx"- output_dir += "/"- print(excel_name, flush=True)- eval_model(- model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,- device=torch_device, excel_file=excel_name,- trust_remote_code=not args.disable_trust_remote_code)-if lm_eval_version >= Version("0.4.2"):if args.eval_bs is None:args.eval_bs = "auto"