Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine model config file for mixed precision quantization (#300)

wenhuach21fccae4bc3e1f
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 513bde7c5..579248876 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -74,7 +74,7 @@ class AutoRound(object):
'group_size': 128,
'sym': True
'act_data_type': None,
- 'act_bits': 32,
+ 'act_bits': 16,
'act_group_size': None,
'act_sym': None,
@@ -105,7 +105,7 @@ class AutoRound(object):
data_type (str): The data type to be used (default is "int").
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_bits (int): Number of bits for activation quantization. Default is 16.
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
@@ -144,7 +144,7 @@ class AutoRound(object):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
- act_bits: int = 32,
+ act_bits: int = 16,
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
@@ -354,7 +354,7 @@ class AutoRound(object):
self.layer_config[n]["data_type"] = "float"
if self.amp_dtype == torch.bfloat16:
self.layer_config[n]["data_type"] = "bfloat"
- self.layer_config[n]["bits"] = 32
+ self.layer_config[n]["bits"] = 16
self.layer_config[n]["group_size"] = None
self.layer_config[n]["sym"] = None
@@ -429,8 +429,8 @@ class AutoRound(object):
layer_config[n] = {}
for key in keys:
layer_config[n][key] = getattr(self, key)
- layer_config[n]["bits"] = 32
- layer_config[n]["act_bits"] = 32
+ layer_config[n]["bits"] = 16
+ layer_config[n]["act_bits"] = 16
for key in keys:
setattr(m, key, layer_config[n][key])
@@ -1375,7 +1375,7 @@ class AutoRoundOPT(AutoRound):
data_type (str): The data type to be used (default is "int").
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_bits (int): Number of bits for activation quantization. Default is 16.
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
@@ -1416,7 +1416,7 @@ class AutoRoundOPT(AutoRound):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
- act_bits: int = 32,
+ act_bits: int = 16,
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
@@ -1541,7 +1541,7 @@ class AutoRoundAdam(AutoRoundOPT):
optimizer: string or object
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_bits (int): Number of bits for activation quantization. Default is 16.
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
@@ -1580,7 +1580,7 @@ class AutoRoundAdam(AutoRoundOPT):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
- act_bits: int = 32,
+ act_bits: int = 16,
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 0ca355c38..aa0ce85a3 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -221,7 +221,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if len(neq_keys) > 0:
extra_config[layer_name] = {}
for key in neq_keys:
- extra_config[layer_name][key] = layer_config[layer_name][key]
+ if layer_config[layer_name][key] is not None:
+ extra_config[layer_name][key] = layer_config[layer_name][key]
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 2623592d2..871e0e410 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -145,7 +145,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
- self.add_argument("--act_bits", default=32, type=int,
+ self.add_argument("--act_bits", default=16, type=int,
help="activation bits")
self.add_argument("--fp_layers_list", default="", type=str,
@@ -322,7 +322,7 @@ def tune(args):
for n, m in model.named_modules():
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
- layer_config[n] = {"bits": 32}
+ layer_config[n] = {"bits": 16}
logger.info(
f"{n} will not be quantized due to its shape not being divisible by 32,"
" resulting in an exporting issue to autogptq")
@@ -332,7 +332,7 @@ def tune(args):
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
name = n.split('.')[-1]
if n in fp_layers_list or name in fp_layers_list:
- layer_config[n] = {"bits": 32}
+ layer_config[n] = {"bits": 16}
logger.info(
f"{n} will not be quantized.")
lm_head_layer_name = "lm_head"
@@ -376,7 +376,11 @@ def tune(args):
if "cpu" not in device_str:
torch.cuda.empty_cache()
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"
+ if model_name.split('/')[-1] == ".":
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+ else:
+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
+
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 3631d867e..3ec7404db 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -282,7 +282,10 @@ def tune(args):
if args.device != "cpu":
torch.cuda.empty_cache()
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
+ if model_name.split('/')[-1] == ".":
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+ else:
+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index d209f04e8..eab5069a1 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -150,7 +150,7 @@ if __name__ == '__main__':
parser.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
- parser.add_argument("--act_bits", default=32, type=int,
+ parser.add_argument("--act_bits", default=16, type=int,
help="activation bits")
parser.add_argument("--fp_layers", default="", type=str,
@@ -288,7 +288,7 @@ if __name__ == '__main__':
for n, m in model.named_modules():
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
- layer_config[n] = {"bits": 32}
+ layer_config[n] = {"bits": 16}
print(
f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
fp_layers = args.fp_layers.split(",")
@@ -297,7 +297,7 @@ if __name__ == '__main__':
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
name = n.split('.')[-1]
if n in fp_layers or name in fp_layers:
- layer_config[n] = {"bits": 32}
+ layer_config[n] = {"bits": 16}
print(
f"{n} will not be quantized.")
lm_head_layer_name = "lm_head"
@@ -344,8 +344,10 @@ if __name__ == '__main__':
if "cpu" not in device_str:
torch.cuda.empty_cache()
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"
- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}-qdq"
+ if model_name.split('/')[-1] == ".":
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+ else:
+ export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
eval_folder = None
if args.format:
@@ -382,6 +384,7 @@ if __name__ == '__main__':
if "cpu" in deployment_device:
autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
if "fake" in deployment_device:
+ output_dir = f'{export_dir}-qdq'
model = model.to("cpu")
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
@@ -416,29 +419,6 @@ if __name__ == '__main__':
if isinstance(tasks, str):
tasks = tasks.split(',')
- if lm_eval_version < Version("0.4.2"):
- if args.eval_bs is None:
- args.eval_bs = 1
- if use_eval_legacy:
- if "mmlu" in tasks:
- tmp_tasks = tasks
- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
- tmp_tasks = tasks
- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
- seen = set()
- tmp_tasks = tasks
- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
-
- excel_name = f"{output_dir}_result.xlsx"
- output_dir += "/"
- print(excel_name, flush=True)
- eval_model(
- model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
- device=torch_device, excel_file=excel_name,
- trust_remote_code=not args.disable_trust_remote_code)
-
if lm_eval_version >= Version("0.4.2"):
if args.eval_bs is None:
args.eval_bs = "auto"