PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Large impact]set the default nsamples to 128 and low_gpu_mem_usage to False (#174)
wenhuach21ab614824b5f9
diff --git a/README.md b/README.mdindex 377560986..72afb77b1 100644--- a/README.md+++ b/README.md@@ -22,6 +22,7 @@ image presents an overview of AutoRound. Check out our updated paper on [arxiv]<div align="left">## What's New+* [2024/07] Important change: the default value of nsamples has been changed from 512 to 128 to reduce the memory usages, which may cause a slight accuracy drop in some scenarios* [2024/06] AutoRound format supports mixed bit-widths and group sizes for inference, resolving the significant performance drop issue with the asymmetric kernel* [2024/05] AutoRound supports lm-head quantization, saving 0.7G for LLaMA3-8B at W4G128.* [2024/05] AutoRound performs well@@ -98,7 +99,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).-- `nsamples (int)`: Number of samples for tuning (default is 512).+- `nsamples (int)`: Number of samples for tuning (default is 128).- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).@@ -113,7 +114,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).-- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is True).+- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is False).- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "@@ -126,6 +127,14 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r</details>+### Tips++1 Consider increasin 'iters' (e.g. 1000) to achieve better results, albeit with increased tuning time.++2 Consider increasing 'nsamples' (e.g. 512) to achieve better results, albeit with more memory(~20G).++3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.+## Model inferencePlease run the quantization code first.@@ -197,11 +206,7 @@ approach achieved superior performance compared to GPTQ, scoring 30/32, AWQ withwith a perfect score of 16/16 across llamv1/llamav2/mistral-7b on W4G-1, W4G128, W3G128, and W2G128, based on theaverage accuracies of 11 zero-shot tasks.-## Tips--1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time.-2 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.## Referencediff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex da040752c..9a57b20d0 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -79,7 +79,7 @@ class AutoRound(object):low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).iters (int): Number of iterations (default is 200).seqlen (int): Data length of the sequence for tuning (default is 2048).- nsamples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 128).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).nblocks (int): Number of blocks (default is 1).@@ -112,10 +112,10 @@ class AutoRound(object):enable_minmax_tuning: bool = True,lr: float = None,minmax_lr: float = None,- low_gpu_mem_usage: bool = True,+ low_gpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,- nsamples: int = 512,+ nsamples: int = 128,sampler: str = "rand",seed: int = 42,nblocks: int = 1,@@ -170,6 +170,7 @@ class AutoRound(object):self.optimizer = self.get_optimizer(None)self.share_attention_mask_flag = Noneself.hidden_dim_flag = None+ self.infer_bs_coeff = 1torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()@@ -788,7 +789,8 @@ class AutoRound(object):Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)"""- output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, self.cache_device)+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,+ self.cache_device)if q_input is not None:input_ids = q_input@@ -904,7 +906,8 @@ class AutoRound(object):if self.enable_quanted_input:q_outputs = self.get_block_outputs(- block, input_ids, input_others, self.train_bs, device, cache_device=self.cache_device+ block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,+ cache_device=self.cache_device)for i in range(len(input_ids)):input_ids[i] = None@@ -1183,10 +1186,10 @@ class AutoOPTRound(AutoRound):enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).lr (float): The learning rate (default is 0.005).minmax_lr (float): The learning rate for min-max tuning (default is None).- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.- nsamples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 128).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).nblocks (int): Number of blocks (default is 1).@@ -1220,10 +1223,10 @@ class AutoOPTRound(AutoRound):enable_minmax_tuning: bool = True,lr: float = None,minmax_lr: float = None,- low_gpu_mem_usage: bool = True,+ low_gpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,- nsamples: int = 512,+ nsamples: int = 128,sampler: str = "rand",seed: int = 42,nblocks: int = 1,@@ -1331,10 +1334,10 @@ class AutoAdamRound(AutoOPTRound):enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).lr (float): The learning rate (default is 0.005).minmax_lr (float): The learning rate for min-max tuning (default is None).- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.- nsamples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 128).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).nblocks (int): Number of blocks (default is 1).@@ -1368,10 +1371,10 @@ class AutoAdamRound(AutoOPTRound):enable_minmax_tuning: bool = True,lr: float = None,minmax_lr: float = None,- low_gpu_mem_usage: bool = True,+ low_gpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,- nsamples: int = 512,+ nsamples: int = 128,sampler: str = "rand",seed: int = 42,nblocks: int = 1,@@ -1415,4 +1418,3 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)-diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 9145b6cc3..14d9b5c60 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -74,11 +74,11 @@ if __name__ == '__main__':parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")- parser.add_argument("--nsamples", default=512, type=int,+ parser.add_argument("--nsamples", default=128, type=int,help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="low_gpu_mem_usage is deprecated")+ help="lower gpu memory but 50%-100% slower")parser.add_argument("--enable_minmax_tuning", action='store_true',help="enable_minmax_tuning is deprecated")@@ -104,9 +104,6 @@ if __name__ == '__main__':parser.add_argument("--disable_amp", action='store_true',help="disable amp")- parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',- help="disable low_gpu_mem_usage")-parser.add_argument("--disable_minmax_tuning", action='store_true',help="whether disable enable weight minmax tuning")@@ -123,9 +120,7 @@ if __name__ == '__main__':help="force to convert the dtype, some backends supports fp16 dtype better")args = parser.parse_args()- if args.low_gpu_mem_usage:- print(- "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")+if args.enable_minmax_tuning:print("enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")@@ -204,7 +199,7 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"if "bloom" in model_name:- args.disable_low_gpu_mem_usage = True+ args.low_gpu_mem_usage = Falseround = AutoRoundif args.adam:@@ -240,15 +235,15 @@ if __name__ == '__main__':error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."raise EnvironmentError(error_message)- if args.quant_lm_head and not args.disable_low_gpu_mem_usage:- print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "+ if args.quant_lm_head and not args.low_gpu_mem_usage:+ print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,amp=not args.disable_amp, nsamples=args.nsamples,- low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,+ low_gpu_mem_usage=args.low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, weight_config=weight_config,enable_minmax_tuning=not args.disable_minmax_tuning)@@ -340,7 +335,7 @@ if __name__ == '__main__':output_dir += "/"print(excel_name, flush=True)eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,- eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,device=torch_device, excel_file=excel_name)if not args.disable_eval and lm_eval_version == "0.4.2":diff --git a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.shindex 7058a1983..a96d03e14 100644--- a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh+++ b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh@@ -5,5 +5,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"diff --git a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.shindex 7af9f6a7e..c4e306cc0 100644--- a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh+++ b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh@@ -6,8 +6,8 @@ python3 main.py \--bits 4 \--iters 1000 \--quant_lm_head \+--nsamples 512 \--deployment_device 'gpu' \---disable_low_gpu_mem_usage \--output_dir "./tmp_autoround"@@ -20,5 +20,5 @@ python3 main.py \--bits 4 \--iters 1000 \--deployment_device 'gpu' \---disable_low_gpu_mem_usage \+--nsamples 512 \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.shindex 762982acb..76680b747 100644--- a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh+++ b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh@@ -4,5 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh b/examples/language-modeling/scripts/Mistral-7B-v0.1.shindex e433806d8..1071f8db9 100644--- a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh+++ b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh@@ -4,5 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.shindex 018969dc5..cd98d5b12 100644--- a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh+++ b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh@@ -4,5 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.shindex 51ef47e3b..185cda732 100644--- a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh+++ b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh@@ -4,6 +4,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \--disable_quanted_input\ No newline at end of filediff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.shindex 84ab60490..a9fcebe21 100644--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh@@ -5,5 +5,6 @@ python3 main.py \--bits 4 \--iters 1000 \--deployment_device 'gpu' \---minmax_lr 2e-3+--minmax_lr 2e-3\+--nsamples 512diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.shindex 32d8123d3..07161e16a 100644--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh@@ -6,4 +6,6 @@ python3 main.py \--iters 200 \--deployment_device gpu \--sym \---minmax_lr 0.01+--nsamples 512 \+--minmax_lr 0.01+diff --git a/examples/language-modeling/scripts/baichuan2-7b-chat.sh b/examples/language-modeling/scripts/baichuan2-7b-chat.shindex 51d674509..74bf14fb6 100644--- a/examples/language-modeling/scripts/baichuan2-7b-chat.sh+++ b/examples/language-modeling/scripts/baichuan2-7b-chat.sh@@ -4,5 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--minmax_lr 2e-3diff --git a/examples/language-modeling/scripts/bloom-3b.sh b/examples/language-modeling/scripts/bloom-3b.shindex a12ee2962..712d1ae09 100644--- a/examples/language-modeling/scripts/bloom-3b.sh+++ b/examples/language-modeling/scripts/bloom-3b.sh@@ -4,5 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"diff --git a/examples/language-modeling/scripts/falcon-7b.sh b/examples/language-modeling/scripts/falcon-7b.shindex 07bbae4d4..adc6b5fba 100644--- a/examples/language-modeling/scripts/falcon-7b.sh+++ b/examples/language-modeling/scripts/falcon-7b.sh@@ -4,7 +4,7 @@ python3 main.py \--group_size 64 \--bits 4 \--iters 1000 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \---disable_quanted_input \---disable_low_gpu_mem_usage\ No newline at end of file+--disable_quanted_input \\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-2b.sh b/examples/language-modeling/scripts/gemma-2b.shindex b7136bfb7..7bf58bc2a 100644--- a/examples/language-modeling/scripts/gemma-2b.sh+++ b/examples/language-modeling/scripts/gemma-2b.sh@@ -8,6 +8,6 @@ python3 main.py \--bits 4 \--iters 400 \--model_dtype "float16" \---disable_low_gpu_mem_usage \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.shindex 5b13c99bd..63e18bcaf 100644--- a/examples/language-modeling/scripts/gemma-7b-it.sh+++ b/examples/language-modeling/scripts/gemma-7b-it.sh@@ -7,4 +7,5 @@ python3 main.py \--minmax_lr 2e-3 \--model_dtype "float16" \--deployment_device 'gpu' \+--nsamples 512 \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.shindex bf89b0704..37a7b9872 100644--- a/examples/language-modeling/scripts/gemma-7b.sh+++ b/examples/language-modeling/scripts/gemma-7b.sh@@ -4,8 +4,8 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--minmax_lr 2e-3 \--deployment_device 'gpu' \--model_dtype "float16" \---disable_low_gpu_mem_usage \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gpt-j-6b.sh b/examples/language-modeling/scripts/gpt-j-6b.shindex e11f21ade..99b90a678 100644--- a/examples/language-modeling/scripts/gpt-j-6b.sh+++ b/examples/language-modeling/scripts/gpt-j-6b.sh@@ -5,5 +5,6 @@ python3 main.py \--bits 4 \--iters 1000 \--disable_quanted_input \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround"diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.shindex 5bc14d124..ff3da6570 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh@@ -5,6 +5,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \+--nsamples 512 \--minmax_lr 2e-3 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.shindex 15a3d4b71..378266df5 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh@@ -5,6 +5,7 @@ python3 main.py \--bits 4 \--iters 1000 \--minmax_lr 2e-3 \+--nsamples 512 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \--disable_quanted_input\ No newline at end of filediff --git a/examples/language-modeling/scripts/opt-2.7b.sh b/examples/language-modeling/scripts/opt-2.7b.shindex bc9bc2ef5..f7954ece8 100644--- a/examples/language-modeling/scripts/opt-2.7b.sh+++ b/examples/language-modeling/scripts/opt-2.7b.sh@@ -5,6 +5,7 @@ python3 main.py \--bits 4 \--iters 1000 \--minmax_lr 0.002 \+--nsamples 512 \--disable_quanted_input \--deployment_device 'gpu' \--output_dir "./tmp_autoround"diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.shindex 46e27226f..8b5a1b821 100644--- a/examples/language-modeling/scripts/phi-2.sh+++ b/examples/language-modeling/scripts/phi-2.sh@@ -5,6 +5,6 @@ python3 main.py \--bits 4 \--iters 1000 \--sym \+--nsamples 512 \--deployment_device 'gpu' \---output_dir "./tmp_autoround" \---disable_low_gpu_mem_usage+--output_dir "./tmp_autoround"\ No newline at end of file