PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Deprecated args in examples (#42)
wenhuach21bb6c681ff51a
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex dd1573452..509e5b334 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -305,8 +305,8 @@ def unwrapper_block(block, vs, min_scales, max_scales):for n, m in block.named_modules():if hasattr(m, "orig_layer"):v = 0- min_scale = 0- max_scale = 0+ min_scale = torch.tensor(0)+ max_scale = torch.tensor(0)if isinstance(vs, dict):v = vs[n]if isinstance(min_scales, dict):@@ -852,6 +852,7 @@ class AutoRound(object):mse_loss = torch.nn.MSELoss().to(device)scaler = self.get_scaler() # pylint: disable=assignment-from-noneinit_loss = None+ best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(0), torch.tensor(0)for i in range(self.iters):if self.sampler == "rand":indices = torch.randperm(n_samples)[:pick_samples]@@ -1094,8 +1095,6 @@ class AutoRound(object):summary_info += f", {unquantized_layers} have not been quantized"logger.info(summary_info)- if len(unquantized_layers) > 0:- logger.info(f"Summary: {unquantized_layers} have not been quantized")self.quantized = Trueself.model = self.model.to(self.model_orig_dtype)diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pyindex 6d0b6ab9b..a50131fc6 100644--- a/examples/code-generation/main.py+++ b/examples/code-generation/main.py@@ -1,6 +1,7 @@import argparseimport randomimport sys+sys.path.insert(0, '../..')parser = argparse.ArgumentParser()import torch@@ -16,8 +17,6 @@ import reos.environ["TOKENIZERS_PARALLELISM"] = "false"--if __name__ == '__main__':parser.add_argument(@@ -38,7 +37,7 @@ if __name__ == '__main__':parser.add_argument("--device", default="auto", type=str,help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -59,7 +58,7 @@ if __name__ == '__main__':help="seed")parser.add_argument("--amp", action='store_true',- help="amp")+ help="amp is deprecated")parser.add_argument("--adam", action='store_true',help="adam")@@ -75,10 +74,10 @@ if __name__ == '__main__':help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="low_gpu_mem_usage")+ help="low_gpu_mem_usage is deprecated")parser.add_argument("--enable_minmax_tuning", action='store_true',- help="whether enable weight minmax tuning")+ help="enable_minmax_tuning is deprecated")parser.add_argument("--deployment_device", default='fake', type=str,help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."@@ -96,32 +95,53 @@ if __name__ == '__main__':parser.add_argument("--output_dir", default="./tmp_autoround", type=str,help="Where to store the final model.")+ parser.add_argument("--disable_amp", action='store_true',+ help="disable amp")++ parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',+ help="disable low_gpu_mem_usage")++ parser.add_argument("--disable_minmax_tuning", action='store_true',+ help="whether disable enable weight minmax tuning") # #not usage+args = parser.parse_args()+ if args.low_gpu_mem_usage:+ print(+ "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")+ if args.enable_minmax_tuning:+ print(+ "enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")+ if args.amp:+ print(+ "amp is deprecated, it has been set to the default, use disable_amp to turn it off")+set_seed(args.seed)tasks = args.tasks-+model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]print(model_name, flush=True)from auto_round.utils import detect_device+device_str = detect_device(args.device)torch_dtype = "auto"if device_str == "hpu":torch_dtype = torch.bfloat16torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))-+if is_glm:model = AutoModel.from_pretrained(model_name, trust_remote_code=True)else:model = AutoModelForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True)-+from auto_round import (AutoRound,- AutoAdamRound)+ AutoAdamRound)+model = model.eval()# align wigh GPTQ to eval pplif "opt" in model_name:@@ -158,9 +178,11 @@ if __name__ == '__main__':autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,+ amp=not args.disable_amp, n_samples=args.n_samples,+ low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, dataset="mbpp", dataset_split=['train', 'validation', 'test']) ##TODO args pass+ scale_dtype=args.scale_dtype, dataset="mbpp", dataset_split=['train', 'validation', 'test'],+ enable_minmax_tuning=not args.disable_minmax_tuning) ##TODO args passmodel, q_config = autoround.quantize()model_name = args.model_name.rstrip("/")model.eval()@@ -178,4 +200,3 @@ if __name__ == '__main__':model = model.to("cpu")model.save_pretrained(output_dir)tokenizer.save_pretrained(output_dir)-diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex cb43e9904..2d2758e18 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -32,11 +32,18 @@ The transformers version required varies across different types of models. Here,| microsoft/phi-2 | 4.36 |-## 2. Prepare Dataset+## 2. Prepare Calibration Dataset-The NeelNanda/pile-10k in huggingface is adopted as the default calibration data and will be downloaded automatically from the datasets Hub. To customize a dataset, please kindly follow our dataset code.+### Default Dataset+The [NeelNanda/pile-10k](https://huggingface.co/datasets/NeelNanda/pile-10k) in huggingface is adopted as the default calibration data and will be downloaded automatically from the datasets Hub. To customize a dataset, please kindly follow our dataset code.See more about loading [huggingface dataset](https://huggingface.co/docs/datasets/main/en/quickstart)+### Customized Dataset+- Following the [code](./main_customized_data.py) to pass list of string or list of inputs to dataloader.++- Register your dataset/dataloader following the [code](../../auto_round/calib_dataset.py) and pass the new dataset&split args to initialize AutoRound object.++<br />## 3. Run Examples@@ -47,25 +54,27 @@ pip install -r requirements.txt- **Default Settings:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --enable_minmax_tuning --use_quant_input+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --use_quant_input```- **Reduced GPU Memory Usage and Adjusted Training Batch Size:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --train_bs 1 --gradient_accumulate_steps 8```- **Utilizing the AdamW Optimizer:**-Include the flag `--adam`. Note that AdamW is less effective than Sign gradient descent in many scenarios we tested.++Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.- **Running the Original SignRound:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --iters 400 --lr 0.0025 --minmax_lr 0.0025+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --iters 400 --lr 0.0025 --disable_minmax_tuning```+- **Running on Intel Gaudi2**```bashbash run_autoround_on_gaudi.sh```- `--enable_minmax_tuning` is strongly recommended+## 4. Evaluationdiff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 833224f4d..34d10306b 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -1,5 +1,6 @@import argparseimport sys+sys.path.insert(0, '../..')parser = argparse.ArgumentParser()import torch@@ -36,7 +37,7 @@ if __name__ == '__main__':parser.add_argument("--device", default="auto", type=str,help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -56,11 +57,13 @@ if __name__ == '__main__':parser.add_argument("--seed", default=42, type=int,help="seed")++parser.add_argument("--eval_fp16_baseline", action='store_true',help="whether to eval FP16 baseline")parser.add_argument("--amp", action='store_true',- help="amp")+ help="amp is deprecated ")parser.add_argument("--adam", action='store_true',help="adam")@@ -76,10 +79,10 @@ if __name__ == '__main__':help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="low_gpu_mem_usage")+ help="low_gpu_mem_usage is deprecated")parser.add_argument("--enable_minmax_tuning", action='store_true',- help="whether enable weight minmax tuning")+ help="enable_minmax_tuning is deprecated")parser.add_argument("--deployment_device", default='fake', type=str,help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."@@ -100,25 +103,53 @@ if __name__ == '__main__':parser.add_argument("--disable_eval", action='store_true',help="Whether to do lmeval evaluation.")+ parser.add_argument("--disable_amp", action='store_true',+ help="disable amp")++ parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',+ help="disable low_gpu_mem_usage")++ parser.add_argument("--disable_minmax_tuning", action='store_true',+ help="whether disable enable weight minmax tuning")++ parser.add_argument("--disable_trust_remote_code", action='store_true',+ help="Whether to disable trust_remote_code")++args = parser.parse_args()+ if args.low_gpu_mem_usage:+ print(+ "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")+ if args.enable_minmax_tuning:+ print(+ "enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")+ if args.amp:+ print(+ "amp is deprecated, it has been set to the default, use disable_amp to turn it off")+set_seed(args.seed)tasks = args.tasksuse_eval_legacy = Falseimport subprocess++def get_library_version(library_name):try:version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]return versionexcept subprocess.CalledProcessError:return "Library not found"++res = get_library_version("lm-eval")if res == "0.3.0":use_eval_legacy = True-+if not use_eval_legacy:from eval import eval_modelelse:from eval_legacy import eval_model+if isinstance(tasks, str):tasks = tasks.split(',')if isinstance(tasks, list):@@ -131,7 +162,7 @@ if __name__ == '__main__':tasks = list(set(tasks))if isinstance(args.tasks, str):tasks = ','.join(tasks)-+if 'fake' in args.deployment_device and not args.disable_eval:if use_eval_legacy:print("Using the legacy lm_eval(0.3.0)")@@ -142,24 +173,26 @@ if __name__ == '__main__':if model_name[-1] == "/":model_name = model_name[:-1]print(model_name, flush=True)-+from auto_round.utils import detect_device+device_str = detect_device(args.device)torch_dtype = "auto"if device_str == "hpu":torch_dtype = torch.bfloat16torch_device = torch.device(device_str)-+is_glm = bool(re.search("chatglm", model_name.lower()))if is_glm:- model = AutoModel.from_pretrained(model_name, trust_remote_code=True)+ model = AutoModel.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)else:model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=not args.disable_trust_remote_code)-+from auto_round import (AutoRound,- AutoAdamRound)+ AutoAdamRound)+model = model.eval()# align with GPTQ to eval pplif "opt" in model_name:@@ -177,7 +210,7 @@ if __name__ == '__main__':if tokenizer.pad_token is None:tokenizer.add_special_tokens({'pad_token': '[PAD]'})else:- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)if hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:@@ -200,11 +233,11 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"if args.eval_fp16_baseline:- if not args.low_gpu_mem_usage:+ if args.disable_low_gpu_mem_usage:model = model.to(torch_device)excel_name += "_fp16.xlsx"eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,+ eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,device=torch_device, excel_file=excel_name)exit()@@ -226,9 +259,11 @@ if __name__ == '__main__':autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,+ amp=not args.disable_amp, n_samples=args.n_samples,+ low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass+ scale_dtype=args.scale_dtype, weight_config=weight_config,+ enable_minmax_tuning=not args.disable_minmax_tuning) ##TODO args passmodel, _ = autoround.quantize()model_name = args.model_name.rstrip("/")@@ -255,6 +290,3 @@ if __name__ == '__main__':eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,device=torch_device, excel_file=excel_name)---diff --git a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.shindex 59b075efd..95616850e 100644--- a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh+++ b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh@@ -5,11 +5,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--use_quant_input \---low_gpu_mem_usage \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp+--output_dir "./tmp_autoround"diff --git a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh b/examples/language-modeling/scripts/Mistral-7B-v0.1.shindex 226b042c8..34f2e0074 100644--- a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh+++ b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh@@ -4,10 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--use_quant_input \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.shindex 175368630..59edc3366 100644--- a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh+++ b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh@@ -4,11 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--use_quant_input \---low_gpu_mem_usage \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.shindex f488beae2..35439ad5c 100644--- a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh+++ b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh@@ -4,10 +4,5 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \---low_gpu_mem_usage \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat.shindex 4161ee24c..390cf1f8d 100644--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh@@ -4,13 +4,9 @@ python3 main.py \--model_name Qwen/Qwen1.5-7B-Chat\--bits 4 \--group_size 128 \---enable_minmax_tuning \---amp \--use_quant_input \---n_samples 512 \--iters 200 \--deployment_device gpu \--sym \---eval_bs 32 \--minmax_lr 0.01diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat_2.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat_2.shindex 43fcb7b66..79e6dae10 100644--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat_2.sh+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat_2.sh@@ -4,12 +4,8 @@ python3 main.py \--model_name Qwen/Qwen1.5-7B-Chat\--bits 4 \--group_size 128 \---enable_minmax_tuning \---amp \--use_quant_input \---n_samples 512 \--iters 1000 \--deployment_device gpu \--sym \---eval_bs 32 \--minmax_lr 0.002diff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.shindex 222387add..14bfb4bd6 100644--- a/examples/language-modeling/scripts/gemma-7b-it.sh+++ b/examples/language-modeling/scripts/gemma-7b-it.sh@@ -4,12 +4,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--use_quant_input \--minmax_lr 2e-3 \---low_gpu_mem_usage \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.shindex 2999a3337..bf4daeabd 100644--- a/examples/language-modeling/scripts/gemma-7b.sh+++ b/examples/language-modeling/scripts/gemma-7b.sh@@ -4,12 +4,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--use_quant_input \--minmax_lr 2e-3 \---low_gpu_mem_usage \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.shindex e3751eed2..7d16a10db 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh@@ -5,10 +5,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--minmax_lr 0.0002 \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.shindex a87fbef27..75dd31f2e 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh@@ -4,10 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--minmax_lr 0.0002 \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp+--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.shindex 3763573c2..69c44b24e 100644--- a/examples/language-modeling/scripts/phi-2.sh+++ b/examples/language-modeling/scripts/phi-2.sh@@ -4,9 +4,6 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---enable_minmax_tuning \--deployment_device 'gpu' \---scale_dtype 'fp32' \---eval_bs 32 \---output_dir "./tmp_autoround" \---amp\ No newline at end of file+--disable_trust_remote_code \+--output_dir "./tmp_autoround"\ No newline at end of file