PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine export API
Zhang, Weiwei1413e0732c4c8
diff --git a/README.md b/README.mdindex 1a081848c..d742ae0b5 100644--- a/README.md+++ b/README.md@@ -38,7 +38,7 @@ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in- `lr (float)`: The learning rate for rounding value (default is None, it will be set to 1.0/iters automatically).- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).- `n_samples (int)`: Number of samples for tuning (default is 512).-- `seqlen (int)`: Data length of the sequence for tuning.+- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).- `bs (int)`: Batch size for training (default is 8).- `amp (bool)`: Whether to use automatic mixed precision (default is True).- `n_blocks (int)`: Packing several blocks as one for tuning together (default is 1).@@ -145,10 +145,8 @@ After the quantization process is complete, you can invoke the export API to per## export for CPU deploymentfrom auto_round import QuantConfig, compress_model, save_compressed_modeloutput_dir = "/PATH/TO/SAVE/COMPRESSED/MODEL/"- compressed_model = compress_model(fake_qdq_model, weight_config)- quantize_config = QuantConfig(bits=4, sym=False, group_size=128)- save_compressed_model(compressed_model, output_dir=output_dir,- quantize_config=quantize_config, tokenizer=tokenizer)+ save_compressed_model(fake_qdq_model, weight_config=weight_config, output_dir=output_dir, tokenizer=tokenizer)+ del weight_config## export for GPU deployment# please install auto-gptq first@@ -171,3 +169,4 @@ If you find SignRound useful for your research, please cite our paper:}```+diff --git a/auto_round/export_to_speed/export.py b/auto_round/export_to_speed/export.pyindex 71c6bce9e..df29400d4 100644--- a/auto_round/export_to_speed/export.py+++ b/auto_round/export_to_speed/export.py@@ -22,6 +22,7 @@ from safetensors.torch import save_file as safe_savefrom os.path import join, isfile, isdirimport copyimport json+from .config import QuantConfigfrom .model_wrapper import WeightOnlyLinearfrom ..utils import quant_weight_w_scale, get_module, set_module@@ -32,7 +33,6 @@ def compress_model(enable_full_range=False,compression_dtype=torch.int32,compression_dim=1,- scale_dtype=torch.float32,device="cpu",use_optimum_format=True,):@@ -46,8 +46,6 @@ def compress_model(Defaults to torch.int32.compression_dim (int, optional): Select from [0, 1], 0 is output channel,1 is input channel. Defaults to 1.- scale_dtype (torch.Tensor, optional): Use float32 or float16.- Defaults to torch.float32.device (str, optional): choose device for compression. Defaults to cpu.use_optimum_format (bool, optional): use the popular huggingface compression format.1: compression_dim: weight = 1, zeros = 0 and both are transposed.@@ -98,22 +96,28 @@ def compress_model()new_module.pack(int_weight, scale, zp, m.bias)set_module(compressed_model, k, new_module)- return compressed_model++ quantize_config = QuantConfig(bits=num_bits, sym=(scheme=="sym"), group_size=group_size)+ return compressed_model, quantize_config-def save_compressed_model(model, output_dir, quantize_config=None, tokenizer=None):+def save_compressed_model(model,+ weight_config:Union[str, dict],+ output_dir,+ tokenizer=None):"""Save configure file and weights for CPU backend inference."""+ compressed_model, quantize_config = compress_model(model, weight_config)if quantize_config is not None:- config = model.config+ config = compressed_model.configsetattr(config, "quantization_config", quantize_config.to_dict())config.save_pretrained(output_dir)quantize_config.save_pretrained(output_dir)try:- model.save_pretrained(output_dir, safe_serialization=True)+ compressed_model.save_pretrained(output_dir, safe_serialization=True)if tokenizer is not None:tokenizer.save_pretrained(output_dir)logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))except IOError as e: # pragma: no cover- logger.error("Fail to save configure file and weights due to {}.".format(e))\ No newline at end of file+ logger.error("Fail to save configure file and weights due to {}.".format(e))diff --git a/examples/main.py b/examples/main.pyindex 95dab31f2..0d8ed2f24 100644--- a/examples/main.py+++ b/examples/main.py@@ -1,11 +1,9 @@import argparseimport sys-sys.path.insert(0, './')+sys.path.insert(0, '../')from auto_round import (AutoRound,AutoAdamRound,- compress_model,- save_compressed_model,- QuantConfig)+ save_compressed_model)parser = argparse.ArgumentParser()import torch@@ -45,7 +43,7 @@ if __name__ == '__main__':parser.add_argument("--device", default=0, type=str,help="device gpu int number, or 'cpu' ")- #+parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -95,10 +93,6 @@ if __name__ == '__main__':parser.add_argument("--scale_dtype", default='fp32',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")-- # parser.add_argument("--tasks",- # default=['lambada_openai'],- # help="lm-eval tasks")parser.add_argument("--tasks",default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',@@ -167,17 +161,6 @@ if __name__ == '__main__':eval_orig_float=True, excel_file=excel_name)exit()- # if args.iters <= 0:- # print("eval rtn", flush=True)- # excel_name += "_rtn.xlsx"- # q_dq_weight(model, bits=args.bits, group_size=args.group_size)- # if not args.low_gpu_mem_usage:- # model = model.to(cuda_device)- # eval_model(output_dir=args.output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \- # eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,- # excel_file=excel_name)- # exit()-if not args.low_gpu_mem_usage:model = model.to(cuda_device)@@ -197,11 +180,8 @@ if __name__ == '__main__':export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"if args.deployment_device == 'cpu':- compressed_model = compress_model(model, q_config)- quantize_config = QuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)- save_compressed_model(compressed_model, output_dir=export_dir, quantize_config=quantize_config, tokenizer=tokenizer)+ save_compressed_model(model, weight_config=q_config, output_dir=export_dir, tokenizer=tokenizer)del q_config- del compressed_modelelif args.deployment_device == 'gpu':autoround.export_to_autogptq(export_dir, use_triton=True)@@ -209,11 +189,9 @@ if __name__ == '__main__':model.eval()output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"- # model.to(cuda_device)- # eval_model(model, model_name, tokenizer, tasks=args.tasks, eval_bs=args.eval_bs)excel_name = f"{output_dir}_result.xlsx"output_dir += "/"print(excel_name, flush=True)eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,- limit=None)\ No newline at end of file+ limit=None)diff --git a/examples/run_autoround.sh b/examples/run_autoround.shindex 2c8b13a9a..66b3f0b44 100644--- a/examples/run_autoround.sh+++ b/examples/run_autoround.sh@@ -1,59 +1,17 @@#!/bin/bashset -x-# export CUBLAS_WORKSPACE_CONFIG=':4096:8'-export HF_HOME="/models/huggingface/"-export https_proxy=http://child-jf.intel.com:912-export http_proxy=http://child-jf.intel.com:912--# model="neural-chat-7b-v3-3"-# for iter in 200 400 1000-# do-# CUDA_VISIBLE_DEVICES=2 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \-# examples/main.py \-# --model_name /models/${model} \-# --bits 4 --group_size 128 \-# --enable_minmax_tuning \-# --use_quant_input \-# --amp \-# --iters ${iter} \-# --deployment_device 'cpu' \-# --scale_dtype 'fp32' \-# --eval_bs 16 \-# --output_dir /data1/zww/autoround/int_llama_${iter} \-# 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt-# done &-model="neural-chat-7b-v3-3"-for iter in 2000-do- CUDA_VISIBLE_DEVICES=4 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \- examples/main.py \++CUDA_VISIBLE_DEVICES=4 python3 main.py \--model_name /models/${model} \--bits 4 --group_size 128 \--enable_minmax_tuning \--use_quant_input \--amp \- --iters ${iter} \- --deployment_device 'fake' \+ --iters 200 \+ --deployment_device 'cpu' \--scale_dtype 'fp32' \--eval_bs 16 \- --output_dir /data1/zww/autoround/int_llama_${iter} \- 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt-done-# model_dir="/data2/zww/gptq_actorder/W4G128_gptq"-# for model in "static_neural-chat-7b-v3-3" "nonstatic_neural-chat-7b-v3-3"-# do-# CUDA_VISIBLE_DEVICES=5 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \-# examples/main.py \-# --model_name ${model_dir}/${model} \-# --bits 4 --group_size 128 \-# --eval_fp16_baseline \-# --enable_minmax_tuning \-# --use_quant_input \-# --amp \-# --scale_dtype 'fp32' \-# --eval_bs 16 \-# 2>&1 | tee -a /data1/zww/optround_logs/${model}_gptq_W4G128.txt-# done+ --output_dir "SAVE/PATH"