Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine export API

Zhang, Weiwei1413e0732c4c8
diff --git a/README.md b/README.md
index 1a081848c..d742ae0b5 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in
- `lr (float)`: The learning rate for rounding value (default is None, it will be set to 1.0/iters automatically).
- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).
- `n_samples (int)`: Number of samples for tuning (default is 512).
-- `seqlen (int)`: Data length of the sequence for tuning.
+- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).
- `bs (int)`: Batch size for training (default is 8).
- `amp (bool)`: Whether to use automatic mixed precision (default is True).
- `n_blocks (int)`: Packing several blocks as one for tuning together (default is 1).
@@ -145,10 +145,8 @@ After the quantization process is complete, you can invoke the export API to per
## export for CPU deployment
from auto_round import QuantConfig, compress_model, save_compressed_model
output_dir = "/PATH/TO/SAVE/COMPRESSED/MODEL/"
- compressed_model = compress_model(fake_qdq_model, weight_config)
- quantize_config = QuantConfig(bits=4, sym=False, group_size=128)
- save_compressed_model(compressed_model, output_dir=output_dir,
- quantize_config=quantize_config, tokenizer=tokenizer)
+ save_compressed_model(fake_qdq_model, weight_config=weight_config, output_dir=output_dir, tokenizer=tokenizer)
+ del weight_config
## export for GPU deployment
# please install auto-gptq first
@@ -171,3 +169,4 @@ If you find SignRound useful for your research, please cite our paper:
}
```
+
diff --git a/auto_round/export_to_speed/export.py b/auto_round/export_to_speed/export.py
index 71c6bce9e..df29400d4 100644
--- a/auto_round/export_to_speed/export.py
+++ b/auto_round/export_to_speed/export.py
@@ -22,6 +22,7 @@ from safetensors.torch import save_file as safe_save
from os.path import join, isfile, isdir
import copy
import json
+from .config import QuantConfig
from .model_wrapper import WeightOnlyLinear
from ..utils import quant_weight_w_scale, get_module, set_module
@@ -32,7 +33,6 @@ def compress_model(
enable_full_range=False,
compression_dtype=torch.int32,
compression_dim=1,
- scale_dtype=torch.float32,
device="cpu",
use_optimum_format=True,
):
@@ -46,8 +46,6 @@ def compress_model(
Defaults to torch.int32.
compression_dim (int, optional): Select from [0, 1], 0 is output channel,
1 is input channel. Defaults to 1.
- scale_dtype (torch.Tensor, optional): Use float32 or float16.
- Defaults to torch.float32.
device (str, optional): choose device for compression. Defaults to cpu.
use_optimum_format (bool, optional): use the popular huggingface compression format.
1: compression_dim: weight = 1, zeros = 0 and both are transposed.
@@ -98,22 +96,28 @@ def compress_model(
)
new_module.pack(int_weight, scale, zp, m.bias)
set_module(compressed_model, k, new_module)
- return compressed_model
+
+ quantize_config = QuantConfig(bits=num_bits, sym=(scheme=="sym"), group_size=group_size)
+ return compressed_model, quantize_config
-def save_compressed_model(model, output_dir, quantize_config=None, tokenizer=None):
+def save_compressed_model(model,
+ weight_config:Union[str, dict],
+ output_dir,
+ tokenizer=None):
"""Save configure file and weights for CPU backend inference."""
+ compressed_model, quantize_config = compress_model(model, weight_config)
if quantize_config is not None:
- config = model.config
+ config = compressed_model.config
setattr(config, "quantization_config", quantize_config.to_dict())
config.save_pretrained(output_dir)
quantize_config.save_pretrained(output_dir)
try:
- model.save_pretrained(output_dir, safe_serialization=True)
+ compressed_model.save_pretrained(output_dir, safe_serialization=True)
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
except IOError as e: # pragma: no cover
- logger.error("Fail to save configure file and weights due to {}.".format(e))
\ No newline at end of file
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
diff --git a/examples/main.py b/examples/main.py
index 95dab31f2..0d8ed2f24 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -1,11 +1,9 @@
import argparse
import sys
-sys.path.insert(0, './')
+sys.path.insert(0, '../')
from auto_round import (AutoRound,
AutoAdamRound,
- compress_model,
- save_compressed_model,
- QuantConfig)
+ save_compressed_model)
parser = argparse.ArgumentParser()
import torch
@@ -45,7 +43,7 @@ if __name__ == '__main__':
parser.add_argument("--device", default=0, type=str,
help="device gpu int number, or 'cpu' ")
- #
+
parser.add_argument("--sym", action='store_true',
help=" sym quantization")
@@ -95,10 +93,6 @@ if __name__ == '__main__':
parser.add_argument("--scale_dtype", default='fp32',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
-
- # parser.add_argument("--tasks",
- # default=['lambada_openai'],
- # help="lm-eval tasks")
parser.add_argument("--tasks",
default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
@@ -167,17 +161,6 @@ if __name__ == '__main__':
eval_orig_float=True, excel_file=excel_name)
exit()
- # if args.iters <= 0:
- # print("eval rtn", flush=True)
- # excel_name += "_rtn.xlsx"
- # q_dq_weight(model, bits=args.bits, group_size=args.group_size)
- # if not args.low_gpu_mem_usage:
- # model = model.to(cuda_device)
- # eval_model(output_dir=args.output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
- # eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,
- # excel_file=excel_name)
- # exit()
-
if not args.low_gpu_mem_usage:
model = model.to(cuda_device)
@@ -197,11 +180,8 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
if args.deployment_device == 'cpu':
- compressed_model = compress_model(model, q_config)
- quantize_config = QuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)
- save_compressed_model(compressed_model, output_dir=export_dir, quantize_config=quantize_config, tokenizer=tokenizer)
+ save_compressed_model(model, weight_config=q_config, output_dir=export_dir, tokenizer=tokenizer)
del q_config
- del compressed_model
elif args.deployment_device == 'gpu':
autoround.export_to_autogptq(export_dir, use_triton=True)
@@ -209,11 +189,9 @@ if __name__ == '__main__':
model.eval()
output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"
- # model.to(cuda_device)
- # eval_model(model, model_name, tokenizer, tasks=args.tasks, eval_bs=args.eval_bs)
excel_name = f"{output_dir}_result.xlsx"
output_dir += "/"
print(excel_name, flush=True)
eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,
- limit=None)
\ No newline at end of file
+ limit=None)
diff --git a/examples/run_autoround.sh b/examples/run_autoround.sh
index 2c8b13a9a..66b3f0b44 100644
--- a/examples/run_autoround.sh
+++ b/examples/run_autoround.sh
@@ -1,59 +1,17 @@
#!/bin/bash
set -x
-# export CUBLAS_WORKSPACE_CONFIG=':4096:8'
-export HF_HOME="/models/huggingface/"
-export https_proxy=http://child-jf.intel.com:912
-export http_proxy=http://child-jf.intel.com:912
-
-# model="neural-chat-7b-v3-3"
-# for iter in 200 400 1000
-# do
-# CUDA_VISIBLE_DEVICES=2 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
-# examples/main.py \
-# --model_name /models/${model} \
-# --bits 4 --group_size 128 \
-# --enable_minmax_tuning \
-# --use_quant_input \
-# --amp \
-# --iters ${iter} \
-# --deployment_device 'cpu' \
-# --scale_dtype 'fp32' \
-# --eval_bs 16 \
-# --output_dir /data1/zww/autoround/int_llama_${iter} \
-# 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt
-# done &
-
model="neural-chat-7b-v3-3"
-for iter in 2000
-do
- CUDA_VISIBLE_DEVICES=4 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
- examples/main.py \
+
+CUDA_VISIBLE_DEVICES=4 python3 main.py \
--model_name /models/${model} \
--bits 4 --group_size 128 \
--enable_minmax_tuning \
--use_quant_input \
--amp \
- --iters ${iter} \
- --deployment_device 'fake' \
+ --iters 200 \
+ --deployment_device 'cpu' \
--scale_dtype 'fp32' \
--eval_bs 16 \
- --output_dir /data1/zww/autoround/int_llama_${iter} \
- 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt
-done
-# model_dir="/data2/zww/gptq_actorder/W4G128_gptq"
-# for model in "static_neural-chat-7b-v3-3" "nonstatic_neural-chat-7b-v3-3"
-# do
-# CUDA_VISIBLE_DEVICES=5 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
-# examples/main.py \
-# --model_name ${model_dir}/${model} \
-# --bits 4 --group_size 128 \
-# --eval_fp16_baseline \
-# --enable_minmax_tuning \
-# --use_quant_input \
-# --amp \
-# --scale_dtype 'fp32' \
-# --eval_bs 16 \
-# 2>&1 | tee -a /data1/zww/optround_logs/${model}_gptq_W4G128.txt
-# done
+ --output_dir "SAVE/PATH"