Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add customized data example (#38)

wenhuach21d6ee2237d8cf
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 3488aa67c..dd1573452 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -651,21 +651,30 @@ class AutoRound(object):
if data is None:
continue
if isinstance(data, torch.Tensor):
- data_new = data.to(self.model.device)
- input_ids = data_new
+ input_ids = data.to(self.model.device)
+
+ elif isinstance(data, str):
+ if self.tokenizer is None:
+ logger.error("for string input, please provide tokenizer")
+ exit()
+ data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = data[key].to(self.model.device)
+ input_ids = data_new["input_ids"]
else:
data_new = {}
for key in data.keys():
data_new[key] = data[key].to(self.model.device)
input_ids = data_new["input_ids"]
- # if input_ids.shape[-1] < self.seqlen:
- # continue
+ if input_ids.shape[-1] < self.seqlen:
+ continue
if total_cnt + input_ids.shape[0] > n_samples:
input_ids = input_ids[: n_samples - total_cnt, ...]
try:
if isinstance(data_new, torch.Tensor):
self.model(data_new)
- elif isinstance(data_new, dict):
+ else:
self.model(**data_new)
except NotImplementedError:
pass
@@ -683,7 +692,7 @@ class AutoRound(object):
elif total_cnt < n_samples:
logger.warning(
f"Insufficient number of samples collected may affect the quantification. "
- f"Effective samples size:{total_cnt}, Target sample size:{n_samples}"
+ f"Valid samples size:{total_cnt}, Target sample size:{n_samples}"
)
@torch.no_grad()
diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.py
new file mode 100644
index 000000000..608e03a61
--- /dev/null
+++ b/examples/language-modeling/main_customized_data.py
@@ -0,0 +1,280 @@
+import argparse
+import sys
+
+sys.path.insert(0, '../..')
+parser = argparse.ArgumentParser()
+import torch
+import os
+import transformers
+
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+torch.use_deterministic_algorithms(True, warn_only=True)
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+
+from transformers import set_seed
+
+import re
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def customized_data():
+ ##Important Notice!!! Autoround will drop data < args.seqlen and truncate data to args.seqlen
+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
+ return data
+
+
+def customized_data_with_tokenizer(tokenizer, seqlen=2048):
+ ##Import notice!!! Autoround will dump data < args.seqlen
+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
+ tokens = []
+ for d in data:
+ token = tokenizer(d, truncation=True, max_length=seqlen, return_tensors="pt").data
+ tokens.append(token)
+ return tokens
+
+
+if __name__ == '__main__':
+
+ parser.add_argument(
+ "--model_name", default="facebook/opt-125m"
+ )
+
+ parser.add_argument("--bits", default=4, type=int,
+ help="number of bits")
+
+ parser.add_argument("--group_size", default=128, type=int,
+ help="group size")
+
+ parser.add_argument("--train_bs", default=8, type=int,
+ help="train batch size")
+
+ parser.add_argument("--eval_bs", default=4, type=int,
+ help="eval batch size")
+
+ parser.add_argument("--device", default="auto", type=str,
+ help="The device to be used for tuning. The default is set to auto/None,"
+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
+
+ parser.add_argument("--sym", action='store_true',
+ help=" sym quantization")
+
+ parser.add_argument("--iters", default=200, type=int,
+ help=" iters")
+
+ parser.add_argument("--use_quant_input", action='store_true',
+ help="whether to use the output of quantized block to tune the next block")
+
+ parser.add_argument("--lr", default=None, type=float,
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
+
+ parser.add_argument("--minmax_lr", default=None, type=float,
+ help="minmax learning rate, if None,it will beset to be the same with lr")
+
+ parser.add_argument("--seed", default=42, type=int,
+ help="seed")
+
+ parser.add_argument("--eval_fp16_baseline", action='store_true',
+ help="whether to eval FP16 baseline")
+
+ parser.add_argument("--amp", action='store_true',
+ help="amp")
+
+ parser.add_argument("--adam", action='store_true',
+ help="adam")
+
+ parser.add_argument("--seqlen", default=2048, type=int,
+ help="sequence length")
+
+ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+
+ parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")
+
+ parser.add_argument("--n_samples", default=512, type=int,
+ help="number of samples")
+
+ parser.add_argument("--low_gpu_mem_usage", action='store_true',
+ help="low_gpu_mem_usage")
+
+ parser.add_argument("--enable_minmax_tuning", action='store_true',
+ help="whether enable weight minmax tuning")
+
+ parser.add_argument("--deployment_device", default='fake', type=str,
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."
+ "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
+
+ parser.add_argument("--scale_dtype", default='fp32',
+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
+
+ parser.add_argument("--tasks",
+ default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
+ "arc_easy", "arc_challenge"],
+ help="lm-eval tasks for lm_eval version 0.4")
+
+ parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
+ help="Where to store the final model.")
+
+ parser.add_argument("--disable_eval", action='store_true',
+ help="Whether to do lmeval evaluation.")
+
+ args = parser.parse_args()
+ set_seed(args.seed)
+ tasks = args.tasks
+ use_eval_legacy = False
+ import subprocess
+
+
+ def get_library_version(library_name):
+ try:
+ version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
+ return version
+ except subprocess.CalledProcessError:
+ return "Library not found"
+
+
+ res = get_library_version("lm-eval")
+ if res == "0.3.0":
+ use_eval_legacy = True
+ if not use_eval_legacy:
+ from eval import eval_model
+ else:
+ from eval_legacy import eval_model
+
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+ if isinstance(tasks, list):
+ if "mmlu" in tasks:
+ tmp_tasks = tasks
+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
+ tmp_tasks = tasks
+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
+ tasks = list(set(tasks))
+ if isinstance(args.tasks, str):
+ tasks = ','.join(tasks)
+
+ model_name = args.model_name
+ if model_name[-1] == "/":
+ model_name = model_name[:-1]
+ print(model_name, flush=True)
+
+ from auto_round.utils import detect_device
+
+ device_str = detect_device(args.device)
+ torch_dtype = "auto"
+ if device_str == "hpu":
+ torch_dtype = torch.bfloat16
+ torch_device = torch.device(device_str)
+
+ is_glm = bool(re.search("chatglm", model_name.lower()))
+ if is_glm:
+ model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
+ else:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True
+ )
+
+ from auto_round import (AutoRound,
+ AutoAdamRound)
+
+ model = model.eval()
+ # align with GPTQ to eval ppl
+ if "opt" in model_name:
+ seqlen = model.config.max_position_embeddings
+ model.seqlen = model.config.max_position_embeddings
+ else:
+ seqlen = 2048
+ model.seqlen = seqlen
+ seqlen = args.seqlen
+
+ if "llama" in model_name:
+ from transformers import LlamaTokenizer
+
+ tokenizer = LlamaTokenizer.from_pretrained(model_name)
+ if tokenizer.pad_token is None:
+ tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+ if hasattr(tokenizer, "model_max_length"):
+ if tokenizer.model_max_length < seqlen:
+ print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
+ flush=True)
+ seqlen = min(seqlen, tokenizer.model_max_length)
+ args.seqlen = seqlen
+
+ pt_dtype = torch.float16
+ if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
+ dtype = 'bfloat16'
+ pt_dtype = torch.bfloat16
+ else:
+ if str(args.device) != "cpu":
+ pt_dtype = torch.float16
+ dtype = 'float16'
+ else:
+ pt_dtype = torch.float32
+ dtype = 'float32'
+
+ excel_name = f"{model_name}_{args.bits}_{args.group_size}"
+ if args.eval_fp16_baseline:
+ if not args.low_gpu_mem_usage:
+ model = model.to(torch_device)
+ excel_name += "_fp16.xlsx"
+ eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
+ device=torch_device, excel_file=excel_name)
+ exit()
+
+ if not args.low_gpu_mem_usage:
+ model = model.to(torch_device)
+
+ round = AutoRound
+ if args.adam:
+ round = AutoAdamRound
+
+ weight_config = {}
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
+ weight_config[n] = {"data_type": "fp"}
+ print(
+ f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
+
+ data = customized_data(tokenizer, args.seqlen)
+
+ autoround = round(model, tokenizer, args.bits, args.group_size, dataloader=data, sym=args.sym,
+ batch_size=args.train_bs,
+ seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
+ minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
+ amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
+ seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
+ scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass
+ model, _ = autoround.quantize()
+ model_name = args.model_name.rstrip("/")
+
+ model.eval()
+ if args.device != "cpu":
+ torch.cuda.empty_cache()
+
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
+ deployment_device = args.deployment_device.split(',')
+ if 'gpu' in deployment_device:
+ autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
+ if "cpu" in deployment_device:
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)
+ if "fake" in deployment_device:
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ tokenizer.save_pretrained(output_dir)
+
+ if not args.disable_eval and "fake" in deployment_device: ##support autogptq real eval later
+ excel_name = f"{output_dir}_result.xlsx"
+ output_dir += "/"
+ print(excel_name, flush=True)
+ eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
+ device=torch_device, excel_file=excel_name)