PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fixtypos (#31)
WeiweiZhang153fb877ec83f
diff --git a/README.md b/README.mdindex 3c0b7ebb5..b6da0240a 100644--- a/README.md+++ b/README.md@@ -23,7 +23,7 @@ python setup.py install```## Usage of Tuning-### On CPU/Gaudi2/ GPU+### On CPU/ Gaudi2/ GPU```pythonimport torch@@ -44,6 +44,48 @@ output_dir = "./tmp_autoround"autoround.save_quantized(output_dir)```+++## Model inference+Please run the tuning code first++++### Intel CPU+```python+# Please save the quantized model in 'itrex' format first, then refer to the ITREX tutorial for more details on inference with the INT4 model.+# (https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed)+from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig+from transformers import AutoTokenizer++quantized_model_path = "./tmp_autoround"+scheme = "sym" if sym else "asym"+woq_config = WeightOnlyQuantConfig(+ group_size=group_size, scheme=scheme, use_autoround=True+) ##only supports 4 bits currently+prompt = "There is a girl who likes adventure,"+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)+inputs = tokenizer(prompt, return_tensors="pt").input_ids+model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, quantization_config=woq_config, trust_remote_code=True, device="cpu"+)+outputs = model.generate(inputs, max_new_tokens=50)+print(tokenizer.decode(outputs[0]))+```+++### GPU+```python+from transformers import AutoModelForCausalLM, AutoTokenizer++quantized_model_path = "./tmp_autoround"+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)+text = "There is a girl who likes adventure,"+inputs = tokenizer(text, return_tensors="pt").to(model.device)+print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+```+<details><summary>Detailed Hyperparameters</summary>@@ -71,7 +113,7 @@ autoround.save_quantized(output_dir)- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).-- `bs (int)`: Batch size for training (default is 8).+- `batch_size (int)`: Batch size for training (default is 8).- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float32"), different kernels have different choices.@@ -91,7 +133,7 @@ autoround.save_quantized(output_dir)- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits or mixed precision.-- `device`: The device to be used for tuning. The default is set to None, allowing for automatic detection.+- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.</details>@@ -123,46 +165,8 @@ autoround.save_quantized(output_dir)| MBZUAI/LaMini-GPT-124M | [example](./examples/language-modeling/) || EleutherAI/gpt-neo-125m | [example](./examples/language-modeling/) || databricks/dolly-v2-3b | [example](./examples/language-modeling/) |-| stabilityai/stablelm-base-alpha-3b | [example](./examples/language-modeling/) |---## Model inference-Please run the tuning code first--+| stabilityai/stablelm-base-alpha-3b | [example](./examples/language-modeling/)-### Intel CPU-```python-# save_quantized to itrex format first-# Please read ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to understand the details-# currently please install neural-speed (https://github.com/intel/neural-speed) from source-from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig-from transformers import AutoTokenizer--quantized_model_path = "./tmp_autoround"-scheme = "sym" if sym else "asym"-woq_config = WeightOnlyQuantConfig(- group_size=group_size, scheme=scheme, use_autoround=True-) ##only supports 4 bits currently-prompt = "There is a girl who likes adventure,"-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)-inputs = tokenizer(prompt, return_tensors="pt").input_ids-model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, quantization_config=woq_config, trust_remote_code=True, device="cpu"-)-outputs = model.generate(inputs, max_new_tokens=50)-```-### GPU-```python-from transformers import AutoModelForCausalLM, AutoTokenizer--quantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)-text = "There is a girl who likes adventure,"-inputs = tokenizer(text, return_tensors="pt").to(model.device)-print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-```diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 79ea084f5..3488aa67c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -481,7 +481,7 @@ class AutoRound(object):self.set_layerwise_config(self.weight_config)self.optimizer = self.get_optimizer(None)self.check_configs()- torch.set_printoptions(precision=5)+ torch.set_printoptions(precision=3, sci_mode=True)def get_optimizer(self, optimizer):"""Returns the specified optimizer. In SignRound, we fix the optimizer.@@ -644,6 +644,8 @@ class AutoRound(object):split=self.dataset_split,dataset_name=self.dataset_name,)++ self.start_time = time.time()total_cnt = 0for data in self.dataloader:if data is None:@@ -902,7 +904,7 @@ class AutoRound(object):last_loss = best_lossbest_iter = last_best_iterdump_info = (- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))}"+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}")logger.info(dump_info)@@ -1014,20 +1016,17 @@ class AutoRound(object):Returns:The quantized model and weight configurations."""- start_time = time.time()# logger.info("cache block input")block_names = get_block_names(self.model)if len(block_names) == 0:logger.warning("could not find blocks, exit with original model")return-if self.amp:self.model = self.model.to(self.amp_dtype)if not self.low_gpu_mem_usage:self.model = self.model.to(self.device)inputs = self.cache_block_input(block_names[0], self.n_samples)del self.inputs-if "input_ids" in inputs.keys():dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))total_samples = inputs["input_ids"].shape[dim]@@ -1068,7 +1067,7 @@ class AutoRound(object):self.weight_config[n]["sym"] = Noneend_time = time.time()- cost_time = end_time - start_time+ cost_time = end_time - self.start_timelogger.info(f"quantization tuning time {cost_time}")## dump a summaryquantized_layers = []@@ -1079,9 +1078,13 @@ class AutoRound(object):unquantized_layers.append(n)else:quantized_layers.append(n)- logger.info(+ summary_info = (f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model")+ if len(unquantized_layers) > 0:+ summary_info += f", {unquantized_layers} have not been quantized"++ logger.info(summary_info)if len(unquantized_layers) > 0:logger.info(f"Summary: {unquantized_layers} have not been quantized")diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pyindex 64d0a253d..6d0b6ab9b 100644--- a/examples/code-generation/main.py+++ b/examples/code-generation/main.py@@ -1,11 +1,7 @@import argparseimport randomimport sys-sys.path.insert(0, '../..')-from auto_round import (AutoRound,- AutoAdamRound)-parser = argparse.ArgumentParser()import torchimport os@@ -40,8 +36,9 @@ if __name__ == '__main__':parser.add_argument("--eval_bs", default=4, type=int,help="eval batch size")- parser.add_argument("--device", default=0, type=str,- help="device gpu int number, or 'cpu' ")+ parser.add_argument("--device", default="auto", type=str,+ help="The device to be used for tuning. The default is set to auto/None,"+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -84,7 +81,7 @@ if __name__ == '__main__':help="whether enable weight minmax tuning")parser.add_argument("--deployment_device", default='fake', type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',"+ help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")parser.add_argument("--scale_dtype", default='fp32',@@ -101,31 +98,30 @@ if __name__ == '__main__':args = parser.parse_args()set_seed(args.seed)-+ tasks = args.tasks+model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]print(model_name, flush=True)- tasks = args.tasks-- if args.device == "cpu":- device_str = "cpu"- else:- device_str = f"cuda:{int(args.device)}"+ from auto_round.utils import detect_device+ device_str = detect_device(args.device)+ torch_dtype = "auto"+ if device_str == "hpu":+ torch_dtype = torch.bfloat16torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))- is_llava = bool(re.search("llava", model_name.lower()))- if is_llava:- from transformers import LlavaForConditionalGeneration-- model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")- elif is_glm:++ if is_glm:model = AutoModel.from_pretrained(model_name, trust_remote_code=True)else:model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True)++ from auto_round import (AutoRound,+ AutoAdamRound)model = model.eval()# align wigh GPTQ to eval pplif "opt" in model_name:@@ -159,38 +155,27 @@ if __name__ == '__main__':round = AutoRoundif args.adam:round = AutoAdamRound- autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, bs=args.train_bs,+ autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, dataset_name="mbpp", dataset_split=['train', 'validation', 'test']) ##TODO args pass+ scale_dtype=args.scale_dtype, dataset="mbpp", dataset_split=['train', 'validation', 'test']) ##TODO args passmodel, q_config = autoround.quantize()model_name = args.model_name.rstrip("/")- export_dir = args.output_dir + "/compressed_" + model_name.split('/')[-1] + "/"- if args.deployment_device == 'cpu':- autoround.export(output_dir=export_dir)- del q_config- elif args.deployment_device == 'gpu':- autoround.export(export_dir, target="auto_gptq", use_triton=True)+ model.eval()if args.device != "cpu":torch.cuda.empty_cache()- model.eval()- output_dir = args.output_dir + "_" + model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"-- import shutil-- if os.path.exists(output_dir):- shutil.rmtree(output_dir)-- if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):- dtype = 'bfloat16'- pt_dtype = torch.bfloat16- else:- pt_dtype = torch.float16- model = model.to(pt_dtype)- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"+ deployment_device = args.deployment_device.split(',')+ if 'gpu' in deployment_device:+ autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)+ if "cpu" in deployment_device:+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)+ if "fake" in deployment_device:+ model = model.to("cpu")+ model.save_pretrained(output_dir)+ tokenizer.save_pretrained(output_dir)diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex 9b34185ea..cb43e9904 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -19,7 +19,7 @@ The transformers version required varies across different types of models. Here,| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36 || mosaicml/mpt-7b | 4.28/4.30/4.34/4.36 || bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36 |-| baichuan-inc/Baichuan-7B | 4.28/4.30 |+| baichuan-inc/Baichuan2-7B-Chat | 4.36 || Qwen/Qwen-7B | 4.28/4.30/4.34/4.36 || THUDM/chatglm3-6b | 4.34/4.36 || mistralai/Mistral-7B-v0.1 | 4.34/4.36 |@@ -28,12 +28,14 @@ The transformers version required varies across different types of models. Here,| databricks/dolly-v2-3b | 4.34 || stabilityai/stablelm-base-alpha-3b | 4.34 || Intel/neural-chat-7b-v3 | 4.34/4.36 |+| rinna/bilingual-gpt-neox-4b | 4.36 |+| microsoft/phi-2 | 4.36 |## 2. Prepare DatasetThe NeelNanda/pile-10k in huggingface is adopted as the default calibration data and will be downloaded automatically from the datasets Hub. To customize a dataset, please kindly follow our dataset code.-See more about loading [huggingface dataset](https://huggingface.co/docs/datasets/loading_datasets.html)+See more about loading [huggingface dataset](https://huggingface.co/docs/datasets/main/en/quickstart)<br />@@ -106,3 +108,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 1020b08bc..5e7b47a4f 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -210,13 +210,12 @@ if __name__ == '__main__':round = AutoAdamRoundweight_config = {}- if 'gpu' in args.deployment_device:- for n, m in model.named_modules():- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- weight_config[n] = {"data_type": "fp"}- print(- f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")+ for n, m in model.named_modules():+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:+ weight_config[n] = {"data_type": "fp"}+ print(+ f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,diff --git a/test/test_autoopt.py b/test/test_autoopt.pynew file mode 100644index 000000000..a95c49cfc--- /dev/null+++ b/test/test_autoopt.py@@ -0,0 +1,51 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, ".")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoOPTRound+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ model_name = "facebook/opt-125m"+ self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_default(self):+ bits, group_size, sym = 4, 128, False+ autoround = AutoOPTRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ dataloader=self.llm_dataloader,+ )+ autoround.quantize()+++if __name__ == "__main__":+ unittest.main()