PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Consolidate dataloader&dataset_split to dataset (#105)
wenhuach211f9cb4f8b8e6
diff --git a/README.md b/README.mdindex 8d2f5d14b..59fc584b7 100644--- a/README.md+++ b/README.md@@ -105,11 +105,7 @@ autoround.save_quantized(output_dir)- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of a little tuning time (default is True).-- `dataset (str)`: The default dataset name for tuning (default is "NeelNanda/pile-10k").--- `dataset_split (str)`: The split of the dataset to be used for tuning (default is "train").--- `dataloader`: The dataloader for tuning data.+- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bitsor mixed precision.@@ -173,7 +169,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))| facebook/opt-2.7b | [accuracy](./docs/opt-2.7b-acc.md), [recipe](./examples/language-modeling/scripts/opt-2.7b.sh), [example](./examples/language-modeling/) || bigscience/bloom-3b | [accuracy](./docs/bloom-3B-acc.md), [recipe](./examples/language-modeling/scripts/bloom-3b.sh), [example](./examples/language-modeling/) || EleutherAI/gpt-j-6b | [accuracy](./docs/gpt-j-6B-acc.md), [recipe](./examples/language-modeling/scripts/gpt-j-6b.sh), [example](./examples/language-modeling/) |-| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |+| Salesforce/codegen25-7b-multi | [example](./examples/language-modeling/) || huggyllama/llama-7b | [example](./examples/language-modeling/) || mosaicml/mpt-7b | [example](./examples/language-modeling/) || THUDM/chatglm3-6b | [example](./examples/language-modeling/) |diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 933e34893..ba3b74a99 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -15,8 +15,10 @@import copyimport time+from typing import Optional, Unionimport torch+import transformersfrom torch import autocastfrom .calib_dataset import get_dataloader@@ -25,7 +27,6 @@ from .utils import (CpuInfo,block_forward,check_is_cpu,- check_memory_availability,check_to_quantized,collect_minmax_scale,collect_round_v,@@ -34,7 +35,6 @@ from .utils import (get_module,get_scale_shape,htcore,- is_local_path,is_optimum_habana_available,logger,move_input_to_device,@@ -277,18 +277,14 @@ def wrapper_block(block, enable_minmax_tuning):set_module(block, n, new_m)quantized_layers.append(n)- try:- import transformers+ if isinstance(m, transformers.modeling_utils.Conv1D):+ if not check_to_quantized(m):+ unquantized_layers.append(n)+ continue+ new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)+ set_module(block, n, new_m)+ quantized_layers.append(n)- if isinstance(m, transformers.modeling_utils.Conv1D):- if not check_to_quantized(m):- unquantized_layers.append(n)- continue- new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)- set_module(block, n, new_m)- quantized_layers.append(n)- except:- passreturn quantized_layers, unquantized_layers@@ -371,9 +367,7 @@ class AutoRound(object):amp (bool): Whether to use automatic mixed precision (default is True).device: The device to be used for tuning (default is "auto").lr_scheduler: The learning rate scheduler to be used.- dataloader: The dataloader for input data (to be supported in future).- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").- dataset_split (str): The split of the dataset to be used (default is "train").+ dataset (str): The default dataset name (default is "NeelNanda/pile-10k").enable_quanted_input (bool): Whether to use the output of the previous quantized block asthe input for the current block (default is True).enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).@@ -410,9 +404,7 @@ class AutoRound(object):amp: bool = True,device=None,lr_scheduler=None,- dataloader=None, ## to support later- dataset: str = "NeelNanda/pile-10k",- dataset_split: str = "train",+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,enable_minmax_tuning: bool = True,lr: float = None,@@ -427,7 +419,7 @@ class AutoRound(object):gradient_accumulate_steps: int = 1,not_use_best_mse: bool = False,dynamic_max_gap: int = -1,- data_type: str = "int", ##only support data_type+ data_type: str = "int", ##only support int for nowscale_dtype: str = "fp32",**kwargs,):@@ -444,15 +436,8 @@ class AutoRound(object):self.sym = symself.low_gpu_mem_usage = low_gpu_mem_usageself.data_type = data_type- self.supported_types = [torch.nn.Linear]- try:- import transformers-- self.supported_types.append(transformers.modeling_utils.Conv1D)- except:- pass+ self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]self.weight_config = weight_config- self.dataset_split = dataset_splitself.seed = seedself.tokenizer = tokenizerself.seqlen = seqlen@@ -476,15 +461,15 @@ class AutoRound(object):if self.device == "cpu" and not CpuInfo().bf16:self.amp = Falseself.model = self.model.to(torch.float32)- logger.warning("amp is set to FALSE as the current" "device does not support the 'bf16' data type.")+ logger.warning(+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."+ )else:self.model = self.model.to(self.amp_dtype)else:self.model = self.model.to(torch.float32)logger.info(f"using {self.model.dtype} for quantization tuning")- self.dataset_name = dataset-- self.dataloader = dataloader+ self.dataset = datasetself.iters = itersif self.iters <= 0:logger.warning("iters must be positive, reset it to 200")@@ -673,16 +658,18 @@ class AutoRound(object):bs (int): The number of samples to use for calibration"""- if self.dataloader is None:+ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "") ##remove all whitespacesself.dataloader = get_dataloader(self.tokenizer,self.seqlen,- self.dataset_name,- self.dataset_split,+ dataset,self.seed,bs,self.n_samples,)+ else:+ self.dataloader = self.datasettotal_cnt = 0for data in self.dataloader:if data is None:@@ -693,7 +680,7 @@ class AutoRound(object):elif isinstance(data, str):if self.tokenizer is None:- logger.error("for string input, please provide tokenizer")+ logger.error("please provide tokenizer for string input")exit()data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").datadata_new = {}@@ -723,7 +710,7 @@ class AutoRound(object):if total_cnt == 0:logger.error(f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "- f"dataloader or decease the sequence length"+ f"dataset or decease the sequence length")exit()elif total_cnt < n_samples:@@ -1247,34 +1234,13 @@ class AutoRound(object):)return compressed_model- #- # @torch.no_grad()- # def gets_layer_names_outside_blocks(self):- # """Gets the names of layers outside blocks in the model.- #- # Returns:- # list: List of layer names outside blocks.- # """- # all_layer_names = set()- # for n, m in self.model.named_modules():- # if isinstance(m, tuple(self.supported_types)):- # m.tmp_name = n- # all_layer_names.add(n)- # block_names = get_block_names(self.model)- # all_layer_names_in_block = set()- # for block_name in block_names:- # block = get_module(self.model, block_name)- # for n, m in block.named_modules():- # if isinstance(m, tuple(self.supported_types)):- # all_layer_names_in_block.add(m.tmp_name)- #- # res = all_layer_names - all_layer_names_in_block- # for n, m in self.model.named_modules():- # if isinstance(m, tuple(self.supported_types)):- # delattr(m, "tmp_name")- # return list(res)-def get_layer_names_in_block(self):+ """Retrieves the names of layers within each block of the model.++ Returns:+ list: A list of strings, where each string is the name of a layer+ within a block of the model.+ """for n, m in self.model.named_modules():if isinstance(m, tuple(self.supported_types)):m.tmp_name = n@@ -1434,9 +1400,7 @@ class AutoOPTRound(AutoRound):amp (bool): Whether to use automatic mixed precision (default is True).device: The device to be used for training (default is "auto").lr_scheduler: The learning rate scheduler to be used.- dataloader: The dataloader for input data (to be supported in future).- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").- dataset_split (str): The split of the dataset to be used (default is "train").+ dataset: The default dataset name (default is "NeelNanda/pile-10k").enable_quanted_input (bool): Whether to use quantized input data (default is True).enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).lr (float): The learning rate (default is 0.005).@@ -1473,9 +1437,7 @@ class AutoOPTRound(AutoRound):amp: bool = True,device="auto",lr_scheduler=None,- dataloader=None,- dataset: str = "NeelNanda/pile-10k",- dataset_split: str = "train",+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,enable_minmax_tuning: bool = True,lr: float = None,@@ -1507,9 +1469,7 @@ class AutoOPTRound(AutoRound):amp,device,lr_scheduler,- dataloader,dataset,- dataset_split,enable_quanted_input,enable_minmax_tuning,lr,@@ -1587,9 +1547,8 @@ class AutoAdamRound(AutoOPTRound):amp (bool): Whether to use automatic mixed precision (default is True).device: The device to be used for training (default is "auto").lr_scheduler: The learning rate scheduler to be used.- dataloader: The dataloader for input data (to be supported in future).- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").- dataset_split (str): The split of the dataset to be used (default is "train").+ dataset (Union[str, list, tuple, torch.utils.data.DataLoader]):+ The default dataset name (default is "NeelNanda/pile-10k").enable_quanted_input (bool): Whether to use quantized input data (default is True).enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).lr (float): The learning rate (default is 0.005).@@ -1626,9 +1585,7 @@ class AutoAdamRound(AutoOPTRound):amp: bool = True,device="auto",lr_scheduler=None,- dataloader=None,- dataset: str = "NeelNanda/pile-10k",- dataset_split: str = "train",+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,enable_minmax_tuning: bool = True,lr: float = None,@@ -1660,9 +1617,7 @@ class AutoAdamRound(AutoOPTRound):amp,device,lr_scheduler,- dataloader,dataset,- dataset_split,enable_quanted_input,enable_minmax_tuning,lr,diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 242604fbb..7b25eef45 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -47,7 +47,7 @@ def get_tokenizer_function(tokenizer, seqlen):@register_dataset("NeelNanda/pile-10k")-def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4):+def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42):"""Returns a dataloader for the specified dataset and split.Args:@@ -56,7 +56,6 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", splitdata_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.- bs: The batch size for the dataloader.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.@@ -74,9 +73,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split@register_dataset("madao33/new-title-chinese")-def get_new_chinese_title_dataset(- tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42, bs=4-):+def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42):"""Returns a dataloader for the specified dataset and split.Args:@@ -85,7 +82,6 @@ def get_new_chinese_title_dataset(data_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.- bs: The batch size for the dataloader.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.@@ -121,7 +117,7 @@ def get_new_chinese_title_dataset(@register_dataset("mbpp")-def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, bs=4):+def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42):"""Returns a dataloader for the specified dataset and split.Args:@@ -130,7 +126,6 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42data_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.- bs: The batch size for the dataloader.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.@@ -144,7 +139,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42if splits is None:splits = ["train", "validation", "test"]if isinstance(splits, str):- splits = [split]+ splits = splits.split("+")for split in splits:dataset = load_dataset(dataset_name, split=split)@@ -160,7 +155,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42@register_dataset("local")-def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, bs=4):+def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42):"""Returns a dataloader for a custom dataset and split.We allow the input of a json or text file containing a processed text sample each line.@@ -170,7 +165,6 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,data_name: The name or path of the dataset, which is a jsonl file.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.- bs: The batch size for the dataloader.Returns:A dataloader for a custom dataset and split, using the provided tokenizer and sequence length.@@ -214,7 +208,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,return calib_dataset-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4, n_samples=512):+def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, n_samples=512):"""Generate a DataLoader for calibration using specified parameters.Args:@@ -234,17 +228,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=Ndataset_names = dataset_name.split(",")- # if len(dataset_names) == 1 and dataset_names[0] == "NeelNanda/pile-10k": ##to guarantee the reproducibility.- # get_dataloader = CALIB_DATASETS.get("legacy-NeelNanda/pile-10k")- # dataloader = get_dataloader(- # tokenizer,- # seqlen,- # seed=seed,- # bs=bs,- # split=split,- # dataset_name=dataset_name,- # )- # return dataloaderdef filter_func(example):if example["input_ids"].shape[-1] < seqlen:return False@@ -256,6 +239,9 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=Ndatasets = []for name in dataset_names:+ split = None+ if ":" in name:+ name, split = name.split(":")if is_local_path(name):get_dataset = CALIB_DATASETS.get("local")else:@@ -264,7 +250,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=Ntokenizer,seqlen,seed=seed,- bs=bs,split=split,dataset_name=name,)@@ -316,50 +301,3 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=Ncalib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader---# ##we keep this for reproduce issue, do not delete, keep it for a while-# @register_dataset("legacy-NeelNanda/pile-10k")-# def get_pile_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):-# """Returns a dataloader for the specified dataset and split.-#-# Args:-# tokenizer: The tokenizer to be used for tokenization.-# seqlen: The maximum sequence length.-# data_name: The name of the dataset.-# split: The data split to be used (e.g., "train", "test").-# seed: The random seed for shuffling the dataset.-# bs: The batch size for the dataloader.-#-# Returns:-# A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.-# """-# from datasets import load_dataset-# from torch.utils.data import DataLoader-#-# tokenizer_function = get_tokenizer_function(tokenizer, seqlen)-#-# @torch.no_grad()-# def collate_batch(batch):-# input_ids_new = []-# for text in batch:-# input_ids = text["input_ids"]-# if input_ids.shape[0] < seqlen:-# continue-# input_ids = input_ids[:seqlen]-# input_ids_list = input_ids.tolist()-# if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:-# continue-# input_ids_new.append(input_ids)-# if len(input_ids_new) == 0:-# return None-# tmp = torch.vstack(input_ids_new)-# res = {"input_ids": tmp}-# return res-#-# calib_dataset = load_dataset(dataset_name, split=split)-# calib_dataset = calib_dataset.shuffle(seed=seed)-# calib_dataset = calib_dataset.map(tokenizer_function, batched=True)-# calib_dataset.set_format(type="torch", columns=["input_ids"])-# calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)-# return calib_dataloaderdiff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex aaf40eef7..73fcc50a9 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -86,7 +86,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarall_to_quantized = Falseelse:modules_in_block_to_quantize.append(n)- modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq+ modules_in_block_to_quantize = [modules_in_block_to_quantize]if all_to_quantized:modules_in_block_to_quantize = None@@ -102,6 +102,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarlogger.warning("triton does not support 3 bits, reset it to False")quantizers = {}for key in weight_config:+ if key == "lm_head": ##TODO remove this after pr 87 is merged+ continueinfo = weight_config[key]if not check_to_quantized(info):continue@@ -119,6 +121,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarelse:quantizers = {}for key in weight_config:+ if key == "lm_head": ##TODO remove this after pr 87 is merged+ continueinfo = weight_config[key]if not check_to_quantized(info):continuediff --git a/examples/code-generation/README.md b/examples/code-generation/README.mddeleted file mode 100644index 312c00359..000000000--- a/examples/code-generation/README.md+++ /dev/null@@ -1,65 +0,0 @@-Step-by-Step-============--This document presents step-by-step instructions for auto-round.---## 2. Prepare Dataset--The mbpp in huggingface is adopted as the default calibration data and will be downloaded automatically from the datasets Hub. To customize a dataset, please kindly follow our dataset code.-See more about loading [huggingface dataset](https://huggingface.co/docs/datasets/loading_datasets.html)--<br />--## 3. Run Examples-Enter into the examples folder-- **Default Settings:**-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --enable_minmax_tuning-```-- **Reduced GPU Memory Usage and Adjusted Training Batch Size:**-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8-```-- **Utilizing the AdamW Optimizer:**-Include the flag `--adam`. Note that AdamW is less effective than Sign gradient descent in many scenarios we tested.--- **Running the Original SignRound:**-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --iters 400 --lr 0.0025 --minmax_lr 0.0025-```- `--enable_minmax_tuning` is strongly recommended---## 4. Evaluation-Please follow https://github.com/bigcode-project/bigcode-evaluation-harness to eval the model, currently we only support fake model evaluation.-<table border="1">- <tr>- <th>Model</th>- <th>Method</th>- <th>HumanEval top1 t=0.2 n_samples=20</th>- </tr>- <tr>- <th rowspan="2">Salesforce/codegen25-7b-multi</th>- <th>FP16 </th>- <th>0.2854</th>- </tr>- <tr>- <th>AutoRound enable_quanted_input=False</th>- <th>0.2841</th>- </tr>-</table>----## Reference-If you find SignRound useful for your research, please cite our paper:-```bash-@article{cheng2023optimize,- title={Optimize Weight Rounding via Signed Gradient Descent for the Quantization of LLMs},- author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao},- journal={arXiv preprint arXiv:2309.05516},- year={2023}-}-```-diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pydeleted file mode 100644index 7d20016d1..000000000--- a/examples/code-generation/main.py+++ /dev/null@@ -1,208 +0,0 @@-import argparse-import random-import sys--sys.path.insert(0, '../..')-parser = argparse.ArgumentParser()-import torch-import os--os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"-torch.use_deterministic_algorithms(True, warn_only=True)-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel--from transformers import set_seed--import re--os.environ["TOKENIZERS_PARALLELISM"] = "false"--if __name__ == '__main__':-- parser.add_argument(- "--model_name", default="facebook/opt-125m"- )-- parser.add_argument("--bits", default=4, type=int,- help="number of bits")-- parser.add_argument("--group_size", default=128, type=int,- help="group size")-- parser.add_argument("--train_bs", default=8, type=int,- help="train batch size")-- parser.add_argument("--eval_bs", default=4, type=int,- help="eval batch size")-- parser.add_argument("--device", default="auto", type=str,- help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")-- parser.add_argument("--sym", action='store_true',- help=" sym quantization")-- parser.add_argument("--iters", default=200, type=int,- help=" iters")-- parser.add_argument("--enable_quanted_input", action='store_true',- help="enable_quanted_input is deprecated.")-- parser.add_argument("--lr", default=None, type=float,- help="learning rate, if None, it will be set to 1.0/iters automatically")-- parser.add_argument("--minmax_lr", default=None, type=float,- help="minmax learning rate, if None,it will beset to be the same with lr")-- parser.add_argument("--seed", default=42, type=int,- help="seed")-- parser.add_argument("--amp", action='store_true',- help="amp is deprecated")-- parser.add_argument("--adam", action='store_true',- help="adam")-- parser.add_argument("--seqlen", default=128, type=int,- help="sequence length")-- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")-- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")-- parser.add_argument("--n_samples", default=512, type=int,- help="number of samples")-- parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="low_gpu_mem_usage is deprecated")-- parser.add_argument("--enable_minmax_tuning", action='store_true',- help="enable_minmax_tuning is deprecated")-- parser.add_argument("--deployment_device", default='fake', type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")-- parser.add_argument("--scale_dtype", default='fp32',- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")-- parser.add_argument("--tasks",- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",- "arc_easy", "arc_challenge"],- help="lm-eval tasks")-- parser.add_argument("--output_dir", default="./tmp_autoround", type=str,- help="Where to store the final model.")-- parser.add_argument("--disable_amp", action='store_true',- help="disable amp")-- parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',- help="disable low_gpu_mem_usage")-- parser.add_argument("--disable_minmax_tuning", action='store_true',- help="whether disable enable weight minmax tuning") # #not usage-- parser.add_argument("--disable_quanted_input", action='store_true',- help="disuse the output of quantized block to tune the next block.")-- args = parser.parse_args()- if args.low_gpu_mem_usage:- print(- "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")- if args.enable_minmax_tuning:- print(- "enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")- if args.amp:- print(- "amp is deprecated, it has been set to the default, use disable_amp to turn it off")- if args.enable_quanted_input:- print(- "enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")-- set_seed(args.seed)- tasks = args.tasks-- model_name = args.model_name- if model_name[-1] == "/":- model_name = model_name[:-1]- print(model_name, flush=True)-- from auto_round.utils import detect_device-- device_str = detect_device(args.device)- torch_dtype = "auto"- if "hpu" in device_str:- torch_dtype = torch.bfloat16- torch_device = torch.device(device_str)- is_glm = bool(re.search("chatglm", model_name.lower()))-- if is_glm:- model = AutoModel.from_pretrained(model_name, trust_remote_code=True)- else:- model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True- )-- from auto_round import (AutoRound,- AutoAdamRound)-- model = model.eval()- # align wigh GPTQ to eval ppl- if "opt" in model_name:- seqlen = model.config.max_position_embeddings- model.seqlen = model.config.max_position_embeddings- else:- seqlen = 2048- model.seqlen = seqlen- seqlen = args.seqlen-- if "llama" in model_name:- from transformers import LlamaTokenizer-- tokenizer = LlamaTokenizer.from_pretrained(model_name)- if tokenizer.pad_token is None:- tokenizer.add_special_tokens({'pad_token': '[PAD]'})- else:- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- if hasattr(tokenizer, "model_max_length"):- if tokenizer.model_max_length < seqlen:- print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",- flush=True)- seqlen = min(seqlen, tokenizer.model_max_length)- args.seqlen = seqlen-- excel_name = f"{model_name}_{args.bits}_{args.group_size}"-- if args.disable_low_gpu_mem_usage:- model = model.to(torch_device)-- round = AutoRound- if args.adam:- round = AutoAdamRound- autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,- seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,- minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,- amp=not args.disable_amp, n_samples=args.n_samples,- low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,- seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, dataset="mbpp", dataset_split=['train', 'validation', 'test'],- enable_minmax_tuning=not args.disable_minmax_tuning) ##TODO args pass- model, q_config = autoround.quantize()- model_name = args.model_name.rstrip("/")- model.eval()- if args.device != "cpu":- torch.cuda.empty_cache()-- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"- deployment_device = args.deployment_device.split(',')- if 'gpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)- if "cpu" in deployment_device:- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)- if "fake" in deployment_device:- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)diff --git a/examples/code-generation/run_autoround.sh b/examples/code-generation/run_autoround.shdeleted file mode 100644index 473ebe942..000000000--- a/examples/code-generation/run_autoround.sh+++ /dev/null@@ -1,17 +0,0 @@-#!/bin/bash-set -x-device=0-model_name="Salesforce/codegen25-7b-multi"--CUDA_VISIBLE_DEVICES=$device \-python3 main.py \---model_name $model_name \---device $device \---group_size 128 \---bits 4 \---iters 200 \---seqlen 128 \---enable_minmax_tuning \---output_dir "./tmp_signround" \---amp \---disable_quanted_input\ No newline at end of filediff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex df370963f..92faa77c4 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -44,10 +44,28 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset### Customized Dataset- Option 1: Pass a local json file path to dataset argument-- Option 2: Register your dataset following the [code](../../auto_round/calib_dataset.py) and pass the new dataset&split args to initialize AutoRound object.-- Option 3: Following the [code](./main_customized_data.py) to pass list of string or list of inputs to dataloader.--Combination of different datasets has been supported, --dataset "./tmp.json,NeelNanda/pile-10k". Please note that samples with sequence length < args.seq will be dropped.+- Option 2: Register your dataset following the [code](../../auto_round/calib_dataset.py) and pass the new dataset&split args to initialize AutoRound object,e.g. autoround=Autoround(dataset="NeelNanda/pile-10k:train",...)+- Option 3: pass list of string or list of input_ids to dataset.+~~~python+def customized_data():+ ##Important Notice!!! Autoround will drop data < args.seqlen and truncate data to args.seqlen+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")+ return data+++def customized_data_with_tokenizer(tokenizer, seqlen=2048):+ ##Import notice!!! Autoround will drop data < args.seqlen+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")+ tokens = []+ for d in data:+ token = tokenizer(d, truncation=True, max_length=seqlen, return_tensors="pt").data+ tokens.append(token)+ return tokens+~~~++Combination of different datasets has been supported, --dataset "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test". Please note that samples with sequence length < args.seq will be dropped.<br />@@ -59,18 +77,18 @@ pip install -r requirements.txt- **Default Settings:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128```- **Reduced GPU Memory Usage:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --train_bs 1 --gradient_accumulate_steps 8+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8```- **Enable quantized lm-head:**--disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --quant_lm_head --disable_low_gpu_mem_usage+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage```- **Utilizing the AdamW Optimizer:**@@ -79,9 +97,15 @@ Include the flag `--adam`. Note that AdamW is less effective than sign gradient- **Running the Original SignRound:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input```+- **Code generation LLM:**++We utilized mbpp for calibration, but your own training dataset is highly recommended. Please note that samples with seqlen < args.seqlen will be dropped in current version.+```bash+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --bits 4 --group_size 128 --dataset "mbpp" --seqlen 128 "+```- **Running on Intel Gaudi2**```bashbash run_autoround_on_gaudi.shdiff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.pydeleted file mode 100644index 021981cad..000000000--- a/examples/language-modeling/main_customized_data.py+++ /dev/null@@ -1,280 +0,0 @@-import argparse-import sys--sys.path.insert(0, '../..')-parser = argparse.ArgumentParser()-import torch-import os-import transformers--os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"-torch.use_deterministic_algorithms(True, warn_only=True)-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel--from transformers import set_seed--import re--os.environ["TOKENIZERS_PARALLELISM"] = "false"---def customized_data():- ##Important Notice!!! Autoround will drop data < args.seqlen and truncate data to args.seqlen- data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]- data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")- return data---def customized_data_with_tokenizer(tokenizer, seqlen=2048):- ##Import notice!!! Autoround will dump data < args.seqlen- data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]- data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")- tokens = []- for d in data:- token = tokenizer(d, truncation=True, max_length=seqlen, return_tensors="pt").data- tokens.append(token)- return tokens---if __name__ == '__main__':-- parser.add_argument(- "--model_name", default="facebook/opt-125m"- )-- parser.add_argument("--bits", default=4, type=int,- help="number of bits")-- parser.add_argument("--group_size", default=128, type=int,- help="group size")-- parser.add_argument("--train_bs", default=8, type=int,- help="train batch size")-- parser.add_argument("--eval_bs", default=4, type=int,- help="eval batch size")-- parser.add_argument("--device", default="auto", type=str,- help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")-- parser.add_argument("--sym", action='store_true',- help=" sym quantization")-- parser.add_argument("--iters", default=200, type=int,- help=" iters")-- parser.add_argument("--enable_quanted_input", action='store_true',- help="whether to use the output of quantized block to tune the next block")-- parser.add_argument("--lr", default=None, type=float,- help="learning rate, if None, it will be set to 1.0/iters automatically")-- parser.add_argument("--minmax_lr", default=None, type=float,- help="minmax learning rate, if None,it will beset to be the same with lr")-- parser.add_argument("--seed", default=42, type=int,- help="seed")-- parser.add_argument("--eval_fp16_baseline", action='store_true',- help="whether to eval FP16 baseline")-- parser.add_argument("--amp", action='store_true',- help="amp")-- parser.add_argument("--adam", action='store_true',- help="adam")-- parser.add_argument("--seqlen", default=2048, type=int,- help="sequence length")-- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")-- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")-- parser.add_argument("--n_samples", default=512, type=int,- help="number of samples")-- parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="low_gpu_mem_usage")-- parser.add_argument("--enable_minmax_tuning", action='store_true',- help="whether enable weight minmax tuning")-- parser.add_argument("--deployment_device", default='fake', type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")-- parser.add_argument("--scale_dtype", default='fp32',- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")-- parser.add_argument("--tasks",- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",- "arc_easy", "arc_challenge"],- help="lm-eval tasks for lm_eval version 0.4")-- parser.add_argument("--output_dir", default="./tmp_autoround", type=str,- help="Where to store the final model.")-- parser.add_argument("--disable_eval", action='store_true',- help="Whether to do lmeval evaluation.")-- args = parser.parse_args()- set_seed(args.seed)- tasks = args.tasks- use_eval_legacy = False- import subprocess--- def get_library_version(library_name):- try:- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]- return version- except subprocess.CalledProcessError:- return "Library not found"--- res = get_library_version("lm-eval")- if res == "0.3.0":- use_eval_legacy = True- if not use_eval_legacy:- from eval import eval_model- else:- from eval_legacy import eval_model-- if isinstance(tasks, str):- tasks = tasks.split(',')- if isinstance(tasks, list):- if "mmlu" in tasks:- tmp_tasks = tasks- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:- tmp_tasks = tasks- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]- tasks = list(set(tasks))- if isinstance(args.tasks, str):- tasks = ','.join(tasks)-- model_name = args.model_name- if model_name[-1] == "/":- model_name = model_name[:-1]- print(model_name, flush=True)-- from auto_round.utils import detect_device-- device_str = detect_device(args.device)- torch_dtype = "auto"- if "hpu" in device_str:- torch_dtype = torch.bfloat16- torch_device = torch.device(device_str)-- is_glm = bool(re.search("chatglm", model_name.lower()))- if is_glm:- model = AutoModel.from_pretrained(model_name, trust_remote_code=True)- else:- model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True- )-- from auto_round import (AutoRound,- AutoAdamRound)-- model = model.eval()- # align with GPTQ to eval ppl- if "opt" in model_name:- seqlen = model.config.max_position_embeddings- model.seqlen = model.config.max_position_embeddings- else:- seqlen = 2048- model.seqlen = seqlen- seqlen = args.seqlen-- if "llama" in model_name:- from transformers import LlamaTokenizer-- tokenizer = LlamaTokenizer.from_pretrained(model_name)- if tokenizer.pad_token is None:- tokenizer.add_special_tokens({'pad_token': '[PAD]'})- else:- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)-- if hasattr(tokenizer, "model_max_length"):- if tokenizer.model_max_length < seqlen:- print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",- flush=True)- seqlen = min(seqlen, tokenizer.model_max_length)- args.seqlen = seqlen-- pt_dtype = torch.float16- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):- dtype = 'bfloat16'- pt_dtype = torch.bfloat16- else:- if str(args.device) != "cpu":- pt_dtype = torch.float16- dtype = 'float16'- else:- pt_dtype = torch.float32- dtype = 'float32'-- excel_name = f"{model_name}_{args.bits}_{args.group_size}"- if args.eval_fp16_baseline:- if not args.low_gpu_mem_usage:- model = model.to(torch_device)- excel_name += "_fp16.xlsx"- eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,- device=torch_device, excel_file=excel_name)- exit()-- if not args.low_gpu_mem_usage:- model = model.to(torch_device)-- round = AutoRound- if args.adam:- round = AutoAdamRound-- weight_config = {}- for n, m in model.named_modules():- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- weight_config[n] = {"data_type": "fp"}- print(- f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")-- data = customized_data()-- autoround = round(model, tokenizer, args.bits, args.group_size, dataloader=data, sym=args.sym,- batch_size=args.train_bs,- seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,- minmax_lr=args.minmax_lr, enable_quanted_input=args.enable_quanted_input, device=device_str,- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,- seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass- model, _ = autoround.quantize()- model_name = args.model_name.rstrip("/")-- model.eval()- if args.device != "cpu":- torch.cuda.empty_cache()-- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"- deployment_device = args.deployment_device.split(',')- if 'gpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)- if "cpu" in deployment_device:- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)- if "fake" in deployment_device:- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)-- if not args.disable_eval and "fake" in deployment_device: ##support autogptq real eval later- excel_name = f"{output_dir}_result.xlsx"- output_dir += "/"- print(excel_name, flush=True)- eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,- device=torch_device, excel_file=excel_name)diff --git a/test/test_autoopt.py b/test/test_autoopt.pyindex d52556bbf..2f500162a 100644--- a/test/test_autoopt.py+++ b/test/test_autoopt.py@@ -43,7 +43,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 0ca68189f..d871705f0 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -43,7 +43,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=2,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()if torch.cuda.is_available():@@ -60,7 +60,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -74,7 +74,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -88,7 +88,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -102,7 +102,7 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -117,7 +117,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=10,enable_quanted_input=False,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -132,7 +132,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=10,enable_minmax_tuning=False,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -148,7 +148,7 @@ class TestAutoRound(unittest.TestCase):seqlen=10,enable_minmax_tuning=False,enable_quanted_input=False,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,)autoround.quantize()@@ -165,7 +165,7 @@ class TestAutoRound(unittest.TestCase):seqlen=10,enable_minmax_tuning=False,enable_quanted_input=False,- dataloader=self.llm_dataloader,+ dataset=self.llm_dataloader,weight_config=weight_config,)autoround.quantize()