PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support combination of calibration datasets (#70)
wenhuach21e8fb5da19806
diff --git a/README.md b/README.mdindex b104f79a4..7adab513d 100644--- a/README.md+++ b/README.md@@ -141,35 +141,35 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))## Support List-| Model | Supported |-|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |-| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |-| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |-| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc), [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/)-| tiiuae/falcon-7b | [HF-int4-model](https://huggingface.co/Intel/falcon-7b-int4-inc), [accuracy](./docs/falcon-7b-acc.md), [recipe](./examples/language-modeling/scripts/falcon-7b.sh), [example](./examples/language-modeling/) |-| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)+| Model | Supported |+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|+| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |+| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |+| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |+| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc), [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/)+| tiiuae/falcon-7b | [HF-int4-model](https://huggingface.co/Intel/falcon-7b-int4-inc), [accuracy](./docs/falcon-7b-acc.md), [recipe](./examples/language-modeling/scripts/falcon-7b.sh), [example](./examples/language-modeling/) |+| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)| mistralai/Mixtral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) |-| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |-| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) (under review), [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) |-| mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |-| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |-| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |-| Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh), [example](./examples/language-modeling/) |-| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |-| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |-| huggyllama/llama-7b | [example](./examples/language-modeling/) |-| meta-llama/Llama-2-7b-hf | [example](./examples/language-modeling/) |-| facebook/opt-6.7b | [example](./examples/language-modeling/) ||-| mosaicml/mpt-7b | [example](./examples/language-modeling/) |-| bigscience/bloom-7b1 | [example](./examples/language-modeling/) |-| baichuan-inc/Baichuan-7B | [example](./examples/language-modeling/) |-| Qwen/Qwen-7B | [example](./examples/language-modeling/) |-| THUDM/chatglm3-6b | [example](./examples/language-modeling/) |-| MBZUAI/LaMini-GPT-124M | [example](./examples/language-modeling/) |-| EleutherAI/gpt-neo-125m | [example](./examples/language-modeling/) |-| databricks/dolly-v2-3b | [example](./examples/language-modeling/) |-| stabilityai/stablelm-base-alpha-3b | [example](./examples/language-modeling/)+| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |+| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) (under review), [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) |+| mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |+| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |+| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |+| Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [sym recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh), [asym recipe ](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh), [example](./examples/language-modeling/) |+| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |+| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |+| huggyllama/llama-7b | [example](./examples/language-modeling/) |+| meta-llama/Llama-2-7b-hf | [example](./examples/language-modeling/) |+| facebook/opt-6.7b | [example](./examples/language-modeling/) ||+| mosaicml/mpt-7b | [example](./examples/language-modeling/) |+| bigscience/bloom-7b1 | [example](./examples/language-modeling/) |+| baichuan-inc/Baichuan-7B | [example](./examples/language-modeling/) |+| Qwen/Qwen-7B | [example](./examples/language-modeling/) |+| THUDM/chatglm3-6b | [example](./examples/language-modeling/) |+| MBZUAI/LaMini-GPT-124M | [example](./examples/language-modeling/) |+| EleutherAI/gpt-neo-125m | [example](./examples/language-modeling/) |+| databricks/dolly-v2-3b | [example](./examples/language-modeling/) |+| stabilityai/stablelm-base-alpha-3b | [example](./examples/language-modeling/)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 2f6fa8c16..e433275d1 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -18,6 +18,7 @@ import timeimport torch+from .calib_dataset import get_dataloaderfrom .special_model_handler import check_hidden_state_dim, check_share_attention_maskfrom .utils import (CpuInfo,@@ -32,6 +33,7 @@ from .utils import (get_scale_shape,htcore,is_hpu_available,+ is_local_path,logger,move_input_to_device,quant_weight,@@ -634,17 +636,16 @@ class AutoRound(object):Args:n_samples (int): The number of samples to use for calibration."""- if self.dataloader is None:- from .calib_dataset import CALIB_DATASETS- get_dataloader = CALIB_DATASETS.get(self.dataset_name, CALIB_DATASETS["NeelNanda/pile-10k"])+ if self.dataloader is None:self.dataloader = get_dataloader(self.tokenizer,self.seqlen,- seed=self.seed,- bs=self.train_bs,- split=self.dataset_split,- dataset_name=self.dataset_name,+ self.dataset_name,+ self.dataset_split,+ self.seed,+ self.train_bs,+ self.n_samples,)self.start_time = time.time()@@ -654,6 +655,7 @@ class AutoRound(object):continueif isinstance(data, torch.Tensor):input_ids = data.to(self.model.device)+ data_new = input_idselif isinstance(data, str):if self.tokenizer is None:@@ -671,8 +673,8 @@ class AutoRound(object):input_ids = data_new["input_ids"]if input_ids.shape[-1] < self.seqlen:continue- if total_cnt + input_ids.shape[0] > n_samples:- input_ids = input_ids[: n_samples - total_cnt, ...]+ # if total_cnt + input_ids.shape[0] > n_samples:+ # input_ids = input_ids[: n_samples - total_cnt, ...]try:if isinstance(data_new, torch.Tensor):self.model(data_new)diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex b1fd535f4..dda325f69 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -1,6 +1,10 @@+import jsonimport randomimport torch+from torch.utils.data import DataLoader++from .utils import is_local_path, loggerCALIB_DATASETS = {}@@ -44,7 +48,7 @@ def get_tokenizer_function(tokenizer, seqlen):@register_dataset("NeelNanda/pile-10k")-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):+def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4):"""Returns a dataloader for the specified dataset and split.Args:@@ -59,38 +63,21 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="A dataloader for the specified dataset and split, using the provided tokenizer and sequence length."""from datasets import load_dataset- from torch.utils.data import DataLoader+ split = "train"tokenizer_function = get_tokenizer_function(tokenizer, seqlen)- @torch.no_grad()- def collate_batch(batch):- input_ids_new = []- for text in batch:- input_ids = text["input_ids"]- if input_ids.shape[0] < seqlen:- continue- input_ids = input_ids[:seqlen]- input_ids_list = input_ids.tolist()- if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:- continue- input_ids_new.append(input_ids)- if len(input_ids_new) == 0:- return None- tmp = torch.vstack(input_ids_new)- res = {"input_ids": tmp}- return res-calib_dataset = load_dataset(dataset_name, split=split)calib_dataset = calib_dataset.shuffle(seed=seed)calib_dataset = calib_dataset.map(tokenizer_function, batched=True)- calib_dataset.set_format(type="torch", columns=["input_ids"])- calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)- return calib_dataloader+ return calib_dataset-@register_dataset("mbpp")-def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=["train", "validation", "test"], seed=42, bs=4):++@register_dataset("madao33/new-title-chinese")+def get_new_chinese_title_dataset(+ tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42, bs=4+):"""Returns a dataloader for the specified dataset and split.Args:@@ -104,38 +91,207 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=["train",Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length."""- from datasets import load_dataset- from torch.utils.data import DataLoader- def get_mbpp_tokenizer_function(tokenizer, seqlen):+ def get_tokenizer_function(tokenizer, seqlen):"""Returns a default tokenizer function.Args:tokenizer: The tokenizer to be used for tokenization.seqlen: The maximum sequence length.- Returns: A default tokenizer function that applies the provided tokenizer with truncation and- a maximum length of seqlen to the "text" field of examples.+ Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length+ of seqlen to the "text" field of examples."""def default_tokenizer_function(examples):- example = tokenizer(examples, truncation=True, max_length=seqlen, return_tensors="pt")- # example = tokenizer(examples, return_tensors="pt")+ example = tokenizer(examples["content"], truncation=True, max_length=seqlen)return examplereturn default_tokenizer_function- tokenizer_function = get_mbpp_tokenizer_function(tokenizer, seqlen)+ split = "train"+ from datasets import load_dataset++ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)++ calib_dataset = load_dataset(dataset_name, split=split)+ calib_dataset = calib_dataset.shuffle(seed=seed)+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)++ return calib_dataset+++@register_dataset("mbpp")+def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, bs=4):+ """Returns a dataloader for the specified dataset and split.++ Args:+ tokenizer: The tokenizer to be used for tokenization.+ seqlen: The maximum sequence length.+ data_name: The name of the dataset.+ split: The data split to be used (e.g., "train", "test").+ seed: The random seed for shuffling the dataset.+ bs: The batch size for the dataloader.++ Returns:+ A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.+ """+ from datasets import load_dataset++ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)++ samples = []+ splits = split+ if splits is None:+ splits = ["train", "validation", "test"]+ if isinstance(splits, str):+ splits = [split]++ for split in splits:+ dataset = load_dataset(dataset_name, split=split)+ for data in dataset:+ samples.append({"text": data["text"] + data["code"]})+ random.Random(seed).shuffle(samples)+ import datasets++ calib_dataset = datasets.Dataset.from_list(samples)+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)++ return calib_dataset+++@register_dataset("local")+def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, bs=4):+ """Returns a dataloader for a custom dataset and split.+ We allow the input of a json or text file containing a processed text sample each line.++ Args:+ tokenizer: The tokenizer to be used for tokenization.+ seqlen: The maximum sequence length.+ data_name: The name or path of the dataset, which is a jsonl file.+ split: The data split to be used (e.g., "train", "test").+ seed: The random seed for shuffling the dataset.+ bs: The batch size for the dataloader.++ Returns:+ A dataloader for a custom dataset and split, using the provided tokenizer and sequence length.+ """+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)++ def load_local_data(data_path):+ if data_path.endswith(".json"):+ with open(data_path, "r") as f:+ data = json.load(f)+ return data+ elif data_path.endswith(".txt"):+ with open(data_path) as f:+ data = [line for line in f]+ return data+ else:+ logger.error("invalid local file type,for now only support json ")++ samples = []+ dataset = load_local_data(dataset_name)+ for data in dataset:+ text = data+ if isinstance(text, str):+ pass+ elif isinstance(data, dict) and len(data.keys()) == 1:+ for item in data.items():+ text = item[1]+ elif isinstance(data, dict) and "text" in data.keys():+ text = data["text"]+ elif isinstance(data, dict) and "input_ids" in data.keys():+ text = data["input_ids"]+ assert isinstance(text, str), "data must be string"+ text = text.rstrip()+ text = text.rstrip("\n")+ samples.append({"text": text})+ random.Random(seed).shuffle(samples)+ import datasets++ calib_dataset = datasets.Dataset.from_list(samples)+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)+ return calib_dataset+++def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4, n_samples=512):+ """Generate a DataLoader for calibration using specified parameters.++ Args:+ tokenizer (Tokenizer): The tokenizer to use for tokenization.+ seqlen (int): The exact sequence length. samples < seqlen will be dropped,+ samples longer than seqlen will be truncated+ dataset_name (str, optional): The name of the dataset or datasets separated by commas.+ Defaults to "NeelNanda/pile-10k".+ split (str, optional): The data split to use. Defaults to None.+ seed (int, optional): The random seed for reproducibility. Defaults to 42.+ bs (int, optional): The batch size. Defaults to 4.+ n_samples (int, optional): The total number of samples to include. Defaults to 512.++ Returns:+ DataLoader: The DataLoader for the calibrated dataset.+ """++ dataset_names = dataset_name.split(",")+ # if len(dataset_names) == 1 and dataset_names[0] == "NeelNanda/pile-10k": ##to guarantee the reproducibility.+ # get_dataloader = CALIB_DATASETS.get("legacy-NeelNanda/pile-10k")+ # dataloader = get_dataloader(+ # tokenizer,+ # seqlen,+ # seed=seed,+ # bs=bs,+ # split=split,+ # dataset_name=dataset_name,+ # )+ # return dataloader++ datasets = []+ for name in dataset_names:+ if is_local_path(name):+ get_dataset = CALIB_DATASETS.get("local")+ else:+ get_dataset = CALIB_DATASETS.get(name)+ dataset = get_dataset(+ tokenizer,+ seqlen,+ seed=seed,+ bs=bs,+ split=split,+ dataset_name=name,+ )+ dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])+ dataset = dataset.filter(lambda example: example["input_ids"].shape[-1] >= seqlen)++ datasets.append(dataset)+ indices = range(len(datasets))+ res = sorted(zip(indices, datasets), key=lambda x: len(x[1]))+ indices = [item[0] for item in res]+ datasets = [item[1] for item in res]+ dataset_names = [dataset_names[index] for index in indices]+ cnt = 0+ dataset_cnt_info = {}+ for i in range(len(datasets)):+ target_cnt = (n_samples - cnt) // (len(datasets) - i)+ target_cnt = min(target_cnt, len(datasets[i]))+ datasets[i] = datasets[i].select(range(target_cnt))+ dataset_cnt_info[dataset_names[i]] = target_cnt+ cnt += target_cnt+ if len(datasets) > 1:+ from datasets import concatenate_datasets++ dataset_final = concatenate_datasets(datasets)+ dataset_final = dataset_final.shuffle(seed=seed)+ logger.info(dataset_cnt_info)+ else:+ dataset_final = datasets[0]@torch.no_grad()def collate_batch(batch):input_ids_new = []attention_mask_new = []for text in batch:- token_text = tokenizer_function(text)- input_ids, attention_mask = token_text["input_ids"], token_text["attention_mask"]- if input_ids.shape[1] < seqlen:- continue+ input_ids, attention_mask = text["input_ids"], text["attention_mask"]input_ids = input_ids[:seqlen]input_ids_list = input_ids.tolist()if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:@@ -150,13 +306,52 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=["train",res = {"input_ids": input_ids_new, "attention_mask": attention_mask_new}return res- samples = []- splits = split- for split in splits:- dataset = load_dataset(dataset_name, split=split)- for data in dataset:- samples.append(data["text"] + data["code"])- random.Random(seed).shuffle(samples)-- calib_dataloader = DataLoader(samples, batch_size=bs, shuffle=False, collate_fn=collate_batch)+ calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader+++# ##we keep this for reproduce issue, do not delete, keep it for a while+# @register_dataset("legacy-NeelNanda/pile-10k")+# def get_pile_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):+# """Returns a dataloader for the specified dataset and split.+#+# Args:+# tokenizer: The tokenizer to be used for tokenization.+# seqlen: The maximum sequence length.+# data_name: The name of the dataset.+# split: The data split to be used (e.g., "train", "test").+# seed: The random seed for shuffling the dataset.+# bs: The batch size for the dataloader.+#+# Returns:+# A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.+# """+# from datasets import load_dataset+# from torch.utils.data import DataLoader+#+# tokenizer_function = get_tokenizer_function(tokenizer, seqlen)+#+# @torch.no_grad()+# def collate_batch(batch):+# input_ids_new = []+# for text in batch:+# input_ids = text["input_ids"]+# if input_ids.shape[0] < seqlen:+# continue+# input_ids = input_ids[:seqlen]+# input_ids_list = input_ids.tolist()+# if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:+# continue+# input_ids_new.append(input_ids)+# if len(input_ids_new) == 0:+# return None+# tmp = torch.vstack(input_ids_new)+# res = {"input_ids": tmp}+# return res+#+# calib_dataset = load_dataset(dataset_name, split=split)+# calib_dataset = calib_dataset.shuffle(seed=seed)+# calib_dataset = calib_dataset.map(tokenizer_function, batched=True)+# calib_dataset.set_format(type="torch", columns=["input_ids"])+# calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)+# return calib_dataloaderdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 6fb78ebbe..d5524c1e9 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -14,6 +14,7 @@import copyimport logging+import osimport subprocessfrom collections import UserDict@@ -584,3 +585,7 @@ class CpuInfo(object):for line in proc.stdout:return int(line.decode("utf-8", errors="ignore").strip())return 0+++def is_local_path(path):+ return os.path.exists(path)diff --git a/docs/Qwen1.5-7B-Chat-acc.md b/docs/Qwen1.5-7B-Chat-acc.mdindex 7e0f3a834..a07463a04 100644--- a/docs/Qwen1.5-7B-Chat-acc.md+++ b/docs/Qwen1.5-7B-Chat-acc.md@@ -2,18 +2,19 @@ Due to licensing restrictions, we are unable to release the model.Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d.-Please use the command below for evaluation.+We used the following command for evaluation.+~~~bash'lm_eval --model hf \--model_args pretrained="gpu_model_path",autogptq=True,gptq_use_triton=True,trust_remote_code=True \- --device cuda:0 --tasks ceval-valid,cmmlu,mmlu --batch_size 16 --num_fewshot 0'+ --device cuda:0 --tasks ceval-valid,cmmlu,mmlu,gsm8k --batch_size 16 --num_fewshot 0'+~~~--| Metric | BF16 | INT4 recipe |-| -------------- | ------ | -----------------|-| Avg. | 0.6231 | 0.6205 |-| ceval | 0.6887 | 0.6761 |-| cmmlu | 0.6959 | 0.6870 |-| mmlu | 0.6020 | 0.5974 |-| gsm8k | 0.5057 | 0.5216 |+| Metric | BF16 | INT4 sym recipe | INT4 asym recipe |+| -------------- | ------ |-----------------|------------------|+| Avg. | 0.6231 | 0.6205 | 0.6186 |+| ceval | 0.6887 | 0.6761 | 0.6820 |+| cmmlu | 0.6959 | 0.6870 | 0.6884 |+| mmlu | 0.6020 | 0.5974 | 0.5946 |+| gsm8k | 0.5057 | 0.5216 | 0.5095 |diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 5e3a65409..b0c9c5448 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -44,6 +44,9 @@ if __name__ == '__main__':parser.add_argument("--iters", default=200, type=int,help=" iters")++ parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,+ help="The dataset for quantization training. It can be a custom one.")parser.add_argument("--use_quant_input", action='store_true',help="whether to use the output of quantized block to tune the next block")@@ -254,7 +257,7 @@ if __name__ == '__main__':f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,- seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,+ dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,amp=not args.disable_amp, n_samples=args.n_samples,low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.shnew file mode 100644index 000000000..1823f3d28--- /dev/null+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh@@ -0,0 +1,9 @@+python3 main.py \+--model_name Qwen/Qwen1.5-7B-Chat\+--group_size 128 \+--bits 4 \+--iters 1000 \+--deployment_device 'gpu' \+--minmax_lr 2e-3 \+--use_quant_input+diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.shsimilarity index 100%rename from examples/language-modeling/scripts/Qwen1.5-7B-Chat.shrename to examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.shdiff --git a/test/test_autoopt.py b/test/test_autoopt.pyindex a95c49cfc..d52556bbf 100644--- a/test/test_autoopt.py+++ b/test/test_autoopt.py@@ -3,7 +3,7 @@ import shutilimport sysimport unittest-sys.path.insert(0, ".")+sys.path.insert(0, "..")import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -42,6 +42,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,dataloader=self.llm_dataloader,)autoround.quantize()diff --git a/test/test_autoround.py b/test/test_autoround.pyindex ba02e6f82..2235fd5e0 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -3,7 +3,7 @@ import shutilimport sysimport unittest-sys.path.insert(0, ".")+sys.path.insert(0, "..")import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -42,6 +42,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=2,dataloader=self.llm_dataloader,)autoround.quantize()@@ -58,6 +59,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,dataloader=self.llm_dataloader,)autoround.quantize()@@ -71,6 +73,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,dataloader=self.llm_dataloader,)autoround.quantize()@@ -84,6 +87,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,dataloader=self.llm_dataloader,)autoround.quantize()@@ -97,6 +101,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,dataloader=self.llm_dataloader,)autoround.quantize()@@ -110,6 +115,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,use_quant_input=False,dataloader=self.llm_dataloader,)@@ -124,6 +130,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,enable_minmax_tuning=False,dataloader=self.llm_dataloader,)@@ -138,6 +145,7 @@ class TestAutoRound(unittest.TestCase):group_size=group_size,sym=sym,iters=2,+ seqlen=10,enable_minmax_tuning=False,use_quant_input=False,dataloader=self.llm_dataloader,diff --git a/test/test_autoround_export.py b/test/test_autoround_export.pyindex 0f4a13233..c49395e8d 100644--- a/test/test_autoround_export.py+++ b/test/test_autoround_export.py@@ -3,7 +3,7 @@ import shutilimport sysimport unittest-sys.path.insert(0, ".")+sys.path.insert(0, "..")import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerdiff --git a/test/test_calib_dataset.py b/test/test_calib_dataset.pynew file mode 100644index 000000000..2e7b887e4--- /dev/null+++ b/test/test_calib_dataset.py@@ -0,0 +1,86 @@+import os+import shutil+import sys+import unittest++sys.path.insert(0, "..")+import json++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestLocalCalibDataset(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ json_data = [{"text": "awefdsfsddfd"}, {"text": "fdfdfsdfdfdfd"}, {"text": "dfdsfsdfdfdfdf"}]+ os.makedirs("./saved", exist_ok=True)+ self.json_file = "./saved/tmp.json"+ with open(self.json_file, "w") as json_file:+ json.dump(json_data, json_file, indent=4)++ self.text_file = "./saved/tmp.txt"+ txt_data = ["awefdsfsddfd", "fdfdfsdfdfdfd", "dfdsfsdfdfdfdf"]+ with open(self.text_file, "w") as text_file:+ for data in txt_data:+ text_file.write(data + "\n")++ model_name = "facebook/opt-125m"+ self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)++ def test_json(self):+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=5,+ dataset=self.json_file,+ )+ autoround.quantize()++ def test_txt(self):+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=5,+ dataset=self.text_file,+ )+ autoround.quantize()++ def test_combine_dataset(self):+ dataset = self.text_file + "," + "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp"+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset+ )+ autoround.quantize()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()