PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support calibration dataset concat (#147)
yintong-lu75e3fde00b36
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 9f8e0c2b7..d4b4d649a 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -199,6 +199,11 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,samples = []dataset = load_local_data(dataset_name)+ if isinstance(dataset, dict):+ new_dataset = []+ for key in dataset.keys():+ new_dataset.append(dataset[key])+ dataset = new_datasetfor data in dataset:text = dataif isinstance(text, str):@@ -241,8 +246,9 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42"""dataset_names = dataset_name.split(",")-def filter_func(example):+ if isinstance(example["input_ids"], list):+ example["input_ids"] = torch.tensor(example["input_ids"])if example["input_ids"].shape[-1] < seqlen:return Falseinput_ids = example["input_ids"][:seqlen]@@ -251,11 +257,64 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42return Falsereturn True- datasets = []+ def concat_dataset_element(dataset):+ input_ids, concat_input_ids = [eg['input_ids'] for eg in dataset], []+ attention_mask_list, attention_mask = [], torch.ones([1, seqlen]).to(torch.int64)+ buffer_input_id = torch.Tensor().to(torch.int64)+ bos_token_id, eos_token_id = tokenizer.bos_token_id, tokenizer.eos_token_id+ os_cnt, have_bos, have_eos = 0, False, False++ for input_id in input_ids:+ if input_id[0] == bos_token_id:+ input_id = input_id[1:]+ os_cnt, have_bos = os_cnt + 1, True+ if input_id[-1] == eos_token_id:+ input_id = input_id[:-1]+ os_cnt, have_eos = os_cnt + 1, True++ if buffer_input_id.shape[-1] + input_id.shape[-1] + os_cnt > seqlen:+ idx_keep = seqlen - buffer_input_id.shape[-1] - os_cnt+ input_id_to_append = [buffer_input_id, input_id[:idx_keep]]+ if have_bos:+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append+ if have_eos:+ input_id_to_append.append(torch.tensor([eos_token_id]))++ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64))+ attention_mask_list.append(attention_mask)+ buffer_input_id = input_id[idx_keep:]+ else:+ buffer_input_id = torch.cat([buffer_input_id, input_id])++ if buffer_input_id.shape[-1] + os_cnt == seqlen:+ input_id_to_append = [buffer_input_id]+ if have_bos:+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append+ if have_eos:+ input_id_to_append.append(torch.tensor([eos_token_id]))+ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64))+ attention_mask_list.append(attention_mask)+ buffer_input_id = torch.Tensor().to(torch.int64)+ data = [{'input_ids': a, 'attention_mask': b} for a, b in zip(concat_input_ids, attention_mask_list)]+ import datasets+ dataset_new = datasets.Dataset.from_list(data)+ return dataset_new++ datasets, data_lens = [], {}for name in dataset_names:split = None+ do_concat = Falseif ":" in name:- name, split = name.split(":")+ split_list = name.split(":")+ name, split_list = name.split(":")[0], name.split(":")[1:]+ for ele in split_list:+ key, values = ele.split('=')[0], ele.split('=')[1:]+ if key == "split":+ split = values[0].split('+')+ if key == "num":+ data_lens[name] = int(values[0])+ if key == "concat":+ do_concat = True if (len(values) > 0 and values[0].lower == 'false') else Trueif is_local_path(name):get_dataset = CALIB_DATASETS.get("local")else:@@ -268,22 +327,33 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42dataset_name=name,)dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])+ if do_concat:+ dataset = concat_dataset_element(dataset)dataset = dataset.filter(filter_func)-+ if name in data_lens:+ dataset = dataset.select(range(data_lens[name]))datasets.append(dataset)indices = range(len(datasets))res = sorted(zip(indices, datasets), key=lambda x: len(x[1]))indices = [item[0] for item in res]datasets = [item[1] for item in res]dataset_names = [dataset_names[index] for index in indices]- cnt = 0+ cnt = 0 if not data_lens else sum(data_lens.values())dataset_cnt_info = {}+ if cnt > n_samples:+ cnt = 0+for i in range(len(datasets)):- target_cnt = (n_samples - cnt) // (len(datasets) - i)- target_cnt = min(target_cnt, len(datasets[i]))+ name = dataset_names[i].split(':')[0]+ if name not in data_lens:+ target_cnt = (n_samples - cnt) // (len(datasets) - len(data_lens)) if data_lens \+ else (n_samples - cnt) // (len(datasets) - i)+ target_cnt = min(target_cnt, len(datasets[i]))+ cnt += target_cnt+ else:+ target_cnt = data_lens[name]datasets[i] = datasets[i].select(range(target_cnt))- dataset_cnt_info[dataset_names[i]] = target_cnt- cnt += target_cnt+ dataset_cnt_info[name] = target_cntif len(datasets) > 1:from datasets import concatenate_datasets@@ -299,6 +369,10 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42attention_mask_new = []for text in batch:input_ids, attention_mask = text["input_ids"], text["attention_mask"]+ if isinstance(input_ids, list):+ input_ids = torch.tensor(input_ids)+ if isinstance(attention_mask, list):+ attention_mask = torch.tensor(attention_mask)input_ids = input_ids[:seqlen]input_ids_list = input_ids.tolist()if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex e10ce211d..da08449af 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -35,7 +35,9 @@ def customized_data_with_tokenizer(tokenizer, seqlen=2048):return tokens~~~-Combination of different datasets has been supported, --dataset "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test". Please note that samples with sequence length < args.seqlen will be dropped.+We support combination of different datasets and parametrization of calibration datasets by using "--dataset ./tmp.json:concat,NeelNanda/pile-10k:split=train+val:num=256,mbpp:concat=True:num=128". Both local calibration file and huggingface dataset are supported. Through parametrization, users could specify splits of a dataset by setting "split=split1+split2". A concatenation option could enable users to merge calibration samples, a process commonly used to enhance calibration reliability. Please note that samples shorter than args.seqlen will be dropped when concatenation option is not enabled.+Please use ',' to split datasets, ':' to split parameters of a dataset and '+' to add values for one targeted parameter.+<br />