PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix filter func issue (#71)
wenhuach21900154bfdd9b
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex dda325f69..41197f5a7 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -234,6 +234,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N"""dataset_names = dataset_name.split(",")+# if len(dataset_names) == 1 and dataset_names[0] == "NeelNanda/pile-10k": ##to guarantee the reproducibility.# get_dataloader = CALIB_DATASETS.get("legacy-NeelNanda/pile-10k")# dataloader = get_dataloader(@@ -245,6 +246,14 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N# dataset_name=dataset_name,# )# return dataloader+ def filter_func(example):+ if example["input_ids"].shape[-1] < seqlen:+ return False+ input_ids = example["input_ids"][:seqlen]+ input_ids_list = input_ids.tolist()+ if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:+ return False+ return Truedatasets = []for name in dataset_names:@@ -261,7 +270,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=Ndataset_name=name,)dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])- dataset = dataset.filter(lambda example: example["input_ids"].shape[-1] >= seqlen)+ dataset = dataset.filter(filter_func)datasets.append(dataset)indices = range(len(datasets))