Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix filter func issue (#71)

wenhuach21900154bfdd9b
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index dda325f69..41197f5a7 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -234,6 +234,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
"""
dataset_names = dataset_name.split(",")
+
# if len(dataset_names) == 1 and dataset_names[0] == "NeelNanda/pile-10k": ##to guarantee the reproducibility.
# get_dataloader = CALIB_DATASETS.get("legacy-NeelNanda/pile-10k")
# dataloader = get_dataloader(
@@ -245,6 +246,14 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
# dataset_name=dataset_name,
# )
# return dataloader
+ def filter_func(example):
+ if example["input_ids"].shape[-1] < seqlen:
+ return False
+ input_ids = example["input_ids"][:seqlen]
+ input_ids_list = input_ids.tolist()
+ if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
+ return False
+ return True
datasets = []
for name in dataset_names:
@@ -261,7 +270,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
dataset_name=name,
)
dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
- dataset = dataset.filter(lambda example: example["input_ids"].shape[-1] >= seqlen)
+ dataset = dataset.filter(filter_func)
datasets.append(dataset)
indices = range(len(datasets))