Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add local file of pile-10k (#198)

WeiweiZhang1084f3db2a8ff
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index b71301f4b..4c01302d9 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -234,7 +234,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
data = [line for line in f]
return data
else:
- logger.error("invalid local file type,for now only support json ")
+ logger.error("invalid local file type, for now only support json format data file.")
samples = []
dataset = load_local_data(dataset_name)
@@ -368,7 +368,14 @@ def get_dataloader(
if is_local_path(name):
get_dataset = CALIB_DATASETS.get("local")
else:
- get_dataset = CALIB_DATASETS.get(name)
+ calib_name = name
+ if name not in CALIB_DATASETS.keys():
+ calib_name = name.split('/')[-1]
+ for key in CALIB_DATASETS.keys():
+ if calib_name in key:
+ calib_name = key
+ break
+ get_dataset = CALIB_DATASETS.get(calib_name)
dataset = get_dataset(
tokenizer,
seqlen,
@@ -440,3 +447,4 @@ def get_dataloader(
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index ff2d473a4..cbbe60eb6 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -558,7 +558,11 @@ def is_local_path(path):
Returns:
bool: True if the path exists locally, False otherwise.
"""
- return os.path.exists(path)
+ format_list = ("json", "txt",)
+ flag = None
+ for x in format_list:
+ flag = True if x in path else flag
+ return flag and os.path.exists(path)
def convert_dtype_str2torch(str_dtype):
diff --git a/examples/language-modeling/calib_data/pile-10k b/examples/language-modeling/calib_data/pile-10k
new file mode 160000
index 000000000..c2115101a
--- /dev/null
+++ b/examples/language-modeling/calib_data/pile-10k
@@ -0,0 +1 @@
+Subproject commit c2115101aa8d63a0c2d3869003c7f3819c14c92e
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 0b0548332..1d23e6520 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -44,7 +44,7 @@ if __name__ == '__main__':
parser.add_argument("--iters", default=200, type=int,
help=" iters")
- parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,
+ parser.add_argument("--dataset", default="calib_data/pile-10k", type=str,
help="The dataset for quantization training. It can be a custom one.")
parser.add_argument("--enable_quanted_input", action='store_true',