PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add local file of pile-10k (#198)
WeiweiZhang1084f3db2a8ff
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex b71301f4b..4c01302d9 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -234,7 +234,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,data = [line for line in f]return dataelse:- logger.error("invalid local file type,for now only support json ")+ logger.error("invalid local file type, for now only support json format data file.")samples = []dataset = load_local_data(dataset_name)@@ -368,7 +368,14 @@ def get_dataloader(if is_local_path(name):get_dataset = CALIB_DATASETS.get("local")else:- get_dataset = CALIB_DATASETS.get(name)+ calib_name = name+ if name not in CALIB_DATASETS.keys():+ calib_name = name.split('/')[-1]+ for key in CALIB_DATASETS.keys():+ if calib_name in key:+ calib_name = key+ break+ get_dataset = CALIB_DATASETS.get(calib_name)dataset = get_dataset(tokenizer,seqlen,@@ -440,3 +447,4 @@ def get_dataloader(calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex ff2d473a4..cbbe60eb6 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -558,7 +558,11 @@ def is_local_path(path):Returns:bool: True if the path exists locally, False otherwise."""- return os.path.exists(path)+ format_list = ("json", "txt",)+ flag = None+ for x in format_list:+ flag = True if x in path else flag+ return flag and os.path.exists(path)def convert_dtype_str2torch(str_dtype):diff --git a/examples/language-modeling/calib_data/pile-10k b/examples/language-modeling/calib_data/pile-10knew file mode 160000index 000000000..c2115101a--- /dev/null+++ b/examples/language-modeling/calib_data/pile-10k@@ -0,0 +1 @@+Subproject commit c2115101aa8d63a0c2d3869003c7f3819c14c92ediff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 0b0548332..1d23e6520 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -44,7 +44,7 @@ if __name__ == '__main__':parser.add_argument("--iters", default=200, type=int,help=" iters")- parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,+ parser.add_argument("--dataset", default="calib_data/pile-10k", type=str,help="The dataset for quantization training. It can be a custom one.")parser.add_argument("--enable_quanted_input", action='store_true',