PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable cn_tasks and fix bugs (#45)
yintong-lu259932543d77
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex cb3806a3d..6681607a5 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -34,6 +34,9 @@ fewshots_dict['paper'] = {"rte": [0],"arc_easy": [0],"arc_challenge": [0],+ "gsm8k": [0],+ "ceval-valid": [0],+ "cmmlu": [0],}fewshots_dict['leadboard'] = {"hellaswag": [10],@@ -375,7 +378,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "pprint.pprint(tmp_results["results"])print(f"\n{sub_name} cost time: {time.time() - task_s}\n")results[sub_name] = {}- if 'mmlu' in tmp_tasks:+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:for cata in ['humanities', 'other', 'stem', 'sociology']:results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']results[sub_name]['avg'] = tmp_results['results'][f'mmlu']diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex c6863a8e8..256fe241c 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -143,13 +143,12 @@ if __name__ == '__main__':if res == "0.3.0":use_eval_legacy = True+ if isinstance(tasks, str):+ tasks = tasks.split(',')if not use_eval_legacy:from eval import eval_modelelse:from eval_legacy import eval_model-- if isinstance(tasks, str):- tasks = tasks.split(',')if isinstance(tasks, list):if "mmlu" in tasks:tmp_tasks = tasks