Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable cn_tasks and fix bugs (#45)

yintong-lu259932543d77
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index cb3806a3d..6681607a5 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -34,6 +34,9 @@ fewshots_dict['paper'] = {
"rte": [0],
"arc_easy": [0],
"arc_challenge": [0],
+ "gsm8k": [0],
+ "ceval-valid": [0],
+ "cmmlu": [0],
}
fewshots_dict['leadboard'] = {
"hellaswag": [10],
@@ -375,7 +378,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
pprint.pprint(tmp_results["results"])
print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
results[sub_name] = {}
- if 'mmlu' in tmp_tasks:
+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
for cata in ['humanities', 'other', 'stem', 'sociology']:
results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']
results[sub_name]['avg'] = tmp_results['results'][f'mmlu']
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index c6863a8e8..256fe241c 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -143,13 +143,12 @@ if __name__ == '__main__':
if res == "0.3.0":
use_eval_legacy = True
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
if not use_eval_legacy:
from eval import eval_model
else:
from eval_legacy import eval_model
-
- if isinstance(tasks, str):
- tasks = tasks.split(',')
if isinstance(tasks, list):
if "mmlu" in tasks:
tmp_tasks = tasks