Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable cmmlu,ceval eval, Qwen acc shell and readme update

Lu, Yintong630385d9dac2
diff --git a/README.md b/README.md
index 7d7e05605..24c5af468 100644
--- a/README.md
+++ b/README.md
@@ -155,6 +155,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |
+| Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh), [example](./examples/language-modeling/) |
| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |
| huggyllama/llama-7b | [example](./examples/language-modeling/) |
diff --git a/docs/Qwen1.5-7B-Chat-acc.md b/docs/Qwen1.5-7B-Chat-acc.md
new file mode 100644
index 000000000..3b362a842
--- /dev/null
+++ b/docs/Qwen1.5-7B-Chat-acc.md
@@ -0,0 +1,14 @@
+Due to licensing restrictions, we are unable to release the model.
+
+Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d.
+
+
+
+| Metric | FP16 | [INT8-gptq](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int8) | [INT4-gptq](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int4) | INT4 recipe1 | INT4 recipe2 |
+| -------------- | ------ | -------- | --------- | -----------------| -----------------|
+| Avg. | 0.6220 | 0.6218 | 0.6064 | 0.6207 | 0.6185 |
+| ceval | 0.6988 | 0.6956 | 0.6763 | 0.6855 | 0.6913 |
+| cmmlu | 0.6879 | 0.6876 | 0.6729 | 0.6783 | 0.6776 |
+| mmlu | 0.6016 | 0.6013 | 0.5902 | 0.5966 | 0.5970 |
+| gsm8k | 0.4996 | 0.5027 | 0.4860 | 0.5224 | 0.5080 |
+
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 9d4824b6b..0a9b55dc1 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -34,6 +34,7 @@ fewshots_dict['paper'] = {
"rte": [0],
"arc_easy": [0],
"arc_challenge": [0],
+ "gsm8k": [0],
}
fewshots_dict['leadboard'] = {
"hellaswag": [10],
@@ -351,9 +352,24 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
results = {}
model = None
lm = None
+ if 'cmmlu' in tasks:
+ tasks.remove('cmmlu')
+ from eval.utils import CMMLU_SUBJECTS
+ for key in CMMLU_SUBJECTS.keys():
+ tasks.append('cmmlu_' + key)
+
+ if 'ceval' in tasks:
+ tasks.remove('ceval')
+ from eval.utils import CEVAL_SUBJECTS
+ for key in CEVAL_SUBJECTS.keys():
+ tasks.append('ceval-valid_' + key)
+
for tmp_tasks in tasks:
try:
- num_fewshot = fewshots_dict[mark][tmp_tasks]
+ if 'cmmlu' in tmp_tasks or 'ceval' in tmp_tasks:
+ num_fewshot = [0]
+ else:
+ num_fewshot = fewshots_dict[mark][tmp_tasks]
print(f'********* {tmp_tasks} evaluate ************')
task_s = time.time()
for shot in num_fewshot:
diff --git a/examples/language-modeling/eval/utils.py b/examples/language-modeling/eval/utils.py
index 2a3a2e4c5..4a2690a00 100644
--- a/examples/language-modeling/eval/utils.py
+++ b/examples/language-modeling/eval/utils.py
@@ -213,3 +213,132 @@ def eval_ppl_same_with_gptq(model, testenc, dev):
if hasattr(model.config, "use_cache"):
model.config.use_cache = use_cache
return ppl.item()
+
+
+
+
+CMMLU_SUBJECTS = {
+ "agronomy": "农学",
+ "anatomy": "解剖学",
+ "ancient_chinese": "古汉语",
+ "arts": "艺术学",
+ "astronomy": "天文学",
+ "business_ethics": "商业伦理",
+ "chinese_civil_service_exam": "中国公务员考试",
+ "chinese_driving_rule": "中国驾驶规则",
+ "chinese_food_culture": "中国饮食文化",
+ "chinese_foreign_policy": "中国外交政策",
+ "chinese_history": "中国历史",
+ "chinese_literature": "中国文学",
+ "chinese_teacher_qualification": "中国教师资格",
+ "clinical_knowledge": "临床知识",
+ "college_actuarial_science": "大学精算学",
+ "college_education": "大学教育学",
+ "college_engineering_hydrology": "大学工程水文学",
+ "college_law": "大学法律",
+ "college_mathematics": "大学数学",
+ "college_medical_statistics": "大学医学统计",
+ "college_medicine": "大学医学",
+ "computer_science": "计算机科学",
+ "computer_security": "计算机安全",
+ "conceptual_physics": "概念物理学",
+ "construction_project_management": "建设工程管理",
+ "economics": "经济学",
+ "education": "教育学",
+ "electrical_engineering": "电气工程",
+ "elementary_chinese": "小学语文",
+ "elementary_commonsense": "小学常识",
+ "elementary_information_and_technology": "小学信息技术",
+ "elementary_mathematics": "初等数学",
+ "ethnology": "民族学",
+ "food_science": "食品科学",
+ "genetics": "遗传学",
+ "global_facts": "全球事实",
+ "high_school_biology": "高中生物",
+ "high_school_chemistry": "高中化学",
+ "high_school_geography": "高中地理",
+ "high_school_mathematics": "高中数学",
+ "high_school_physics": "高中物理学",
+ "high_school_politics": "高中政治",
+ "human_sexuality": "人类性行为",
+ "international_law": "国际法学",
+ "journalism": "新闻学",
+ "jurisprudence": "法理学",
+ "legal_and_moral_basis": "法律与道德基础",
+ "logical": "逻辑学",
+ "machine_learning": "机器学习",
+ "management": "管理学",
+ "marketing": "市场营销",
+ "marxist_theory": "马克思主义理论",
+ "modern_chinese": "现代汉语",
+ "nutrition": "营养学",
+ "philosophy": "哲学",
+ "professional_accounting": "专业会计",
+ "professional_law": "专业法学",
+ "professional_medicine": "专业医学",
+ "professional_psychology": "专业心理学",
+ "public_relations": "公共关系",
+ "security_study": "安全研究",
+ "sociology": "社会学",
+ "sports_science": "体育学",
+ "traditional_chinese_medicine": "中医中药",
+ "virology": "病毒学",
+ "world_history": "世界历史",
+ "world_religions": "世界宗教",
+}
+
+
+CEVAL_SUBJECTS = {
+ "computer_network": "计算机网络",
+ "operating_system": "操作系统",
+ "computer_architecture": "计算机组成",
+ "college_programming": "大学编程",
+ "college_physics": "大学物理",
+ "college_chemistry": "大学化学",
+ "advanced_mathematics": "高等数学",
+ "probability_and_statistics": "概率统计",
+ "discrete_mathematics": "离散数学",
+ "electrical_engineer": "注册电气工程师",
+ "metrology_engineer": "注册计量师",
+ "high_school_mathematics": "高中数学",
+ "high_school_physics": "高中物理",
+ "high_school_chemistry": "高中化学",
+ "high_school_biology": "高中生物",
+ "middle_school_mathematics": "初中数学",
+ "middle_school_biology": "初中生物",
+ "middle_school_physics": "初中物理",
+ "middle_school_chemistry": "初中化学",
+ "veterinary_medicine": "兽医学",
+ "college_economics": "大学经济学",
+ "business_administration": "工商管理",
+ "marxism": "马克思主义基本原理",
+ "mao_zedong_thought": "毛泽东思想和中国特色社会主义理论体系概论",
+ "education_science": "教育学",
+ "teacher_qualification": "教师资格",
+ "high_school_politics": "高中政治",
+ "high_school_geography": "高中地理",
+ "middle_school_politics": "初中政治",
+ "middle_school_geography": "初中地理",
+ "modern_chinese_history": "近代史纲要",
+ "ideological_and_moral_cultivation": "思想道德修养与法律基础",
+ "logic": "逻辑学",
+ "law": "法学",
+ "chinese_language_and_literature": "中国语言文学",
+ "art_studies": "艺术学",
+ "professional_tour_guide": "导游资格",
+ "legal_professional": "法律职业资格",
+ "high_school_chinese": "高中语文",
+ "high_school_history": "高中历史",
+ "middle_school_history": "初中历史",
+ "civil_servant": "公务员",
+ "sports_science": "体育学",
+ "plant_protection": "植物保护",
+ "basic_medicine": "基础医学",
+ "clinical_medicine": "临床医学",
+ "urban_and_rural_planner": "注册城乡规划师",
+ "accountant": "注册会计师",
+ "fire_engineer": "注册消防工程师",
+ "environmental_impact_assessment_engineer": "环境影响评价工程师",
+ "tax_accountant": "税务师",
+ "physician": "医师资格",
+}
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh
new file mode 100644
index 000000000..036a07aae
--- /dev/null
+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh
@@ -0,0 +1,34 @@
+## Due to licensing restrictions, we are unable to release the model.
+#Recipe 1
+python3 main.py \
+--model_name Qwen/Qwen1.5-7B-Chat\
+--bits 4 \
+--group_size 128 \
+--enable_minmax_tuning \
+--amp \
+--use_quant_input \
+--n_samples 512 \
+--iters 200 \
+--deployment_device fake,gpu \
+--tasks mmlu,ceval,cmmlu,gsm8k \
+--sym \
+--eval_bs 32 \
+--minmax_lr 0.01
+
+
+
+# Recipe 2
+python3 main.py \
+--model_name Qwen/Qwen1.5-7B-Chat\
+--bits 4 \
+--group_size 128 \
+--enable_minmax_tuning \
+--amp \
+--use_quant_input \
+--n_samples 512 \
+--iters 1000 \
+--deployment_device fake,gpu \
+--tasks mmlu,ceval,cmmlu,gsm8k \
+--sym \
+--eval_bs 32 \
+--minmax_lr 0.002