Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Revert "enable cmmlu,ceval eval, Qwen acc shell and readme update"

chensuyue89f22b0afdd1
diff --git a/README.md b/README.md
index 24c5af468..7d7e05605 100644
--- a/README.md
+++ b/README.md
@@ -155,7 +155,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |
-| Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh), [example](./examples/language-modeling/) |
| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |
| huggyllama/llama-7b | [example](./examples/language-modeling/) |
diff --git a/docs/Qwen1.5-7B-Chat-acc.md b/docs/Qwen1.5-7B-Chat-acc.md
deleted file mode 100644
index 3b362a842..000000000
--- a/docs/Qwen1.5-7B-Chat-acc.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Due to licensing restrictions, we are unable to release the model.
-
-Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d.
-
-
-
-| Metric | FP16 | [INT8-gptq](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int8) | [INT4-gptq](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int4) | INT4 recipe1 | INT4 recipe2 |
-| -------------- | ------ | -------- | --------- | -----------------| -----------------|
-| Avg. | 0.6220 | 0.6218 | 0.6064 | 0.6207 | 0.6185 |
-| ceval | 0.6988 | 0.6956 | 0.6763 | 0.6855 | 0.6913 |
-| cmmlu | 0.6879 | 0.6876 | 0.6729 | 0.6783 | 0.6776 |
-| mmlu | 0.6016 | 0.6013 | 0.5902 | 0.5966 | 0.5970 |
-| gsm8k | 0.4996 | 0.5027 | 0.4860 | 0.5224 | 0.5080 |
-
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 0a9b55dc1..9d4824b6b 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -34,7 +34,6 @@ fewshots_dict['paper'] = {
"rte": [0],
"arc_easy": [0],
"arc_challenge": [0],
- "gsm8k": [0],
}
fewshots_dict['leadboard'] = {
"hellaswag": [10],
@@ -352,24 +351,9 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
results = {}
model = None
lm = None
- if 'cmmlu' in tasks:
- tasks.remove('cmmlu')
- from eval.utils import CMMLU_SUBJECTS
- for key in CMMLU_SUBJECTS.keys():
- tasks.append('cmmlu_' + key)
-
- if 'ceval' in tasks:
- tasks.remove('ceval')
- from eval.utils import CEVAL_SUBJECTS
- for key in CEVAL_SUBJECTS.keys():
- tasks.append('ceval-valid_' + key)
-
for tmp_tasks in tasks:
try:
- if 'cmmlu' in tmp_tasks or 'ceval' in tmp_tasks:
- num_fewshot = [0]
- else:
- num_fewshot = fewshots_dict[mark][tmp_tasks]
+ num_fewshot = fewshots_dict[mark][tmp_tasks]
print(f'********* {tmp_tasks} evaluate ************')
task_s = time.time()
for shot in num_fewshot:
diff --git a/examples/language-modeling/eval/utils.py b/examples/language-modeling/eval/utils.py
index 4a2690a00..2a3a2e4c5 100644
--- a/examples/language-modeling/eval/utils.py
+++ b/examples/language-modeling/eval/utils.py
@@ -213,132 +213,3 @@ def eval_ppl_same_with_gptq(model, testenc, dev):
if hasattr(model.config, "use_cache"):
model.config.use_cache = use_cache
return ppl.item()
-
-
-
-
-CMMLU_SUBJECTS = {
- "agronomy": "农学",
- "anatomy": "解剖学",
- "ancient_chinese": "古汉语",
- "arts": "艺术学",
- "astronomy": "天文学",
- "business_ethics": "商业伦理",
- "chinese_civil_service_exam": "中国公务员考试",
- "chinese_driving_rule": "中国驾驶规则",
- "chinese_food_culture": "中国饮食文化",
- "chinese_foreign_policy": "中国外交政策",
- "chinese_history": "中国历史",
- "chinese_literature": "中国文学",
- "chinese_teacher_qualification": "中国教师资格",
- "clinical_knowledge": "临床知识",
- "college_actuarial_science": "大学精算学",
- "college_education": "大学教育学",
- "college_engineering_hydrology": "大学工程水文学",
- "college_law": "大学法律",
- "college_mathematics": "大学数学",
- "college_medical_statistics": "大学医学统计",
- "college_medicine": "大学医学",
- "computer_science": "计算机科学",
- "computer_security": "计算机安全",
- "conceptual_physics": "概念物理学",
- "construction_project_management": "建设工程管理",
- "economics": "经济学",
- "education": "教育学",
- "electrical_engineering": "电气工程",
- "elementary_chinese": "小学语文",
- "elementary_commonsense": "小学常识",
- "elementary_information_and_technology": "小学信息技术",
- "elementary_mathematics": "初等数学",
- "ethnology": "民族学",
- "food_science": "食品科学",
- "genetics": "遗传学",
- "global_facts": "全球事实",
- "high_school_biology": "高中生物",
- "high_school_chemistry": "高中化学",
- "high_school_geography": "高中地理",
- "high_school_mathematics": "高中数学",
- "high_school_physics": "高中物理学",
- "high_school_politics": "高中政治",
- "human_sexuality": "人类性行为",
- "international_law": "国际法学",
- "journalism": "新闻学",
- "jurisprudence": "法理学",
- "legal_and_moral_basis": "法律与道德基础",
- "logical": "逻辑学",
- "machine_learning": "机器学习",
- "management": "管理学",
- "marketing": "市场营销",
- "marxist_theory": "马克思主义理论",
- "modern_chinese": "现代汉语",
- "nutrition": "营养学",
- "philosophy": "哲学",
- "professional_accounting": "专业会计",
- "professional_law": "专业法学",
- "professional_medicine": "专业医学",
- "professional_psychology": "专业心理学",
- "public_relations": "公共关系",
- "security_study": "安全研究",
- "sociology": "社会学",
- "sports_science": "体育学",
- "traditional_chinese_medicine": "中医中药",
- "virology": "病毒学",
- "world_history": "世界历史",
- "world_religions": "世界宗教",
-}
-
-
-CEVAL_SUBJECTS = {
- "computer_network": "计算机网络",
- "operating_system": "操作系统",
- "computer_architecture": "计算机组成",
- "college_programming": "大学编程",
- "college_physics": "大学物理",
- "college_chemistry": "大学化学",
- "advanced_mathematics": "高等数学",
- "probability_and_statistics": "概率统计",
- "discrete_mathematics": "离散数学",
- "electrical_engineer": "注册电气工程师",
- "metrology_engineer": "注册计量师",
- "high_school_mathematics": "高中数学",
- "high_school_physics": "高中物理",
- "high_school_chemistry": "高中化学",
- "high_school_biology": "高中生物",
- "middle_school_mathematics": "初中数学",
- "middle_school_biology": "初中生物",
- "middle_school_physics": "初中物理",
- "middle_school_chemistry": "初中化学",
- "veterinary_medicine": "兽医学",
- "college_economics": "大学经济学",
- "business_administration": "工商管理",
- "marxism": "马克思主义基本原理",
- "mao_zedong_thought": "毛泽东思想和中国特色社会主义理论体系概论",
- "education_science": "教育学",
- "teacher_qualification": "教师资格",
- "high_school_politics": "高中政治",
- "high_school_geography": "高中地理",
- "middle_school_politics": "初中政治",
- "middle_school_geography": "初中地理",
- "modern_chinese_history": "近代史纲要",
- "ideological_and_moral_cultivation": "思想道德修养与法律基础",
- "logic": "逻辑学",
- "law": "法学",
- "chinese_language_and_literature": "中国语言文学",
- "art_studies": "艺术学",
- "professional_tour_guide": "导游资格",
- "legal_professional": "法律职业资格",
- "high_school_chinese": "高中语文",
- "high_school_history": "高中历史",
- "middle_school_history": "初中历史",
- "civil_servant": "公务员",
- "sports_science": "体育学",
- "plant_protection": "植物保护",
- "basic_medicine": "基础医学",
- "clinical_medicine": "临床医学",
- "urban_and_rural_planner": "注册城乡规划师",
- "accountant": "注册会计师",
- "fire_engineer": "注册消防工程师",
- "environmental_impact_assessment_engineer": "环境影响评价工程师",
- "tax_accountant": "税务师",
- "physician": "医师资格",
-}
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh
deleted file mode 100644
index 036a07aae..000000000
--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-## Due to licensing restrictions, we are unable to release the model.
-#Recipe 1
-python3 main.py \
---model_name Qwen/Qwen1.5-7B-Chat\
---bits 4 \
---group_size 128 \
---enable_minmax_tuning \
---amp \
---use_quant_input \
---n_samples 512 \
---iters 200 \
---deployment_device fake,gpu \
---tasks mmlu,ceval,cmmlu,gsm8k \
---sym \
---eval_bs 32 \
---minmax_lr 0.01
-
-
-
-# Recipe 2
-python3 main.py \
---model_name Qwen/Qwen1.5-7B-Chat\
---bits 4 \
---group_size 128 \
---enable_minmax_tuning \
---amp \
---use_quant_input \
---n_samples 512 \
---iters 1000 \
---deployment_device fake,gpu \
---tasks mmlu,ceval,cmmlu,gsm8k \
---sym \
---eval_bs 32 \
---minmax_lr 0.002