PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Adjust the default evaluation data type by selecting from the model path configuration (#107)
WeiweiZhang1c42eaa92dc43
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex 33505088c..2e498a260 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -326,7 +326,7 @@ def simple_evaluate(def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],- eval_bs=32, use_accelerate=True, dtype="float16", limit=None,+ eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):print("evaluation with official lm-eval", flush=True)try:@@ -339,9 +339,18 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "except:raise ImportError("""follow requirements to install dependencies.""")- import collections+org_s = time.time()+ if dtype == None:+ from eval.utils import convert_dtype_torch2str_hf+ from transformers import AutoConfig+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)+ if hasattr(config, "torch_dtype"):+ dtype = convert_dtype_torch2str_hf(config.torch_dtype)+ else:+ dtype = "float16"print(f"Using {dtype} as evaluation data type.")+external_tasks = []if isinstance(tasks, str):tasks = tasks.split(',')@@ -361,9 +370,9 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "task_s = time.time()for shot in num_fewshot:model_type = "hf"- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'if 'gpu' in model_path:- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code=True'+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code={trust_remote_code}'if use_accelerate: # bool(re.search("chatglm", model_path.lower()))model_args += f',parallelize=True'@@ -513,3 +522,4 @@ if __name__ == "__main__":eval_bs=args.eval_bs, limit=None, excel_file=excel_name)print("cost time: ", time.time() - s)+diff --git a/examples/language-modeling/eval/utils.py b/examples/language-modeling/eval/utils.pyindex 37a43e8f5..9514c811c 100644--- a/examples/language-modeling/eval/utils.py+++ b/examples/language-modeling/eval/utils.py@@ -215,3 +215,84 @@ def eval_ppl_same_with_gptq(model, testenc, dev):return ppl.item()+def convert_dtype_str2torch(str_dtype):+ """Converts a string dtype to its corresponding PyTorch dtype.++ Args:+ str_dtype (str): The string representation of the dtype.++ Returns:+ torch.dtype: The PyTorch dtype.++ Raises:+ AssertionError: If the input str_dtype is unsupported.+ """+ if isinstance(str_dtype, torch.dtype) or str_dtype is None:+ return str_dtype+ if str_dtype == "int8":+ return torch.int8+ elif str_dtype == "fp32" or str_dtype == "auto":+ return torch.float+ elif str_dtype == "fp16":+ return torch.float16+ elif str_dtype == "bf16":+ return torch.bfloat16+ else:+ assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)+++def convert_dtype_torch2str(dtype):+ """Converts a PyTorch dtype to its corresponding string representation.++ Args:+ dtype: PyTorch dtype or str. The dtype to convert.++ Returns:+ str: The string representation of the dtype.++ Raises:+ AssertionError: If the input dtype is unsupported.+ """+ if isinstance(dtype, str) or dtype is None:+ return dtype+ if dtype == torch.int8:+ return "int8"+ elif dtype == torch.float:+ return "fp32"+ elif dtype == torch.float16:+ return "fp16"+ elif dtype == torch.bfloat16:+ return "bf16"+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:+ return dtype+ else:+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+++def convert_dtype_torch2str_hf(dtype):+ """Converts a PyTorch dtype to its corresponding huggingface string dtype, e.g. torch.float32 -> 'float32'.++ Args:+ dtype: PyTorch dtype or str. The dtype to convert.++ Returns:+ str: The string representation of the dtype.++ Raises:+ AssertionError: If the input str_dtype is unsupported.+ """+ if dtype is None:+ return dtype+ if isinstance(dtype, str):+ if "float" not in dtype and "int" not in dtype:+ dtype = convert_dtype_str2torch(dtype)+ else:+ return dtype+ str_dtype = str(dtype)+ if "." not in str_dtype:+ assert False, "Unsupported pytorch dtype {} to huggingface str dtype".format(dtype)+ str_dtype = str_dtype.split(".")[1]+ return str_dtype+++diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.pyindex 1bb2593e6..c9f5b88e5 100644--- a/examples/language-modeling/eval_legacy/evaluation.py+++ b/examples/language-modeling/eval_legacy/evaluation.py@@ -202,17 +202,28 @@ def simple_evaluate(def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],- eval_bs=32, use_accelerate=True, dtype="float16", limit=None,+ eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):print("evaluation with official lm-eval", flush=True)try:import lm_evalfrom lm_eval import evaluatorfrom lm_eval.tasks import ALL_TASKS, get_task_dict+ import pandas as pdexcept:raise ImportError("""follow requirements to install dependencies.""")- print(f"Using {dtype} as evaluation data type.")+org_s = time.time()+ if dtype == None:+ from eval.utils import convert_dtype_torch2str_hf+ from transformers import AutoConfig+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)+ if hasattr(config, "torch_dtype"):+ dtype = convert_dtype_torch2str_hf(config.torch_dtype)+ else:+ dtype = "float16"+ print(f"Using {dtype} as evaluation data type.")+external_tasks = []if isinstance(tasks, str):@@ -233,10 +244,10 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winograndtask_s = time.time()for shot in num_fewshot:if bool(re.search("chatglm", model_path.lower())):- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'model_type = "hf-causal"else:- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code={trust_remote_code}'model_type = "hf-causal-experimental"if "wikitext" in task_names:@@ -323,7 +334,6 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand# continuenew_dict[new_key] = data[sub_key][sub_sub_key]- import pandas as pddf = pd.DataFrame(data=new_dict, index=[0])df.to_excel(excel_file)@@ -360,3 +370,4 @@ if __name__ == "__main__":+