Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Adjust the default evaluation data type by selecting from the model path configuration (#107)

WeiweiZhang1c42eaa92dc43
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 33505088c..2e498a260 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -326,7 +326,7 @@ def simple_evaluate(
def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
- eval_bs=32, use_accelerate=True, dtype="float16", limit=None,
+ eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,
device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
@@ -339,9 +339,18 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
except:
raise ImportError("""follow requirements to install dependencies.""")
- import collections
+
org_s = time.time()
+ if dtype == None:
+ from eval.utils import convert_dtype_torch2str_hf
+ from transformers import AutoConfig
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
+ if hasattr(config, "torch_dtype"):
+ dtype = convert_dtype_torch2str_hf(config.torch_dtype)
+ else:
+ dtype = "float16"
print(f"Using {dtype} as evaluation data type.")
+
external_tasks = []
if isinstance(tasks, str):
tasks = tasks.split(',')
@@ -361,9 +370,9 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
task_s = time.time()
for shot in num_fewshot:
model_type = "hf"
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
if 'gpu' in model_path:
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code=True'
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code={trust_remote_code}'
if use_accelerate: # bool(re.search("chatglm", model_path.lower()))
model_args += f',parallelize=True'
@@ -513,3 +522,4 @@ if __name__ == "__main__":
eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
print("cost time: ", time.time() - s)
+
diff --git a/examples/language-modeling/eval/utils.py b/examples/language-modeling/eval/utils.py
index 37a43e8f5..9514c811c 100644
--- a/examples/language-modeling/eval/utils.py
+++ b/examples/language-modeling/eval/utils.py
@@ -215,3 +215,84 @@ def eval_ppl_same_with_gptq(model, testenc, dev):
return ppl.item()
+def convert_dtype_str2torch(str_dtype):
+ """Converts a string dtype to its corresponding PyTorch dtype.
+
+ Args:
+ str_dtype (str): The string representation of the dtype.
+
+ Returns:
+ torch.dtype: The PyTorch dtype.
+
+ Raises:
+ AssertionError: If the input str_dtype is unsupported.
+ """
+ if isinstance(str_dtype, torch.dtype) or str_dtype is None:
+ return str_dtype
+ if str_dtype == "int8":
+ return torch.int8
+ elif str_dtype == "fp32" or str_dtype == "auto":
+ return torch.float
+ elif str_dtype == "fp16":
+ return torch.float16
+ elif str_dtype == "bf16":
+ return torch.bfloat16
+ else:
+ assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)
+
+
+def convert_dtype_torch2str(dtype):
+ """Converts a PyTorch dtype to its corresponding string representation.
+
+ Args:
+ dtype: PyTorch dtype or str. The dtype to convert.
+
+ Returns:
+ str: The string representation of the dtype.
+
+ Raises:
+ AssertionError: If the input dtype is unsupported.
+ """
+ if isinstance(dtype, str) or dtype is None:
+ return dtype
+ if dtype == torch.int8:
+ return "int8"
+ elif dtype == torch.float:
+ return "fp32"
+ elif dtype == torch.float16:
+ return "fp16"
+ elif dtype == torch.bfloat16:
+ return "bf16"
+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
+ return dtype
+ else:
+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)
+
+
+def convert_dtype_torch2str_hf(dtype):
+ """Converts a PyTorch dtype to its corresponding huggingface string dtype, e.g. torch.float32 -> 'float32'.
+
+ Args:
+ dtype: PyTorch dtype or str. The dtype to convert.
+
+ Returns:
+ str: The string representation of the dtype.
+
+ Raises:
+ AssertionError: If the input str_dtype is unsupported.
+ """
+ if dtype is None:
+ return dtype
+ if isinstance(dtype, str):
+ if "float" not in dtype and "int" not in dtype:
+ dtype = convert_dtype_str2torch(dtype)
+ else:
+ return dtype
+ str_dtype = str(dtype)
+ if "." not in str_dtype:
+ assert False, "Unsupported pytorch dtype {} to huggingface str dtype".format(dtype)
+ str_dtype = str_dtype.split(".")[1]
+ return str_dtype
+
+
+
diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.py
index 1bb2593e6..c9f5b88e5 100644
--- a/examples/language-modeling/eval_legacy/evaluation.py
+++ b/examples/language-modeling/eval_legacy/evaluation.py
@@ -202,17 +202,28 @@ def simple_evaluate(
def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
- eval_bs=32, use_accelerate=True, dtype="float16", limit=None,
+ eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,
device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
import lm_eval
from lm_eval import evaluator
from lm_eval.tasks import ALL_TASKS, get_task_dict
+ import pandas as pd
except:
raise ImportError("""follow requirements to install dependencies.""")
- print(f"Using {dtype} as evaluation data type.")
+
org_s = time.time()
+ if dtype == None:
+ from eval.utils import convert_dtype_torch2str_hf
+ from transformers import AutoConfig
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
+ if hasattr(config, "torch_dtype"):
+ dtype = convert_dtype_torch2str_hf(config.torch_dtype)
+ else:
+ dtype = "float16"
+ print(f"Using {dtype} as evaluation data type.")
+
external_tasks = []
if isinstance(tasks, str):
@@ -233,10 +244,10 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
task_s = time.time()
for shot in num_fewshot:
if bool(re.search("chatglm", model_path.lower())):
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
model_type = "hf-causal"
else:
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code={trust_remote_code}'
model_type = "hf-causal-experimental"
if "wikitext" in task_names:
@@ -323,7 +334,6 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
# continue
new_dict[new_key] = data[sub_key][sub_sub_key]
- import pandas as pd
df = pd.DataFrame(data=new_dict, index=[0])
df.to_excel(excel_file)
@@ -360,3 +370,4 @@ if __name__ == "__main__":
+