Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

upgrade lm_eval

Zhang, Weiwei16ba09c6775e1
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index b0a423993..000e12c4b 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -20,9 +20,10 @@ fewshots_dict['paper'] = {
"hellaswag": [0],
"winogrande": [0],
"piqa": [0],
- "hendrycksTest-*": [0],
+ "mmlu": [0],
"wikitext": [0],
- "truthfulqa_mc": [0],
+ "truthfulqa_mc1": [0],
+ "truthfulqa_mc2": [0],
"openbookqa": [0],
"boolq": [0],
"rte": [0],
@@ -34,7 +35,7 @@ fewshots_dict['leadboard'] = {
"winogrande": [5],
"arc_easy": [25],
"arc_challenge": [25],
- "hendrycksTest-*": [5],
+ "mmlu": [5],
"drop": [3],
"gsm8k": [5],
}
@@ -44,13 +45,14 @@ fewshots_dict['all'] = {
"winogrande": [0, 5],
"piqa": [0],
"coqa": [], ## coqa is not enabled in llamav1 models
- "truthfulqa_mc": [0],
+ "truthfulqa_mc1": [0],
+ "truthfulqa_mc2": [0],
"openbookqa": [0],
"boolq": [0],
"rte": [0],
"arc_easy": [0, 25],
"arc_challenge": [0, 25],
- "hendrycksTest-*": [0, 5],
+ "mmlu": [0, 5],
"wikitext": [0],
"drop": [3],
"gsm8k": [5]
@@ -58,27 +60,27 @@ fewshots_dict['all'] = {
def simple_evaluate(
- model,
- model_args=None,
- tasks=[],
- num_fewshot=0,
- batch_size=None,
- max_batch_size=None,
- device=None,
- no_cache=True,
- limit=None,
- bootstrap_iters=100000,
- description_dict=None,
- check_integrity=False,
- decontamination_ngrams_path=None,
- write_out=False,
- output_base_path=None,
- lm=None
+ model,
+ model_args=None,
+ tasks=[],
+ num_fewshot=None,
+ batch_size=None,
+ max_batch_size=None,
+ device=None,
+ use_cache=None,
+ limit=None,
+ bootstrap_iters: int = 100000,
+ check_integrity: bool = False,
+ decontamination_ngrams_path=None,
+ write_out: bool = False,
+ log_samples: bool = True,
+ gen_kwargs: str = None,
+ lm=None
):
"""Instantiate and evaluate a model on a list of tasks.
:param model: Union[str, LM]
- Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model
+ Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model
:param model_args: Optional[str]
String arguments for each model class, see LM.create_from_arg_string.
Ignored if `model` argument is a LM object.
@@ -92,46 +94,67 @@ def simple_evaluate(
Maximal batch size to try with automatic batch size detection
:param device: str, optional
PyTorch device (e.g. "cpu" or "cuda:0") for running models
- :param no_cache: bool
- Whether or not to cache
+ :param use_cache: str, optional
+ A path to a sqlite db file for caching model responses. `None` if not caching.
:param limit: int or float, optional
Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
:param bootstrap_iters:
Number of iterations for bootstrap statistics
- :param description_dict: dict[str, str]
- Dictionary of custom task descriptions of the form: `task_name: description`
:param check_integrity: bool
Whether to run the relevant part of the test suite for the tasks
:param write_out: bool
- If True, write details about prompts and logits to json for all tasks
- :param output_base_path: str, optional
- Directory to which detailed eval info will be written. Defaults to present working dir.
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :param gen_kwargs: str
+ String arguments for model generation
+ Ignored for all tasks with loglikelihood output_type
:return
Dictionary of results
"""
- import collections
- import itertools
import random
-
- import lm_eval.metrics
- import lm_eval.models
- import lm_eval.tasks
- import lm_eval.base
- from lm_eval.utils import positional_deprecated, run_task_tests
- from lm_eval.models.gpt2 import HFLM
-
import numpy as np
import transformers
+ import lm_eval.api
+ import lm_eval.tasks
+ import lm_eval.models
+ import lm_eval.api.metrics
+ import lm_eval.api.registry
from lm_eval.evaluator import evaluate
+ lm_eval.tasks.initialize_tasks()
+
+ from lm_eval.utils import (
+ positional_deprecated,
+ run_task_tests,
+ make_table,
+ get_git_commit_hash,
+ simple_parse_args_string,
+ eval_logger,
+ )
+
random.seed(1234)
np.random.seed(1234)
+ torch.manual_seed(
+ 1234
+ ) # TODO: this may affect training runs that are run with evaluation mid-run.
+
+ assert (
+ tasks != []
+ ), "No tasks specified, or no tasks found. Please verify the task names."
- assert tasks != [], "No tasks specified"
if lm == None:
+ if gen_kwargs is not None:
+ gen_kwargs = simple_parse_args_string(gen_kwargs)
+ eval_logger.warning(
+ f"generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks."
+ )
+ if gen_kwargs == "":
+ gen_kwargs = None
+
if isinstance(model, str):
if model_args is None:
model_args = ""
- lm = lm_eval.models.get_model(model).create_from_arg_string(
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
model_args,
{
"batch_size": batch_size,
@@ -140,18 +163,19 @@ def simple_evaluate(
},
)
elif isinstance(model, transformers.PreTrainedModel):
- lm = lm_eval.models.get_model("hf-causal")(
+ lm = lm_eval.api.registry.get_model("hf-causal")(
pretrained=model,
batch_size=batch_size,
max_batch_size=max_batch_size,
)
- no_cache = True
+ use_cache = None
else:
- assert isinstance(model, lm_eval.base.LM)
+ assert isinstance(model, lm_eval.api.model.LM)
lm = model
- if not no_cache:
- lm = lm_eval.base.CachingLM(
+ if use_cache is not None:
+ print(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
+ lm = lm_eval.api.model.CachingLM(
lm,
"lm_cache/"
+ (model if isinstance(model, str) else model.model.config._name_or_path)
@@ -161,6 +185,31 @@ def simple_evaluate(
)
task_dict = lm_eval.tasks.get_task_dict(tasks)
+ for task_name in task_dict.keys():
+ task_obj = task_dict[task_name]
+ if type(task_obj) == tuple:
+ group, task_obj = task_obj
+ if task_obj is None:
+ continue
+
+ config = task_obj._config
+ if config["output_type"] == "generate_until" and gen_kwargs is not None:
+ config["generation_kwargs"].update(gen_kwargs)
+
+ if num_fewshot is not None:
+ if config["num_fewshot"] == 0:
+ eval_logger.info(
+ f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
+ )
+ else:
+ # default_num_fewshot = config["num_fewshot"]
+ # eval_logger.warning(
+ # f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
+ # )
+
+ task_obj._config["num_fewshot"] = num_fewshot
+ else:
+ num_fewshot = config["num_fewshot"]
if check_integrity:
run_task_tests(task_list=tasks)
@@ -168,37 +217,35 @@ def simple_evaluate(
results = evaluate(
lm=lm,
task_dict=task_dict,
- num_fewshot=num_fewshot,
limit=limit,
bootstrap_iters=bootstrap_iters,
- description_dict=description_dict,
decontamination_ngrams_path=decontamination_ngrams_path,
write_out=write_out,
- output_base_path=output_base_path,
+ log_samples=log_samples,
)
- # add info about the model and few shot config
- model_name = None
- if isinstance(model, str):
- model_name = model
- elif isinstance(model, transformers.PreTrainedModel):
- model_name = "pretrained=" + model.config._name_or_path
- results["config"] = {
- "model": model_name,
- "model_args": model_args,
- "num_fewshot": num_fewshot,
- "batch_size": batch_size,
- "batch_sizes": list(lm.batch_sizes.values())
- if hasattr(lm, "batch_sizes")
- else [],
- "device": device,
- "no_cache": no_cache,
- "limit": limit,
- "bootstrap_iters": bootstrap_iters,
- "description_dict": description_dict,
- }
-
- return results, lm
+ if lm.rank == 0:
+ # add info about the model and few shot config
+ results["config"] = {
+ "model": model
+ if isinstance(model, str)
+ else model.model.config._name_or_path,
+ "model_args": model_args,
+ "batch_size": batch_size,
+ "batch_sizes": list(lm.batch_sizes.values())
+ if hasattr(lm, "batch_sizes")
+ else [],
+ "device": device,
+ "use_cache": use_cache,
+ "limit": limit,
+ "num_fewshot": num_fewshot,
+ "bootstrap_iters": bootstrap_iters,
+ "gen_kwargs": gen_kwargs,
+ }
+ results["git_hash"] = get_git_commit_hash()
+ return results, lm
+ else:
+ return None
def eval_model(output_dir=None, model=None, tokenizer=None,
@@ -207,12 +254,16 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
- import lm_eval
- from lm_eval import evaluator
- from lm_eval.tasks import ALL_TASKS, get_task_dict
+ import lm_eval.api
+ import lm_eval.tasks
+ import lm_eval.models
+ import lm_eval.api.metrics
+ import lm_eval.api.registry
+ from lm_eval.evaluator import evaluate
+
except:
raise ImportError("""follow requirements to install dependencies.""")
-
+ import collections
org_s = time.time()
##save model
if output_dir is None:
@@ -238,8 +289,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
-
-
external_tasks = []
for each in EXT_TASKS:
if each in tasks:
@@ -252,29 +301,32 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
for tmp_tasks in tasks:
try:
num_fewshot = fewshots_dict[mark][tmp_tasks]
- task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)
print(f'********* {tmp_tasks} evaluate ************')
task_s = time.time()
for shot in num_fewshot:
- if bool(re.search("chatglm", output_dir.lower())):
- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'
- model_type = "hf-causal"
- else:
- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'
- model_type = "hf-causal-experimental"
+ model_type = "hf"
+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'
+ if use_accelerate: # bool(re.search("chatglm", output_dir.lower()))
+ model_args += f',parallelize=True'
- if "wikitext" in task_names:
+ if "wikitext" in tmp_tasks:
tmp_eval_bs = 1
else:
tmp_eval_bs = eval_bs
- tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,
+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,
num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
sub_name = f'{tmp_tasks} {shot}-shot'
print(f'{sub_name}: ')
pprint.pprint(tmp_results["results"])
print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
- results[sub_name] = tmp_results
+ results[sub_name] = {}
+ if 'mmlu' in tmp_tasks:
+ for cata in ['humanities', 'other', 'stem', 'sociology']:
+ results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']
+ results[sub_name]['avg'] = tmp_results['results'][f'mmlu']
+ else:
+ results[sub_name] = tmp_results['results']
except Exception as e:
print(f'********* {tmp_tasks} ERROR ************')
print(str(e))
@@ -314,7 +366,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
continue
print(results, flush=True)
- new_results = result_parser(results)
print("cost time: ", time.time() - org_s)
import pickle
from collections import OrderedDict
@@ -328,23 +379,22 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
new_dict["wikitext 0-shot_word_perplexity"] = 0
new_dict["wikitext 0-shot_byte_perplexity"] = 0
new_dict["wikitext 0-shot_bits_per_byte"] = 0
- for key in new_results.keys():
+ for key in results.keys():
if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
continue
- data = new_results[key]
+ data = results[key]
if not isinstance(data, dict):
- new_dict[key] = new_results[key]
+ new_dict[key] = results[key]
continue
for sub_key in data.keys():
for sub_sub_key in data[sub_key].keys():
- if "hendry" in key:
+ if "mmlu" in key:
new_key = key + "-" + sub_key + "-" + sub_sub_key
else:
new_key = key + "_" + sub_sub_key
- if "std" in new_key:
+ if "std" in new_key or "alias" in new_key:
continue
- # if "norm" in new_key:
- # continue
+ new_key = new_key.rstrip(",none")
new_dict[new_key] = data[sub_key][sub_sub_key]
import pandas as pd
@@ -367,8 +417,8 @@ if __name__ == "__main__":
s = time.time()
test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",
- "arc_challenge"]
+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
+ "arc_easy", "arc_challenge"]
model_name = args.model_name.rstrip('/')
excel_name = model_name.split('/')[-1] + ".xlsx"
eval_model(output_dir=args.model_name,
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index d0f4c5804..7835995fa 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -1,6 +1,5 @@
import argparse
import sys
-
sys.path.insert(0, '../..')
from auto_round import (AutoRound,
AutoAdamRound)
@@ -11,7 +10,6 @@ import os
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=True)
-import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
from transformers import set_seed
@@ -22,6 +20,8 @@ import re
os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
if __name__ == '__main__':
parser.add_argument(
@@ -42,7 +42,7 @@ if __name__ == '__main__':
parser.add_argument("--device", default=0, type=str,
help="device gpu int number, or 'cpu' ")
-
+
parser.add_argument("--sym", action='store_true',
help=" sym quantization")
@@ -85,23 +85,24 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="whether enable weight minmax tuning")
-
+
parser.add_argument("--deployment_device", default='fake', type=str,
help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',"
"default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
-
+
parser.add_argument("--scale_dtype", default='fp32',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
-
+
parser.add_argument("--tasks",
default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",
+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
"arc_easy", "arc_challenge"],
help="lm-eval tasks")
parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
help="Where to store the final model.")
+
args = parser.parse_args()
set_seed(args.seed)
@@ -118,7 +119,11 @@ if __name__ == '__main__':
device_str = f"cuda:{int(args.device)}"
torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
- if is_glm:
+ is_llava = bool(re.search("llava", model_name.lower()))
+ if is_llava:
+ from transformers import LlavaForConditionalGeneration
+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")
+ elif is_glm:
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(