PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
upgrade lm_eval
Zhang, Weiwei16ba09c6775e1
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex b0a423993..000e12c4b 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -20,9 +20,10 @@ fewshots_dict['paper'] = {"hellaswag": [0],"winogrande": [0],"piqa": [0],- "hendrycksTest-*": [0],+ "mmlu": [0],"wikitext": [0],- "truthfulqa_mc": [0],+ "truthfulqa_mc1": [0],+ "truthfulqa_mc2": [0],"openbookqa": [0],"boolq": [0],"rte": [0],@@ -34,7 +35,7 @@ fewshots_dict['leadboard'] = {"winogrande": [5],"arc_easy": [25],"arc_challenge": [25],- "hendrycksTest-*": [5],+ "mmlu": [5],"drop": [3],"gsm8k": [5],}@@ -44,13 +45,14 @@ fewshots_dict['all'] = {"winogrande": [0, 5],"piqa": [0],"coqa": [], ## coqa is not enabled in llamav1 models- "truthfulqa_mc": [0],+ "truthfulqa_mc1": [0],+ "truthfulqa_mc2": [0],"openbookqa": [0],"boolq": [0],"rte": [0],"arc_easy": [0, 25],"arc_challenge": [0, 25],- "hendrycksTest-*": [0, 5],+ "mmlu": [0, 5],"wikitext": [0],"drop": [3],"gsm8k": [5]@@ -58,27 +60,27 @@ fewshots_dict['all'] = {def simple_evaluate(- model,- model_args=None,- tasks=[],- num_fewshot=0,- batch_size=None,- max_batch_size=None,- device=None,- no_cache=True,- limit=None,- bootstrap_iters=100000,- description_dict=None,- check_integrity=False,- decontamination_ngrams_path=None,- write_out=False,- output_base_path=None,- lm=None+ model,+ model_args=None,+ tasks=[],+ num_fewshot=None,+ batch_size=None,+ max_batch_size=None,+ device=None,+ use_cache=None,+ limit=None,+ bootstrap_iters: int = 100000,+ check_integrity: bool = False,+ decontamination_ngrams_path=None,+ write_out: bool = False,+ log_samples: bool = True,+ gen_kwargs: str = None,+ lm=None):"""Instantiate and evaluate a model on a list of tasks.:param model: Union[str, LM]- Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model+ Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model:param model_args: Optional[str]String arguments for each model class, see LM.create_from_arg_string.Ignored if `model` argument is a LM object.@@ -92,46 +94,67 @@ def simple_evaluate(Maximal batch size to try with automatic batch size detection:param device: str, optionalPyTorch device (e.g. "cpu" or "cuda:0") for running models- :param no_cache: bool- Whether or not to cache+ :param use_cache: str, optional+ A path to a sqlite db file for caching model responses. `None` if not caching.:param limit: int or float, optionalLimit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.:param bootstrap_iters:Number of iterations for bootstrap statistics- :param description_dict: dict[str, str]- Dictionary of custom task descriptions of the form: `task_name: description`:param check_integrity: boolWhether to run the relevant part of the test suite for the tasks:param write_out: bool- If True, write details about prompts and logits to json for all tasks- :param output_base_path: str, optional- Directory to which detailed eval info will be written. Defaults to present working dir.+ If True, write out an example document and model input for checking task integrity+ :param log_samples: bool+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis+ :param gen_kwargs: str+ String arguments for model generation+ Ignored for all tasks with loglikelihood output_type:returnDictionary of results"""- import collections- import itertoolsimport random-- import lm_eval.metrics- import lm_eval.models- import lm_eval.tasks- import lm_eval.base- from lm_eval.utils import positional_deprecated, run_task_tests- from lm_eval.models.gpt2 import HFLM-import numpy as npimport transformers+ import lm_eval.api+ import lm_eval.tasks+ import lm_eval.models+ import lm_eval.api.metrics+ import lm_eval.api.registryfrom lm_eval.evaluator import evaluate+ lm_eval.tasks.initialize_tasks()++ from lm_eval.utils import (+ positional_deprecated,+ run_task_tests,+ make_table,+ get_git_commit_hash,+ simple_parse_args_string,+ eval_logger,+ )+random.seed(1234)np.random.seed(1234)+ torch.manual_seed(+ 1234+ ) # TODO: this may affect training runs that are run with evaluation mid-run.++ assert (+ tasks != []+ ), "No tasks specified, or no tasks found. Please verify the task names."- assert tasks != [], "No tasks specified"if lm == None:+ if gen_kwargs is not None:+ gen_kwargs = simple_parse_args_string(gen_kwargs)+ eval_logger.warning(+ f"generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks."+ )+ if gen_kwargs == "":+ gen_kwargs = None+if isinstance(model, str):if model_args is None:model_args = ""- lm = lm_eval.models.get_model(model).create_from_arg_string(+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(model_args,{"batch_size": batch_size,@@ -140,18 +163,19 @@ def simple_evaluate(},)elif isinstance(model, transformers.PreTrainedModel):- lm = lm_eval.models.get_model("hf-causal")(+ lm = lm_eval.api.registry.get_model("hf-causal")(pretrained=model,batch_size=batch_size,max_batch_size=max_batch_size,)- no_cache = True+ use_cache = Noneelse:- assert isinstance(model, lm_eval.base.LM)+ assert isinstance(model, lm_eval.api.model.LM)lm = model- if not no_cache:- lm = lm_eval.base.CachingLM(+ if use_cache is not None:+ print(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")+ lm = lm_eval.api.model.CachingLM(lm,"lm_cache/"+ (model if isinstance(model, str) else model.model.config._name_or_path)@@ -161,6 +185,31 @@ def simple_evaluate()task_dict = lm_eval.tasks.get_task_dict(tasks)+ for task_name in task_dict.keys():+ task_obj = task_dict[task_name]+ if type(task_obj) == tuple:+ group, task_obj = task_obj+ if task_obj is None:+ continue++ config = task_obj._config+ if config["output_type"] == "generate_until" and gen_kwargs is not None:+ config["generation_kwargs"].update(gen_kwargs)++ if num_fewshot is not None:+ if config["num_fewshot"] == 0:+ eval_logger.info(+ f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."+ )+ else:+ # default_num_fewshot = config["num_fewshot"]+ # eval_logger.warning(+ # f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"+ # )++ task_obj._config["num_fewshot"] = num_fewshot+ else:+ num_fewshot = config["num_fewshot"]if check_integrity:run_task_tests(task_list=tasks)@@ -168,37 +217,35 @@ def simple_evaluate(results = evaluate(lm=lm,task_dict=task_dict,- num_fewshot=num_fewshot,limit=limit,bootstrap_iters=bootstrap_iters,- description_dict=description_dict,decontamination_ngrams_path=decontamination_ngrams_path,write_out=write_out,- output_base_path=output_base_path,+ log_samples=log_samples,)- # add info about the model and few shot config- model_name = None- if isinstance(model, str):- model_name = model- elif isinstance(model, transformers.PreTrainedModel):- model_name = "pretrained=" + model.config._name_or_path- results["config"] = {- "model": model_name,- "model_args": model_args,- "num_fewshot": num_fewshot,- "batch_size": batch_size,- "batch_sizes": list(lm.batch_sizes.values())- if hasattr(lm, "batch_sizes")- else [],- "device": device,- "no_cache": no_cache,- "limit": limit,- "bootstrap_iters": bootstrap_iters,- "description_dict": description_dict,- }-- return results, lm+ if lm.rank == 0:+ # add info about the model and few shot config+ results["config"] = {+ "model": model+ if isinstance(model, str)+ else model.model.config._name_or_path,+ "model_args": model_args,+ "batch_size": batch_size,+ "batch_sizes": list(lm.batch_sizes.values())+ if hasattr(lm, "batch_sizes")+ else [],+ "device": device,+ "use_cache": use_cache,+ "limit": limit,+ "num_fewshot": num_fewshot,+ "bootstrap_iters": bootstrap_iters,+ "gen_kwargs": gen_kwargs,+ }+ results["git_hash"] = get_git_commit_hash()+ return results, lm+ else:+ return Nonedef eval_model(output_dir=None, model=None, tokenizer=None,@@ -207,12 +254,16 @@ def eval_model(output_dir=None, model=None, tokenizer=None,device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):print("evaluation with official lm-eval", flush=True)try:- import lm_eval- from lm_eval import evaluator- from lm_eval.tasks import ALL_TASKS, get_task_dict+ import lm_eval.api+ import lm_eval.tasks+ import lm_eval.models+ import lm_eval.api.metrics+ import lm_eval.api.registry+ from lm_eval.evaluator import evaluate+except:raise ImportError("""follow requirements to install dependencies.""")-+ import collectionsorg_s = time.time()##save modelif output_dir is None:@@ -238,8 +289,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,model.save_pretrained(output_dir)tokenizer.save_pretrained(output_dir)--external_tasks = []for each in EXT_TASKS:if each in tasks:@@ -252,29 +301,32 @@ def eval_model(output_dir=None, model=None, tokenizer=None,for tmp_tasks in tasks:try:num_fewshot = fewshots_dict[mark][tmp_tasks]- task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)print(f'********* {tmp_tasks} evaluate ************')task_s = time.time()for shot in num_fewshot:- if bool(re.search("chatglm", output_dir.lower())):- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'- model_type = "hf-causal"- else:- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'- model_type = "hf-causal-experimental"+ model_type = "hf"+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'+ if use_accelerate: # bool(re.search("chatglm", output_dir.lower()))+ model_args += f',parallelize=True'- if "wikitext" in task_names:+ if "wikitext" in tmp_tasks:tmp_eval_bs = 1else:tmp_eval_bs = eval_bs- tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,max_batch_size=tmp_eval_bs, lm=lm, device=str(device))sub_name = f'{tmp_tasks} {shot}-shot'print(f'{sub_name}: ')pprint.pprint(tmp_results["results"])print(f"\n{sub_name} cost time: {time.time() - task_s}\n")- results[sub_name] = tmp_results+ results[sub_name] = {}+ if 'mmlu' in tmp_tasks:+ for cata in ['humanities', 'other', 'stem', 'sociology']:+ results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']+ results[sub_name]['avg'] = tmp_results['results'][f'mmlu']+ else:+ results[sub_name] = tmp_results['results']except Exception as e:print(f'********* {tmp_tasks} ERROR ************')print(str(e))@@ -314,7 +366,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,continueprint(results, flush=True)- new_results = result_parser(results)print("cost time: ", time.time() - org_s)import picklefrom collections import OrderedDict@@ -328,23 +379,22 @@ def eval_model(output_dir=None, model=None, tokenizer=None,new_dict["wikitext 0-shot_word_perplexity"] = 0new_dict["wikitext 0-shot_byte_perplexity"] = 0new_dict["wikitext 0-shot_bits_per_byte"] = 0- for key in new_results.keys():+ for key in results.keys():if key == "model" or key == "paper-avg" or key == "leaderboard-avg":continue- data = new_results[key]+ data = results[key]if not isinstance(data, dict):- new_dict[key] = new_results[key]+ new_dict[key] = results[key]continuefor sub_key in data.keys():for sub_sub_key in data[sub_key].keys():- if "hendry" in key:+ if "mmlu" in key:new_key = key + "-" + sub_key + "-" + sub_sub_keyelse:new_key = key + "_" + sub_sub_key- if "std" in new_key:+ if "std" in new_key or "alias" in new_key:continue- # if "norm" in new_key:- # continue+ new_key = new_key.rstrip(",none")new_dict[new_key] = data[sub_key][sub_sub_key]import pandas as pd@@ -367,8 +417,8 @@ if __name__ == "__main__":s = time.time()test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",- "arc_challenge"]+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",+ "arc_easy", "arc_challenge"]model_name = args.model_name.rstrip('/')excel_name = model_name.split('/')[-1] + ".xlsx"eval_model(output_dir=args.model_name,diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex d0f4c5804..7835995fa 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -1,6 +1,5 @@import argparseimport sys-sys.path.insert(0, '../..')from auto_round import (AutoRound,AutoAdamRound)@@ -11,7 +10,6 @@ import osos.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"torch.use_deterministic_algorithms(True, warn_only=True)-import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer, AutoModelfrom transformers import set_seed@@ -22,6 +20,8 @@ import reos.environ["TOKENIZERS_PARALLELISM"] = "false"++if __name__ == '__main__':parser.add_argument(@@ -42,7 +42,7 @@ if __name__ == '__main__':parser.add_argument("--device", default=0, type=str,help="device gpu int number, or 'cpu' ")-+parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -85,23 +85,24 @@ if __name__ == '__main__':parser.add_argument("--enable_minmax_tuning", action='store_true',help="whether enable weight minmax tuning")-+parser.add_argument("--deployment_device", default='fake', type=str,help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")-+parser.add_argument("--scale_dtype", default='fp32',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")-+parser.add_argument("--tasks",default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte","arc_easy", "arc_challenge"],help="lm-eval tasks")parser.add_argument("--output_dir", default="./tmp_autoround", type=str,help="Where to store the final model.")+args = parser.parse_args()set_seed(args.seed)@@ -118,7 +119,11 @@ if __name__ == '__main__':device_str = f"cuda:{int(args.device)}"torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))- if is_glm:+ is_llava = bool(re.search("llava", model_name.lower()))+ if is_llava:+ from transformers import LlavaForConditionalGeneration+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")+ elif is_glm:model = AutoModel.from_pretrained(model_name, trust_remote_code=True)else:model = AutoModelForCausalLM.from_pretrained(