PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix old eval tasks order (#78)
WeiweiZhang1257c5be2cded
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 54abf173c..ef8dc6cbd 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -34,8 +34,8 @@ from .utils import (get_module,get_scale_shape,htcore,- is_hpu_available,is_local_path,+ is_optimum_habana_available,logger,move_input_to_device,quant_weight,@@ -43,10 +43,6 @@ from .utils import (set_module,)-if is_hpu_available:- import habana_frameworks.torch.core as htcore # pylint: disable=E0401- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401-class WrapperLinear(torch.nn.Module):def __init__(self, orig_layer, enable_minmax_tuning=True):@@ -513,6 +509,10 @@ class AutoRound(object):self.share_attention_mask_flag = Noneself.hidden_dim_flag = Nonetorch.set_printoptions(precision=3, sci_mode=True)+ if is_optimum_habana_available():+ logger.info("Optimum Habana is available, import htcore explicitly.")+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401def get_optimizer(self, optimizer):"""Returns the specified optimizer. In SignRound, we fix the optimizer.@@ -543,7 +543,7 @@ class AutoRound(object):"""scale_loss = loss * 1000scale_loss.backward()- if is_hpu_available:+ if is_optimum_habana_available():htcore.mark_step()return scale_loss@@ -560,7 +560,7 @@ class AutoRound(object):"""optimizer.step()# for hpu- if is_hpu_available:+ if is_optimum_habana_available():htcore.mark_step()optimizer.zero_grad()lr_schedule.step()@@ -1566,7 +1566,7 @@ class AutoOPTRound(AutoRound):loss = scaler.scale(loss)loss.backward()- if is_hpu_available:+ if is_optimum_habana_available():htcore.mark_step()return loss@@ -1580,7 +1580,7 @@ class AutoOPTRound(AutoRound):optimizer.step()optimizer.zero_grad()lr_schedule.step()- if is_hpu_available:+ if is_optimum_habana_available():htcore.mark_step()diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f53939daa..d7cf32a42 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -33,27 +33,49 @@ fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(linefh.setFormatter(fh_formatter)logger.addHandler(fh)+import importlib-def is_optimum_habana_available():- import importlib- from transformers.utils.import_utils import is_optimum_available+class LazyImport(object):+ """Lazy import python module till use."""- return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not None+ def __init__(self, module_name):+ """Init LazyImport object.+ Args:+ module_name (string): The name of module imported later+ """+ self.module_name = module_name+ self.module = None-try:- import habana_frameworks.torch.core as htcore- import habana_frameworks.torch.hpu as hthpu+ def __getattr__(self, name):+ """Get the attributes of the module by name."""+ try:+ self.module = importlib.import_module(self.module_name)+ mod = getattr(self.module, name)+ except:+ spec = importlib.util.find_spec(str(self.module_name + "." + name))+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ return mod- if is_optimum_habana_available():- is_hpu_available = True- else:- print("Should install optimum-habana when the environment has habana frameworks")- is_hpu_available = False-except ImportError:- is_hpu_available = False- htcore = None+ def __call__(self, *args, **kwargs):+ """Call the function in that module."""+ function_name = self.module_name.split(".")[-1]+ module_name = self.module_name.split(f".{function_name}")[0]+ self.module = importlib.import_module(module_name)+ function = getattr(self.module, function_name)+ return function(*args, **kwargs)+++auto_gptq = LazyImport("auto_gptq")+htcore = LazyImport("habana_frameworks.torch.core")+++def is_optimum_habana_available():+ from transformers.utils.import_utils import is_optimum_available++ return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not Nonedef round_ste(x: torch.Tensor):@@ -507,7 +529,7 @@ def detect_device(device=None):if torch.cuda.is_available():device = torch.device("cuda")logger.info("Using GPU device")- elif is_hpu_available:+ elif is_optimum_habana_available():device = torch.device("hpu")logger.info("Using HPU device")# Use CPU as a fallbackdiff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex adc6bdef3..9c3b62f31 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -10,9 +10,6 @@ import shutilimport transformersfrom typing import TYPE_CHECKING, Optional, Unionimport time---if __name__ == "__main__":import sys@@ -344,7 +341,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "raise ImportError("""follow requirements to install dependencies.""")import collectionsorg_s = time.time()-+ print(f"Using {dtype} as evaluation data type.")external_tasks = []for each in EXT_TASKS:if each in tasks:@@ -375,7 +372,10 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,max_batch_size=tmp_eval_bs, lm=lm, device=str(device))- sub_name = f'{tmp_tasks} {shot}-shot'+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:+ sub_name = f'hendrycksTest-* {shot}-shot'+ else:+ sub_name = f'{tmp_tasks} {shot}-shot'print(f'{sub_name}: ')pprint.pprint(tmp_results["results"])print(f"\n{sub_name} cost time: {time.time() - task_s}\n")@@ -438,6 +438,22 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "new_dict["wikitext 0-shot_word_perplexity"] = 0new_dict["wikitext 0-shot_byte_perplexity"] = 0new_dict["wikitext 0-shot_bits_per_byte"] = 0++ # Special handling of mmlu for compatibility with the old excel results sequence+ search_str = 'hendrycksTest'+ mmlu_matching_keys = [key for key in results.keys() if search_str.lower() in key.lower()]+ for key in mmlu_matching_keys:+ data = results.pop(key)+ for sub_key in data.keys():+ for sub_sub_key in data[sub_key].keys():+ new_key = key + "-" + sub_key + "-" + sub_sub_key+ if "std" in new_key or "alias" in new_key:+ continue+ new_key = new_key.split(",")[0]+ new_dict[new_key] = data[sub_key][sub_sub_key]+ new_key = new_key + "_norm"+ new_dict[new_key] = data[sub_key][sub_sub_key]+for key in results.keys():if key == "model" or key == "paper-avg" or key == "leaderboard-avg":continue@@ -447,14 +463,12 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "continuefor sub_key in data.keys():for sub_sub_key in data[sub_key].keys():- if "mmlu" in key:- new_key = key + "-" + sub_key + "-" + sub_sub_key- else:- new_key = key + "_" + sub_sub_key+ new_key = key + "_" + sub_sub_keyif "std" in new_key or "alias" in new_key:continue- new_key = new_key.rstrip(",none")+ new_key = new_key.split(",")[0]new_dict[new_key] = data[sub_key][sub_sub_key]+import pandas as pddf = pd.DataFrame(data=new_dict, index=[0])@@ -497,7 +511,3 @@ if __name__ == "__main__":eval_bs=args.eval_bs, limit=None, excel_file=excel_name)print("cost time: ", time.time() - s)----diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.pyindex 2ba6a29f9..bafff6c59 100644--- a/examples/language-modeling/eval_legacy/evaluation.py+++ b/examples/language-modeling/eval_legacy/evaluation.py@@ -211,7 +211,7 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winograndfrom lm_eval.tasks import ALL_TASKS, get_task_dictexcept:raise ImportError("""follow requirements to install dependencies.""")-+ print(f"Using {dtype} as evaluation data type.")org_s = time.time()external_tasks = []@@ -353,3 +353,5 @@ if __name__ == "__main__":++diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 3a65084c7..dea8b935f 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -93,9 +93,8 @@ if __name__ == '__main__':help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")parser.add_argument("--tasks",- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",- "arc_easy", "arc_challenge"],+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge,wikitext2,ptb-new,c4-new",help="lm-eval tasks for lm_eval version 0.4")parser.add_argument("--output_dir", default="./tmp_autoround", type=str,@@ -162,7 +161,9 @@ if __name__ == '__main__':if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:tmp_tasks = taskstasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]- tasks = list(set(tasks))+ seen = set()+ tmp_tasks = tasks+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]if isinstance(args.tasks, str):tasks = ','.join(tasks)@@ -302,3 +303,4 @@ if __name__ == '__main__':eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,device=torch_device, excel_file=excel_name)+