PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update eval (#450)
Heng Guod51bd5e0f65d
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex ec4026fd5..246b66fa1 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -14,15 +14,20 @@import sysdef run_eval():- if "--native" in sys.argv:- sys.argv.remove("--native")- from auto_round.script.llm import setup_eval_parser, eval- args = setup_eval_parser()- eval(args)+ from auto_round.script.llm import setup_eval_parser+ args = setup_eval_parser()+ if args.eval_task_by_task:+ from auto_round.script.llm import eval_task_by_task+ eval_task_by_task(+ model=args.model,+ device=args.device,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ trust_remote_code=not args.disable_trust_remote_code)else:- from auto_round.script.llm import setup_eval_parser, eval_sequence- args = setup_eval_parser()- eval_sequence(args)+ from auto_round.script.llm import eval+ eval(args)+def run():if "--eval" in sys.argv:diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ceab10a26..dc9b4b3ff 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -718,7 +718,7 @@ class AutoRound(object):self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()except RuntimeError as e:- if "CUDA out of memory" in str(e):+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):logger.info("switch to cpu to cache block inputs")if (("lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] < 16) orself.__class__.__name__ == "AutoRoundMLLM"):diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex db9edba9c..054960df3 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -109,6 +109,11 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--disable_eval", action='store_true', help="whether to do lm-eval evaluation after tuning")+ self.add_argument(+ "--eval_task_by_task",+ action="store_true",+ help="whether to eval task by task.")+self.add_argument("--disable_amp", action='store_true', help="disable amp")self.add_argument(@@ -194,17 +199,21 @@ class EvalArgumentParser(argparse.ArgumentParser):default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")- self.add_argument("--tasks",- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \- "openbookqa,boolq,arc_easy,arc_challenge",- help="lm-eval tasks")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.")+ self.add_argument(+ "--tasks",+ "--task",+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"+ "openbookqa,boolq,arc_easy,arc_challenge",+ help="lm-eval tasks")self.add_argument("--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")+ self.add_argument("--eval_task_by_task", action='store_true', help="whether to eval task by task.")+def setup_parser():@@ -566,8 +575,6 @@ def tune(args):logger.info(f"Using lm-eval version {lm_eval_version}")- tasks, model_args, device_str = _eval_init(args.tasks, eval_folder, args.device, args.disable_trust_remote_code)-if args.act_bits <= 8:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_model@@ -578,16 +585,25 @@ def tune(args):device_str = detect_device(device_str)user_model = model.to(device_str)- if args.eval_bs is None or args.eval_bs == "auto":- args.eval_bs = 16- from auto_round.eval.evaluation import simple_evaluate_user_model- res = simple_evaluate_user_model(- user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs, device=device_str)+ if args.eval_task_by_task:+ eval_task_by_task(user_model, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)+ else:+ if args.eval_bs is None:+ args.eval_bs = "auto"+ from auto_round.eval.evaluation import simple_evaluate_user_model+ res = simple_evaluate_user_model(+ user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs, device=device_str)+ print(make_table(res))else:- from auto_round.eval.evaluation import simple_evaluate- res = simple_evaluate(- model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)- print(make_table(res))+ if args.eval_task_by_task:+ eval_task_by_task(eval_folder, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)+ else:+ from auto_round.eval.evaluation import simple_evaluate+ tasks, model_args, device_str = _eval_init(+ args.tasks, eval_folder, args.device, args.disable_trust_remote_code)+ res = simple_evaluate(+ model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)+ print(make_table(res))def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):@@ -614,21 +630,56 @@ def eval(args):print(make_table(res))-def eval_sequence(args):- tasks, model_args, device_str = _eval_init(args.tasks, args.model, args.device, args.disable_trust_remote_code)+def eval_task_by_task(model, device, tasks, batch_size=None, max_batch_size=64, trust_remote_code=True):+ set_cuda_visible_devices(device)+ device_str, parallelism = get_device_and_parallelism(device)# load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES- from auto_round.eval.evaluation import simple_evaluate+ import traceback+ from auto_round.utils import logger+ from lm_eval import simple_evaluate as lm_simple_evaluate+ from lm_eval.models.huggingface import HFLM++ # from auto_round import AutoRoundConfig+ if batch_size is None:+ batch_size = "auto"+ if not isinstance(model, str):+ parallelism = False+ hflm = HFLM(+ pretrained=model,+ device=device_str,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ parallelize=parallelism,+ trust_remote_code=trust_remote_code)++ if isinstance(tasks, str):+ tasks = tasks.replace(" ", "").split(",")from lm_eval.utils import make_table # pylint: disable=E0401res_all = {}res_keys = ["results", "versions", "n-shot", "higher_is_better"]for task in tasks:- res = simple_evaluate(model="hf", model_args=model_args, tasks=task, device=device_str, batch_size=args.eval_bs)+ try:+ res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ try:+ logger.warning("Out of memory, reset batch_size to 1 and re-try.")+ res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)+ except Exception as e:+ traceback.print_exc()+ continue+ else:+ traceback.print_exc()+ continue+ except Exception as e:+ traceback.print_exc()+ continue+if not res_all:res_all = reselse:for key in res_keys:res_all[key].update(res[key])print(make_table(res_all))-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f863cd3a4..45de7573e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1164,10 +1164,10 @@ def get_device_and_parallelism(device):device = "cuda"parallelism = Trueelif device == "auto":- device = detect_device(device)- parallelism = True+ device = detect_device(device)+ parallelism = Trueelse:- device = detect_device(device)+ device = detect_device(device)parallelism = Falsereturn device, parallelism@@ -1198,4 +1198,3 @@ def is_debug_mode():bool: True if debugging is enabled, False otherwise."""return sys.gettrace() is not None or sys.flags.debug == 1-diff --git a/test/test_basic_usage.py b/test/test_basic_usage.pyindex 78b0a4f47..af93af4bf 100644--- a/test/test_basic_usage.py+++ b/test/test_basic_usage.py@@ -26,9 +26,13 @@ class TestAutoRoundCmd(unittest.TestCase):assert False, "cmd line test fail, please have a check"res = os.system(- f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")+ f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa,openbookqa")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -m auto_round --model './saved' --eval_task_by_task --tasks piqa,openbookqa --bs 32"+ )# test mllm script