PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support to use lm_eval for vlm (#670)
Heng Guo16b8a26235dd
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex d1c5af73f..1a8c885d7 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -58,10 +58,14 @@ def run_fast():def run_mllm():if "--eval" in sys.argv:- from auto_round.script.mllm import setup_lmeval_parser, eval+ from auto_round.script.llm import setup_eval_parser, evalsys.argv.remove("--eval")- args = setup_lmeval_parser()+ args = setup_eval_parser()+ args.mllm = Trueeval(args)+ elif "--vlmeval" in sys.argv:+ sys.argv.remove("--vlmeval")+ run_vlmeavl()elif "--lmms" in sys.argv:sys.argv.remove("--lmms")run_lmms()@@ -76,6 +80,12 @@ def run_lmms():args = setup_lmms_parser()lmms_eval(args)+def run_vlmeavl():+ from auto_round.script.mllm import setup_lmeval_parser, vlmeval+ args = setup_lmeval_parser()+ vlmeval(args)++def switch():if "--mllm" in sys.argv:sys.argv.remove("--mllm")diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex a7fd648ed..efa2579d3 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -210,6 +210,9 @@ class EvalArgumentParser(argparse.ArgumentParser):super().__init__(*args, **kwargs)self.add_argument("--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")+ self.add_argument(+ "--mllm", default=False, help="whether to eval multi-modal model."+ )self.add_argument("--device","--devices",@@ -672,8 +675,15 @@ def eval(args):print("evaluation running time=%ds" % (time.time() - st))else:st = time.time()+ if "auto" in str(batch_size) and args.mllm:+ logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")+ batch_size = 16res = simple_evaluate(- model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=batch_size)+ model="hf" if not args.mllm else "hf-multimodal",+ model_args=model_args,+ tasks=tasks,+ device=device_str,+ batch_size=batch_size)from lm_eval.utils import make_table # pylint: disable=E0401print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex c25c6180b..676a2bbf8 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -474,7 +474,7 @@ def tune(args):clear_memory()-def eval(args):+def vlmeval(args):set_cuda_visible_devices(args.device)device_str, parallelism = get_device_and_parallelism(args.device)if parallelism:diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex d6cd50391..ece1c8c1a 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -23,9 +23,9 @@ class LLMDataLoader:class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):- model_name = "Qwen/Qwen2.5-0.5B-Instruct"- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_name = "Qwen/Qwen2.5-0.5B-Instruct"+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@classmethod@@ -56,6 +56,7 @@ class TestAutoRound(unittest.TestCase):llm = Llama(f"saved/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)+ shutil.rmtree("./saved", ignore_errors=True)save_dir = os.path.join(os.path.dirname(__file__), "saved")model_path = "Qwen/Qwen2.5-0.5B-Instruct"@@ -67,10 +68,11 @@ class TestAutoRound(unittest.TestCase):self.assertFalse(res > 0 or res == -1, msg="qwen2 tuning fail")from llama_cpp import Llama- gguf_file = os.listdir("saved/Qwen2.5-0.5B-Instruct-w4g32")[0]- llm = Llama(f"saved/Qwen2.5-0.5B-Instruct-w4g32/{gguf_file}", n_gpu_layers=-1)+ gguf_file = os.listdir("saved/Qwen2.5-0.5B-Instruct-gguf")[0]+ llm = Llama(f"saved/Qwen2.5-0.5B-Instruct-gguf/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)+ shutil.rmtree("./saved", ignore_errors=True)@require_ggufdef test_q2_k_export(self):@@ -111,7 +113,7 @@ class TestAutoRound(unittest.TestCase):python_path = sys.executableres = os.system(f"cd ../.. && {python_path} -m auto_round --model {self.model_name} --eval_task_by_task"- f" --tasks piqa,openbookqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0"+ f" --tasks piqa,openbookqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0 --eval_model_dtype bf16")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"@@ -141,7 +143,7 @@ class TestAutoRound(unittest.TestCase):from auto_round.eval.evaluation import simple_evaluate_user_modelresult = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")- self.assertGreater(result['results']['piqa']['acc,none'], 0.55)+ self.assertAlmostEqual(result['results']['piqa']['acc,none'], 0.55, delta=0.01)shutil.rmtree("./saved", ignore_errors=True)@require_gguf@@ -168,7 +170,7 @@ class TestAutoRound(unittest.TestCase):from auto_round.eval.evaluation import simple_evaluate_user_modelresult = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")- self.assertGreater(result['results']['piqa']['acc,none'], 0.55)+ self.assertAlmostEqual(result['results']['piqa']['acc,none'], 0.55, delta=0.01)shutil.rmtree("./saved", ignore_errors=True)@require_gguf@@ -244,7 +246,7 @@ class TestAutoRound(unittest.TestCase):model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"from auto_round.mllm.autoround_mllm import AutoRoundMLLMfrom auto_round.utils import mllm_load_model- model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ model, processor, tokenizer, image_processor = mllm_load_model(model_name, use_auto_mapping=False)autoround = AutoRoundMLLM(model,tokenizer=tokenizer,