PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add limit arg for eval (#764)
Heng Guoadd749ac515a
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex f426a8a11..9722b6696 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -26,6 +26,7 @@ def simple_evaluate_user_model(user_model,tokenizer,batch_size: Optional[int] = 1,+ limit: Optional[Union[int, float]] = None,max_batch_size: Optional[int] = 64,eval_model_dtype="auto",add_bos_token: bool = False,@@ -40,7 +41,7 @@ def simple_evaluate_user_model(add_bos_token=add_bos_token,)return lm_simple_evaluate(- model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs+ model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, limit=limit, **kwargs)@@ -48,6 +49,7 @@ def simple_evaluate(model,model_args: Optional[Union[str, dict]] = None,batch_size: Optional[int] = None,+ limit: Optional[Union[int, float]] = None,max_batch_size: Optional[int] = None,device: Optional[str] = None,**kwargs@@ -61,6 +63,7 @@ def simple_evaluate(model=model,model_args=model_args,batch_size=batch_size,+ limit=limit,max_batch_size=max_batch_size,device=device,**kwargsdiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex c4d2fb306..c65dd9b23 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -80,8 +80,6 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")-self.add_argument("--device_map","--device",@@ -125,26 +123,10 @@ class BasicArgumentParser(argparse.ArgumentParser):help="scale data type to use for quantization",)- self.add_argument(- "--tasks",- "--task",- nargs="?",- const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"- "openbookqa,boolq,arc_easy,arc_challenge",- default=None,- help="lm-eval tasks",- )-self.add_argument("--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model")- self.add_argument(- "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"- )-- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")-self.add_argument("--disable_amp", action="store_true", help="disable amp")self.add_argument(@@ -220,10 +202,6 @@ class BasicArgumentParser(argparse.ArgumentParser):"--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms.")- self.add_argument(- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."- )-self.add_argument("--disable_opt_rtn",action="store_true",@@ -255,6 +233,38 @@ class BasicArgumentParser(argparse.ArgumentParser):help="the template for building training dataset. It can be a custom one.",)+ ## ======================= eval =======================+ self.add_argument(+ "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"+ )++ self.add_argument(+ "--tasks",+ "--task",+ nargs="?",+ const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"+ "openbookqa,boolq,arc_easy,arc_challenge",+ default=None,+ help="lm-eval tasks",+ )++ self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")++ self.add_argument(+ "--limit",+ type=float,+ default=None,+ metavar="N|0<N<1",+ help="Limit the number of examples per task. "+ "If <1, limit is a percentage of the total number of examples.",+ )++ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")++ self.add_argument(+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."+ )+class EvalArgumentParser(argparse.ArgumentParser):@@ -292,6 +302,14 @@ class EvalArgumentParser(argparse.ArgumentParser):self.add_argument("--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation.")+ self.add_argument(+ "--limit",+ type=float,+ default=None,+ metavar="N|0<N<1",+ help="Limit the number of examples per task. "+ "If <1, limit is a percentage of the total number of examples.",+ )def setup_parser():@@ -683,6 +701,7 @@ def tune(args):tokenizer=tokenizer,device=device_str,tasks=args.tasks,+ limit=args.limit,batch_size=args.eval_bs,eval_model_dtype=eval_model_dtype,)@@ -701,6 +720,7 @@ def tune(args):tokenizer,tasks=tasks,batch_size=args.eval_bs,+ limit=args.limit,device=device_str,eval_model_dtype=eval_model_dtype,add_bos_token=add_bos_token,@@ -714,6 +734,7 @@ def tune(args):device=device_str,tasks=args.tasks,batch_size=args.eval_bs,+ limit=args.limit,eval_model_dtype=eval_model_dtype,)else:@@ -726,7 +747,12 @@ def tune(args):if "llama" in args.model.lower():model_args += ",add_bos_token=True"res = simple_evaluate(- model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs+ model="hf",+ model_args=model_args,+ tasks=tasks,+ device=device_str,+ batch_size=args.eval_bs,+ limit=args.limit,)print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))@@ -788,7 +814,9 @@ def eval(args):)model.eval()st = time.time()- res = simple_evaluate_user_model(model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)+ res = simple_evaluate_user_model(+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit+ )print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))else:@@ -802,6 +830,7 @@ def eval(args):tasks=tasks,device=device_str,batch_size=batch_size,+ limit=args.limit,)from lm_eval.utils import make_table # pylint: disable=E0401@@ -815,6 +844,7 @@ def eval_task_by_task(tasks=None,tokenizer=None,batch_size=None,+ limit=None,max_batch_size=64,trust_remote_code=True,eval_model_dtype=None,@@ -887,7 +917,7 @@ def eval_task_by_task(while retry_times:try:res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit)breakexcept Exception as e:@@ -899,7 +929,7 @@ def eval_task_by_task(hflm.batch_sizes[k] = max(v // 2, 1)logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit)hflm.batch_sizes = ori_batch_sizesexcept Exception as e:diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex ce5a625b2..a82abef00 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -3,6 +3,8 @@ import shutilimport sysimport unittest+from parameterized import parameterized+sys.path.insert(0, "../..")import torch@@ -46,6 +48,7 @@ class TestAutoRound(unittest.TestCase):raise ValueError(f"Expected bits to be 8, but got {module.bits}")def test_remove_whole_block(self):+ model_name = "facebook/opt-125m"layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},"model.decoder.layers.0.self_attn.v_proj": {"bits": 32},@@ -56,8 +59,7 @@ class TestAutoRound(unittest.TestCase):}bits, group_size, sym = 4, 128, Falseautoround = AutoRound(- self.model,- self.tokenizer,+ model_name,bits=bits,group_size=group_size,sym=sym,@@ -97,19 +99,46 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()def test_mx_fp4(self):+ model_name = "facebook/opt-125m"bits, group_size, sym = 4, 32, Falseautoround = AutoRound(- self.model,- self.tokenizer,+ model_name,+ bits=bits,+ act_bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ nsamples=2,+ seqlen=128,+ data_type="mx_fp4",+ act_data_type="mx_fp_rceil",+ )+ model, _ = autoround.quantize()+ result = simple_evaluate_user_model(+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32+ )+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.3) # 0.375++ def test_nv_fp4(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 16, False+ autoround = AutoRound(+ model_name,bits=bits,group_size=group_size,sym=sym,iters=2,seqlen=2,dataset=self.llm_dataloader,- data_type="mx_fp4",+ data_type="nv_fp4",)- autoround.quantize()+ model, _ = autoround.quantize()+ result = simple_evaluate_user_model(+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32+ )+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)def test_default(self):bits, group_size, sym = 4, 128, False@@ -134,10 +163,10 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir="./saved", inplace=False)def test_w4g1(self):+ model_name = "facebook/opt-125m"bits, group_size, sym = 4, -1, Trueautoround = AutoRound(- self.model,- self.tokenizer,+ model_name,bits=bits,group_size=group_size,sym=sym,@@ -147,11 +176,12 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()- def test_w3g128(self):- bits, group_size, sym = 3, 128, True+ @parameterized.expand([(2,), (3,), (4,)])+ def test_g128(self, bits):+ model_name = "facebook/opt-125m"+ group_size, sym = 128, Trueautoround = AutoRound(- self.model,- self.tokenizer,+ model_name,bits=bits,group_size=group_size,sym=sym,@@ -159,10 +189,16 @@ class TestAutoRound(unittest.TestCase):seqlen=10,dataset=self.llm_dataloader,)- autoround.quantize()+ model, _ = autoround.quantize()+ if bits > 2:+ result = simple_evaluate_user_model(+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32+ )+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.3)- def test_w2g128(self):- bits, group_size, sym = 2, 128, True+ def test_disable_quanted_input(self):+ bits, group_size, sym = 4, -1, Trueautoround = AutoRound(self.model,self.tokenizer,@@ -171,21 +207,25 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,+ enable_quanted_input=False,dataset=self.llm_dataloader,)autoround.quantize()- def test_disable_quanted_input(self):- bits, group_size, sym = 4, -1, True+ def test_enable_norm_bias_tuning_qwen3(self):+ bits, group_size, sym = 4, 128, True+ model_name = "Qwen/Qwen3-0.6B"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)autoround = AutoRound(- self.model,- self.tokenizer,+ model,+ tokenizer,bits=bits,group_size=group_size,sym=sym,iters=2,seqlen=10,- enable_quanted_input=False,+ enable_norm_bias_tuning=True,dataset=self.llm_dataloader,)autoround.quantize()@@ -206,24 +246,6 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()- def test_enable_norm_bias_tuning_qwen3(self):- bits, group_size, sym = 4, 128, True- model_name = "Qwen/Qwen3-0.6B"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=10,- enable_norm_bias_tuning=True,- dataset=self.llm_dataloader,- )- autoround.quantize()-def test_disable_minmax_tuning(self):bits, group_size, sym = 4, -1, Trueautoround = AutoRound(@@ -241,10 +263,10 @@ class TestAutoRound(unittest.TestCase):#def test_signround(self):+ model_name = "facebook/opt-125m"bits, group_size, sym = 4, -1, Falseautoround = AutoRound(- self.model,- self.tokenizer,+ model_name,bits=bits,group_size=group_size,sym=sym,@@ -275,17 +297,17 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()def test_wa_quant(self):+ model_name = "facebook/opt-125m"bits, group_size, sym, act_bits = 4, 128, False, 4autoround = AutoRound(- self.model,- self.tokenizer,+ model_name,bits=bits,group_size=group_size,sym=sym,iters=2,seqlen=2,dataset=self.llm_dataloader,- act_bits=4,+ act_bits=act_bits,)autoround.quantize()@@ -473,7 +495,7 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},}autoround = AutoRound(@@ -509,7 +531,7 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}}autoround = AutoRound(@@ -545,8 +567,8 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},- "model.decoder.layers.1.self_attn.k_proj": {"bits": 16},+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ r"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},}autoround = AutoRound(model,