Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add limit arg for eval (#764)

Heng Guoadd749ac515a
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index f426a8a11..9722b6696 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -26,6 +26,7 @@ def simple_evaluate_user_model(
user_model,
tokenizer,
batch_size: Optional[int] = 1,
+ limit: Optional[Union[int, float]] = None,
max_batch_size: Optional[int] = 64,
eval_model_dtype="auto",
add_bos_token: bool = False,
@@ -40,7 +41,7 @@ def simple_evaluate_user_model(
add_bos_token=add_bos_token,
)
return lm_simple_evaluate(
- model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs
+ model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, limit=limit, **kwargs
)
@@ -48,6 +49,7 @@ def simple_evaluate(
model,
model_args: Optional[Union[str, dict]] = None,
batch_size: Optional[int] = None,
+ limit: Optional[Union[int, float]] = None,
max_batch_size: Optional[int] = None,
device: Optional[str] = None,
**kwargs
@@ -61,6 +63,7 @@ def simple_evaluate(
model=model,
model_args=model_args,
batch_size=batch_size,
+ limit=limit,
max_batch_size=max_batch_size,
device=device,
**kwargs
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index c4d2fb306..c65dd9b23 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -80,8 +80,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
-
self.add_argument(
"--device_map",
"--device",
@@ -125,26 +123,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="scale data type to use for quantization",
)
- self.add_argument(
- "--tasks",
- "--task",
- nargs="?",
- const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
- "openbookqa,boolq,arc_easy,arc_challenge",
- default=None,
- help="lm-eval tasks",
- )
-
self.add_argument(
"--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"
)
- self.add_argument(
- "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"
- )
-
- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
-
self.add_argument("--disable_amp", action="store_true", help="disable amp")
self.add_argument(
@@ -220,10 +202,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
"--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms."
)
- self.add_argument(
- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
- )
-
self.add_argument(
"--disable_opt_rtn",
action="store_true",
@@ -255,6 +233,38 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="the template for building training dataset. It can be a custom one.",
)
+ ## ======================= eval =======================
+ self.add_argument(
+ "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"
+ )
+
+ self.add_argument(
+ "--tasks",
+ "--task",
+ nargs="?",
+ const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
+ "openbookqa,boolq,arc_easy,arc_challenge",
+ default=None,
+ help="lm-eval tasks",
+ )
+
+ self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
+
+ self.add_argument(
+ "--limit",
+ type=float,
+ default=None,
+ metavar="N|0<N<1",
+ help="Limit the number of examples per task. "
+ "If <1, limit is a percentage of the total number of examples.",
+ )
+
+ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
+
+ self.add_argument(
+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
+ )
+
class EvalArgumentParser(argparse.ArgumentParser):
@@ -292,6 +302,14 @@ class EvalArgumentParser(argparse.ArgumentParser):
self.add_argument(
"--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
)
+ self.add_argument(
+ "--limit",
+ type=float,
+ default=None,
+ metavar="N|0<N<1",
+ help="Limit the number of examples per task. "
+ "If <1, limit is a percentage of the total number of examples.",
+ )
def setup_parser():
@@ -683,6 +701,7 @@ def tune(args):
tokenizer=tokenizer,
device=device_str,
tasks=args.tasks,
+ limit=args.limit,
batch_size=args.eval_bs,
eval_model_dtype=eval_model_dtype,
)
@@ -701,6 +720,7 @@ def tune(args):
tokenizer,
tasks=tasks,
batch_size=args.eval_bs,
+ limit=args.limit,
device=device_str,
eval_model_dtype=eval_model_dtype,
add_bos_token=add_bos_token,
@@ -714,6 +734,7 @@ def tune(args):
device=device_str,
tasks=args.tasks,
batch_size=args.eval_bs,
+ limit=args.limit,
eval_model_dtype=eval_model_dtype,
)
else:
@@ -726,7 +747,12 @@ def tune(args):
if "llama" in args.model.lower():
model_args += ",add_bos_token=True"
res = simple_evaluate(
- model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs
+ model="hf",
+ model_args=model_args,
+ tasks=tasks,
+ device=device_str,
+ batch_size=args.eval_bs,
+ limit=args.limit,
)
print(make_table(res))
print("evaluation running time=%ds" % (time.time() - st))
@@ -788,7 +814,9 @@ def eval(args):
)
model.eval()
st = time.time()
- res = simple_evaluate_user_model(model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)
+ res = simple_evaluate_user_model(
+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit
+ )
print(make_table(res))
print("evaluation running time=%ds" % (time.time() - st))
else:
@@ -802,6 +830,7 @@ def eval(args):
tasks=tasks,
device=device_str,
batch_size=batch_size,
+ limit=args.limit,
)
from lm_eval.utils import make_table # pylint: disable=E0401
@@ -815,6 +844,7 @@ def eval_task_by_task(
tasks=None,
tokenizer=None,
batch_size=None,
+ limit=None,
max_batch_size=64,
trust_remote_code=True,
eval_model_dtype=None,
@@ -887,7 +917,7 @@ def eval_task_by_task(
while retry_times:
try:
res = lm_simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit
)
break
except Exception as e:
@@ -899,7 +929,7 @@ def eval_task_by_task(
hflm.batch_sizes[k] = max(v // 2, 1)
logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
res = lm_simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit
)
hflm.batch_sizes = ori_batch_sizes
except Exception as e:
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index ce5a625b2..a82abef00 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -3,6 +3,8 @@ import shutil
import sys
import unittest
+from parameterized import parameterized
+
sys.path.insert(0, "../..")
import torch
@@ -46,6 +48,7 @@ class TestAutoRound(unittest.TestCase):
raise ValueError(f"Expected bits to be 8, but got {module.bits}")
def test_remove_whole_block(self):
+ model_name = "facebook/opt-125m"
layer_config = {
"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},
"model.decoder.layers.0.self_attn.v_proj": {"bits": 32},
@@ -56,8 +59,7 @@ class TestAutoRound(unittest.TestCase):
}
bits, group_size, sym = 4, 128, False
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
@@ -97,19 +99,46 @@ class TestAutoRound(unittest.TestCase):
autoround.quantize()
def test_mx_fp4(self):
+ model_name = "facebook/opt-125m"
bits, group_size, sym = 4, 32, False
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
+ bits=bits,
+ act_bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ nsamples=2,
+ seqlen=128,
+ data_type="mx_fp4",
+ act_data_type="mx_fp_rceil",
+ )
+ model, _ = autoround.quantize()
+ result = simple_evaluate_user_model(
+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32
+ )
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.3) # 0.375
+
+ def test_nv_fp4(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 16, False
+ autoround = AutoRound(
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
iters=2,
seqlen=2,
dataset=self.llm_dataloader,
- data_type="mx_fp4",
+ data_type="nv_fp4",
)
- autoround.quantize()
+ model, _ = autoround.quantize()
+ result = simple_evaluate_user_model(
+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32
+ )
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)
def test_default(self):
bits, group_size, sym = 4, 128, False
@@ -134,10 +163,10 @@ class TestAutoRound(unittest.TestCase):
autoround.save_quantized(output_dir="./saved", inplace=False)
def test_w4g1(self):
+ model_name = "facebook/opt-125m"
bits, group_size, sym = 4, -1, True
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
@@ -147,11 +176,12 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
- def test_w3g128(self):
- bits, group_size, sym = 3, 128, True
+ @parameterized.expand([(2,), (3,), (4,)])
+ def test_g128(self, bits):
+ model_name = "facebook/opt-125m"
+ group_size, sym = 128, True
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
@@ -159,10 +189,16 @@ class TestAutoRound(unittest.TestCase):
seqlen=10,
dataset=self.llm_dataloader,
)
- autoround.quantize()
+ model, _ = autoround.quantize()
+ if bits > 2:
+ result = simple_evaluate_user_model(
+ model, self.tokenizer, batch_size="auto:8", tasks="lambada_openai", limit=32
+ )
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.3)
- def test_w2g128(self):
- bits, group_size, sym = 2, 128, True
+ def test_disable_quanted_input(self):
+ bits, group_size, sym = 4, -1, True
autoround = AutoRound(
self.model,
self.tokenizer,
@@ -171,21 +207,25 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
+ enable_quanted_input=False,
dataset=self.llm_dataloader,
)
autoround.quantize()
- def test_disable_quanted_input(self):
- bits, group_size, sym = 4, -1, True
+ def test_enable_norm_bias_tuning_qwen3(self):
+ bits, group_size, sym = 4, 128, True
+ model_name = "Qwen/Qwen3-0.6B"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model,
+ tokenizer,
bits=bits,
group_size=group_size,
sym=sym,
iters=2,
seqlen=10,
- enable_quanted_input=False,
+ enable_norm_bias_tuning=True,
dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -206,24 +246,6 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
- def test_enable_norm_bias_tuning_qwen3(self):
- bits, group_size, sym = 4, 128, True
- model_name = "Qwen/Qwen3-0.6B"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=10,
- enable_norm_bias_tuning=True,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
-
def test_disable_minmax_tuning(self):
bits, group_size, sym = 4, -1, True
autoround = AutoRound(
@@ -241,10 +263,10 @@ class TestAutoRound(unittest.TestCase):
#
def test_signround(self):
+ model_name = "facebook/opt-125m"
bits, group_size, sym = 4, -1, False
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
@@ -275,17 +297,17 @@ class TestAutoRound(unittest.TestCase):
autoround.quantize()
def test_wa_quant(self):
+ model_name = "facebook/opt-125m"
bits, group_size, sym, act_bits = 4, 128, False, 4
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ model_name,
bits=bits,
group_size=group_size,
sym=sym,
iters=2,
seqlen=2,
dataset=self.llm_dataloader,
- act_bits=4,
+ act_bits=act_bits,
)
autoround.quantize()
@@ -473,7 +495,7 @@ class TestAutoRound(unittest.TestCase):
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
layer_config = {
- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},
}
autoround = AutoRound(
@@ -509,7 +531,7 @@ class TestAutoRound(unittest.TestCase):
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
layer_config = {
- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}
}
autoround = AutoRound(
@@ -545,8 +567,8 @@ class TestAutoRound(unittest.TestCase):
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
layer_config = {
- "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
- "model.decoder.layers.1.self_attn.k_proj": {"bits": 16},
+ r"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ r"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},
}
autoround = AutoRound(
model,