Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

make the tuning deterministic and move infrequently used arguments to kwargs (#726)

Wenhua Cheng478aedbf3b49
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index ed2ab46bd..9e7b15294 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -154,40 +154,45 @@ class AutoRound(object):
batch_size: int = 8,
amp: bool = True,
device: Union[str, torch.device, int] = 0,
- lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: int = 0,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
- sampler: str = "rand",
seed: int = 42,
- nblocks: int = 1,
gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
data_type: str = "int",
- scale_dtype: Union[str, torch.dtype] = "fp16",
act_bits: int = 16,
act_group_size: int = None,
act_sym: bool = None,
act_data_type: str = None,
act_dynamic: bool = True,
- to_quant_block_names: Union[str, list] = None,
- enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = False,
device_map: Union[str, dict] = None,
- super_bits: int = None,
- super_group_size: int = None,
disable_opt_rtn: bool = False,
- model_kwargs: dict = None,
**kwargs,
):
+ ## to ensure backward compatibility, move infrequently used arguments to kwargs arguments.
+ ## major version releases may be pack them with extra configuration options
+ lr_scheduler = kwargs.pop("lr_scheduler", None)
+ sampler = kwargs.pop("sampler", "rand")
+ not_use_best_mse = kwargs.pop("not_use_best_mse", False)
+ dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)
+ super_group_size = kwargs.pop("super_group_size", None)
+ super_bits = kwargs.pop("super_bits", None)
+ scale_dtype = kwargs.pop("scale_dtype", "fp16")
+ nblocks = kwargs.pop("nblocks", 1)
+ low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)
+ to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
+ enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)
+ enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
+ disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)
+ if not disable_deterministic_algorithms:
+ torch.use_deterministic_algorithms(True, warn_only=False)
+
self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 5e10a5590..520c13610 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -431,7 +431,7 @@ def tune(args):
import torch
if not args.disable_deterministic_algorithms:
- torch.use_deterministic_algorithms(True, warn_only=True)
+ torch.use_deterministic_algorithms(True, warn_only=False)
# logger.info("`torch.use_deterministic_algorithms` is enabled by default for reproducibility "
# "and can be disabled using the `--disable_deterministic_algorithms` argument.")
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 97fa82a9d..3864d4abc 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -332,7 +332,7 @@ def tune(args):
import torch
if not args.disable_deterministic_algorithms:
- torch.use_deterministic_algorithms(True, warn_only=True)
+ torch.use_deterministic_algorithms(True, warn_only=False)
print(
"'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "
"and can be turned off by setting the '--disable_deterministic_algorithms' parameter."
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index f1a7d9ccd..a937c9f50 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -54,7 +54,7 @@ class TestGGUF(unittest.TestCase):
def test_q4_0(self):
bits, group_size, sym = 4, 32, True
autoround = AutoRound(
- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int"
+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1
)
quantized_model_path = "./saved"
@@ -74,7 +74,7 @@ class TestGGUF(unittest.TestCase):
def test_q4_1(self):
bits, group_size, sym = 4, 32, False
autoround = AutoRound(
- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int"
+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1
)
quantized_model_path = "./saved"
@@ -100,6 +100,7 @@ class TestGGUF(unittest.TestCase):
# group_size=group_size,
# sym=sym,
iters=1,
+ nsamples=1,
# data_type="int"
)
quantized_model_path = "./saved"
@@ -122,6 +123,7 @@ class TestGGUF(unittest.TestCase):
group_size=16,
sym=True,
iters=1,
+ nsamples=1,
data_type="int_sym_dq",
super_group_size=16,
super_bits=6,
@@ -145,6 +147,7 @@ class TestGGUF(unittest.TestCase):
group_size=32,
sym=False,
iters=1,
+ nsamples=1,
data_type="int_asym_dq",
super_group_size=8,
super_bits=6,
@@ -168,6 +171,7 @@ class TestGGUF(unittest.TestCase):
group_size=16,
sym=True,
iters=1,
+ nsamples=1,
data_type="int_sym_dq",
super_group_size=16,
super_bits=8,
@@ -191,6 +195,7 @@ class TestGGUF(unittest.TestCase):
group_size=16,
sym=True,
iters=0,
+ nsamples=8,
data_type="rtn_int_sym_dq",
super_group_size=16,
super_bits=6,
@@ -212,6 +217,7 @@ class TestGGUF(unittest.TestCase):
group_size=32,
sym=True,
iters=0,
+ nsamples=8,
data_type="int_asym_dq",
super_group_size=8,
super_bits=6,
@@ -250,6 +256,7 @@ class TestGGUF(unittest.TestCase):
layer_config=layer_config,
iters=0,
seqlen=1,
+ nsamples=8,
dataset=self.llm_dataloader,
disable_opt_rtn=True,
)
@@ -305,6 +312,7 @@ class TestGGUF(unittest.TestCase):
processor=processor,
image_processor=image_processor,
iters=0,
+ nsamples=8,
)
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")