PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix mutable default value (#272)
wenhuach218bf63f39e9f7
diff --git a/README.md b/README.mdindex c6cb4c581..8c6e556c8 100644--- a/README.md+++ b/README.md@@ -121,7 +121,7 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"-- `layer_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits+- `layer_config (dict)`: Configuration for weight quantization (default is None), mainly for mixed bitsor mixed precision.- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 44cbf92e6..142090451 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -68,7 +68,7 @@ class AutoRound(object):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (bool): Whether symmetric quantization is to be used (default is False).- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config (dict): Configuration for weight quantization (default is None).layer_config={'layer1':##layer_name{@@ -125,7 +125,7 @@ class AutoRound(object):bits: int = 4,group_size: int = 128,sym: bool = False,- layer_config: dict = {},+ layer_config: dict = None,enable_full_range: bool = False, ##for symmetric, TODO support laterbatch_size: int = 8,amp: bool = True,@@ -182,7 +182,7 @@ class AutoRound(object):self.low_gpu_mem_usage = low_gpu_mem_usageself.data_type = data_typeself.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]- self.layer_config = layer_config+ self.layer_config = {} if layer_config is None else layer_configself.seed = seedset_seed(self.seed)self.tokenizer = tokenizer@@ -1346,7 +1346,7 @@ class AutoOPTRound(AutoRound):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (bool): Whether sym to be used (default is False).- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config (dict): Configuration for weight quantization (default is None).enable_full_range (bool): Whether to enable full range quantization (default is False).batch_size (int): Batch size for training (default is 8).amp (bool): Whether to use automatic mixed precision (default is True).@@ -1389,7 +1389,7 @@ class AutoOPTRound(AutoRound):bits: int = 4,group_size: int = 128,sym: bool = False,- layer_config: dict = {},+ layer_config=None,enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,@@ -1459,6 +1459,8 @@ class AutoOPTRound(AutoRound):**kwargs,)+ if layer_config is None:+ layer_config = {}self.optimizer = self.get_optimizer(optimizer)def get_optimizer(self, optimizer):@@ -1511,7 +1513,7 @@ class AutoAdamRound(AutoOPTRound):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (str): Whether symmetric quantization to be used (default is False).- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config (dict): Configuration for weight quantization (default is None).enable_full_range (bool): Whether to enable full range quantization (default is False).batch_size (int): Batch size for training (default is 8).amp (bool): Whether to use automatic mixed precision (default is True).@@ -1554,7 +1556,7 @@ class AutoAdamRound(AutoOPTRound):bits: int = 4,group_size: int = 128,sym: bool = False,- layer_config: dict = {},+ layer_config=None,enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,diff --git a/test/test_autoround.py b/test/test_autoround.pyindex a1b871f5c..db29bf38d 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -55,6 +55,36 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_consective_quant(self):+ bits, group_size, sym = 4, -1, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()++ model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2", torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+++def test_mx_fp4(self):bits, group_size, sym = 4, 32, Falseautoround = AutoRound(