Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix mutable default value (#272)

wenhuach218bf63f39e9f7
diff --git a/README.md b/README.md
index c6cb4c581..8c6e556c8 100644
--- a/README.md
+++ b/README.md
@@ -121,7 +121,7 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)
NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "
./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"
-- `layer_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits
+- `layer_config (dict)`: Configuration for weight quantization (default is None), mainly for mixed bits
or mixed precision.
- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 44cbf92e6..142090451 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -68,7 +68,7 @@ class AutoRound(object):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (bool): Whether symmetric quantization is to be used (default is False).
- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config (dict): Configuration for weight quantization (default is None).
layer_config={
'layer1':##layer_name
{
@@ -125,7 +125,7 @@ class AutoRound(object):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- layer_config: dict = {},
+ layer_config: dict = None,
enable_full_range: bool = False, ##for symmetric, TODO support later
batch_size: int = 8,
amp: bool = True,
@@ -182,7 +182,7 @@ class AutoRound(object):
self.low_gpu_mem_usage = low_gpu_mem_usage
self.data_type = data_type
self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]
- self.layer_config = layer_config
+ self.layer_config = {} if layer_config is None else layer_config
self.seed = seed
set_seed(self.seed)
self.tokenizer = tokenizer
@@ -1346,7 +1346,7 @@ class AutoOPTRound(AutoRound):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (bool): Whether sym to be used (default is False).
- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config (dict): Configuration for weight quantization (default is None).
enable_full_range (bool): Whether to enable full range quantization (default is False).
batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
@@ -1389,7 +1389,7 @@ class AutoOPTRound(AutoRound):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- layer_config: dict = {},
+ layer_config=None,
enable_full_range: bool = False,
batch_size: int = 8,
amp: bool = True,
@@ -1459,6 +1459,8 @@ class AutoOPTRound(AutoRound):
**kwargs,
)
+ if layer_config is None:
+ layer_config = {}
self.optimizer = self.get_optimizer(optimizer)
def get_optimizer(self, optimizer):
@@ -1511,7 +1513,7 @@ class AutoAdamRound(AutoOPTRound):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (str): Whether symmetric quantization to be used (default is False).
- layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config (dict): Configuration for weight quantization (default is None).
enable_full_range (bool): Whether to enable full range quantization (default is False).
batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
@@ -1554,7 +1556,7 @@ class AutoAdamRound(AutoOPTRound):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- layer_config: dict = {},
+ layer_config=None,
enable_full_range: bool = False,
batch_size: int = 8,
amp: bool = True,
diff --git a/test/test_autoround.py b/test/test_autoround.py
index a1b871f5c..db29bf38d 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -55,6 +55,36 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_consective_quant(self):
+ bits, group_size, sym = 4, -1, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2", torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+
+
def test_mx_fp4(self):
bits, group_size, sym = 4, 32, False
autoround = AutoRound(