Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

set default lr to None

wenhuach215f40492bc0ba
diff --git a/README.md b/README.md
index b1298aa3a..915a8f690 100644
--- a/README.md
+++ b/README.md
@@ -57,8 +57,8 @@ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in
- `use_quant_input (bool)`: Whether to use the output of the previous quantized block as the input for the current block (default is True).
- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).
- `iters (int)`: Number of tuning iterations (default is 200).
-- `lr (float)`: The learning rate for rounding value (default is 0.005).
-- `minmax_lr (float)`: The learning rate for min-max tuning (default is None).
+- `lr (float)`: The learning rate for rounding value (default is None, it will be set to 1.0/iters automatically).
+- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).
- `n_samples (int)`: Number of samples for tuning (default is 512).
- `seqlen (int)`: Data length of the sequence for tuning.
- `bs (int)`: Batch size for training (default is 8).
@@ -89,7 +89,7 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bi
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8
```
- **Utilizing the AdamW Optimizer:**
-Include the flag `--adam`. Note that AdamW may be less effective than Sign gradient descent in many scenarios.
+Include the flag `--adam`. Note that AdamW is less effective than Sign gradient descent in many scenarios we tested.
- **Running the Original SignRound:**
```bash
@@ -118,7 +118,7 @@ Random issues in tuning Qwen models. ChatGlm-V1 is not supported
| W4G128 | MMLU | Lamb. | Hella. | Wino. | Piqa | Truth. | Open. | Boolq | RTE | ARC-e | ARC-c. | AVG. |
|-------------------------------------------------|-------|-------|--------|-------|-------|--------|-------|-------|-------|-------|--------|-------|
-| mistralai/Mixtral-8x7B-v0.1 FP16 | 69.83 | 78.44 | 64.89 | 76.40 | 82.43 | 34.15 | 35.40 | 84.98 | 71.12 | 84.22 | 56.91 | 67.16 |
+| mistralai/Mixtral-8x7B-v0.1 BF16 | 69.83 | 78.44 | 64.89 | 76.40 | 82.43 | 34.15 | 35.40 | 84.98 | 71.12 | 84.22 | 56.91 | 67.16 |
| mistralai/Mixtral-8x7B-v0.1 Signround+ | 68.90 | 78.11 | 64.31 | 74.27 | 82.10 | 30.97 | 34.20 | 84.57 | 67.87 | 83.96 | 56.57 | 65.98 |
| mistralai/Mixtral-8x7B-v0.1 Signround+ iter800 | 68.84 | 77.99 | 64.18 | 75.30 | 81.82 | 31.21 | 35.80 | 85.41 | 68.95 | 83.75 | 55.38 | 66.24 |
| microsoft/phi-2 | 56.40 | 62.78 | 55.83 | 75.77 | 78.67 | 31.21 | 40.40 | 83.36 | 62.45 | 80.05 | 52.90 | 61.80 |
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 05fd2da2b..5835daf64 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -964,7 +964,7 @@ class AutoRound(object):
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
- lr: float = 0.005,
+ lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = True,
iters: int = 200,
@@ -1028,11 +1028,17 @@ class AutoRound(object):
)
else:
self.dataloader = dataloader
+ self.iters = iters
+ if self.iters <= 0:
+ logger.warning("iters must be positive, reset it to 200")
+ self.iters = 200
self.lr = lr
+ if self.lr is None:
+ self.lr = 1.0 / self.iters
self.minmax_lr = minmax_lr
if self.minmax_lr is None:
self.minmax_lr = self.lr
- self.iters = iters
+
self.sampler = sampler
self.gradient_accumulate_steps = gradient_accumulate_steps
self.not_use_mse = not_use_mse
@@ -1513,7 +1519,7 @@ class AutoOPTRound(AutoRound):
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
- lr: float = 0.005,
+ lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = True,
iters: int = 200,
@@ -1661,7 +1667,7 @@ class AutoAdamRound(AutoOPTRound):
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
- lr: float = 0.01,
+ lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = True,
iters: int = 200,
diff --git a/examples/main.py b/examples/main.py
index 98429dd54..63d32b006 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -51,11 +51,11 @@ if __name__ == '__main__':
parser.add_argument("--use_quant_input", action='store_true',
help="whether to use the output of quantized block to tune the next block")
- parser.add_argument("--lr", default=0.005, type=float,
- help="step size")
+ parser.add_argument("--lr", default=None, type=float,
+ help="learning rate, if None, it will be set to 1.0/iters automatially")
parser.add_argument("--minmax_lr", default=None, type=float,
- help="minmax learning rate, if None,it will set to be the same with lr")
+ help="minmax learning rate, if None,it will beset to be the same with lr")
parser.add_argument("--seed", default=42, type=int,
help="seed")