Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Large impact]set the default nsamples to 128 and low_gpu_mem_usage to False (#174)

wenhuach21ab614824b5f9
diff --git a/README.md b/README.md
index 377560986..72afb77b1 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ image presents an overview of AutoRound. Check out our updated paper on [arxiv]
<div align="left">
## What's New
+* [2024/07] Important change: the default value of nsamples has been changed from 512 to 128 to reduce the memory usages, which may cause a slight accuracy drop in some scenarios
* [2024/06] AutoRound format supports mixed bit-widths and group sizes for inference, resolving the significant performance drop issue with the asymmetric kernel
* [2024/05] AutoRound supports lm-head quantization, saving 0.7G for LLaMA3-8B at W4G128.
* [2024/05] AutoRound performs well
@@ -98,7 +99,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).
-- `nsamples (int)`: Number of samples for tuning (default is 512).
+- `nsamples (int)`: Number of samples for tuning (default is 128).
- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).
@@ -113,7 +114,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).
-- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is True).
+- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is False).
- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "
NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "
@@ -126,6 +127,14 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
</details>
+### Tips
+
+1 Consider increasin 'iters' (e.g. 1000) to achieve better results, albeit with increased tuning time.
+
+2 Consider increasing 'nsamples' (e.g. 512) to achieve better results, albeit with more memory(~20G).
+
+3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
+
## Model inference
Please run the quantization code first.
@@ -197,11 +206,7 @@ approach achieved superior performance compared to GPTQ, scoring 30/32, AWQ with
with a perfect score of 16/16 across llamv1/llamav2/mistral-7b on W4G-1, W4G128, W3G128, and W2G128, based on the
average accuracies of 11 zero-shot tasks.
-## Tips
-
-1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time.
-2 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
## Reference
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index da040752c..9a57b20d0 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -79,7 +79,7 @@ class AutoRound(object):
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
iters (int): Number of iterations (default is 200).
seqlen (int): Data length of the sequence for tuning (default is 2048).
- nsamples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 128).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
nblocks (int): Number of blocks (default is 1).
@@ -112,10 +112,10 @@ class AutoRound(object):
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
+ low_gpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
- nsamples: int = 512,
+ nsamples: int = 128,
sampler: str = "rand",
seed: int = 42,
nblocks: int = 1,
@@ -170,6 +170,7 @@ class AutoRound(object):
self.optimizer = self.get_optimizer(None)
self.share_attention_mask_flag = None
self.hidden_dim_flag = None
+ self.infer_bs_coeff = 1
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
@@ -788,7 +789,8 @@ class AutoRound(object):
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
- output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, self.cache_device)
+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,
+ self.cache_device)
if q_input is not None:
input_ids = q_input
@@ -904,7 +906,8 @@ class AutoRound(object):
if self.enable_quanted_input:
q_outputs = self.get_block_outputs(
- block, input_ids, input_others, self.train_bs, device, cache_device=self.cache_device
+ block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,
+ cache_device=self.cache_device
)
for i in range(len(input_ids)):
input_ids[i] = None
@@ -1183,10 +1186,10 @@ class AutoOPTRound(AutoRound):
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 128).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
nblocks (int): Number of blocks (default is 1).
@@ -1220,10 +1223,10 @@ class AutoOPTRound(AutoRound):
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
+ low_gpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
- nsamples: int = 512,
+ nsamples: int = 128,
sampler: str = "rand",
seed: int = 42,
nblocks: int = 1,
@@ -1331,10 +1334,10 @@ class AutoAdamRound(AutoOPTRound):
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 128).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
nblocks (int): Number of blocks (default is 1).
@@ -1368,10 +1371,10 @@ class AutoAdamRound(AutoOPTRound):
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
+ low_gpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
- nsamples: int = 512,
+ nsamples: int = 128,
sampler: str = "rand",
seed: int = 42,
nblocks: int = 1,
@@ -1415,4 +1418,3 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
-
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 9145b6cc3..14d9b5c60 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -74,11 +74,11 @@ if __name__ == '__main__':
parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
- parser.add_argument("--nsamples", default=512, type=int,
+ parser.add_argument("--nsamples", default=128, type=int,
help="number of samples")
parser.add_argument("--low_gpu_mem_usage", action='store_true',
- help="low_gpu_mem_usage is deprecated")
+ help="lower gpu memory but 50%-100% slower")
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
@@ -104,9 +104,6 @@ if __name__ == '__main__':
parser.add_argument("--disable_amp", action='store_true',
help="disable amp")
- parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',
- help="disable low_gpu_mem_usage")
-
parser.add_argument("--disable_minmax_tuning", action='store_true',
help="whether disable enable weight minmax tuning")
@@ -123,9 +120,7 @@ if __name__ == '__main__':
help="force to convert the dtype, some backends supports fp16 dtype better")
args = parser.parse_args()
- if args.low_gpu_mem_usage:
- print(
- "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")
+
if args.enable_minmax_tuning:
print(
"enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")
@@ -204,7 +199,7 @@ if __name__ == '__main__':
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
if "bloom" in model_name:
- args.disable_low_gpu_mem_usage = True
+ args.low_gpu_mem_usage = False
round = AutoRound
if args.adam:
@@ -240,15 +235,15 @@ if __name__ == '__main__':
error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
raise EnvironmentError(error_message)
- if args.quant_lm_head and not args.disable_low_gpu_mem_usage:
- print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "
+ if args.quant_lm_head and not args.low_gpu_mem_usage:
+ print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
amp=not args.disable_amp, nsamples=args.nsamples,
- low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,
+ low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, weight_config=weight_config,
enable_minmax_tuning=not args.disable_minmax_tuning)
@@ -340,7 +335,7 @@ if __name__ == '__main__':
output_dir += "/"
print(excel_name, flush=True)
eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
- eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
device=torch_device, excel_file=excel_name)
if not args.disable_eval and lm_eval_version == "0.4.2":
diff --git a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
index 7058a1983..a96d03e14 100644
--- a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
+++ b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
@@ -5,5 +5,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
index 7af9f6a7e..c4e306cc0 100644
--- a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
+++ b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
@@ -6,8 +6,8 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--quant_lm_head \
+--nsamples 512 \
--deployment_device 'gpu' \
---disable_low_gpu_mem_usage \
--output_dir "./tmp_autoround"
@@ -20,5 +20,5 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--deployment_device 'gpu' \
---disable_low_gpu_mem_usage \
+--nsamples 512 \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
index 762982acb..76680b747 100644
--- a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
+++ b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
index e433806d8..1071f8db9 100644
--- a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
+++ b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
index 018969dc5..cd98d5b12 100644
--- a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
+++ b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
index 51ef47e3b..185cda732 100644
--- a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
+++ b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
@@ -4,6 +4,7 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
--disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
index 84ab60490..a9fcebe21 100644
--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
@@ -5,5 +5,6 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--deployment_device 'gpu' \
---minmax_lr 2e-3
+--minmax_lr 2e-3\
+--nsamples 512
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
index 32d8123d3..07161e16a 100644
--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
@@ -6,4 +6,6 @@ python3 main.py \
--iters 200 \
--deployment_device gpu \
--sym \
---minmax_lr 0.01
+--nsamples 512 \
+--minmax_lr 0.01
+
diff --git a/examples/language-modeling/scripts/baichuan2-7b-chat.sh b/examples/language-modeling/scripts/baichuan2-7b-chat.sh
index 51d674509..74bf14fb6 100644
--- a/examples/language-modeling/scripts/baichuan2-7b-chat.sh
+++ b/examples/language-modeling/scripts/baichuan2-7b-chat.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--minmax_lr 2e-3
diff --git a/examples/language-modeling/scripts/bloom-3b.sh b/examples/language-modeling/scripts/bloom-3b.sh
index a12ee2962..712d1ae09 100644
--- a/examples/language-modeling/scripts/bloom-3b.sh
+++ b/examples/language-modeling/scripts/bloom-3b.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/falcon-7b.sh b/examples/language-modeling/scripts/falcon-7b.sh
index 07bbae4d4..adc6b5fba 100644
--- a/examples/language-modeling/scripts/falcon-7b.sh
+++ b/examples/language-modeling/scripts/falcon-7b.sh
@@ -4,7 +4,7 @@ python3 main.py \
--group_size 64 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
---disable_quanted_input \
---disable_low_gpu_mem_usage
\ No newline at end of file
+--disable_quanted_input \
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-2b.sh b/examples/language-modeling/scripts/gemma-2b.sh
index b7136bfb7..7bf58bc2a 100644
--- a/examples/language-modeling/scripts/gemma-2b.sh
+++ b/examples/language-modeling/scripts/gemma-2b.sh
@@ -8,6 +8,6 @@ python3 main.py \
--bits 4 \
--iters 400 \
--model_dtype "float16" \
---disable_low_gpu_mem_usage \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.sh
index 5b13c99bd..63e18bcaf 100644
--- a/examples/language-modeling/scripts/gemma-7b-it.sh
+++ b/examples/language-modeling/scripts/gemma-7b-it.sh
@@ -7,4 +7,5 @@ python3 main.py \
--minmax_lr 2e-3 \
--model_dtype "float16" \
--deployment_device 'gpu' \
+--nsamples 512 \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.sh
index bf89b0704..37a7b9872 100644
--- a/examples/language-modeling/scripts/gemma-7b.sh
+++ b/examples/language-modeling/scripts/gemma-7b.sh
@@ -4,8 +4,8 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--model_dtype "float16" \
---disable_low_gpu_mem_usage \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gpt-j-6b.sh b/examples/language-modeling/scripts/gpt-j-6b.sh
index e11f21ade..99b90a678 100644
--- a/examples/language-modeling/scripts/gpt-j-6b.sh
+++ b/examples/language-modeling/scripts/gpt-j-6b.sh
@@ -5,5 +5,6 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--disable_quanted_input \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
index 5bc14d124..ff3da6570 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
@@ -5,6 +5,7 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--nsamples 512 \
--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
index 15a3d4b71..378266df5 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
@@ -5,6 +5,7 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--minmax_lr 2e-3 \
+--nsamples 512 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
--disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/opt-2.7b.sh b/examples/language-modeling/scripts/opt-2.7b.sh
index bc9bc2ef5..f7954ece8 100644
--- a/examples/language-modeling/scripts/opt-2.7b.sh
+++ b/examples/language-modeling/scripts/opt-2.7b.sh
@@ -5,6 +5,7 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--minmax_lr 0.002 \
+--nsamples 512 \
--disable_quanted_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.sh
index 46e27226f..8b5a1b821 100644
--- a/examples/language-modeling/scripts/phi-2.sh
+++ b/examples/language-modeling/scripts/phi-2.sh
@@ -5,6 +5,6 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--sym \
+--nsamples 512 \
--deployment_device 'gpu' \
---output_dir "./tmp_autoround" \
---disable_low_gpu_mem_usage
+--output_dir "./tmp_autoround"
\ No newline at end of file