Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

better align gradient_accumulate_steps for varied length input (#309)

wenhuach21103e7e6c9ea3
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 839448adb..ec093ff2e 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -14,7 +14,7 @@
import sys
def run():
- from .script.llm import setup_parser, tune, eval
+ from auto_round.script.llm import setup_parser, tune, eval
args = setup_parser()
if args.eval:
eval(args)
@@ -22,18 +22,18 @@ def run():
tune(args)
def run_best():
- from .script.llm import setup_best_parser, tune
+ from auto_round.script.llm import setup_best_parser, tune
args = setup_best_parser()
tune(args)
def run_fast():
- from .script.llm import setup_fast_parser, tune
+ from auto_round.script.llm import setup_fast_parser, tune
args = setup_fast_parser()
tune(args)
def run_mllm():
- from .script.mllm import setup_parser, tune, eval
+ from auto_round.script.mllm import setup_parser, tune, eval
args = setup_parser()
if args.eval:
eval(args)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 564ba6fa3..6d5e6f973 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -875,10 +875,22 @@ class AutoRound(object):
if self.sampler != "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
total_loss = 0
+ num_elm = 1
+ mse_reduction = "mean"
+ if gradient_accumulate_steps != 1:
+ mse_reduction = "sum"
+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
+
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
+ if gradient_accumulate_steps != 1:
+ if q_inputs is not None:
+ current_input = [q_inputs[i] for i in whole_indices]
+ else:
+ current_input = [inputs[i] for i in whole_indices]
+ num_elm = sum(id.numel() for id in current_input)
for tmp_step in range(gradient_accumulate_steps):
indices = whole_indices[tmp_step * batch_size: (tmp_step + 1) * batch_size]
if q_inputs is not None:
@@ -902,7 +914,7 @@ class AutoRound(object):
loss = mse_loss( # pylint: disable=not-callable
output_q.to(torch.float32), current_output.to(torch.float32)
)
- total_loss += loss.item() / gradient_accumulate_steps
+ total_loss += loss.item() / num_elm
self.scale_loss_and_backward(scaler, loss)
if i == 0:
@@ -996,15 +1008,24 @@ class AutoRound(object):
whole_indices = torch.randperm(nsamples)[:pick_samples]
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
- mse_loss = torch.nn.MSELoss().to(device)
+ num_elm = 1
+ mse_reduction = "mean"
+ if self.gradient_accumulate_steps != 1:
+ mse_reduction = "sum"
+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
scaler = self.get_scaler() # pylint: disable=assignment-from-none
init_loss = None
best_params = {}
total_loss = 0
+
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
+ ##we assume the block input and output shape is same
+ if self.gradient_accumulate_steps!=1:
+ current_input_ids = [input_ids[i] for i in whole_indices]
+ num_elm = sum(id.numel() for id in current_input_ids)
for tmp_step in range(self.gradient_accumulate_steps):
indices = whole_indices[tmp_step * self.batch_size: (tmp_step + 1) * self.batch_size]
current_input_ids, current_input_others = sampling_inputs(
@@ -1031,7 +1052,7 @@ class AutoRound(object):
output_q.to(torch.float32), current_output.to(torch.float32)
)
- total_loss += loss.item() / self.gradient_accumulate_steps
+ total_loss += loss.item() / num_elm
self.scale_loss_and_backward(scaler, loss)
if i == 0:
init_loss = total_loss
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 181fd53ef..7bf5163a8 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -91,10 +91,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--low_gpu_mem_usage", action='store_true',
help="offload intermediate features to cpu")
- self.add_argument("--format", default=None, type=str,
- choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
- "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
- "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],
+ self.add_argument("--format", default="auto_round", type=str,
help="the format to save the model"
)
@@ -192,7 +189,7 @@ def setup_best_parser():
parser.add_argument("--batch_size", "--train_bs", default=8, type=int,
help="train batch size")
- parser.add_argument("--iters","--iter", default=1000, type=int,
+ parser.add_argument("--iters", "--iter", default=1000, type=int,
help="iterations to tune each block")
parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
@@ -234,6 +231,14 @@ def tune(args):
tasks = args.tasks
if args.format is None:
args.format = "auto_round"
+ supported_formats = ["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"]
+ formats = args.format.replace(' ', '').split(",")
+ for format in formats:
+ if format not in supported_formats:
+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")
+
if "auto_gptq" in args.format and args.asym is True:
print(
"warning: The auto_gptq kernel has issues with asymmetric quantization. "
@@ -359,10 +364,12 @@ def tune(args):
break
if args.quant_lm_head:
layer_config[lm_head_layer_name] = {"bits": args.bits}
- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
- if transformers_version[0] == 4 and transformers_version[1] < 38:
- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
- raise EnvironmentError(error_message)
+ for format in formats:
+ if "auto_round" not in format:
+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]
+ raise ValueError(
+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")
+
autoround = round(
model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index aba755584..e62d72e28 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -75,8 +75,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="offload intermediate features to cpu")
self.add_argument("--format", default="auto_round", type=str,
- choices=["auto_round", "auto_round:gptq", "auto_round:auto_gptq", "auto_round:awq",
- "auto_round:auto_awq", "fake"],
help="the format to save the model"
)
@@ -134,7 +132,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--not_use_best_mse", action='store_true',
help="whether to use the iter of best mes loss in the tuning phase")
-
## ======================= VLM =======================
self.add_argument("--quant_nontext_module", action='store_true',
help="whether to quantize non-text module, e.g. vision component")
@@ -205,6 +202,13 @@ def setup_parser():
def tune(args):
if args.format is None:
args.format = "auto_round"
+ supported_formats = ["auto_round", "auto_round:gptq", "auto_round:auto_gptq",
+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
+ "auto_round:awq"]
+ formats = args.format.replace(' ', '').split(",")
+ for format in formats:
+ if format not in supported_formats:
+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")
model_name = args.model
if model_name[-1] == "/":
@@ -268,10 +272,12 @@ def tune(args):
break
if args.quant_lm_head:
layer_config[lm_head_layer_name] = {"bits": args.bits}
- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
- if transformers_version[0] == 4 and transformers_version[1] < 38:
- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
- raise EnvironmentError(error_message)
+ for format in formats:
+ if "auto_round" not in format:
+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]
+ raise ValueError(
+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")
+
if args.quant_lm_head and args.low_gpu_mem_usage:
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
diff --git a/auto_round/utils.py b/auto_round/utils.py
index d12d2164e..111493d44 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -362,7 +362,9 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,
current_input_others: The sampled other input data.
"""
current_input_ids = [input_ids[i] for i in indices]
+
current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
+
current_input_others = {"positional_inputs": input_others["positional_inputs"]}
for key in input_others.keys():
if "positional_inputs" in key:
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index f55551d33..25b50e101 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -88,17 +88,13 @@ if __name__ == '__main__':
help="number of samples")
parser.add_argument("--low_gpu_mem_usage", action='store_true',
- help="offload intermediate features to cpu")
+ help="offload intermediate features to cpu")
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
-
- parser.add_argument("--format", default=None, type=str,
- choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
- "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
- "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],
- help="The format in which to save the model. "
+ parser.add_argument("--format", default="auto_round", type=str,
+ help="the format to save the quantized model"
)
parser.add_argument("--data_type", "--dtype", default='int',
@@ -170,6 +166,17 @@ if __name__ == '__main__':
print(
"Warning, activation quantization is an experiment feature")
+ if args.format is None:
+ args.format = "auto_round"
+ supported_formats = ["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"]
+ formats = args.format.replace(' ', '').split(",")
+ for format in formats:
+ if format not in supported_formats:
+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")
+
+
tasks = args.tasks
use_eval_legacy = False
@@ -300,6 +307,11 @@ if __name__ == '__main__':
break
if args.quant_lm_head:
layer_config[lm_head_layer_name] = {"bits": args.bits}
+ for format in formats:
+ if "auto_round" not in format:
+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]
+ raise ValueError(
+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")
autoround = round(model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,
dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
@@ -323,7 +335,7 @@ if __name__ == '__main__':
torch.cuda.empty_cache()
if model_name.split('/')[-1].strip(".") == "":
- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
else:
export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")