Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable export without dir (#23)

WeiweiZhang1b65830f3f6cb
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5c5f53cd3..cf044c520 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -863,11 +863,11 @@ class AutoRound(object):
)
if self.amp and not check_is_cpu(device):
with autocast(device_type="cuda", dtype=self.amp_dtype):
- # pylint: disable=not-callable
- loss = mse_loss(output_q, current_output)
+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
else:
- # pylint: disable=not-callable
- loss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))
+ loss = mse_loss( # pylint: disable=not-callable
+ output_q.to(torch.float32), current_output.to(torch.float32)
+ )
total_loss += loss.item() / self.gradient_accumulate_steps
self.scale_loss_and_backward(scaler, loss)
@@ -956,15 +956,23 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def save_quantized(self, output_dir, format="itrex", **kwargs):
- compress_model = None
+ def save_quantized(self, output_dir=None, format="itrex", **kwargs):
+ compressed_model = None
if format == "itrex":
- compress_model = self.save_quantized_as_itrex(output_dir, **kwargs)
+ from auto_round.export.export_to_itrex import compress_model
+
+ inplace = kwargs["inplace"] if "inplace" in kwargs.keys() else True
+ compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
+ if "use_triton" in kwargs.keys():
+ kwargs.pop("use_triton")
+ if output_dir is not None:
+ self.save_quantized_as_itrex(output_dir, compressed_model, **kwargs)
+
elif format == "auto_gptq":
self.save_quantized_as_autogptq(output_dir, **kwargs)
else:
logger.error("export only supports itrex and auto_gptq now")
- return compress_model
+ return compressed_model
def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):
"""Export the model to autogptq format to easily leverage cuda kernel."""
@@ -1058,12 +1066,11 @@ class AutoRound(object):
modules_in_block_to_quantize=modules_in_block_to_quantize,
)
- def save_quantized_as_itrex(self, output_dir, inplace=True):
+ def save_quantized_as_itrex(self, output_dir, compressed_model, **kwargs):
"""Save configure file and weights for CPU backend inference."""
- from auto_round.export.export_to_itrex import QuantConfig, compress_model
-
- compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
sym = self.scheme == "sym"
+ from auto_round.export.export_to_itrex import QuantConfig
+
quantize_config = QuantConfig(
bits=self.bits,
group_size=self.group_size,
@@ -1080,7 +1087,6 @@ class AutoRound(object):
setattr(config, "quantization_config", quantize_config.to_dict())
config.save_pretrained(output_dir)
quantize_config.save_pretrained(output_dir)
-
try:
compressed_model.save_pretrained(output_dir, safe_serialization=True)
if self.tokenizer is not None:
@@ -1088,7 +1094,6 @@ class AutoRound(object):
logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
except IOError as e: # pragma: no cover
logger.error("Fail to save configure file and weights due to {}.".format(e))
- return compressed_model
def quantize(self):
"""Quantize the model and return the quantized model along with weight configurations.
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 141fe9315..b4b224b12 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -41,7 +41,7 @@ if __name__ == '__main__':
parser.add_argument("--device", default=0, type=str,
help="device gpu int number, or 'cpu' ")
-
+
parser.add_argument("--sym", action='store_true',
help=" sym quantization")
@@ -84,14 +84,14 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="whether enable weight minmax tuning")
-
+
parser.add_argument("--deployment_device", default='fake', type=str,
help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."
"default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
-
+
parser.add_argument("--scale_dtype", default='fp32',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
-
+
parser.add_argument("--tasks",
default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
"mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
@@ -107,6 +107,7 @@ if __name__ == '__main__':
parser.add_argument("--disable_lmeval", action='store_true',
help="Whether to do lmeval evaluation.")
+
args = parser.parse_args()
set_seed(args.seed)