Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add export format register (#27)

WeiweiZhang1e24e4e461a8a
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b47271d63..e6a1f1a9d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -961,144 +961,38 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def save_quantized(self, output_dir=None, format="itrex", **kwargs):
+ def save_quantized(self, output_dir=None, format="itrex", inplace=True, **kwargs):
compressed_model = None
- if format == "itrex":
+ if format == "itrex" and output_dir is None:
from auto_round.export.export_to_itrex import compress_model
- inplace = kwargs["inplace"] if "inplace" in kwargs.keys() else True
compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
- if "use_triton" in kwargs.keys():
- kwargs.pop("use_triton")
- if output_dir is not None:
- self.save_quantized_as_itrex(output_dir, compressed_model, **kwargs)
-
- elif format == "auto_gptq":
- self.save_quantized_as_autogptq(output_dir, **kwargs)
- else:
- logger.error("export only supports itrex and auto_gptq now")
- return compressed_model
-
- def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):
- """Export the model to autogptq format to easily leverage cuda kernel."""
- if not self.quantized:
- logger.warning("please run autoround.quantize first")
- return
- logger.info("Saving quantized model to autogptq format, this may take a while...")
- if self.tokenizer is not None:
- self.tokenizer.save_pretrained(output_dir)
- ##check module quantized in block, this may have bug for mixed precision quantization
- block_name = get_block_names(self.model)[0]
- first_block = get_module(self.model, block_name)
- all_to_quantized = True
- modules_in_block_to_quantize = []
- for n, m in first_block.named_modules():
- is_supported_type = False
- for supported_type in self.supported_types:
- if isinstance(m, supported_type):
- is_supported_type = True
- break
- if not is_supported_type:
- continue
- if not check_to_quantized(m):
- all_to_quantized = False
- else:
- modules_in_block_to_quantize.append(n)
- modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
- if all_to_quantized:
- modules_in_block_to_quantize = None
-
- if inplace:
- model = self.model.to("cpu")
else:
- model = copy.deepcopy(self.model.to("cpu"))
-
- from auto_gptq.modeling._utils import pack_model
-
- sym = self.scheme == "sym"
- if self.bits == 3 or use_triton is False:
- if self.bits == 3 and use_triton is True:
- logger.warning("triton does not support 3 bits, reset it to False")
- quantizers = {}
- for key in self.weight_config:
- info = self.weight_config[key]
- if not check_to_quantized(info):
- continue
- quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])
- pack_model(
- model,
- quantizers,
- self.bits,
- self.group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=False,
+ if not self.quantized:
+ logger.warning("please run autoround.quantize first")
+ return
+ from .export import EXPORT_FORMAT
+
+ save_quantized_as_format = EXPORT_FORMAT.get(format)
+ save_quantized_as_format(
+ output_dir,
+ model=self.model,
+ weight_config=self.weight_config,
+ inplace=inplace,
+ bits=self.bits,
+ group_size=self.group_size,
+ scheme=self.scheme,
+ iters=self.iters,
+ lr=self.lr,
+ minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning,
+ use_quant_input=self.use_quant_input,
+ scale_dtype=self.scale_dtype,
+ tokenizer=self.tokenizer,
+ supported_types=self.supported_types,
+ **kwargs,
)
- else:
- quantizers = {}
- for key in self.weight_config:
- info = self.weight_config[key]
- if not check_to_quantized(info):
- continue
- info["zp"] = info["zp"].to(torch.float32)
- quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])
- pack_model(
- model,
- quantizers,
- self.bits,
- self.group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=True,
- )
- from auto_round import save_quantized_to_autogptq
-
- save_quantized_to_autogptq(
- model,
- output_dir,
- bits=self.bits,
- group_size=self.group_size,
- sym=sym,
- iters=self.iters,
- lr=self.lr,
- minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning,
- use_quant_input=self.use_quant_input,
- scale_dtype=self.scale_dtype,
- use_safetensors=True,
- modules_in_block_to_quantize=modules_in_block_to_quantize,
- )
-
- def save_quantized_as_itrex(self, output_dir, compressed_model, **kwargs):
- """Save configure file and weights for CPU backend inference."""
- sym = self.scheme == "sym"
- from auto_round.export.export_to_itrex import QuantConfig
-
- quantize_config = QuantConfig(
- bits=self.bits,
- group_size=self.group_size,
- sym=sym,
- iters=self.iters,
- lr=self.lr,
- minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning,
- use_quant_input=self.use_quant_input,
- scale_dtype=str(self.scale_dtype),
- )
- if quantize_config is not None:
- config = compressed_model.config
- setattr(config, "quantization_config", quantize_config.to_dict())
- config.save_pretrained(output_dir)
- quantize_config.save_pretrained(output_dir)
- try:
- compressed_model.save_pretrained(output_dir, safe_serialization=True)
- if self.tokenizer is not None:
- self.tokenizer.save_pretrained(output_dir)
- logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
- except IOError as e: # pragma: no cover
- logger.error("Fail to save configure file and weights due to {}.".format(e))
+ return compressed_model
def quantize(self):
"""Quantize the model and return the quantized model along with weight configurations.
@@ -1117,12 +1011,12 @@ class AutoRound(object):
self.model = self.model.to(self.amp_dtype)
if not self.low_gpu_mem_usage:
self.model = self.model.to(self.device)
-
save_input_actor = SaveInputs(self.model, self.dataloader, self.seqlen, block_names[0])
inputs = save_input_actor.get_inputs(n_samples=self.n_samples)
del save_input_actor
if "input_ids" in inputs.keys():
- total_samples = inputs["input_ids"].shape[0]
+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))
+ total_samples = inputs["input_ids"].shape[dim]
self.n_samples = total_samples
if total_samples < self.train_bs:
self.train_bs = total_samples
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index e69de29bb..d8a3671e1 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -0,0 +1,191 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+import torch
+from ..utils import (
+ logger,
+ get_block_names,
+ get_module,
+ check_to_quantized,
+)
+
+EXPORT_FORMAT = {}
+
+
+def register_format(name):
+ """Class decorator to register a EXPORT subclass to the registry.
+
+ Decorator function used before a Pattern subclass.
+
+ Args:
+ cls (class): The subclass of register.
+ name: A string. Define the export type.
+
+ Returns:
+ cls: The class of register.
+ """
+
+ def register(format):
+ EXPORT_FORMAT[name] = format
+ return format
+
+ return register
+
+
+@register_format("auto_gptq")
+def save_quantized_as_autogptq(output_dir, use_triton=False, inplace=True, **kwargs):
+ """Export the model to autogptq format to easily leverage cuda kernel."""
+ model = kwargs["model"]
+ weight_config = kwargs["weight_config"]
+ sym = kwargs["scheme"] == "sym"
+ bits = kwargs["bits"]
+ group_size = kwargs["group_size"]
+ iters = kwargs["iters"]
+ lr = kwargs["lr"]
+ minmax_lr = kwargs["minmax_lr"]
+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]
+ use_quant_input = kwargs["use_quant_input"]
+ scale_dtype = kwargs["scale_dtype"]
+ tokenizer = kwargs["tokenizer"]
+ supported_types = kwargs["supported_types"]
+
+ logger.info("Saving quantized model to autogptq format, this may take a while...")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ ##check module quantized in block, this may have bug for mixed precision quantization
+ block_name = get_block_names(model)[0]
+ first_block = get_module(model, block_name)
+ all_to_quantized = True
+ modules_in_block_to_quantize = []
+ for n, m in first_block.named_modules():
+ is_supported_type = False
+ for supported_type in supported_types:
+ if isinstance(m, supported_type):
+ is_supported_type = True
+ break
+ if not is_supported_type:
+ continue
+ if not check_to_quantized(m):
+ all_to_quantized = False
+ else:
+ modules_in_block_to_quantize.append(n)
+ modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
+ if all_to_quantized:
+ modules_in_block_to_quantize = None
+
+ if inplace:
+ compressed_model = model.to("cpu")
+ else:
+ compressed_model = copy.deepcopy(model.to("cpu"))
+
+ from auto_gptq.modeling._utils import pack_model
+
+ if bits == 3 or use_triton is False:
+ if bits == 3 and use_triton is True:
+ logger.warning("triton does not support 3 bits, reset it to False")
+ quantizers = {}
+ for key in weight_config:
+ info = weight_config[key]
+ if not check_to_quantized(info):
+ continue
+ quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])
+ pack_model(
+ compressed_model,
+ quantizers,
+ bits,
+ group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=False,
+ )
+ else:
+ quantizers = {}
+ for key in weight_config:
+ info = weight_config[key]
+ if not check_to_quantized(info):
+ continue
+ info["zp"] = info["zp"].to(torch.float32)
+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])
+ pack_model(
+ compressed_model,
+ quantizers,
+ bits,
+ group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=True,
+ )
+ from auto_round import save_quantized_to_autogptq
+
+ save_quantized_to_autogptq(
+ compressed_model,
+ output_dir,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=iters,
+ lr=lr,
+ minmax_lr=minmax_lr,
+ enable_minmax_tuning=enable_minmax_tuning,
+ use_quant_input=use_quant_input,
+ scale_dtype=scale_dtype,
+ use_safetensors=True,
+ modules_in_block_to_quantize=modules_in_block_to_quantize,
+ )
+
+
+@register_format("itrex")
+def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
+ """Save configure file and weights for CPU backend inference."""
+ model = kwargs["model"]
+ weight_config = kwargs["weight_config"]
+ sym = kwargs["scheme"] == "sym"
+ bits = kwargs["bits"]
+ group_size = kwargs["group_size"]
+ iters = kwargs["iters"]
+ lr = kwargs["lr"]
+ minmax_lr = kwargs["minmax_lr"]
+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]
+ use_quant_input = kwargs["use_quant_input"]
+ scale_dtype = kwargs["scale_dtype"]
+ tokenizer = kwargs["tokenizer"]
+
+ from auto_round.export.export_to_itrex import compress_model, QuantConfig
+
+ compressed_model = compress_model(model, weight_config, inplace=inplace)
+ quantize_config = QuantConfig(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=iters,
+ lr=lr,
+ minmax_lr=minmax_lr,
+ enable_minmax_tuning=enable_minmax_tuning,
+ use_quant_input=use_quant_input,
+ scale_dtype=str(scale_dtype),
+ )
+ if quantize_config is not None:
+ config = compressed_model.config
+ setattr(config, "quantization_config", quantize_config.to_dict())
+ config.save_pretrained(output_dir)
+ quantize_config.save_pretrained(output_dir)
+ try:
+ compressed_model.save_pretrained(output_dir, safe_serialization=True)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
+ except IOError as e: # pragma: no cover
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 1026701da..bf91960aa 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -66,7 +66,7 @@ def compress_model(
else:
q_config = weight_config
for k, v in q_config.items():
- logger.info(f"Compressing {k} on device {device}")
+ logger.info(f"Packing {k}")
if "float" in v["data_type"]:
continue
dtype = v["data_type"]
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index db634efef..ce5693d4e 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -265,3 +265,4 @@ if __name__ == '__main__':
eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
device=torch_device, excel_file=excel_name)
+