PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add export format register (#27)
WeiweiZhang1e24e4e461a8a
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b47271d63..e6a1f1a9d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -961,144 +961,38 @@ class AutoRound(object):torch.cuda.empty_cache()- def save_quantized(self, output_dir=None, format="itrex", **kwargs):+ def save_quantized(self, output_dir=None, format="itrex", inplace=True, **kwargs):compressed_model = None- if format == "itrex":+ if format == "itrex" and output_dir is None:from auto_round.export.export_to_itrex import compress_model- inplace = kwargs["inplace"] if "inplace" in kwargs.keys() else Truecompressed_model = compress_model(self.model, self.weight_config, inplace=inplace)- if "use_triton" in kwargs.keys():- kwargs.pop("use_triton")- if output_dir is not None:- self.save_quantized_as_itrex(output_dir, compressed_model, **kwargs)-- elif format == "auto_gptq":- self.save_quantized_as_autogptq(output_dir, **kwargs)- else:- logger.error("export only supports itrex and auto_gptq now")- return compressed_model-- def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):- """Export the model to autogptq format to easily leverage cuda kernel."""- if not self.quantized:- logger.warning("please run autoround.quantize first")- return- logger.info("Saving quantized model to autogptq format, this may take a while...")- if self.tokenizer is not None:- self.tokenizer.save_pretrained(output_dir)- ##check module quantized in block, this may have bug for mixed precision quantization- block_name = get_block_names(self.model)[0]- first_block = get_module(self.model, block_name)- all_to_quantized = True- modules_in_block_to_quantize = []- for n, m in first_block.named_modules():- is_supported_type = False- for supported_type in self.supported_types:- if isinstance(m, supported_type):- is_supported_type = True- break- if not is_supported_type:- continue- if not check_to_quantized(m):- all_to_quantized = False- else:- modules_in_block_to_quantize.append(n)- modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq- if all_to_quantized:- modules_in_block_to_quantize = None-- if inplace:- model = self.model.to("cpu")else:- model = copy.deepcopy(self.model.to("cpu"))-- from auto_gptq.modeling._utils import pack_model-- sym = self.scheme == "sym"- if self.bits == 3 or use_triton is False:- if self.bits == 3 and use_triton is True:- logger.warning("triton does not support 3 bits, reset it to False")- quantizers = {}- for key in self.weight_config:- info = self.weight_config[key]- if not check_to_quantized(info):- continue- quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])- pack_model(- model,- quantizers,- self.bits,- self.group_size,- use_cuda_fp16=True,- desc_act=False,- force_layer_back_to_cpu=True,- use_triton=False,+ if not self.quantized:+ logger.warning("please run autoround.quantize first")+ return+ from .export import EXPORT_FORMAT++ save_quantized_as_format = EXPORT_FORMAT.get(format)+ save_quantized_as_format(+ output_dir,+ model=self.model,+ weight_config=self.weight_config,+ inplace=inplace,+ bits=self.bits,+ group_size=self.group_size,+ scheme=self.scheme,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ use_quant_input=self.use_quant_input,+ scale_dtype=self.scale_dtype,+ tokenizer=self.tokenizer,+ supported_types=self.supported_types,+ **kwargs,)- else:- quantizers = {}- for key in self.weight_config:- info = self.weight_config[key]- if not check_to_quantized(info):- continue- info["zp"] = info["zp"].to(torch.float32)- quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])- pack_model(- model,- quantizers,- self.bits,- self.group_size,- use_cuda_fp16=True,- desc_act=False,- force_layer_back_to_cpu=True,- use_triton=True,- )- from auto_round import save_quantized_to_autogptq-- save_quantized_to_autogptq(- model,- output_dir,- bits=self.bits,- group_size=self.group_size,- sym=sym,- iters=self.iters,- lr=self.lr,- minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning,- use_quant_input=self.use_quant_input,- scale_dtype=self.scale_dtype,- use_safetensors=True,- modules_in_block_to_quantize=modules_in_block_to_quantize,- )-- def save_quantized_as_itrex(self, output_dir, compressed_model, **kwargs):- """Save configure file and weights for CPU backend inference."""- sym = self.scheme == "sym"- from auto_round.export.export_to_itrex import QuantConfig-- quantize_config = QuantConfig(- bits=self.bits,- group_size=self.group_size,- sym=sym,- iters=self.iters,- lr=self.lr,- minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning,- use_quant_input=self.use_quant_input,- scale_dtype=str(self.scale_dtype),- )- if quantize_config is not None:- config = compressed_model.config- setattr(config, "quantization_config", quantize_config.to_dict())- config.save_pretrained(output_dir)- quantize_config.save_pretrained(output_dir)- try:- compressed_model.save_pretrained(output_dir, safe_serialization=True)- if self.tokenizer is not None:- self.tokenizer.save_pretrained(output_dir)- logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))- except IOError as e: # pragma: no cover- logger.error("Fail to save configure file and weights due to {}.".format(e))+ return compressed_modeldef quantize(self):"""Quantize the model and return the quantized model along with weight configurations.@@ -1117,12 +1011,12 @@ class AutoRound(object):self.model = self.model.to(self.amp_dtype)if not self.low_gpu_mem_usage:self.model = self.model.to(self.device)-save_input_actor = SaveInputs(self.model, self.dataloader, self.seqlen, block_names[0])inputs = save_input_actor.get_inputs(n_samples=self.n_samples)del save_input_actorif "input_ids" in inputs.keys():- total_samples = inputs["input_ids"].shape[0]+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))+ total_samples = inputs["input_ids"].shape[dim]self.n_samples = total_samplesif total_samples < self.train_bs:self.train_bs = total_samplesdiff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex e69de29bb..d8a3671e1 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -0,0 +1,191 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+import torch+from ..utils import (+ logger,+ get_block_names,+ get_module,+ check_to_quantized,+)++EXPORT_FORMAT = {}+++def register_format(name):+ """Class decorator to register a EXPORT subclass to the registry.++ Decorator function used before a Pattern subclass.++ Args:+ cls (class): The subclass of register.+ name: A string. Define the export type.++ Returns:+ cls: The class of register.+ """++ def register(format):+ EXPORT_FORMAT[name] = format+ return format++ return register+++@register_format("auto_gptq")+def save_quantized_as_autogptq(output_dir, use_triton=False, inplace=True, **kwargs):+ """Export the model to autogptq format to easily leverage cuda kernel."""+ model = kwargs["model"]+ weight_config = kwargs["weight_config"]+ sym = kwargs["scheme"] == "sym"+ bits = kwargs["bits"]+ group_size = kwargs["group_size"]+ iters = kwargs["iters"]+ lr = kwargs["lr"]+ minmax_lr = kwargs["minmax_lr"]+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]+ use_quant_input = kwargs["use_quant_input"]+ scale_dtype = kwargs["scale_dtype"]+ tokenizer = kwargs["tokenizer"]+ supported_types = kwargs["supported_types"]++ logger.info("Saving quantized model to autogptq format, this may take a while...")+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ ##check module quantized in block, this may have bug for mixed precision quantization+ block_name = get_block_names(model)[0]+ first_block = get_module(model, block_name)+ all_to_quantized = True+ modules_in_block_to_quantize = []+ for n, m in first_block.named_modules():+ is_supported_type = False+ for supported_type in supported_types:+ if isinstance(m, supported_type):+ is_supported_type = True+ break+ if not is_supported_type:+ continue+ if not check_to_quantized(m):+ all_to_quantized = False+ else:+ modules_in_block_to_quantize.append(n)+ modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq+ if all_to_quantized:+ modules_in_block_to_quantize = None++ if inplace:+ compressed_model = model.to("cpu")+ else:+ compressed_model = copy.deepcopy(model.to("cpu"))++ from auto_gptq.modeling._utils import pack_model++ if bits == 3 or use_triton is False:+ if bits == 3 and use_triton is True:+ logger.warning("triton does not support 3 bits, reset it to False")+ quantizers = {}+ for key in weight_config:+ info = weight_config[key]+ if not check_to_quantized(info):+ continue+ quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])+ pack_model(+ compressed_model,+ quantizers,+ bits,+ group_size,+ use_cuda_fp16=True,+ desc_act=False,+ force_layer_back_to_cpu=True,+ use_triton=False,+ )+ else:+ quantizers = {}+ for key in weight_config:+ info = weight_config[key]+ if not check_to_quantized(info):+ continue+ info["zp"] = info["zp"].to(torch.float32)+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])+ pack_model(+ compressed_model,+ quantizers,+ bits,+ group_size,+ use_cuda_fp16=True,+ desc_act=False,+ force_layer_back_to_cpu=True,+ use_triton=True,+ )+ from auto_round import save_quantized_to_autogptq++ save_quantized_to_autogptq(+ compressed_model,+ output_dir,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=iters,+ lr=lr,+ minmax_lr=minmax_lr,+ enable_minmax_tuning=enable_minmax_tuning,+ use_quant_input=use_quant_input,+ scale_dtype=scale_dtype,+ use_safetensors=True,+ modules_in_block_to_quantize=modules_in_block_to_quantize,+ )+++@register_format("itrex")+def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):+ """Save configure file and weights for CPU backend inference."""+ model = kwargs["model"]+ weight_config = kwargs["weight_config"]+ sym = kwargs["scheme"] == "sym"+ bits = kwargs["bits"]+ group_size = kwargs["group_size"]+ iters = kwargs["iters"]+ lr = kwargs["lr"]+ minmax_lr = kwargs["minmax_lr"]+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]+ use_quant_input = kwargs["use_quant_input"]+ scale_dtype = kwargs["scale_dtype"]+ tokenizer = kwargs["tokenizer"]++ from auto_round.export.export_to_itrex import compress_model, QuantConfig++ compressed_model = compress_model(model, weight_config, inplace=inplace)+ quantize_config = QuantConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=iters,+ lr=lr,+ minmax_lr=minmax_lr,+ enable_minmax_tuning=enable_minmax_tuning,+ use_quant_input=use_quant_input,+ scale_dtype=str(scale_dtype),+ )+ if quantize_config is not None:+ config = compressed_model.config+ setattr(config, "quantization_config", quantize_config.to_dict())+ config.save_pretrained(output_dir)+ quantize_config.save_pretrained(output_dir)+ try:+ compressed_model.save_pretrained(output_dir, safe_serialization=True)+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))+ except IOError as e: # pragma: no cover+ logger.error("Fail to save configure file and weights due to {}.".format(e))diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 1026701da..bf91960aa 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -66,7 +66,7 @@ def compress_model(else:q_config = weight_configfor k, v in q_config.items():- logger.info(f"Compressing {k} on device {device}")+ logger.info(f"Packing {k}")if "float" in v["data_type"]:continuedtype = v["data_type"]diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex db634efef..ce5693d4e 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -265,3 +265,4 @@ if __name__ == '__main__':eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,device=torch_device, excel_file=excel_name)+