PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine export API
Zhang, Weiwei1ee6e785e019e
diff --git a/README.md b/README.mdindex d742ae0b5..ddca169f0 100644--- a/README.md+++ b/README.md@@ -31,7 +31,7 @@ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in- `tokenizer`: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.- `bits (int)`: Number of bits for quantization (default is 4).- `group_size (int)`: Size of the quantization group (default is 128).-- `scheme (str)`: The quantization scheme (symmetric/asymmetric) to be used (default is "asymmetric").+- `scheme (str)`: The quantization scheme (symmetric/asymmetric) to be used (default is "asym").- `use_quant_input (bool)`: Whether to use the output of the previous quantized block as the input for the current block (default is True).- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).- `iters (int)`: Number of tuning iterations (default is 200).@@ -143,9 +143,8 @@ After the quantization process is complete, you can invoke the export API to perfake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict## export for CPU deployment- from auto_round import QuantConfig, compress_model, save_compressed_modeloutput_dir = "/PATH/TO/SAVE/COMPRESSED/MODEL/"- save_compressed_model(fake_qdq_model, weight_config=weight_config, output_dir=output_dir, tokenizer=tokenizer)+ autoround.export_to_speed(output_dir=output_dir)del weight_config## export for GPU deploymentdiff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 757876436..47a925552 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -13,6 +13,6 @@# limitations under the License.from .autoround import AutoRound, AutoAdamRound, AutoOPTRoundfrom .version import __version__-from .export_to_speed.export import compress_model, save_compressed_model+from .export_to_speed import compress_model, QuantConfigfrom .export_to_autogptq import save_quantized_to_autogptq-from .export_to_speed.config import QuantConfig+diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex eb7138343..2771b6968 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -450,7 +450,7 @@ class AutoRound(object):Args:model: The PyTorch model to be quantized.- tokenizer: An optional tokenizer for processing input data. If none, a dataloader must be provided.+ tokenizer: Tokenizer for processing input data. Temporarily set as a mandatory parameter.bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).scheme (str): The quantization scheme to be used (default is "asym").@@ -497,7 +497,7 @@ class AutoRound(object):def __init__(self,model,- tokenizer=None,+ tokenizer,bits: int = 4,group_size: int = 128,scheme: str = "asym",@@ -548,7 +548,7 @@ class AutoRound(object):except:passself.weight_config = weight_config- assert dataloader is not None or tokenizer is not None+ assert tokenizer is not Noneself.dataset_split = dataset_splitself.seed = seedself.tokenizer = tokenizer@@ -966,6 +966,24 @@ class AutoRound(object):iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,use_safetensors=True)++ def export_to_speed(self, output_dir):+ """Save configure file and weights for CPU backend inference."""+ from .export_to_speed import compress_model+ compressed_model, quantize_config = compress_model(self.model, self.weight_config)+ if quantize_config is not None:+ config = compressed_model.config+ setattr(config, "quantization_config", quantize_config.to_dict())+ config.save_pretrained(output_dir)+ quantize_config.save_pretrained(output_dir)++ try:+ compressed_model.save_pretrained(output_dir, safe_serialization=True)+ if self.tokenizer is not None:+ self.tokenizer.save_pretrained(output_dir)+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))+ except IOError as e: # pragma: no cover+ logger.error("Fail to save configure file and weights due to {}.".format(e))def quantize(self):"""Quantize the model and return the quantized model along with weight configurations.@@ -1282,3 +1300,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/auto_round/export_to_speed/__init__.py b/auto_round/export_to_speed/__init__.pynew file mode 100644index 000000000..0f9fc8525--- /dev/null+++ b/auto_round/export_to_speed/__init__.py@@ -0,0 +1,16 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from .export import compress_model+from .config import QuantConfig+diff --git a/auto_round/export_to_speed/export.py b/auto_round/export_to_speed/export.pyindex df29400d4..1177cd17f 100644--- a/auto_round/export_to_speed/export.py+++ b/auto_round/export_to_speed/export.py@@ -99,25 +99,3 @@ def compress_model(quantize_config = QuantConfig(bits=num_bits, sym=(scheme=="sym"), group_size=group_size)return compressed_model, quantize_config---def save_compressed_model(model,- weight_config:Union[str, dict],- output_dir,- tokenizer=None):- """Save configure file and weights for CPU backend inference."""-- compressed_model, quantize_config = compress_model(model, weight_config)- if quantize_config is not None:- config = compressed_model.config- setattr(config, "quantization_config", quantize_config.to_dict())- config.save_pretrained(output_dir)- quantize_config.save_pretrained(output_dir)-- try:- compressed_model.save_pretrained(output_dir, safe_serialization=True)- if tokenizer is not None:- tokenizer.save_pretrained(output_dir)- logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))- except IOError as e: # pragma: no cover- logger.error("Fail to save configure file and weights due to {}.".format(e))diff --git a/examples/main.py b/examples/main.pyindex 0d8ed2f24..dc3b5a065 100644--- a/examples/main.py+++ b/examples/main.py@@ -1,9 +1,8 @@import argparseimport sys-sys.path.insert(0, '../')+sys.path.insert(0, './')from auto_round import (AutoRound,- AutoAdamRound,- save_compressed_model)+ AutoAdamRound)parser = argparse.ArgumentParser()import torch@@ -180,7 +179,7 @@ if __name__ == '__main__':export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"if args.deployment_device == 'cpu':- save_compressed_model(model, weight_config=q_config, output_dir=export_dir, tokenizer=tokenizer)+ autoround.export_to_speed(output_dir=export_dir)del q_configelif args.deployment_device == 'gpu':autoround.export_to_autogptq(export_dir, use_triton=True)diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 9d87ba35b..97a8ef34c 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -9,9 +9,7 @@ import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import (AutoRound,AutoAdamRound,- compress_model,- save_compressed_model,- AutoroundQuantConfig)+ QuantConfig)class SimpleDataLoader:@@ -60,21 +58,20 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):def test_autoround_int_quant(self):model = copy.deepcopy(self.gptj)- # device = "cuda:0"device = "cpu"- model=model.to(device)- out1 = model(self.lm_input.to(device))+ model=model+ out1 = model(self.lm_input)round = AutoRound- optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=True, seqlen=10, iters=10, scale_dtype='fp16')+ optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype='fp32')q_model, weight_config1 = optq_1.quantize()- q_model = q_model.to(device)- compressed_model = compress_model(q_model, weight_config1, device=device)- compressed_model = compressed_model.to(device)- q_model = q_model.to(device)- model = model.to(device)- out2 = model(self.lm_input.to(device))- out3 = q_model(self.lm_input.to(q_model.device))- out4 = compressed_model(self.lm_input.to(compressed_model.device))+ q_model = q_model+ from auto_round.export_to_speed import compress_model+ compressed_model,_ = compress_model(q_model, weight_config1)+ q_model = q_model+ model = model+ out2 = model(self.lm_input)+ out3 = q_model(self.lm_input)+ out4 = compressed_model(self.lm_input)self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))self.assertFalse(torch.all(out1[0] == out2[0]))self.assertTrue(torch.all(out2[0] == out3[0]))@@ -94,9 +91,10 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):def test_config(self):- config = AutoroundQuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")+ from auto_round import QuantConfig+ config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")config.save_pretrained("quantization_config_dir")- loaded_config = AutoroundQuantConfig.from_pretrained("quantization_config_dir")+ loaded_config = QuantConfig.from_pretrained("quantization_config_dir")self.assertEqual(config.group_size, loaded_config.group_size)self.assertEqual(config.true_sequential, loaded_config.true_sequential)@@ -105,3 +103,4 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):if __name__ == "__main__":unittest.main()+