Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine export API

Zhang, Weiwei1ee6e785e019e
diff --git a/README.md b/README.md
index d742ae0b5..ddca169f0 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in
- `tokenizer`: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.
- `bits (int)`: Number of bits for quantization (default is 4).
- `group_size (int)`: Size of the quantization group (default is 128).
-- `scheme (str)`: The quantization scheme (symmetric/asymmetric) to be used (default is "asymmetric").
+- `scheme (str)`: The quantization scheme (symmetric/asymmetric) to be used (default is "asym").
- `use_quant_input (bool)`: Whether to use the output of the previous quantized block as the input for the current block (default is True).
- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).
- `iters (int)`: Number of tuning iterations (default is 200).
@@ -143,9 +143,8 @@ After the quantization process is complete, you can invoke the export API to per
fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict
## export for CPU deployment
- from auto_round import QuantConfig, compress_model, save_compressed_model
output_dir = "/PATH/TO/SAVE/COMPRESSED/MODEL/"
- save_compressed_model(fake_qdq_model, weight_config=weight_config, output_dir=output_dir, tokenizer=tokenizer)
+ autoround.export_to_speed(output_dir=output_dir)
del weight_config
## export for GPU deployment
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 757876436..47a925552 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -13,6 +13,6 @@
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
from .version import __version__
-from .export_to_speed.export import compress_model, save_compressed_model
+from .export_to_speed import compress_model, QuantConfig
from .export_to_autogptq import save_quantized_to_autogptq
-from .export_to_speed.config import QuantConfig
+
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index eb7138343..2771b6968 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -450,7 +450,7 @@ class AutoRound(object):
Args:
model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data. If none, a dataloader must be provided.
+ tokenizer: Tokenizer for processing input data. Temporarily set as a mandatory parameter.
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
scheme (str): The quantization scheme to be used (default is "asym").
@@ -497,7 +497,7 @@ class AutoRound(object):
def __init__(
self,
model,
- tokenizer=None,
+ tokenizer,
bits: int = 4,
group_size: int = 128,
scheme: str = "asym",
@@ -548,7 +548,7 @@ class AutoRound(object):
except:
pass
self.weight_config = weight_config
- assert dataloader is not None or tokenizer is not None
+ assert tokenizer is not None
self.dataset_split = dataset_split
self.seed = seed
self.tokenizer = tokenizer
@@ -966,6 +966,24 @@ class AutoRound(object):
iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
use_safetensors=True)
+
+ def export_to_speed(self, output_dir):
+ """Save configure file and weights for CPU backend inference."""
+ from .export_to_speed import compress_model
+ compressed_model, quantize_config = compress_model(self.model, self.weight_config)
+ if quantize_config is not None:
+ config = compressed_model.config
+ setattr(config, "quantization_config", quantize_config.to_dict())
+ config.save_pretrained(output_dir)
+ quantize_config.save_pretrained(output_dir)
+
+ try:
+ compressed_model.save_pretrained(output_dir, safe_serialization=True)
+ if self.tokenizer is not None:
+ self.tokenizer.save_pretrained(output_dir)
+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
+ except IOError as e: # pragma: no cover
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
def quantize(self):
"""Quantize the model and return the quantized model along with weight configurations.
@@ -1282,3 +1300,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/auto_round/export_to_speed/__init__.py b/auto_round/export_to_speed/__init__.py
new file mode 100644
index 000000000..0f9fc8525
--- /dev/null
+++ b/auto_round/export_to_speed/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .export import compress_model
+from .config import QuantConfig
+
diff --git a/auto_round/export_to_speed/export.py b/auto_round/export_to_speed/export.py
index df29400d4..1177cd17f 100644
--- a/auto_round/export_to_speed/export.py
+++ b/auto_round/export_to_speed/export.py
@@ -99,25 +99,3 @@ def compress_model(
quantize_config = QuantConfig(bits=num_bits, sym=(scheme=="sym"), group_size=group_size)
return compressed_model, quantize_config
-
-
-def save_compressed_model(model,
- weight_config:Union[str, dict],
- output_dir,
- tokenizer=None):
- """Save configure file and weights for CPU backend inference."""
-
- compressed_model, quantize_config = compress_model(model, weight_config)
- if quantize_config is not None:
- config = compressed_model.config
- setattr(config, "quantization_config", quantize_config.to_dict())
- config.save_pretrained(output_dir)
- quantize_config.save_pretrained(output_dir)
-
- try:
- compressed_model.save_pretrained(output_dir, safe_serialization=True)
- if tokenizer is not None:
- tokenizer.save_pretrained(output_dir)
- logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
- except IOError as e: # pragma: no cover
- logger.error("Fail to save configure file and weights due to {}.".format(e))
diff --git a/examples/main.py b/examples/main.py
index 0d8ed2f24..dc3b5a065 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -1,9 +1,8 @@
import argparse
import sys
-sys.path.insert(0, '../')
+sys.path.insert(0, './')
from auto_round import (AutoRound,
- AutoAdamRound,
- save_compressed_model)
+ AutoAdamRound)
parser = argparse.ArgumentParser()
import torch
@@ -180,7 +179,7 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
if args.deployment_device == 'cpu':
- save_compressed_model(model, weight_config=q_config, output_dir=export_dir, tokenizer=tokenizer)
+ autoround.export_to_speed(output_dir=export_dir)
del q_config
elif args.deployment_device == 'gpu':
autoround.export_to_autogptq(export_dir, use_triton=True)
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 9d87ba35b..97a8ef34c 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -9,9 +9,7 @@ import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import (AutoRound,
AutoAdamRound,
- compress_model,
- save_compressed_model,
- AutoroundQuantConfig)
+ QuantConfig)
class SimpleDataLoader:
@@ -60,21 +58,20 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
def test_autoround_int_quant(self):
model = copy.deepcopy(self.gptj)
- # device = "cuda:0"
device = "cpu"
- model=model.to(device)
- out1 = model(self.lm_input.to(device))
+ model=model
+ out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=True, seqlen=10, iters=10, scale_dtype='fp16')
+ optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype='fp32')
q_model, weight_config1 = optq_1.quantize()
- q_model = q_model.to(device)
- compressed_model = compress_model(q_model, weight_config1, device=device)
- compressed_model = compressed_model.to(device)
- q_model = q_model.to(device)
- model = model.to(device)
- out2 = model(self.lm_input.to(device))
- out3 = q_model(self.lm_input.to(q_model.device))
- out4 = compressed_model(self.lm_input.to(compressed_model.device))
+ q_model = q_model
+ from auto_round.export_to_speed import compress_model
+ compressed_model,_ = compress_model(q_model, weight_config1)
+ q_model = q_model
+ model = model
+ out2 = model(self.lm_input)
+ out3 = q_model(self.lm_input)
+ out4 = compressed_model(self.lm_input)
self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))
self.assertFalse(torch.all(out1[0] == out2[0]))
self.assertTrue(torch.all(out2[0] == out3[0]))
@@ -94,9 +91,10 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
def test_config(self):
- config = AutoroundQuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
+ from auto_round import QuantConfig
+ config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
config.save_pretrained("quantization_config_dir")
- loaded_config = AutoroundQuantConfig.from_pretrained("quantization_config_dir")
+ loaded_config = QuantConfig.from_pretrained("quantization_config_dir")
self.assertEqual(config.group_size, loaded_config.group_size)
self.assertEqual(config.true_sequential, loaded_config.true_sequential)
@@ -105,3 +103,4 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+