Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add model compress and export for cpu

Zhang, Weiwei1574c6c3c7477
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 1b2514237..43059f6cb 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -13,3 +13,7 @@
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
from .version import __version__
+from .export import compress_model, save_compressed_model, save_quantized_to_autogptq
+from .config import AutoroundQuantConfig
+from .model_wrapper import WeightOnlyLinear
+
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index ba0e6535e..6e170b001 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -33,7 +33,7 @@ from torch.amp import autocast
from torch.functional import F
-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
+def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
"""Quantizes and dequantizes weight asymmetrically.
Args:
@@ -47,8 +47,8 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** num_bits - 1)
- ##zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=weight.dtype)##force scale to fp16
- zeros = torch.zeros(weight.shape[0], device=weight.device)
+ zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
+ # zeros = torch.zeros(weight.shape[0], device=weight.device)
if isinstance(min_scale, torch.Tensor):
wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
@@ -59,10 +59,11 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
else:
wmin = torch.minimum(weight.min(1)[0], zeros)
wmax = torch.maximum(weight.max(1)[0], zeros)
+
tmp = (wmin == 0) & (wmax == 0)
wmin[tmp] = -1
wmax[tmp] = +1
- scale = (wmax - wmin) / maxq
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
zp = round_ste(-wmin / scale)
scale = scale.unsqueeze(dim=-1)
zp = zp.unsqueeze(dim=-1)
@@ -71,7 +72,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
return scale * (q - zp), scale, zp
-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
+def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
"""Quantizes and dequantizes weight symmetrically.
Args:
@@ -94,13 +95,13 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
wmax *= 1 + max_scale
tmp = wmax == 0
wmax[tmp] = +1
- scale = wmax / ((maxq - minq) / 2)
+ scale = (wmax / ((maxq - minq) / 2)).to(scale_dtype)
scale.unsqueeze_(dim=-1)
q = torch.clamp(round_ste(weight / scale + v), minq, maxq)
return scale * q, scale, None
-def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):
+def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):
"""Quantizes and dequantizes weight symmetrically or asymmetrically .
Args:
@@ -116,12 +117,12 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):
"""
assert num_bits > 0, "num_bits should be larger than 0"
if scheme == "sym":
- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale)
+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
else:
- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale)
+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
-def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0):
+def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
"""Quantizes and dequantizes weight, handing the group size issue .
Args:
@@ -137,7 +138,8 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
Quantized and dequantized weight, scale, zero-point
"""
if group_size == -1 or weight.shape[1] < group_size:
- return quant_weight_actor(weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale)
+ return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,
+ min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)
orig_shape = weight.shape
if weight.shape[1] % group_size == 0:
weight = weight.reshape(-1, group_size)
@@ -145,7 +147,7 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
v = v.reshape(-1, group_size)
weight, scale, zp = quant_weight_actor(
- weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale
+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
)
weight = weight.reshape(orig_shape)
scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d
@@ -161,7 +163,7 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
if isinstance(v, torch.Tensor):
v = v.reshape(-1, group_size)
weight_new, scale, zp = quant_weight_actor(
- weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale
+ weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
)
weight_new = weight_new.reshape(orig_shape[0], -1)
@@ -170,6 +172,40 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
if zp is not None:
zp = zp.reshape(weight_new.shape[0], -1)
return weight_new, scale, zp
+
+
+def quant_weight_w_scale(weight, scale, zp, group_size=-1):
+ """Quant and dequant tensor with group size.
+
+ Args:
+ weight: input weight
+ scale: scale
+ zp: zero point
+ group_size (int, optional): how many elements share one scale/zp. Defaults to -1.
+
+ Returns:
+ output: int weight.
+ """
+ device = weight.device
+ scale = scale.to(device)
+ if zp is not None:
+ zp = zp.to(device)
+ if group_size == -1:
+ return torch.round(weight / scale) if zp is None else torch.round(weight / scale + zp)
+ int_weight = torch.zeros(weight.shape).to(device)
+ leng = weight.shape[1] // group_size
+ tail_flag = False if weight.shape[1] % group_size == 0 else True
+ for i in range(leng):
+ int_weight_tmp = weight[:, i * group_size : (i + 1) * group_size] / scale[:, i].unsqueeze(1)
+ if zp is not None:
+ int_weight_tmp += zp[:, i].unsqueeze(1)
+ int_weight[:, i * group_size : (i + 1) * group_size] = torch.round(int_weight_tmp)
+ if tail_flag:
+ int_weight_tmp = weight[:, leng * group_size :] / scale[:, -1].unsqueeze(1)
+ if zp is not None:
+ int_weight_tmp += zp[:, -1].unsqueeze(1)
+ int_weight[:, leng * group_size :] = torch.round(int_weight_tmp)
+ return int_weight
def round_ste(x: torch.Tensor):
@@ -400,7 +436,8 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer = orig_layer
self.num_bits = self.orig_layer.bits
self.group_size = self.orig_layer.group_size
- self.scheme = (self.orig_layer.scheme,)
+ self.scale_dtype = self.orig_layer.scale_dtype
+ self.scheme = self.orig_layer.scheme
self.value = torch.nn.Parameter(
torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True
)
@@ -440,11 +477,12 @@ class WrapperLinear(torch.nn.Module):
v,
min_scale,
max_scale,
+ self.scale_dtype,
)
self.orig_layer.weight.data.copy_(q_dq_weight)
self.orig_layer.weight.grad = None ##clear grad
self.orig_layer.scale = scale.to("cpu")
- self.orig_layer.zp = zp.to("cpu")
+ self.orig_layer.zp = zp.to("cpu") if zp is not None else None
return self.orig_layer
def forward(self, x):
@@ -460,7 +498,7 @@ class WrapperLinear(torch.nn.Module):
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))
weight_q, _, _ = quant_weight(
- weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale
+ weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale, self.scale_dtype
)
weight_q = weight_q.to(weight.dtype)
return F.linear(x, weight_q, self.orig_layer.bias)
@@ -527,6 +565,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
v,
min_scale,
max_scale,
+ self.scale_dtype
)
self.orig_layer.weight.data.copy_(weight_q.t())
self.orig_layer.weight.grad = None
@@ -554,6 +593,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.value,
self.min_scale,
self.max_scale,
+ self.scale_dtype
)
weight_q = weight_q.to(self.weight_t.dtype)
size_out = x.size()[:-1] + (self.orig_layer.nf,)
@@ -977,6 +1017,7 @@ class AutoRound(object):
not_use_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int", ##only support data_type
+ scale_dtype = "fp16",
**kwargs,
):
self.model = model
@@ -1007,6 +1048,14 @@ class AutoRound(object):
self.train_bs = bs
self.n_blocks = n_blocks
self.device = device
+
+ if scale_dtype == 'fp16':
+ self.scale_dtype = torch.float16
+ elif scale_dtype == 'bf16':
+ self.scale_dtype = torch.bfloat16
+ else:
+ self.scale_dtype = torch.float32
+
self.amp_dtype = torch.float16
if self.model.dtype != torch.float32:
self.amp_dtype = self.model.dtype
@@ -1143,10 +1192,13 @@ class AutoRound(object):
weight_config[n]["group_size"] = self.group_size
if "scheme" not in weight_config[n].keys():
weight_config[n]["scheme"] = self.scheme
+ weight_config[n]["scale_dtype"] = self.scale_dtype
+
m.data_type = weight_config[n]["data_type"]
m.bits = weight_config[n]["bits"]
m.group_size = weight_config[n]["group_size"]
m.scheme = weight_config[n]["scheme"]
+ m.scale_dtype = weight_config[n]["scale_dtype"]
def get_batch_dim(self, input_others):
"""Get the batch dimension of the input tensor.
@@ -1722,3 +1774,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/auto_round/config.py b/auto_round/config.py
new file mode 100644
index 000000000..8d43ceaed
--- /dev/null
+++ b/auto_round/config.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configs for Autoround quantization."""
+try:
+ from neural_compressor.utils.utility import LazyImport
+ torch = LazyImport("torch")
+ from neural_compressor.utils import logger
+except: # pragma: no cover
+ import logging
+ import torch
+ logger = logging.getLogger()
+
+import copy
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, Tuple, Union
+from transformers import BitsAndBytesConfig, PretrainedConfig
+
+QUANT_CONFIG = "quantize_config.json"
+
+class AutoroundQuantConfig(PretrainedConfig):
+ def __init__(
+ self,
+ bits=4,
+ scale_dtype="fp32",
+ group_size=128,
+ sym=False,
+ quant_method="Autoround",
+ model_name_or_path=None,
+ model_file_base_name='model',
+ **kwargs,
+ ):
+ self.bits = bits
+ self.group_size = group_size
+ self.scale_dtype = scale_dtype ## reserved parameter
+ self.sym = sym
+ self.quant_method = quant_method
+ self.model_name_or_path = model_name_or_path
+ self.model_file_base_name = model_file_base_name
+
+ ### Redundant parameters, will be removed later. ###
+ self.damp_percent = 0.01
+ self.desc_act = False
+ self.true_sequential = False
+ self.quant_method = "gptq"
+
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+
+ if self.scale_dtype not in ["fp32", "fp16"]:
+ raise ValueError("scale_dtype must be 'fp32', 'fp16'.")
+
+ if self.group_size not in [-1, 32, 128]:
+ raise ValueError("group_size must be an integer in [-1, 32, 128]")
+
+
+ def quantization_method(self):
+ r"""
+ This method returns the quantization method used for the model.
+ """
+ return self.quant_method
+
+
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ """
+ Instantiates a [`AutoroundQuantConfig`] from a Python dictionary of parameters.
+ Args:
+ config_dict (`Dict[str, Any]`):
+ Dictionary that will be used to instantiate the configuration object.
+ return_unused_kwargs (`bool`):
+ Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in
+ `PreTrainedModel`.
+ kwargs (`Dict[str, Any]`):
+ Additional parameters from which to initialize the configuration object.
+ Returns:
+ [`AutoroundQuantConfig`]: The configuration object instantiated from those parameters.
+ """
+
+ config = cls(**config_dict)
+
+ to_remove = []
+ for key, value in kwargs.items():
+ if hasattr(config, key):
+ setattr(config, key, value)
+ to_remove.append(key)
+ for key in to_remove:
+ kwargs.pop(key, None)
+
+ if return_unused_kwargs:
+ return config, kwargs
+ else:
+ return config
+
+
+ @classmethod
+ def from_json_file(cls, json_file_path, return_unused_kwargs, **kwargs):
+ with open(json_file_path, "r", encoding="utf-8") as f:
+ config_dict = json.load(f)
+ return cls.from_dict(config_dict, return_unused_kwargs, **kwargs)
+
+
+ def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):
+ """
+ Save this instance to a JSON file.
+ Args:
+ json_file_path (`str` or `os.PathLike`):
+ Path to the JSON file in which this configuration instance's parameters will be saved.
+ """
+ with open(json_file_path, "w", encoding="utf-8") as writer:
+ writer.write(self.to_json_string(use_diff=use_diff))
+
+ def to_dict(self) -> Dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary. Returns:
+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+
+ output = copy.deepcopy(self.__dict__)
+ return output
+
+ def __repr__(self):
+ return f"{self.__class__.__name__} {self.to_json_string()}"
+
+
+ def to_json_string(self, use_diff: bool = True) -> str:
+ """
+ Serializes this instance to a JSON string.
+ Args:
+ use_diff (`bool`, *optional*, defaults to `True`):
+ If set to `True`, only the difference between the config instance and the default
+ `AutoroundQuantConfig()`
+ is serialized to JSON string.
+ Returns:
+ `str`: String containing all the attributes that make up this configuration instance in JSON format.
+ """
+ if use_diff is True:
+ config_dict = self.to_diff_dict()
+ else:
+ config_dict = self.to_dict()
+
+ return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
+
+ def to_diff_dict(self) -> Dict[str, Any]:
+ """
+ Removes all attributes from config which correspond to the default config attributes for better readability and
+ serializes to a Python dictionary.
+ Returns:
+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ config_dict = self.to_dict()
+
+ # get the default config dict
+ default_config_dict = AutoroundQuantConfig().to_dict()
+
+ serializable_config_dict = {}
+
+ # only serialize values that differ from the default config
+ for key, value in config_dict.items():
+ if value != default_config_dict[key]:
+ serializable_config_dict[key] = value
+
+ return serializable_config_dict
+
+ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
+ """
+ Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
+ [`~PretrainedConfig.from_pretrained`] class method.
+ Args:
+ save_directory (`str` or `os.PathLike`):
+ Directory where the configuration JSON file will be saved (will be created if it does not exist).
+ push_to_hub (`bool`, *optional*, defaults to `False`):
+ Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
+ repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
+ namespace).
+ kwargs (`Dict[str, Any]`, *optional*):
+ Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
+ """
+ self._set_token_in_kwargs(kwargs)
+
+ if os.path.isfile(save_directory):
+ raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
+
+ os.makedirs(save_directory, exist_ok=True)
+
+ if push_to_hub:
+ commit_message = kwargs.pop("commit_message", None)
+ repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
+ repo_id = self._create_repo(repo_id, **kwargs)
+ files_timestamps = self._get_files_timestamps(save_directory)
+
+ # If we save using the predefined names, we can load using `from_pretrained`
+ output_config_file = os.path.join(save_directory, QUANT_CONFIG)
+
+ self.to_json_file(output_config_file, use_diff=False)
+ logger.info(f"Configuration saved in {output_config_file}")
+
+ if push_to_hub:
+ self._upload_modified_files(
+ save_directory,
+ repo_id,
+ files_timestamps,
+ commit_message=commit_message,
+ token=kwargs.get("token", None),
+ )
+
+ @classmethod
+ def get_config_dict(
+ cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
+ ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+ return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)
\ No newline at end of file
diff --git a/auto_round/export.py b/auto_round/export.py
index ac44f1e38..bb5a552b5 100644
--- a/auto_round/export.py
+++ b/auto_round/export.py
@@ -42,13 +42,104 @@ import os
from typing import Dict, List, Optional, Union
from safetensors.torch import save_file as safe_save
from os.path import join, isfile, isdir
+import copy
+import json
+
+
+def compress_model(
+ model,
+ weight_config:Union[str, dict],
+ enable_full_range=False,
+ compression_dtype=torch.int32,
+ compression_dim=1,
+ scale_dtype=torch.float32,
+ device="cpu",
+ use_optimum_format=True,
+ ):
+ """Convert Linear to WeightOnlyLinear for low memory inference.
+
+ Args:
+ weight_config (str|dict): qconfig dict or Path of qconfig.json.
+ enable_full_range (bool, optional): Whether to leverage the full compression range
+ under symmetric quantization. Defaults to False.
+ compression_dtype (torch.Tensor, optional): The target dtype after comoression.
+ Defaults to torch.int32.
+ compression_dim (int, optional): Select from [0, 1], 0 is output channel,
+ 1 is input channel. Defaults to 1.
+ scale_dtype (torch.Tensor, optional): Use float32 or float16.
+ Defaults to torch.float32.
+ device (str, optional): choose device for compression. Defaults to cpu.
+ use_optimum_format (bool, optional): use the popular huggingface compression format.
+ 1: compression_dim: weight = 1, zeros = 0 and both are transposed.
+ 2: zeros -= 1 before compression. Why we need it?
+ 3: g_idx: use same number for one group instead of recording the channel order.
+ 4. parameter name changed, such as 'packed_weight' -> 'qweight'.
+ 5. zeros is always needed even for sym.
+ """
+ from .model_wrapper import WeightOnlyLinear
+ from .autoround import quant_weight_w_scale, get_module, set_module
+ compressed_model = copy.deepcopy(model)
+ if isinstance(weight_config, str):
+ with open(weight_config, "r") as f:
+ q_config = json.load(f)
+ else:
+ q_config = weight_config
+ for k, v in q_config.items():
+ logger.info(f"Compressing {k} on device {device}")
+ if "float" in v["data_type"]:
+ continue
+ dtype = v["data_type"]
+ num_bits = v["bits"]
+ group_size = v["group_size"]
+ scheme = v["scheme"]
+ scale_dtype = v["scale_dtype"]
+ m = get_module(compressed_model, k)
+ fp_weight = m.weight.data
+ scale = torch.tensor(v["scale"], dtype=torch.float32)
+
+ zp = None if scheme == "sym" else torch.tensor(v["zp"], dtype=torch.int32)
+ int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size)
+ int_weight = int_weight.type(torch.int32)
+ new_module = WeightOnlyLinear(
+ m.in_features,
+ m.out_features,
+ num_bits,
+ group_size,
+ dtype=dtype,
+ scale_dtype=scale_dtype,
+ zp=zp is not None,
+ bias=m.bias is not None,
+ device=device,
+ use_optimum_format=True,
+ )
+ new_module.pack(int_weight, scale, zp, m.bias)
+ set_module(compressed_model, k, new_module)
+ return compressed_model
+
+
+def save_compressed_model(model, output_dir, quantize_config=None, tokenizer=None):
+ """Save configure file and weights for CPU backend inference."""
+
+ if quantize_config is not None:
+ config = model.config
+ setattr(config, "quantization_config", quantize_config.to_dict())
+ config.save_pretrained(output_dir)
+ quantize_config.save_pretrained(output_dir)
+
+ try:
+ model.save_pretrained(output_dir, safe_serialization=True)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
+ except IOError as e: # pragma: no cover
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,
minmax_lr=5e-3,
enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,
safetensors_metadata: Optional[Dict[str, str]] = None):
- """save quantized model and configs to local disk"""
+ """save quantized model and configs to local disk for cuda """
os.makedirs(save_dir, exist_ok=True)
model.to("cpu")
diff --git a/auto_round/model_wrapper.py b/auto_round/model_wrapper.py
new file mode 100644
index 000000000..4bddd6942
--- /dev/null
+++ b/auto_round/model_wrapper.py
@@ -0,0 +1,345 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Torch.nn.Module Class Definition."""
+# Note: Do not import this file unless you have already imported torch,
+# since the model classes inherit torch.nn.Module.
+import math
+
+import torch
+from packaging.version import Version
+from torch.autograd import Function
+from torch.nn import functional as F
+import logging
+logger = logging.getLogger()
+
+
+NF4 = [
+ -1.0,
+ -0.6961928009986877,
+ -0.5250730514526367,
+ -0.39491748809814453,
+ -0.28444138169288635,
+ -0.18477343022823334,
+ -0.09105003625154495,
+ 0.0,
+ 0.07958029955625534,
+ 0.16093020141124725,
+ 0.24611230194568634,
+ 0.33791524171829224,
+ 0.44070982933044434,
+ 0.5626170039176941,
+ 0.7229568362236023,
+ 1.0,
+]
+FP4_BNB = [-12.0, -8.0, -6.0, -4.0, -3.0, -2.0, -0.0625, 0, 0.0625, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0]
+FP4_E2M1 = [-6.0, -4.0, -3.0, -2.0, -1.5, -1.0, -0.0625, 0, 0.0625, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
+
+# the order is the same as float list, bit value range is [-7, 7]
+# 1111 = -1, 1110 = -2, 1101= -3, ...
+
+NF4_BIT = [7, 1, 2, 3, 4, 5, 6, 0, -8, -7, -6, -5, -4, -3, -2, -1]
+FP4_BNB_BIT = [-5, -6, -3, -4, -1, -2, -7, 0, 1, 6, 7, 4, 5, 2, 3]
+FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]
+
+FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}
+INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}
+
+def get_torch_version():
+ try:
+ torch_version = torch.__version__.split("+")[0]
+ except ValueError as e: # pragma: no cover
+ assert False, "Got an unknown version of torch: {}".format(e)
+ version = Version(torch_version)
+ return version
+
+
+PT_VERSION = get_torch_version().release
+
+class WeightOnlyLinear(torch.nn.Module):
+ def __init__(
+ self,
+ in_features,
+ out_features,
+ bits,
+ groupsize,
+ dtype="int",
+ zp=False,
+ bias=False,
+ scale_dtype=torch.float16,
+ compression_dtype=torch.int32,
+ compression_dim=1,
+ device="cpu",
+ use_optimum_format=True,
+ ):
+ super().__init__()
+ self.use_optimum_format = use_optimum_format
+ self.dtype = dtype
+ if "int" not in self.dtype: # for nf4, fp4
+
+ float_list = FLOAT_MAPPING[self.dtype]
+ int_list = INT_MAPPING[self.dtype]
+ self.int2float_mapping = {}
+ for k, v in zip(int_list, float_list):
+ self.int2float_mapping[k] = v
+ self.device = device
+ self.in_features = in_features
+ self.out_features = out_features
+ self.bits = bits
+ self.groupsize = groupsize if groupsize != -1 else in_features
+ self.compression_dim = compression_dim
+ assert compression_dtype in [
+ torch.int8,
+ torch.int16,
+ torch.int32,
+ torch.int64,
+ ], "Only support torch.int8|16|32|64 as compressed dtype."
+ dtype_bits_mapping = {torch.int8: 8, torch.int16: 16, torch.int32: 32, torch.int64: 64}
+ self.compress_bits = dtype_bits_mapping[compression_dtype]
+ self.n_pack = self.compress_bits // self.bits
+ # K is input channel, N is output channel
+ assert compression_dim in [0, 1], (
+ "Only support 0 or 1 as compression dimension, " + "0 is output channel, 1 is input channel."
+ )
+ if self.use_optimum_format:
+ self.float_type = scale_dtype
+ self.compression_dtype = torch.int32
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(in_features / self.groupsize), out_features),
+ dtype=self.float_type,
+ ).to(device),
+ )
+ self.register_buffer(
+ "qweight",
+ torch.zeros(
+ (math.ceil(in_features / self.n_pack), out_features),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (math.ceil(self.in_features / self.groupsize), math.ceil(self.out_features / self.n_pack)),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))
+ else:
+ self.compression_dtype = compression_dtype
+ self.float_type = scale_dtype
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (out_features, math.ceil(in_features / self.groupsize)),
+ dtype=self.float_type,
+ ).to(device),
+ )
+ if compression_dim == 1:
+ self.register_buffer(
+ "qweight",
+ torch.zeros(
+ (out_features, math.ceil(in_features / self.n_pack)),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ if zp:
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (self.out_features, math.ceil(self.in_features / self.groupsize / self.n_pack)),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ else:
+ self.register_buffer(
+ "qweight",
+ torch.zeros(
+ (math.ceil(out_features / self.n_pack), in_features),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ if zp:
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (math.ceil(self.out_features / self.n_pack), math.ceil(self.in_features / self.groupsize)),
+ dtype=self.compression_dtype,
+ ).to(device),
+ )
+ if bias:
+ self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))
+ else:
+ self.bias = None
+
+ def pack(self, int_weight, scale, zp, bias):
+ if self.use_optimum_format:
+ self.scales = self.scales.t_().contiguous()
+ self.qweight = self.qweight.t_().contiguous()
+ self.qzeros = self.qzeros.t_().contiguous()
+
+ int_weight = int_weight.to(self.device)
+ if self.use_optimum_format and zp is None:
+ # to avoid overflow
+ int_weight = int_weight.type(torch.int32)
+ shift_bias = 2 ** (self.bits - 1)
+ int_weight += shift_bias
+ zp = torch.zeros_like(scale, dtype=torch.uint8) + shift_bias
+ if bias is not None:
+ assert hasattr(self, "bias"), "bias is not set when initializing."
+ self.bias = bias.type(self.float_type).to(self.device)
+ assert scale.shape == self.scales.shape, "Scale shape is mismatched."
+ self.scales = scale.type(self.float_type).to(self.device)
+ if not self.use_optimum_format and self.compression_dim == 0:
+ int_weight = int_weight.t_().contiguous()
+ self.qweight = self.qweight.t_().contiguous()
+ origin_shape = int_weight.shape
+ target_shape = self.qweight.shape
+ assert origin_shape[0] == target_shape[0], "output channels mismatch, please check."
+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)
+
+ # pack weight
+ for j in range(target_shape[1]):
+ start = self.n_pack * j
+ end = self.n_pack * (j + 1)
+ tmp = int_weight[:, start:end].type(self.compression_dtype)
+ for e in range(tmp.shape[1]):
+ tmp[:, e] &= mask
+ tmp[:, e] = tmp[:, e] << (self.bits * e)
+ self.qweight[:, j] |= tmp[:, e]
+ if not self.use_optimum_format and self.compression_dim == 0:
+ self.qweight = self.qweight.t_().contiguous()
+
+ if zp is not None:
+ zp = zp.to(self.device)
+ if self.use_optimum_format:
+ zp -= 1
+ if self.use_optimum_format or self.compression_dim == 0:
+ zp = zp.t_().contiguous()
+ self.qzeros = self.qzeros.t_().contiguous()
+ assert hasattr(self, "qzeros"), "zp is not set when initializing."
+ target_shape = self.qzeros.shape
+ for j in range(target_shape[1]):
+ start = self.n_pack * j
+ end = self.n_pack * (j + 1)
+ tmp = zp[:, start:end].type(self.compression_dtype)
+ for e in range(tmp.shape[1]):
+ tmp[:, e] &= mask
+ tmp[:, e] = tmp[:, e] << (self.bits * e)
+ self.qzeros[:, j] |= tmp[:, e]
+ if self.use_optimum_format or self.compression_dim == 0:
+ self.qzeros = self.qzeros.t_().contiguous()
+ if self.use_optimum_format:
+ self.scales = self.scales.t_().contiguous()
+ self.qweight = self.qweight.t_().contiguous()
+ self.qzeros = self.qzeros.t_().contiguous()
+
+ def recover(self):
+ logger.debug(f"Recovering {self} weight")
+ scales = self.scales.t_().contiguous() if self.use_optimum_format else self.scales
+ qweight = self.qweight.t_().contiguous() if self.use_optimum_format else self.qweight
+
+ device = scales.device
+ fp32_weight = torch.zeros(self.out_features, self.in_features, dtype=self.float_type).to(device)
+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(device)
+ if hasattr(self, "qzeros"):
+ weight_dtype = torch.uint8
+ else:
+ weight_dtype = torch.int8
+ # unpack weight
+ weight = torch.zeros(self.out_features, self.in_features, dtype=weight_dtype).to(device)
+ if not self.use_optimum_format and self.compression_dim == 0:
+ weight = weight.t_().contiguous()
+ qweight = qweight.t_().contiguous()
+ origin_shape = weight.shape
+ target_shape = qweight.shape
+ for j in range(target_shape[1]):
+ for e in range(self.n_pack):
+ index = j * self.n_pack + e
+ if index >= origin_shape[1]:
+ continue
+ tmp = qweight[:, j]
+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))
+ tmp = tmp >> self.compress_bits - self.bits
+ if weight_dtype == torch.uint8:
+ tmp &= mask # remove sign bit
+ weight[:, index] = tmp.type(weight_dtype)
+ if not self.use_optimum_format and self.compression_dim == 0:
+ weight = weight.t_().contiguous()
+ if "int" not in self.dtype:
+ new_weight = torch.zeros(self.out_features, self.in_features).to(device)
+ for k, v in self.int2float_mapping.items():
+ new_weight += torch.where(weight == k, v, 0)
+ weight = new_weight
+ # unpack zero_point
+ if hasattr(self, "qzeros"):
+ zp_dtype = self.compression_dtype # to avoid overflow when weight-zp
+ zp = torch.zeros(scales.shape, dtype=zp_dtype).to(device)
+ qzeros = self.qzeros.t_().contiguous() if self.use_optimum_format else self.qzeros
+ if self.use_optimum_format or self.compression_dim == 0:
+ zp = zp.t_().contiguous()
+ qzeros = qzeros.t_().contiguous()
+ origin_shape = zp.shape
+ target_shape = qzeros.shape
+ for j in range(target_shape[1]):
+ for e in range(self.n_pack):
+ index = j * self.n_pack + e
+ if index >= origin_shape[1]:
+ continue
+ tmp = qzeros[:, j]
+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))
+ tmp = tmp >> self.compress_bits - self.bits
+ tmp &= mask
+ zp[:, index] = tmp.type(zp_dtype)
+ if self.use_optimum_format or self.compression_dim == 0:
+ zp = zp.t_().contiguous()
+ if self.use_optimum_format:
+ # zp -= 1 may cause zp == -1, after recover it becomes 2**self.bits - 1
+ zp += 1
+ zp = torch.where(zp > (2**self.bits - 1), 0, zp)
+ # recover fp32 weight with int_weight, scale, and zero_point
+ for idx in range(self.in_features):
+ g_idx = idx // self.groupsize
+ fp32_weight[:, idx] = (weight[:, idx] - zp[:, g_idx]) * scales[:, g_idx]
+ else:
+ # recover fp32 weight with int_weight, scale
+ for idx in range(self.in_features):
+ g_idx = idx // self.groupsize
+ fp32_weight[:, idx] = weight[:, idx] * scales[:, g_idx]
+ return fp32_weight
+
+ def forward(self, input):
+ weight = self.recover()
+ device = self.scales.device
+ if weight.dtype == torch.float16 and device.type == "cpu":
+ weight = weight.float()
+ self.bias = self.bias.float() if self.bias is not None else None
+ input = input.type(weight.dtype)
+ return F.linear(input, weight, self.bias)
+
+ def extra_repr(self) -> str:
+ tmp_str = "in_features={}, out_features={}, bits={}, group_size={}, bias={}".format(
+ self.in_features,
+ self.out_features,
+ self.bits,
+ self.groupsize,
+ self.bias is not None,
+ )
+ if self.use_optimum_format:
+ tmp_str += ", use_optimum_format=True"
+ return tmp_str
+
diff --git a/examples/main.py b/examples/main.py
index 63d32b006..f1a4dd768 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -1,7 +1,11 @@
import argparse
import sys
-sys.path.insert(0, '../')
-from auto_round import AutoRound, AutoAdamRound
+sys.path.insert(0, './')
+from auto_round import (AutoRound,
+ AutoAdamRound,
+ compress_model,
+ save_compressed_model,
+ AutoroundQuantConfig)
parser = argparse.ArgumentParser()
import torch
@@ -84,26 +88,19 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="whether enable weight minmax tuning")
+
+ parser.add_argument("--target_platform", default='cpu',
+ help="targeted inference acceleration platform, 'cpu' or 'gpu'")
+
+ parser.add_argument("--scale_dtype", default='fp32',
+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
- # parser.add_argument("--tasks", default=["lambada_openai", "hellaswag", "winogrande", "piqa"],
- # help="lm-eval tasks")
-
- # parser.add_argument("--tasks", default=["lambada_openai","wikitext2",'hendrycksTest-*'],
- # help="lm-eval tasks")
-
- # parser.add_argument("--tasks",
- # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- # 'coqa', 'truthfulqa_mc', 'openbookqa', 'boolq', 'rte', 'arc_easy', 'arc_challenge',
- # 'hendrycksTest-*', 'wikitext', 'drop', 'gsm8k'],##all
parser.add_argument("--tasks",
default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
"hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",
"arc_easy", "arc_challenge"],
help="lm-eval tasks")
- # parser.add_argument("--tasks", default=["lambada_openai"],
- # help="lm-eval tasks")
-
parser.add_argument("--output_dir", default="./tmp_signround", type=str,
help="Where to store the final model.")
@@ -189,8 +186,17 @@ if __name__ == '__main__':
autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,
- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps) ##TODO args pass
- autoround.quantize()
+ amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
+ seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args pass
+ model, q_config = autoround.quantize()
+
+ if args.target_platform == 'cpu':
+ output_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
+ compressed_model = compress_model(model, q_config)
+ quantize_config = AutoroundQuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)
+ save_compressed_model(compressed_model, output_dir=output_dir, quantize_config=quantize_config, tokenizer=tokenizer)
+ del q_config
+ del compressed_model
torch.cuda.empty_cache()
model.eval()
diff --git a/examples/requirements.txt b/examples/requirements.txt
index 04d132ae6..a971eef6d 100644
--- a/examples/requirements.txt
+++ b/examples/requirements.txt
@@ -8,4 +8,6 @@ sentencepiece
einops
accelerate
datasets
+protobuf
+