PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add model compress and export for cpu
Zhang, Weiwei1574c6c3c7477
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 1b2514237..43059f6cb 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -13,3 +13,7 @@# limitations under the License.from .autoround import AutoRound, AutoAdamRound, AutoOPTRoundfrom .version import __version__+from .export import compress_model, save_compressed_model, save_quantized_to_autogptq+from .config import AutoroundQuantConfig+from .model_wrapper import WeightOnlyLinear+diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ba0e6535e..6e170b001 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -33,7 +33,7 @@ from torch.amp import autocastfrom torch.functional import F-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):+def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):"""Quantizes and dequantizes weight asymmetrically.Args:@@ -47,8 +47,8 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):Quantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** num_bits - 1)- ##zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=weight.dtype)##force scale to fp16- zeros = torch.zeros(weight.shape[0], device=weight.device)+ zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)+ # zeros = torch.zeros(weight.shape[0], device=weight.device)if isinstance(min_scale, torch.Tensor):wmin_tmp = torch.minimum(weight.min(1)[0], zeros)wmax_tmp = torch.maximum(weight.max(1)[0], zeros)@@ -59,10 +59,11 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):else:wmin = torch.minimum(weight.min(1)[0], zeros)wmax = torch.maximum(weight.max(1)[0], zeros)+tmp = (wmin == 0) & (wmax == 0)wmin[tmp] = -1wmax[tmp] = +1- scale = (wmax - wmin) / maxq+ scale = ((wmax - wmin) / maxq).to(scale_dtype)zp = round_ste(-wmin / scale)scale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)@@ -71,7 +72,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):return scale * (q - zp), scale, zp-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):+def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):"""Quantizes and dequantizes weight symmetrically.Args:@@ -94,13 +95,13 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):wmax *= 1 + max_scaletmp = wmax == 0wmax[tmp] = +1- scale = wmax / ((maxq - minq) / 2)+ scale = (wmax / ((maxq - minq) / 2)).to(scale_dtype)scale.unsqueeze_(dim=-1)q = torch.clamp(round_ste(weight / scale + v), minq, maxq)return scale * q, scale, None-def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):+def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):"""Quantizes and dequantizes weight symmetrically or asymmetrically .Args:@@ -116,12 +117,12 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):"""assert num_bits > 0, "num_bits should be larger than 0"if scheme == "sym":- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale)+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)else:- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale)+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)-def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0):+def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):"""Quantizes and dequantizes weight, handing the group size issue .Args:@@ -137,7 +138,8 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalQuantized and dequantized weight, scale, zero-point"""if group_size == -1 or weight.shape[1] < group_size:- return quant_weight_actor(weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale)+ return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,+ min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)orig_shape = weight.shapeif weight.shape[1] % group_size == 0:weight = weight.reshape(-1, group_size)@@ -145,7 +147,7 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalv = v.reshape(-1, group_size)weight, scale, zp = quant_weight_actor(- weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)weight = weight.reshape(orig_shape)scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d@@ -161,7 +163,7 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalif isinstance(v, torch.Tensor):v = v.reshape(-1, group_size)weight_new, scale, zp = quant_weight_actor(- weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale+ weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)weight_new = weight_new.reshape(orig_shape[0], -1)@@ -170,6 +172,40 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalif zp is not None:zp = zp.reshape(weight_new.shape[0], -1)return weight_new, scale, zp+++def quant_weight_w_scale(weight, scale, zp, group_size=-1):+ """Quant and dequant tensor with group size.++ Args:+ weight: input weight+ scale: scale+ zp: zero point+ group_size (int, optional): how many elements share one scale/zp. Defaults to -1.++ Returns:+ output: int weight.+ """+ device = weight.device+ scale = scale.to(device)+ if zp is not None:+ zp = zp.to(device)+ if group_size == -1:+ return torch.round(weight / scale) if zp is None else torch.round(weight / scale + zp)+ int_weight = torch.zeros(weight.shape).to(device)+ leng = weight.shape[1] // group_size+ tail_flag = False if weight.shape[1] % group_size == 0 else True+ for i in range(leng):+ int_weight_tmp = weight[:, i * group_size : (i + 1) * group_size] / scale[:, i].unsqueeze(1)+ if zp is not None:+ int_weight_tmp += zp[:, i].unsqueeze(1)+ int_weight[:, i * group_size : (i + 1) * group_size] = torch.round(int_weight_tmp)+ if tail_flag:+ int_weight_tmp = weight[:, leng * group_size :] / scale[:, -1].unsqueeze(1)+ if zp is not None:+ int_weight_tmp += zp[:, -1].unsqueeze(1)+ int_weight[:, leng * group_size :] = torch.round(int_weight_tmp)+ return int_weightdef round_ste(x: torch.Tensor):@@ -400,7 +436,8 @@ class WrapperLinear(torch.nn.Module):self.orig_layer = orig_layerself.num_bits = self.orig_layer.bitsself.group_size = self.orig_layer.group_size- self.scheme = (self.orig_layer.scheme,)+ self.scale_dtype = self.orig_layer.scale_dtype+ self.scheme = self.orig_layer.schemeself.value = torch.nn.Parameter(torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True)@@ -440,11 +477,12 @@ class WrapperLinear(torch.nn.Module):v,min_scale,max_scale,+ self.scale_dtype,)self.orig_layer.weight.data.copy_(q_dq_weight)self.orig_layer.weight.grad = None ##clear gradself.orig_layer.scale = scale.to("cpu")- self.orig_layer.zp = zp.to("cpu")+ self.orig_layer.zp = zp.to("cpu") if zp is not None else Nonereturn self.orig_layerdef forward(self, x):@@ -460,7 +498,7 @@ class WrapperLinear(torch.nn.Module):self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))weight_q, _, _ = quant_weight(- weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale+ weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale, self.scale_dtype)weight_q = weight_q.to(weight.dtype)return F.linear(x, weight_q, self.orig_layer.bias)@@ -527,6 +565,7 @@ class WrapperTransformerConv1d(torch.nn.Module):v,min_scale,max_scale,+ self.scale_dtype)self.orig_layer.weight.data.copy_(weight_q.t())self.orig_layer.weight.grad = None@@ -554,6 +593,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.value,self.min_scale,self.max_scale,+ self.scale_dtype)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)@@ -977,6 +1017,7 @@ class AutoRound(object):not_use_mse: bool = False,dynamic_max_gap: int = -1,data_type: str = "int", ##only support data_type+ scale_dtype = "fp16",**kwargs,):self.model = model@@ -1007,6 +1048,14 @@ class AutoRound(object):self.train_bs = bsself.n_blocks = n_blocksself.device = device++ if scale_dtype == 'fp16':+ self.scale_dtype = torch.float16+ elif scale_dtype == 'bf16':+ self.scale_dtype = torch.bfloat16+ else:+ self.scale_dtype = torch.float32+self.amp_dtype = torch.float16if self.model.dtype != torch.float32:self.amp_dtype = self.model.dtype@@ -1143,10 +1192,13 @@ class AutoRound(object):weight_config[n]["group_size"] = self.group_sizeif "scheme" not in weight_config[n].keys():weight_config[n]["scheme"] = self.scheme+ weight_config[n]["scale_dtype"] = self.scale_dtype+m.data_type = weight_config[n]["data_type"]m.bits = weight_config[n]["bits"]m.group_size = weight_config[n]["group_size"]m.scheme = weight_config[n]["scheme"]+ m.scale_dtype = weight_config[n]["scale_dtype"]def get_batch_dim(self, input_others):"""Get the batch dimension of the input tensor.@@ -1722,3 +1774,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/auto_round/config.py b/auto_round/config.pynew file mode 100644index 000000000..8d43ceaed--- /dev/null+++ b/auto_round/config.py@@ -0,0 +1,229 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Configs for Autoround quantization."""+try:+ from neural_compressor.utils.utility import LazyImport+ torch = LazyImport("torch")+ from neural_compressor.utils import logger+except: # pragma: no cover+ import logging+ import torch+ logger = logging.getLogger()++import copy+import json+import os+from dataclasses import dataclass, field+from typing import Any, Dict, Tuple, Union+from transformers import BitsAndBytesConfig, PretrainedConfig++QUANT_CONFIG = "quantize_config.json"++class AutoroundQuantConfig(PretrainedConfig):+ def __init__(+ self,+ bits=4,+ scale_dtype="fp32",+ group_size=128,+ sym=False,+ quant_method="Autoround",+ model_name_or_path=None,+ model_file_base_name='model',+ **kwargs,+ ):+ self.bits = bits+ self.group_size = group_size+ self.scale_dtype = scale_dtype ## reserved parameter+ self.sym = sym+ self.quant_method = quant_method+ self.model_name_or_path = model_name_or_path+ self.model_file_base_name = model_file_base_name++ ### Redundant parameters, will be removed later. ###+ self.damp_percent = 0.01+ self.desc_act = False+ self.true_sequential = False+ self.quant_method = "gptq"+++ def post_init(self):+ r"""+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.+ """++ if self.scale_dtype not in ["fp32", "fp16"]:+ raise ValueError("scale_dtype must be 'fp32', 'fp16'.")++ if self.group_size not in [-1, 32, 128]:+ raise ValueError("group_size must be an integer in [-1, 32, 128]")+++ def quantization_method(self):+ r"""+ This method returns the quantization method used for the model.+ """+ return self.quant_method+++ @classmethod+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):+ """+ Instantiates a [`AutoroundQuantConfig`] from a Python dictionary of parameters.+ Args:+ config_dict (`Dict[str, Any]`):+ Dictionary that will be used to instantiate the configuration object.+ return_unused_kwargs (`bool`):+ Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in+ `PreTrainedModel`.+ kwargs (`Dict[str, Any]`):+ Additional parameters from which to initialize the configuration object.+ Returns:+ [`AutoroundQuantConfig`]: The configuration object instantiated from those parameters.+ """++ config = cls(**config_dict)++ to_remove = []+ for key, value in kwargs.items():+ if hasattr(config, key):+ setattr(config, key, value)+ to_remove.append(key)+ for key in to_remove:+ kwargs.pop(key, None)++ if return_unused_kwargs:+ return config, kwargs+ else:+ return config+++ @classmethod+ def from_json_file(cls, json_file_path, return_unused_kwargs, **kwargs):+ with open(json_file_path, "r", encoding="utf-8") as f:+ config_dict = json.load(f)+ return cls.from_dict(config_dict, return_unused_kwargs, **kwargs)+++ def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):+ """+ Save this instance to a JSON file.+ Args:+ json_file_path (`str` or `os.PathLike`):+ Path to the JSON file in which this configuration instance's parameters will be saved.+ """+ with open(json_file_path, "w", encoding="utf-8") as writer:+ writer.write(self.to_json_string(use_diff=use_diff))++ def to_dict(self) -> Dict[str, Any]:+ """+ Serializes this instance to a Python dictionary. Returns:+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.+ """++ output = copy.deepcopy(self.__dict__)+ return output++ def __repr__(self):+ return f"{self.__class__.__name__} {self.to_json_string()}"+++ def to_json_string(self, use_diff: bool = True) -> str:+ """+ Serializes this instance to a JSON string.+ Args:+ use_diff (`bool`, *optional*, defaults to `True`):+ If set to `True`, only the difference between the config instance and the default+ `AutoroundQuantConfig()`+ is serialized to JSON string.+ Returns:+ `str`: String containing all the attributes that make up this configuration instance in JSON format.+ """+ if use_diff is True:+ config_dict = self.to_diff_dict()+ else:+ config_dict = self.to_dict()++ return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"++ def to_diff_dict(self) -> Dict[str, Any]:+ """+ Removes all attributes from config which correspond to the default config attributes for better readability and+ serializes to a Python dictionary.+ Returns:+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,+ """+ config_dict = self.to_dict()++ # get the default config dict+ default_config_dict = AutoroundQuantConfig().to_dict()++ serializable_config_dict = {}++ # only serialize values that differ from the default config+ for key, value in config_dict.items():+ if value != default_config_dict[key]:+ serializable_config_dict[key] = value++ return serializable_config_dict++ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):+ """+ Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the+ [`~PretrainedConfig.from_pretrained`] class method.+ Args:+ save_directory (`str` or `os.PathLike`):+ Directory where the configuration JSON file will be saved (will be created if it does not exist).+ push_to_hub (`bool`, *optional*, defaults to `False`):+ Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the+ repository you want to push to with `repo_id` (will default to the name of `save_directory` in your+ namespace).+ kwargs (`Dict[str, Any]`, *optional*):+ Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.+ """+ self._set_token_in_kwargs(kwargs)++ if os.path.isfile(save_directory):+ raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")++ os.makedirs(save_directory, exist_ok=True)++ if push_to_hub:+ commit_message = kwargs.pop("commit_message", None)+ repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])+ repo_id = self._create_repo(repo_id, **kwargs)+ files_timestamps = self._get_files_timestamps(save_directory)++ # If we save using the predefined names, we can load using `from_pretrained`+ output_config_file = os.path.join(save_directory, QUANT_CONFIG)++ self.to_json_file(output_config_file, use_diff=False)+ logger.info(f"Configuration saved in {output_config_file}")++ if push_to_hub:+ self._upload_modified_files(+ save_directory,+ repo_id,+ files_timestamps,+ commit_message=commit_message,+ token=kwargs.get("token", None),+ )++ @classmethod+ def get_config_dict(+ cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs+ ) -> Tuple[Dict[str, Any], Dict[str, Any]]:+ return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)\ No newline at end of filediff --git a/auto_round/export.py b/auto_round/export.pyindex ac44f1e38..bb5a552b5 100644--- a/auto_round/export.py+++ b/auto_round/export.py@@ -42,13 +42,104 @@ import osfrom typing import Dict, List, Optional, Unionfrom safetensors.torch import save_file as safe_savefrom os.path import join, isfile, isdir+import copy+import json+++def compress_model(+ model,+ weight_config:Union[str, dict],+ enable_full_range=False,+ compression_dtype=torch.int32,+ compression_dim=1,+ scale_dtype=torch.float32,+ device="cpu",+ use_optimum_format=True,+ ):+ """Convert Linear to WeightOnlyLinear for low memory inference.++ Args:+ weight_config (str|dict): qconfig dict or Path of qconfig.json.+ enable_full_range (bool, optional): Whether to leverage the full compression range+ under symmetric quantization. Defaults to False.+ compression_dtype (torch.Tensor, optional): The target dtype after comoression.+ Defaults to torch.int32.+ compression_dim (int, optional): Select from [0, 1], 0 is output channel,+ 1 is input channel. Defaults to 1.+ scale_dtype (torch.Tensor, optional): Use float32 or float16.+ Defaults to torch.float32.+ device (str, optional): choose device for compression. Defaults to cpu.+ use_optimum_format (bool, optional): use the popular huggingface compression format.+ 1: compression_dim: weight = 1, zeros = 0 and both are transposed.+ 2: zeros -= 1 before compression. Why we need it?+ 3: g_idx: use same number for one group instead of recording the channel order.+ 4. parameter name changed, such as 'packed_weight' -> 'qweight'.+ 5. zeros is always needed even for sym.+ """+ from .model_wrapper import WeightOnlyLinear+ from .autoround import quant_weight_w_scale, get_module, set_module+ compressed_model = copy.deepcopy(model)+ if isinstance(weight_config, str):+ with open(weight_config, "r") as f:+ q_config = json.load(f)+ else:+ q_config = weight_config+ for k, v in q_config.items():+ logger.info(f"Compressing {k} on device {device}")+ if "float" in v["data_type"]:+ continue+ dtype = v["data_type"]+ num_bits = v["bits"]+ group_size = v["group_size"]+ scheme = v["scheme"]+ scale_dtype = v["scale_dtype"]+ m = get_module(compressed_model, k)+ fp_weight = m.weight.data+ scale = torch.tensor(v["scale"], dtype=torch.float32)++ zp = None if scheme == "sym" else torch.tensor(v["zp"], dtype=torch.int32)+ int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size)+ int_weight = int_weight.type(torch.int32)+ new_module = WeightOnlyLinear(+ m.in_features,+ m.out_features,+ num_bits,+ group_size,+ dtype=dtype,+ scale_dtype=scale_dtype,+ zp=zp is not None,+ bias=m.bias is not None,+ device=device,+ use_optimum_format=True,+ )+ new_module.pack(int_weight, scale, zp, m.bias)+ set_module(compressed_model, k, new_module)+ return compressed_model+++def save_compressed_model(model, output_dir, quantize_config=None, tokenizer=None):+ """Save configure file and weights for CPU backend inference."""++ if quantize_config is not None:+ config = model.config+ setattr(config, "quantization_config", quantize_config.to_dict())+ config.save_pretrained(output_dir)+ quantize_config.save_pretrained(output_dir)++ try:+ model.save_pretrained(output_dir, safe_serialization=True)+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))+ except IOError as e: # pragma: no cover+ logger.error("Fail to save configure file and weights due to {}.".format(e))def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,minmax_lr=5e-3,enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,safetensors_metadata: Optional[Dict[str, str]] = None):- """save quantized model and configs to local disk"""+ """save quantized model and configs to local disk for cuda """os.makedirs(save_dir, exist_ok=True)model.to("cpu")diff --git a/auto_round/model_wrapper.py b/auto_round/model_wrapper.pynew file mode 100644index 000000000..4bddd6942--- /dev/null+++ b/auto_round/model_wrapper.py@@ -0,0 +1,345 @@+#+# -*- coding: utf-8 -*-+#+# Copyright (c) 2021 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Torch.nn.Module Class Definition."""+# Note: Do not import this file unless you have already imported torch,+# since the model classes inherit torch.nn.Module.+import math++import torch+from packaging.version import Version+from torch.autograd import Function+from torch.nn import functional as F+import logging+logger = logging.getLogger()+++NF4 = [+ -1.0,+ -0.6961928009986877,+ -0.5250730514526367,+ -0.39491748809814453,+ -0.28444138169288635,+ -0.18477343022823334,+ -0.09105003625154495,+ 0.0,+ 0.07958029955625534,+ 0.16093020141124725,+ 0.24611230194568634,+ 0.33791524171829224,+ 0.44070982933044434,+ 0.5626170039176941,+ 0.7229568362236023,+ 1.0,+]+FP4_BNB = [-12.0, -8.0, -6.0, -4.0, -3.0, -2.0, -0.0625, 0, 0.0625, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0]+FP4_E2M1 = [-6.0, -4.0, -3.0, -2.0, -1.5, -1.0, -0.0625, 0, 0.0625, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]++# the order is the same as float list, bit value range is [-7, 7]+# 1111 = -1, 1110 = -2, 1101= -3, ...++NF4_BIT = [7, 1, 2, 3, 4, 5, 6, 0, -8, -7, -6, -5, -4, -3, -2, -1]+FP4_BNB_BIT = [-5, -6, -3, -4, -1, -2, -7, 0, 1, 6, 7, 4, 5, 2, 3]+FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]++FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}+INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}++def get_torch_version():+ try:+ torch_version = torch.__version__.split("+")[0]+ except ValueError as e: # pragma: no cover+ assert False, "Got an unknown version of torch: {}".format(e)+ version = Version(torch_version)+ return version+++PT_VERSION = get_torch_version().release++class WeightOnlyLinear(torch.nn.Module):+ def __init__(+ self,+ in_features,+ out_features,+ bits,+ groupsize,+ dtype="int",+ zp=False,+ bias=False,+ scale_dtype=torch.float16,+ compression_dtype=torch.int32,+ compression_dim=1,+ device="cpu",+ use_optimum_format=True,+ ):+ super().__init__()+ self.use_optimum_format = use_optimum_format+ self.dtype = dtype+ if "int" not in self.dtype: # for nf4, fp4++ float_list = FLOAT_MAPPING[self.dtype]+ int_list = INT_MAPPING[self.dtype]+ self.int2float_mapping = {}+ for k, v in zip(int_list, float_list):+ self.int2float_mapping[k] = v+ self.device = device+ self.in_features = in_features+ self.out_features = out_features+ self.bits = bits+ self.groupsize = groupsize if groupsize != -1 else in_features+ self.compression_dim = compression_dim+ assert compression_dtype in [+ torch.int8,+ torch.int16,+ torch.int32,+ torch.int64,+ ], "Only support torch.int8|16|32|64 as compressed dtype."+ dtype_bits_mapping = {torch.int8: 8, torch.int16: 16, torch.int32: 32, torch.int64: 64}+ self.compress_bits = dtype_bits_mapping[compression_dtype]+ self.n_pack = self.compress_bits // self.bits+ # K is input channel, N is output channel+ assert compression_dim in [0, 1], (+ "Only support 0 or 1 as compression dimension, " + "0 is output channel, 1 is input channel."+ )+ if self.use_optimum_format:+ self.float_type = scale_dtype+ self.compression_dtype = torch.int32+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(in_features / self.groupsize), out_features),+ dtype=self.float_type,+ ).to(device),+ )+ self.register_buffer(+ "qweight",+ torch.zeros(+ (math.ceil(in_features / self.n_pack), out_features),+ dtype=self.compression_dtype,+ ).to(device),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (math.ceil(self.in_features / self.groupsize), math.ceil(self.out_features / self.n_pack)),+ dtype=self.compression_dtype,+ ).to(device),+ )+ self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))+ else:+ self.compression_dtype = compression_dtype+ self.float_type = scale_dtype+ self.register_buffer(+ "scales",+ torch.zeros(+ (out_features, math.ceil(in_features / self.groupsize)),+ dtype=self.float_type,+ ).to(device),+ )+ if compression_dim == 1:+ self.register_buffer(+ "qweight",+ torch.zeros(+ (out_features, math.ceil(in_features / self.n_pack)),+ dtype=self.compression_dtype,+ ).to(device),+ )+ if zp:+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (self.out_features, math.ceil(self.in_features / self.groupsize / self.n_pack)),+ dtype=self.compression_dtype,+ ).to(device),+ )+ else:+ self.register_buffer(+ "qweight",+ torch.zeros(+ (math.ceil(out_features / self.n_pack), in_features),+ dtype=self.compression_dtype,+ ).to(device),+ )+ if zp:+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (math.ceil(self.out_features / self.n_pack), math.ceil(self.in_features / self.groupsize)),+ dtype=self.compression_dtype,+ ).to(device),+ )+ if bias:+ self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))+ else:+ self.bias = None++ def pack(self, int_weight, scale, zp, bias):+ if self.use_optimum_format:+ self.scales = self.scales.t_().contiguous()+ self.qweight = self.qweight.t_().contiguous()+ self.qzeros = self.qzeros.t_().contiguous()++ int_weight = int_weight.to(self.device)+ if self.use_optimum_format and zp is None:+ # to avoid overflow+ int_weight = int_weight.type(torch.int32)+ shift_bias = 2 ** (self.bits - 1)+ int_weight += shift_bias+ zp = torch.zeros_like(scale, dtype=torch.uint8) + shift_bias+ if bias is not None:+ assert hasattr(self, "bias"), "bias is not set when initializing."+ self.bias = bias.type(self.float_type).to(self.device)+ assert scale.shape == self.scales.shape, "Scale shape is mismatched."+ self.scales = scale.type(self.float_type).to(self.device)+ if not self.use_optimum_format and self.compression_dim == 0:+ int_weight = int_weight.t_().contiguous()+ self.qweight = self.qweight.t_().contiguous()+ origin_shape = int_weight.shape+ target_shape = self.qweight.shape+ assert origin_shape[0] == target_shape[0], "output channels mismatch, please check."+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)++ # pack weight+ for j in range(target_shape[1]):+ start = self.n_pack * j+ end = self.n_pack * (j + 1)+ tmp = int_weight[:, start:end].type(self.compression_dtype)+ for e in range(tmp.shape[1]):+ tmp[:, e] &= mask+ tmp[:, e] = tmp[:, e] << (self.bits * e)+ self.qweight[:, j] |= tmp[:, e]+ if not self.use_optimum_format and self.compression_dim == 0:+ self.qweight = self.qweight.t_().contiguous()++ if zp is not None:+ zp = zp.to(self.device)+ if self.use_optimum_format:+ zp -= 1+ if self.use_optimum_format or self.compression_dim == 0:+ zp = zp.t_().contiguous()+ self.qzeros = self.qzeros.t_().contiguous()+ assert hasattr(self, "qzeros"), "zp is not set when initializing."+ target_shape = self.qzeros.shape+ for j in range(target_shape[1]):+ start = self.n_pack * j+ end = self.n_pack * (j + 1)+ tmp = zp[:, start:end].type(self.compression_dtype)+ for e in range(tmp.shape[1]):+ tmp[:, e] &= mask+ tmp[:, e] = tmp[:, e] << (self.bits * e)+ self.qzeros[:, j] |= tmp[:, e]+ if self.use_optimum_format or self.compression_dim == 0:+ self.qzeros = self.qzeros.t_().contiguous()+ if self.use_optimum_format:+ self.scales = self.scales.t_().contiguous()+ self.qweight = self.qweight.t_().contiguous()+ self.qzeros = self.qzeros.t_().contiguous()++ def recover(self):+ logger.debug(f"Recovering {self} weight")+ scales = self.scales.t_().contiguous() if self.use_optimum_format else self.scales+ qweight = self.qweight.t_().contiguous() if self.use_optimum_format else self.qweight++ device = scales.device+ fp32_weight = torch.zeros(self.out_features, self.in_features, dtype=self.float_type).to(device)+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(device)+ if hasattr(self, "qzeros"):+ weight_dtype = torch.uint8+ else:+ weight_dtype = torch.int8+ # unpack weight+ weight = torch.zeros(self.out_features, self.in_features, dtype=weight_dtype).to(device)+ if not self.use_optimum_format and self.compression_dim == 0:+ weight = weight.t_().contiguous()+ qweight = qweight.t_().contiguous()+ origin_shape = weight.shape+ target_shape = qweight.shape+ for j in range(target_shape[1]):+ for e in range(self.n_pack):+ index = j * self.n_pack + e+ if index >= origin_shape[1]:+ continue+ tmp = qweight[:, j]+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))+ tmp = tmp >> self.compress_bits - self.bits+ if weight_dtype == torch.uint8:+ tmp &= mask # remove sign bit+ weight[:, index] = tmp.type(weight_dtype)+ if not self.use_optimum_format and self.compression_dim == 0:+ weight = weight.t_().contiguous()+ if "int" not in self.dtype:+ new_weight = torch.zeros(self.out_features, self.in_features).to(device)+ for k, v in self.int2float_mapping.items():+ new_weight += torch.where(weight == k, v, 0)+ weight = new_weight+ # unpack zero_point+ if hasattr(self, "qzeros"):+ zp_dtype = self.compression_dtype # to avoid overflow when weight-zp+ zp = torch.zeros(scales.shape, dtype=zp_dtype).to(device)+ qzeros = self.qzeros.t_().contiguous() if self.use_optimum_format else self.qzeros+ if self.use_optimum_format or self.compression_dim == 0:+ zp = zp.t_().contiguous()+ qzeros = qzeros.t_().contiguous()+ origin_shape = zp.shape+ target_shape = qzeros.shape+ for j in range(target_shape[1]):+ for e in range(self.n_pack):+ index = j * self.n_pack + e+ if index >= origin_shape[1]:+ continue+ tmp = qzeros[:, j]+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))+ tmp = tmp >> self.compress_bits - self.bits+ tmp &= mask+ zp[:, index] = tmp.type(zp_dtype)+ if self.use_optimum_format or self.compression_dim == 0:+ zp = zp.t_().contiguous()+ if self.use_optimum_format:+ # zp -= 1 may cause zp == -1, after recover it becomes 2**self.bits - 1+ zp += 1+ zp = torch.where(zp > (2**self.bits - 1), 0, zp)+ # recover fp32 weight with int_weight, scale, and zero_point+ for idx in range(self.in_features):+ g_idx = idx // self.groupsize+ fp32_weight[:, idx] = (weight[:, idx] - zp[:, g_idx]) * scales[:, g_idx]+ else:+ # recover fp32 weight with int_weight, scale+ for idx in range(self.in_features):+ g_idx = idx // self.groupsize+ fp32_weight[:, idx] = weight[:, idx] * scales[:, g_idx]+ return fp32_weight++ def forward(self, input):+ weight = self.recover()+ device = self.scales.device+ if weight.dtype == torch.float16 and device.type == "cpu":+ weight = weight.float()+ self.bias = self.bias.float() if self.bias is not None else None+ input = input.type(weight.dtype)+ return F.linear(input, weight, self.bias)++ def extra_repr(self) -> str:+ tmp_str = "in_features={}, out_features={}, bits={}, group_size={}, bias={}".format(+ self.in_features,+ self.out_features,+ self.bits,+ self.groupsize,+ self.bias is not None,+ )+ if self.use_optimum_format:+ tmp_str += ", use_optimum_format=True"+ return tmp_str+diff --git a/examples/main.py b/examples/main.pyindex 63d32b006..f1a4dd768 100644--- a/examples/main.py+++ b/examples/main.py@@ -1,7 +1,11 @@import argparseimport sys-sys.path.insert(0, '../')-from auto_round import AutoRound, AutoAdamRound+sys.path.insert(0, './')+from auto_round import (AutoRound,+ AutoAdamRound,+ compress_model,+ save_compressed_model,+ AutoroundQuantConfig)parser = argparse.ArgumentParser()import torch@@ -84,26 +88,19 @@ if __name__ == '__main__':parser.add_argument("--enable_minmax_tuning", action='store_true',help="whether enable weight minmax tuning")++ parser.add_argument("--target_platform", default='cpu',+ help="targeted inference acceleration platform, 'cpu' or 'gpu'")++ parser.add_argument("--scale_dtype", default='fp32',+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")- # parser.add_argument("--tasks", default=["lambada_openai", "hellaswag", "winogrande", "piqa"],- # help="lm-eval tasks")-- # parser.add_argument("--tasks", default=["lambada_openai","wikitext2",'hendrycksTest-*'],- # help="lm-eval tasks")-- # parser.add_argument("--tasks",- # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- # 'coqa', 'truthfulqa_mc', 'openbookqa', 'boolq', 'rte', 'arc_easy', 'arc_challenge',- # 'hendrycksTest-*', 'wikitext', 'drop', 'gsm8k'],##allparser.add_argument("--tasks",default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',"hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte","arc_easy", "arc_challenge"],help="lm-eval tasks")- # parser.add_argument("--tasks", default=["lambada_openai"],- # help="lm-eval tasks")-parser.add_argument("--output_dir", default="./tmp_signround", type=str,help="Where to store the final model.")@@ -189,8 +186,17 @@ if __name__ == '__main__':autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps) ##TODO args pass- autoround.quantize()+ amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,+ seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args pass+ model, q_config = autoround.quantize()++ if args.target_platform == 'cpu':+ output_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"+ compressed_model = compress_model(model, q_config)+ quantize_config = AutoroundQuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)+ save_compressed_model(compressed_model, output_dir=output_dir, quantize_config=quantize_config, tokenizer=tokenizer)+ del q_config+ del compressed_modeltorch.cuda.empty_cache()model.eval()diff --git a/examples/requirements.txt b/examples/requirements.txtindex 04d132ae6..a971eef6d 100644--- a/examples/requirements.txt+++ b/examples/requirements.txt@@ -8,4 +8,6 @@ sentencepieceeinopsacceleratedatasets+protobuf+