Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add initial support for activation quantization (#176)

wenhuach215f67048c878b
diff --git a/README.md b/README.md
index 72afb77b1..ea6b26e8c 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
AutoRound
===========================
-<h3> Advanced Weight-Only Quantization Algorithm for LLMs</h3>
+<h3> Advanced Quantization Algorithm for LLMs</h3>
[![python](https://img.shields.io/badge/python-3.8%2B-blue)](https://github.com/intel/auto-round)
[![version](https://img.shields.io/badge/release-0.2-green)](https://github.com/intel/auto-round)
@@ -10,7 +10,7 @@ AutoRound
---
<div align="left">
-AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference. It's tailored for a wide range
+AutoRound is an advanced quantization algorithm for low-bits LLM inference. It's tailored for a wide range
of models. Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,
which competes impressively against recent methods without introducing any additional inference overhead. The below
image presents an overview of AutoRound. Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)
@@ -120,7 +120,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "
./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"
-- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits
+- `layer_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits
or mixed precision.
- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index dab5c7f21..2ddd1f1f0 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -224,7 +224,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
group_size: int = 128,
sym: bool = False,
backend="autoround:exllamav2",
- weight_config: dict = None,
+ layer_config: dict = None,
**kwargs,
):
@@ -234,7 +234,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
self.group_size = group_size
self.sym = sym
self.backend = backend
- self.weight_config = weight_config
+ self.layer_config = layer_config
if kwargs is not None:
for key in kwargs.keys():
setattr(self, key, kwargs[key])
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 51159c14c..85421f2ec 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -54,14 +54,19 @@ class AutoRound(object):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (bool): Whether symmetric quantization is to be used (default is False).
- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
- weight_config={
+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config={
'layer1':##layer_name
{
'data_type': 'int',
'bits': 4,
- 'group_size': 32,
+ 'group_size': 128,
'sym': False
+ 'act_data_type': None,
+ 'act_bits': 32,
+ 'group_size': None,
+ 'sym': None,
+
}
...
}
@@ -89,6 +94,10 @@ class AutoRound(object):
data_type (str): The data type to be used (default is "int").
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
+ act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_group_size (int): Group size for activation quantization. Default is None.
+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.
+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
Returns:
The quantized model.
@@ -101,11 +110,11 @@ class AutoRound(object):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- weight_config: dict = {},
+ layer_config: dict = {},
enable_full_range: bool = False, ##for symmetric, TODO support later
batch_size: int = 8,
amp: bool = True,
- device=None,
+ device: str = None,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
@@ -122,8 +131,12 @@ class AutoRound(object):
gradient_accumulate_steps: int = 1,
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
- data_type: str = "int", ##only support int for now
+ data_type: str = "int",
scale_dtype: str = "fp16",
+ act_bits: int = 32,
+ act_group_size: int = None,
+ act_sym: bool = None,
+ act_dynamic: bool = True,
**kwargs,
):
self.quantized = False
@@ -140,7 +153,7 @@ class AutoRound(object):
self.low_gpu_mem_usage = low_gpu_mem_usage
self.data_type = data_type
self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]
- self.weight_config = weight_config
+ self.layer_config = layer_config
self.seed = seed
self.tokenizer = tokenizer
self.seqlen = seqlen
@@ -165,11 +178,15 @@ class AutoRound(object):
self.dynamic_max_gap = dynamic_max_gap
self.enable_full_range = enable_full_range
self.lr_scheduler = lr_scheduler
- self.set_layerwise_config(self.weight_config)
self.optimizer = self.get_optimizer(None)
self.share_attention_mask_flag = None
self.hidden_dim_flag = None
self.infer_bs_coeff = 1
+ self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size
+ self.act_bits = act_bits if not (act_bits is None) else self.bits
+ self.act_sym = act_sym if not (act_sym is None) else self.sym
+ self.act_dynamic = act_dynamic
+ self.set_layerwise_config(self.layer_config)
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
logger.info(f"using {self.model.dtype} for quantization tuning")
@@ -186,27 +203,31 @@ class AutoRound(object):
"""
assert isinstance(self.model, torch.nn.Module)
assert self.bits > 0, "bits must be positive"
+ assert self.act_bits > 0, "bits must be positive"
assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
+ assert self.act_group_size == -1 or self.act_group_size >= 1,\
+ "only supports positive group_size or -1(per channel)"
assert self.train_bs > 0, "batch size must be positive"
assert self.iters > 0, "iters must be positive"
assert self.seqlen > 0, "seqlen must be positive"
assert self.nblocks > 0, "nblocks must be positive"
assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
assert self.enable_full_range is False, "only support enable_full_range=False currently"
+ assert self.act_dynamic is True, "only support dynamic quantization for activation currently"
# assert self.tokenizer != None or self.dataloader != None
def quantize(self):
- """Quantize the model and return the quantized model along with weight configurations.
+ """Quantize the model and return the quantized model along with layer configurations.
the entry of AutoRound.
Returns:
- The quantized model and weight configurations.
+ The quantized model and layer configurations.
"""
# logger.info("cache block input")
block_names = get_block_names(self.model)
if len(block_names) == 0:
logger.warning("could not find blocks, exit with original model")
- return self.model, self.weight_config
+ return self.model, self.layer_config
if self.amp:
self.model = self.model.to(self.amp_dtype)
@@ -239,7 +260,7 @@ class AutoRound(object):
self.quant_layers(layer_names, all_inputs)
- self.dump_data_to_weight_config()
+ self.dump_qinfo_to_layer_config()
end_time = time.time()
cost_time = end_time - self.start_time
@@ -250,7 +271,7 @@ class AutoRound(object):
unquantized_layers = []
for n, m in self.model.named_modules():
if isinstance(m, tuple(self.supported_types)):
- if self.weight_config[n]["bits"] == 16:
+ if m.bits > 8:
unquantized_layers.append(n)
else:
quantized_layers.append(n)
@@ -263,39 +284,45 @@ class AutoRound(object):
self.quantized = True
##self.model = self.model.to(self.model_orig_dtype)##keep it as amp dtype
- return self.model, self.weight_config
+ return self.model, self.layer_config
- def dump_data_to_weight_config(self):
+ def dump_qinfo_to_layer_config(self):
"""
- dump quantization scale and zp to weight configuration
+ dump quantization scale and zp to layer configuration
Args:
Returns:
None
"""
for n, m in self.model.named_modules():
- if n not in self.weight_config.keys():
+ if n not in self.layer_config.keys():
continue
if hasattr(m, "scale"):
- self.weight_config[n]["scale"] = m.scale
- self.weight_config[n]["zp"] = m.zp
+ self.layer_config[n]["scale"] = m.scale
+ self.layer_config[n]["zp"] = m.zp
+ if isinstance(m, transformers.modeling_utils.Conv1D):
+ weight = m.weight.t()
+ else:
+ weight = m.weight
+
if self.group_size <= 0:
- self.weight_config[n]["g_idx"] = torch.tensor(
- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+
+ self.layer_config[n]["g_idx"] = torch.tensor(
+ [0 for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"
)
else:
- self.weight_config[n]["g_idx"] = torch.tensor(
- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+ self.layer_config[n]["g_idx"] = torch.tensor(
+ [i // self.group_size for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"
)
delattr(m, "scale")
delattr(m, "zp")
else:
- self.weight_config[n]["data_type"] = "float"
+ self.layer_config[n]["data_type"] = "float"
if self.amp_dtype == torch.bfloat16:
- self.weight_config[n]["data_type"] = "bfloat"
- self.weight_config[n]["bits"] = 16
- self.weight_config[n]["group_size"] = None
- self.weight_config[n]["sym"] = None
+ self.layer_config[n]["data_type"] = "bfloat"
+ self.layer_config[n]["bits"] = 16
+ self.layer_config[n]["group_size"] = None
+ self.layer_config[n]["sym"] = None
def quant_layers(self, layer_names, layer_inputs):
"""Quantizes specified layers based on inputs and configuration.
@@ -328,51 +355,41 @@ class AutoRound(object):
q_layer_input[i] = None
torch.cuda.empty_cache()
- def set_layerwise_config(self, weight_config):
- """Sets the layer-wise configuration based on the provided weight_config.
+ def set_layerwise_config(self, layer_config):
+ """Sets the layer-wise configuration based on the provided layer_config.
By default, only quantize layers in blocks.
Args:
- weight_config: The weight configuration.
+ layer_config: The layer configuration.
Returns:
None
"""
layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types)
+ keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",
+ "act_dynamic"]
for n, m in self.model.named_modules():
if not isinstance(m, tuple(self.supported_types)):
continue
- if n not in weight_config.keys() and n in layers_in_blocks:
- weight_config[n] = {}
- weight_config[n]["data_type"] = self.data_type
- weight_config[n]["bits"] = self.bits
- weight_config[n]["group_size"] = self.group_size
- weight_config[n]["sym"] = self.sym
- weight_config[n]["scale_dtype"] = self.scale_dtype
- elif n in weight_config.keys():
- if "data_type" not in weight_config[n].keys():
- weight_config[n]["data_type"] = self.data_type
- if "bits" not in weight_config[n].keys():
- weight_config[n]["bits"] = self.bits
- if "group_size" not in weight_config[n].keys():
- weight_config[n]["group_size"] = self.group_size
- if "sym" not in weight_config[n].keys():
- weight_config[n]["sym"] = self.sym
- if "scale_dtype" not in weight_config[n].keys():
- weight_config[n]["scale_dtype"] = self.scale_dtype
- else:
- weight_config[n] = {}
- weight_config[n]["data_type"] = "float"
- weight_config[n]["bits"] = 16
- weight_config[n]["group_size"] = self.group_size
- weight_config[n]["sym"] = self.sym
- weight_config[n]["scale_dtype"] = self.scale_dtype
-
- m.data_type = weight_config[n]["data_type"]
- m.bits = weight_config[n]["bits"]
- m.group_size = weight_config[n]["group_size"]
- m.sym = weight_config[n]["sym"]
- m.scale_dtype = weight_config[n]["scale_dtype"]
+ ##not set in layer config, so use the default values
+ if n not in layer_config.keys() and n in layers_in_blocks:
+ layer_config[n] = {}
+ for key in keys:
+ layer_config[n][key] = getattr(self, key)
+ elif n in layer_config.keys(): ## partly set
+ for key in keys:
+ if key not in layer_config[n].keys():
+ layer_config[n][key] = getattr(self, key)
+ else: ##not in layer_config and layers in block,
+ layer_config[n] = {}
+ for key in keys:
+ layer_config[n][key] = getattr(self, key)
+ layer_config[n]["bits"] = 32
+ layer_config[n]["act_bits"] = 32
+
+ for key in keys:
+ setattr(m, key, layer_config[n][key])
+ tmp=1
@torch.no_grad()
def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device):
@@ -1009,6 +1026,13 @@ class AutoRound(object):
if not self.quantized:
logger.warning("please run autoround.quantize first")
return
+ if format == "fake" or format == "qdq" or self.act_bits <= 8: ##TODO fix act quantizaiton later
+ self.model = self.model.to("cpu")
+ self.model.save_pretrained(output_dir)
+ if self.tokenizer is not None:
+ self.tokenizer.save_pretrained(output_dir)
+ return
+
from auto_round.export import EXPORT_FORMAT
backend = format
format = format.split(":")[0]
@@ -1049,7 +1073,7 @@ class AutoRound(object):
compressed_model = save_quantized_as_format( ##TODO refine the code
output_dir,
model=self.model,
- weight_config=self.weight_config,
+ layer_config=self.layer_config,
inplace=inplace,
bits=self.bits,
group_size=self.group_size,
@@ -1075,20 +1099,20 @@ class AutoRound(object):
Returns:
list: List of layer names outside blocks.
"""
- if self.weight_config is None or len(self.weight_config) == 0:
+ if self.layer_config is None or len(self.layer_config) == 0:
return []
layer_names = []
all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types)
- for key in self.weight_config.keys():
+ for key in self.layer_config.keys():
if key in all_layers_in_block:
continue
layer = get_module(self.model, key)
if layer is None:
logger.error(f"could not find layer {key} in the model, exit...")
exit()
- if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.weight_config[key]):
+ if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):
layer_names.append(key)
return layer_names
@@ -1174,7 +1198,7 @@ class AutoOPTRound(AutoRound):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (bool): Whether sym to be used (default is False).
- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
enable_full_range (bool): Whether to enable full range quantization (default is False).
batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
@@ -1198,6 +1222,11 @@ class AutoOPTRound(AutoRound):
data_type (str): The data type to be used (default is "int").
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
+ act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_group_size (int): Group size for activation quantization. Default is None.
+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.
+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
+
**kwargs: Additional keyword arguments.
Returns:
@@ -1211,7 +1240,7 @@ class AutoOPTRound(AutoRound):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- weight_config: dict = {},
+ layer_config: dict = {},
enable_full_range: bool = False,
batch_size: int = 8,
amp: bool = True,
@@ -1234,6 +1263,10 @@ class AutoOPTRound(AutoRound):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
+ act_bits: int = 32,
+ act_group_size: int = None,
+ act_sym: bool = None,
+ act_dynamic: bool = True,
optimizer="AdamW",
**kwargs,
):
@@ -1243,7 +1276,7 @@ class AutoOPTRound(AutoRound):
bits,
group_size,
sym,
- weight_config,
+ layer_config,
enable_full_range,
batch_size,
amp,
@@ -1266,6 +1299,10 @@ class AutoOPTRound(AutoRound):
dynamic_max_gap,
data_type,
scale_dtype,
+ act_bits,
+ act_group_size,
+ act_sym,
+ act_dynamic,
**kwargs,
)
@@ -1321,7 +1358,7 @@ class AutoAdamRound(AutoOPTRound):
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (str): Whether symmetric quantization to be used (default is False).
- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).
enable_full_range (bool): Whether to enable full range quantization (default is False).
batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
@@ -1347,6 +1384,10 @@ class AutoAdamRound(AutoOPTRound):
optimizer: string or object
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
+ act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_group_size (int): Group size for activation quantization. Default is None.
+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.
+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
Returns:
The quantized model.
@@ -1359,7 +1400,7 @@ class AutoAdamRound(AutoOPTRound):
bits: int = 4,
group_size: int = 128,
sym: bool = False,
- weight_config: dict = {},
+ layer_config: dict = {},
enable_full_range: bool = False,
batch_size: int = 8,
amp: bool = True,
@@ -1382,6 +1423,10 @@ class AutoAdamRound(AutoOPTRound):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
+ act_bits: int = 32,
+ act_group_size: int = None,
+ act_sym: bool = None,
+ act_dynamic: bool = True,
optimizer="AdamW",
**kwargs,
):
@@ -1391,7 +1436,7 @@ class AutoAdamRound(AutoOPTRound):
bits,
group_size,
sym,
- weight_config,
+ layer_config,
enable_full_range,
batch_size,
amp,
@@ -1414,6 +1459,10 @@ class AutoAdamRound(AutoOPTRound):
dynamic_max_gap,
data_type,
scale_dtype,
+ act_bits,
+ act_group_size,
+ act_sym,
+ act_dynamic,
optimizer,
**kwargs,
)
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index acff4fe4a..a80015823 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -57,7 +57,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
except ImportError:
raise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")
model = kwargs["model"]
- weight_config = kwargs["weight_config"]
+ layer_config = kwargs["layer_config"]
sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
@@ -105,10 +105,10 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
if bits == 3 and use_triton is True:
logger.warning("triton does not support 3 bits, reset it to False")
quantizers = {}
- for key in weight_config:
+ for key in layer_config:
if key == "lm_head": ##TODO remove this after pr 87 is merged
continue
- info = weight_config[key]
+ info = layer_config[key]
if not check_to_quantized(info):
continue
##force to float32 to be compatible with torch 2.0
@@ -125,10 +125,10 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
)
else:
quantizers = {}
- for key in weight_config:
+ for key in layer_config:
if key == "lm_head": ##TODO remove this after pr 87 is merged
continue
- info = weight_config[key]
+ info = layer_config[key]
if not check_to_quantized(info):
continue
quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 68aa4585f..f7c5e535d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -147,7 +147,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exl
Default is "autoround:exllamav2".
**kwargs: Additional keyword arguments including:
- model (nn.Module): The model to be quantized.
- - weight_config (dict): The weight configuration for each layer.
+ - layer_config (dict): The layer configuration for each layer.
- serialization_dict (dict): The serialization configuration.
- tokenizer (Tokenizer, optional): The tokenizer to be saved.
@@ -171,12 +171,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exl
model = copy.deepcopy(model.to("cpu"))
layer_names_in_block = get_layer_names_in_block(model)
- weight_config = kwargs["weight_config"]
+ layer_config = kwargs["layer_config"]
with tctl.threadpool_limits(limits=1):
- for name in weight_config.keys():
+ for name in layer_config.keys():
- config = kwargs["weight_config"][name]
+ config = kwargs["layer_config"][name]
if config["data_type"] != "int" and config["bits"] >= 16:
continue
logger.info(f"packing {name}")
@@ -208,8 +208,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exl
new_layer.device = device
set_module(model, name, new_layer)
qlayer = new_layer
- scale = weight_config[name]["scale"]
- zero = weight_config[name]["zp"]
+ scale = layer_config[name]["scale"]
+ zero = layer_config[name]["zp"]
# so far can only pack layer on CPU
qlayer.to("cpu")
##force to float32 to be compatible with torch 2.0
@@ -225,18 +225,18 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exl
quantization_config["quant_method"] = "intel/auto-round"
quantization_config["backend"] = backend
extra_config = {}
- for layer_name in weight_config:
- if weight_config[layer_name]["bits"] >= 16:
+ for layer_name in layer_config:
+ if layer_config[layer_name]["bits"] >= 16:
continue
if layer_name not in layer_names_in_block:
extra_config[layer_name] = {}
- extra_config[layer_name]["bits"] = weight_config[layer_name]["bits"]
- extra_config[layer_name]["data_type"] = weight_config[layer_name]["data_type"]
- extra_config[layer_name]["group_size"] = weight_config[layer_name]["group_size"]
- extra_config[layer_name]["sym"] = weight_config[layer_name]["sym"]
+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]
+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
else:
neq_keys = check_neq_config(
- weight_config[layer_name],
+ layer_config[layer_name],
data_type=quantization_config["data_type"],
bits=quantization_config["bits"],
group_size=quantization_config["group_size"],
@@ -245,7 +245,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exl
if len(neq_keys) > 0:
extra_config[layer_name] = {}
for key in neq_keys:
- extra_config[layer_name][key] = weight_config[layer_name][key]
+ extra_config[layer_name][key] = layer_config[layer_name][key]
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
if hasattr(model, "config"):
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index d6d6fe679..d7220a78d 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -64,7 +64,7 @@ def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):
def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
"""Save configure file and weights for CPU backend inference."""
model = kwargs["model"]
- weight_config = kwargs["weight_config"]
+ layer_config = kwargs["layer_config"]
sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
@@ -76,7 +76,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
- compressed_model = pack_model(model, weight_config, inplace=inplace)
+ compressed_model = pack_model(model, layer_config, inplace=inplace)
if output_dir is None:
return compressed_model
quantize_config = QuantConfig(
@@ -110,7 +110,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):
"""Save configure file and weights for CPU backend inference."""
model = kwargs["model"]
- weight_config = kwargs["weight_config"]
+ layer_config = kwargs["layer_config"]
sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
@@ -156,7 +156,7 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):
def pack_model(
model,
- weight_config: Union[str, dict],
+ layer_config: Union[str, dict],
enable_full_range=False,
compression_dtype=torch.int32,
compression_dim=1,
@@ -168,7 +168,7 @@ def pack_model(
"""Convert Linear to WeightOnlyLinear for low memory inference.
Args:
- weight_config (str|dict): qconfig dict or Path of qconfig.json.
+ layer_config (str|dict): qconfig dict or Path of qconfig.json.
enable_full_range (bool, optional): Whether to leverage the full compression range
under symmetric quantization. Defaults to False.
compression_dtype (torch.Tensor, optional): The target dtype after comoression.
@@ -195,11 +195,11 @@ def pack_model(
compressed_model = model
else:
compressed_model = copy.deepcopy(model)
- if isinstance(weight_config, str):
- with open(weight_config, "r") as f:
+ if isinstance(layer_config, str):
+ with open(layer_config, "r") as f:
q_config = json.load(f)
else:
- q_config = weight_config
+ q_config = layer_config
for k, v in q_config.items():
logger.info(f"Packing {k}")
if "float" in v["data_type"]:
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 9d8738331..151d14266 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -35,8 +35,8 @@ def round_ste(x: torch.Tensor):
return (x.round() - x).detach() + x
-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- weight_min=None, weight_max=None):
+def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
+ weight_min=None, weight_max=None, q_scale_thresh=0.0):
"""Quantizes and dequantizes weight asymmetrically.
Args:
@@ -75,6 +75,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, sca
wmin[tmp] = -1
wmax[tmp] = +1
scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
zp = round_ste(-wmin / scale) # pylint: disable=E1130
scale = scale.unsqueeze(dim=-1)
zp = zp.unsqueeze(dim=-1)
@@ -83,8 +84,8 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, sca
return scale * (q - zp), scale, zp
-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
- weight_max=None):
+def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
+ weight_max=None, q_scale_thresh=0.0):
"""Quantizes and dequantizes weight symmetrically.
Args:
@@ -129,7 +130,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scal
wmin_new[tmp] = -1
wmax_new[tmp] = +1
scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
-
+ scale = torch.clamp(scale, min=q_scale_thresh)
scale = scale.unsqueeze(dim=-1)
zp = torch.full_like(scale, (maxq + 1) / 2)
@@ -138,8 +139,8 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scal
return scale * (q - zp), scale, zp
-def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,
- weight_max=None):
+def quant_tensor_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,
+ weight_max=None, q_scale_thresh=0.0):
"""Quantizes and dequantizes weight symmetrically or asymmetrically .
Args:
@@ -157,9 +158,11 @@ def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dty
"""
assert num_bits > 0, "num_bits should be larger than 0"
if sym:
- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)
+ return quant_tensor_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,
+ q_scale_thresh)
else:
- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)
+ return quant_tensor_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,
+ q_scale_thresh)
def reshape_tensor(v, group_size=-1):
@@ -182,14 +185,14 @@ def reshape_tensor(v, group_size=-1):
return v
-def quant_weight(
- weight, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- weight_min=None, weight_max=None
+def quant_tensor(
+ data, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
+ weight_min=None, weight_max=None, q_scale_thresh=0.0
):
"""Quantizes and dequantizes weight, handing the group size issue .
Args:
- weight: Tensor containing the weight to be quantized
+ data: Tensor containing the weight to be quantized
num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
group_size: The number of elements shares scale and zero point
sym: Sym or asym
@@ -202,32 +205,47 @@ def quant_weight(
Returns:
Quantized and dequantized weight, scale, zero-point
"""
- if group_size == -1 or weight.shape[1] < group_size:
- return quant_weight_actor(
- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,
- weight_min=weight_min, weight_max=weight_max
- )
- orig_shape = weight.shape
- if weight.shape[1] % group_size == 0:
- weight = weight.reshape(-1, group_size)
- weight, scale, zp = quant_weight_actor(
- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,
- weight_min=weight_min, weight_max=weight_max
- )
- weight = weight.reshape(orig_shape)
- return weight, scale, zp
+ orig_shape = data.shape
+ if len(data.shape) > 2:
+ data = data.reshape(-1, orig_shape[-1])
+ if group_size == -1 or data.shape[1] < group_size:
+ data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,
+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
+ q_scale_thresh=q_scale_thresh)
+ data = data.reshape(orig_shape)
+ return data, scale, zp
+
+ if data.shape[1] % group_size == 0:
+ data = data.reshape(-1, group_size)
+ data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,
+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
+ q_scale_thresh=q_scale_thresh)
+ data = data.reshape(orig_shape)
+ return data, scale, zp
else:
- pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]
- weight_new = torch.nn.functional.pad(weight, (0, pad_len))
- weight_new = weight_new.reshape(-1, group_size)
- weight_new, scale, zp = quant_weight_actor(
- weight_new, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,
- weight_min=weight_min, weight_max=weight_max
- )
- weight_new = weight_new.reshape(orig_shape[0], -1)
- weight_new = weight_new[:, :-pad_len]
- return weight_new, scale, zp
+ tmp_shape = data.shape
+ pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]
+ data_new = torch.nn.functional.pad(data, (0, pad_len))
+ data_new = data_new.reshape(-1, group_size)
+ data_new, scale, zp = quant_tensor_actor(data_new, num_bits, sym=sym, v=v, min_scale=min_scale,
+ max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,
+ weight_max=weight_max, q_scale_thresh=q_scale_thresh)
+ data_new = data_new.reshape(tmp_shape[0], -1)
+ data_new = data_new[:, :-pad_len]
+ data_new = data_new.reshape(orig_shape)
+ return data_new, scale, zp
+
+
+class WrapperWALayer(torch.nn.Module):
+ def __init__(self, orig_layer):
+ super(WrapperWALayer, self).__init__()
+ self.orig_layer = orig_layer
+
+ def forward(self, x):
+ x, _, _ = quant_tensor(x, self.orig_layer.act_bits, self.orig_layer.group_size, self.orig_layer.sym,
+ scale_dtype=self.orig_layer.scale_dtype, q_scale_thresh=self.orig_layer.q_scale_thresh)
+ return self.orig_layer.forward(x)
class WrapperLinear(torch.nn.Module):
@@ -254,6 +272,13 @@ class WrapperLinear(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.scale_dtype = self.orig_layer.scale_dtype
self.sym = self.orig_layer.sym
+ self.act_bits = self.orig_layer.act_bits
+ self.act_group_size = self.orig_layer.act_group_size
+ self.act_sym = self.orig_layer.act_sym
+ self.act_dynamic = self.orig_layer.act_dynamic
+ self.act_quant = self.act_bits <= 8
+ self.q_scale_thresh = 1e-5
+
weight_dtype = torch.float32
self.value = torch.nn.Parameter(
reshape_tensor(
@@ -291,25 +316,19 @@ class WrapperLinear(torch.nn.Module):
min_scale.clamp_(0, 1.0)
max_scale.clamp_(0, 1.0)
- qdq_weight, scale, zp = quant_weight(
- self.orig_layer.weight,
- self.num_bits,
- self.group_size,
- self.sym,
- v,
- min_scale,
- max_scale,
- self.scale_dtype,
- self.weight_min,
- self.weight_max,
- )
+ qdq_weight, scale, zp = quant_tensor(self.orig_layer.weight, self.num_bits, self.group_size, self.sym, v,
+ min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
self.orig_layer.weight.data.copy_(qdq_weight)
- self.orig_layer.weight.grad = None ##clear grad
+ self.orig_layer.weight.grad = None
self.orig_layer.scale = scale.to("cpu")
self.orig_layer.zp = zp.to("cpu") if zp is not None else None
+ self.orig_layer.q_scale_thresh = self.q_scale_thresh
+ if self.act_quant:
+ wrapper_layer = WrapperWALayer(self.orig_layer)
+ return wrapper_layer
return self.orig_layer
def forward(self, x):
@@ -326,19 +345,12 @@ class WrapperLinear(torch.nn.Module):
weight = self.orig_layer.weight
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, 0, 1.0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, 0, 1.0))
- weight_q, _, _ = quant_weight(
- weight,
- self.num_bits,
- self.group_size,
- self.sym,
- self.value,
- self.min_scale,
- self.max_scale,
- self.scale_dtype,
- self.weight_min,
- self.weight_max
- )
+ weight_q, _, _ = quant_tensor(weight, self.num_bits, self.group_size, self.sym, self.value, self.min_scale,
+ self.max_scale, self.scale_dtype, self.weight_min, self.weight_max)
weight_q = weight_q.to(weight.dtype)
+ if self.act_quant:
+ x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,
+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)
# pylint: disable=not-callable
return F.linear(x, weight_q, self.orig_layer.bias)
@@ -372,6 +384,12 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.sym = self.orig_layer.sym
self.scale_dtype = self.orig_layer.scale_dtype
+ self.act_bits = self.orig_layer.act_bits
+ self.act_group_size = self.orig_layer.act_group_size
+ self.act_sym = self.orig_layer.act_sym
+ self.act_dynamic = self.orig_layer.act_dynamic
+ self.act_quant = self.act_bits <= 8
+ self.q_scale_thresh = 1e-5
weight_dtype = torch.float32
device = self.orig_layer.weight.device
self.weight_t = self.orig_layer.weight.t()
@@ -410,10 +428,8 @@ class WrapperTransformerConv1d(torch.nn.Module):
"""
min_scale.clamp_(0, 1.0)
max_scale.clamp_(0, 1.0)
- qdq_weight, scale, zp = quant_weight(
- self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale, max_scale, self.scale_dtype,
- self.weight_min, self.weight_max
- )
+ qdq_weight, scale, zp = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale,
+ max_scale, self.scale_dtype, self.weight_min, self.weight_max)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
@@ -421,6 +437,10 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.orig_layer.weight.grad = None
self.orig_layer.scale = scale.to("cpu")
self.orig_layer.zp = zp.to("cpu")
+ self.orig_layer.q_scale_thresh = self.q_scale_thresh
+ if self.act_quant:
+ wrapper_layer = WrapperWALayer(self.orig_layer)
+ return wrapper_layer
return self.orig_layer
def forward(self, x):
@@ -435,20 +455,14 @@ class WrapperTransformerConv1d(torch.nn.Module):
with torch.no_grad():
self.min_scale.clamp_(0, 1.0)
self.max_scale.clamp_(0, 1.0)
- weight_q, _, _ = quant_weight(
- self.weight_t,
- self.num_bits,
- self.group_size,
- self.sym,
- self.value,
- self.min_scale,
- self.max_scale,
- self.scale_dtype,
- self.weight_min,
- self.weight_max
- )
+ weight_q, _, _ = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, self.value,
+ self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,
+ self.weight_max)
weight_q = weight_q.to(self.weight_t.dtype)
size_out = x.size()[:-1] + (self.orig_layer.nf,)
+ if self.act_quant:
+ x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,
+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)
x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())
x = x.view(*size_out)
return x
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 8b98f97d5..7dab5830e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -307,11 +307,11 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo
def check_to_quantized(config):
if isinstance(config, dict):
- if config["bits"] > 8 or "fp" in config["data_type"] or "float" in config["data_type"]:
+ if config["bits"] > 8:
return False
return True
else:
- if config.bits > 8 or "fp" in config.data_type or "float" in config.data_type:
+ if config.bits > 8:
return False
return True
diff --git a/examples/language-modeling/eval_042/evaluation.py b/examples/language-modeling/eval_042/evaluation.py
index b14a67c82..fa1a6f4f0 100644
--- a/examples/language-modeling/eval_042/evaluation.py
+++ b/examples/language-modeling/eval_042/evaluation.py
@@ -54,6 +54,7 @@ def simple_evaluate(
random_seed: int = 0,
numpy_random_seed: int = 1234,
torch_random_seed: int = 1234,
+ user_model = None, ##user model does not support tensor parallelism
):
"""Instantiate and evaluate a model on a list of tasks.
@@ -185,6 +186,8 @@ def simple_evaluate(
+ str(lm.rank)
+ ".db",
)
+ if user_model is not None:
+ lm._model = user_model
if task_manager is None:
task_manager = TaskManager(verbosity)
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 14d9b5c60..5ec9dcb23 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -119,6 +119,9 @@ if __name__ == '__main__':
parser.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
+ parser.add_argument("--act_bits", default=32, type=int,
+ help="activation bits")
+
args = parser.parse_args()
if args.enable_minmax_tuning:
@@ -131,6 +134,10 @@ if __name__ == '__main__':
print(
"enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")
+ if args.act_bits <= 8:
+ print(
+ "Warning, activation quantization is an experiment feature")
+
set_seed(args.seed)
tasks = args.tasks
use_eval_legacy = False
@@ -138,6 +145,9 @@ if __name__ == '__main__':
if "marlin" in args.deployment_device and args.sym == False:
assert False, "marlin backend only supports sym quantization, please set --sym"
+ if args.act_bits <= 8 and args.deployment_device != "fake":
+ assert False, "only support fake mode for activation quantization currently"
+
model_name = args.model_name
if model_name[-1] == "/":
model_name = model_name[:-1]
@@ -205,11 +215,11 @@ if __name__ == '__main__':
if args.adam:
round = AutoAdamRound
- weight_config = {}
+ layer_config = {}
for n, m in model.named_modules():
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
- weight_config[n] = {"data_type": "fp"}
+ layer_config[n] = {"bits": 32}
print(
f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
lm_head_layer_name = "lm_head"
@@ -229,7 +239,7 @@ if __name__ == '__main__':
f"supported currently")
break
if args.quant_lm_head:
- weight_config[lm_head_layer_name] = {"data_type": "int"}
+ layer_config[lm_head_layer_name] = {"bits": args.bits}
transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
if transformers_version[0] == 4 and transformers_version[1] < 38:
error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
@@ -245,8 +255,8 @@ if __name__ == '__main__':
amp=not args.disable_amp, nsamples=args.nsamples,
low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype, weight_config=weight_config,
- enable_minmax_tuning=not args.disable_minmax_tuning)
+ scale_dtype=args.scale_dtype, layer_config=layer_config,
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits)
model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
@@ -264,7 +274,7 @@ if __name__ == '__main__':
gpu_formats.append(item)
if 'gpu' in deployment_device:
- if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":
+ if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":
gpu_formats.append("auto_round")
else:
gpu_formats.append("auto_gptq")
@@ -349,10 +359,13 @@ if __name__ == '__main__':
model_args = f"pretrained={eval_folder}"
else:
exit() ## does not support cpu,xpu model eval
+ user_model = None
+ if args.act_bits <= 8:
+ user_model = model.to(device_str)
res = simple_evaluate(model="hf", model_args=model_args,
tasks=tasks,
- batch_size=args.eval_bs)
+ batch_size=args.eval_bs, user_model=user_model)
from lm_eval.utils import make_table
print(make_table(res))
diff --git a/test/test_autoround.py b/test/test_autoround.py
index aff2f53f0..279f5dee9 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -34,7 +34,7 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("runs", ignore_errors=True)
def test_remove_whole_block(self):
- weight_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},
+ layer_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},
"model.decoder.layers.0.self_attn.v_proj": {"data_type": "float"},
"model.decoder.layers.0.self_attn.q_proj": {"data_type": "float"},
"model.decoder.layers.0.self_attn.out_proj": {"data_type": "float"},
@@ -51,7 +51,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=2,
dataset=self.llm_dataloader,
- weight_config=weight_config
+ layer_config=layer_config
)
autoround.quantize()
@@ -177,7 +177,7 @@ class TestAutoRound(unittest.TestCase):
def test_lm_head(self):
bits, group_size, sym = 4, -1, False
- weight_config = {"lm_head": {"data_type": "int"}}
+ layer_config = {"lm_head": {"data_type": "int"}}
autoround = AutoRound(
self.model,
self.tokenizer,
@@ -189,9 +189,24 @@ class TestAutoRound(unittest.TestCase):
enable_minmax_tuning=False,
enable_quanted_input=False,
dataset=self.llm_dataloader,
- weight_config=weight_config,
+ layer_config=layer_config,
)
autoround.quantize()
+ def test_wa_quant(self):
+ bits, group_size, sym, act_bits = 4, 128, False, 4
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ act_bits=4,
+ )
+ autoround.quantize()
+
if __name__ == "__main__":
diff --git a/test/test_autoround_export.py b/test/test_autoround_export.py
index d90ca15f1..3c005d224 100644
--- a/test/test_autoround_export.py
+++ b/test/test_autoround_export.py
@@ -57,10 +57,10 @@ class TestAutoroundExport(unittest.TestCase):
out1 = model(self.lm_input)
round = AutoRound
optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)
- q_model, weight_config1 = optq_1.quantize()
+ q_model, layer_config1 = optq_1.quantize()
from auto_round.export.export_to_itrex import pack_model
- compressed_model = pack_model(model=q_model, weight_config=weight_config1)
+ compressed_model = pack_model(model=q_model, layer_config=layer_config1)
out2 = model(self.lm_input)
out3 = q_model(self.lm_input)
out4 = compressed_model(self.lm_input)
@@ -73,8 +73,8 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out6 = model(self.lm_input)
optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)
- q_model, weight_config2 = optq_2.quantize()
- compressed_model = pack_model(model=q_model, weight_config=weight_config2, inplace=False)
+ q_model, layer_config2 = optq_2.quantize()
+ compressed_model = pack_model(model=q_model, layer_config=layer_config2, inplace=False)
out4 = q_model(self.lm_input)
out5 = compressed_model(self.lm_input)
self.assertTrue(torch.all(out1[0] == out6[0]))
diff --git a/test/test_conv1d.py b/test/test_conv1d.py
new file mode 100644
index 000000000..b2e51a24e
--- /dev/null
+++ b/test/test_conv1d.py
@@ -0,0 +1,52 @@
+import copy
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "..")
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestQuantizationConv1d(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "MBZUAI/LaMini-GPT-124M"
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ def test_wa_quant(self):
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+
+ )
+ autoround.quantize()
+ autoround.save_quantized("./saved")
+
+
+if __name__ == "__main__":
+ unittest.main()