Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

files refector, bug fix, refine doc

Zhang, Weiwei1d8cd8ca73f7a
diff --git a/README.md b/README.md
index 3c8b36a74..1a081848c 100644
--- a/README.md
+++ b/README.md
@@ -25,11 +25,6 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")
fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict
-## export to gpu
-## please install autogptq first
-# packed_folder = "./tmp_autoround_packed"
-# autoround.export_to_autogptq(packed_folder, use_triton=True) ## Utilizing Triton for 2-bit and 4-bit scenarios
-
```
### Detailed Hyperparameters
- `model`: The PyTorch model to be quantized.
@@ -132,6 +127,38 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bi
| mistralai/Mistral-7B-v0.1 | 4.34/4.36 |
+### Deployment
+The quantized models support export to Int4 precision and can be run on Intel CPU using [neural-speed](examples/huggingface/neural_speed/runtime_acc.py), it also supports deployment on GPU using autogptq.
+
+After the quantization process is complete, you can invoke the export API to perform the conversion and save the model along with the quantization configuration file.
+```python
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ from auto_round import AutoRound
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")
+ fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict
+
+ ## export for CPU deployment
+ from auto_round import QuantConfig, compress_model, save_compressed_model
+ output_dir = "/PATH/TO/SAVE/COMPRESSED/MODEL/"
+ compressed_model = compress_model(fake_qdq_model, weight_config)
+ quantize_config = QuantConfig(bits=4, sym=False, group_size=128)
+ save_compressed_model(compressed_model, output_dir=output_dir,
+ quantize_config=quantize_config, tokenizer=tokenizer)
+
+ ## export for GPU deployment
+ # please install auto-gptq first
+ packed_folder = "./tmp_autoround_packed"
+ autoround.export_to_autogptq(packed_folder, use_triton=True) ## Utilizing Triton for 2-bit and 4-bit scenarios
+
+```
+
+
+
## Reference
If you find SignRound useful for your research, please cite our paper:
@@ -144,5 +171,3 @@ If you find SignRound useful for your research, please cite our paper:
}
```
-
-
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 43059f6cb..757876436 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -13,7 +13,6 @@
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
from .version import __version__
-from .export import compress_model, save_compressed_model, save_quantized_to_autogptq
-from .config import AutoroundQuantConfig
-from .model_wrapper import WeightOnlyLinear
-
+from .export_to_speed.export import compress_model, save_compressed_model
+from .export_to_autogptq import save_quantized_to_autogptq
+from .export_to_speed.config import QuantConfig
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 6e170b001..eb7138343 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -26,199 +26,12 @@ logger.addHandler(fh)
import copy
import time
-from collections import UserDict
-from functools import partial
-
from torch.amp import autocast
+from functools import partial
from torch.functional import F
-
-
-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
- """Quantizes and dequantizes weight asymmetrically.
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- maxq = torch.tensor(2 ** num_bits - 1)
- zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
- # zeros = torch.zeros(weight.shape[0], device=weight.device)
- if isinstance(min_scale, torch.Tensor):
- wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
- wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
- wmin_tmp *= min_scale + 1.0
- wmax_tmp *= max_scale + 1.0
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
- else:
- wmin = torch.minimum(weight.min(1)[0], zeros)
- wmax = torch.maximum(weight.max(1)[0], zeros)
-
- tmp = (wmin == 0) & (wmax == 0)
- wmin[tmp] = -1
- wmax[tmp] = +1
- scale = ((wmax - wmin) / maxq).to(scale_dtype)
- zp = round_ste(-wmin / scale)
- scale = scale.unsqueeze(dim=-1)
- zp = zp.unsqueeze(dim=-1)
- int_w = round_ste(weight / scale + v)
- q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
-
-
-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
- """Quantizes and dequantizes weight symmetrically.
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- maxq = torch.tensor(2 ** (num_bits - 1) - 1).to(weight.device)
- minq = torch.tensor(-(2 ** (num_bits - 1))).to(weight.device)
- if num_bits == 1:
- maxq = torch.tensor(2 ** (num_bits - 1))
- minq = torch.tensor(2 ** (num_bits - 1) - 1)
-
- wmax = torch.abs(weight).max(1)[0]
- wmax *= 1 + max_scale
- tmp = wmax == 0
- wmax[tmp] = +1
- scale = (wmax / ((maxq - minq) / 2)).to(scale_dtype)
- scale.unsqueeze_(dim=-1)
- q = torch.clamp(round_ste(weight / scale + v), minq, maxq)
- return scale * q, scale, None
-
-
-def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):
- """Quantizes and dequantizes weight symmetrically or asymmetrically .
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- scheme: Sym or asym
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- assert num_bits > 0, "num_bits should be larger than 0"
- if scheme == "sym":
- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
- else:
- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
-
-
-def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
- """Quantizes and dequantizes weight, handing the group size issue .
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- group_size: The number of elements shares scale and zero point
- scheme: Sym or asym
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- if group_size == -1 or weight.shape[1] < group_size:
- return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,
- min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)
- orig_shape = weight.shape
- if weight.shape[1] % group_size == 0:
- weight = weight.reshape(-1, group_size)
- if isinstance(v, torch.Tensor):
- v = v.reshape(-1, group_size)
-
- weight, scale, zp = quant_weight_actor(
- weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
- )
- weight = weight.reshape(orig_shape)
- scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d
- if zp is not None:
- zp = zp.reshape(weight.shape[0], -1)
- return weight, scale, zp
-
- else:
- pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]
- weight_new = torch.nn.functional.pad(weight, (0, pad_len))
- v = torch.nn.functional.pad(v, (0, pad_len))
- weight_new = weight_new.reshape(-1, group_size)
- if isinstance(v, torch.Tensor):
- v = v.reshape(-1, group_size)
- weight_new, scale, zp = quant_weight_actor(
- weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
- )
- weight_new = weight_new.reshape(orig_shape[0], -1)
-
- weight_new = weight_new[:, :-pad_len]
- scale = scale.reshape(weight_new.shape[0], -1) ##only for linear, conv1d
- if zp is not None:
- zp = zp.reshape(weight_new.shape[0], -1)
- return weight_new, scale, zp
-
-
-def quant_weight_w_scale(weight, scale, zp, group_size=-1):
- """Quant and dequant tensor with group size.
-
- Args:
- weight: input weight
- scale: scale
- zp: zero point
- group_size (int, optional): how many elements share one scale/zp. Defaults to -1.
-
- Returns:
- output: int weight.
- """
- device = weight.device
- scale = scale.to(device)
- if zp is not None:
- zp = zp.to(device)
- if group_size == -1:
- return torch.round(weight / scale) if zp is None else torch.round(weight / scale + zp)
- int_weight = torch.zeros(weight.shape).to(device)
- leng = weight.shape[1] // group_size
- tail_flag = False if weight.shape[1] % group_size == 0 else True
- for i in range(leng):
- int_weight_tmp = weight[:, i * group_size : (i + 1) * group_size] / scale[:, i].unsqueeze(1)
- if zp is not None:
- int_weight_tmp += zp[:, i].unsqueeze(1)
- int_weight[:, i * group_size : (i + 1) * group_size] = torch.round(int_weight_tmp)
- if tail_flag:
- int_weight_tmp = weight[:, leng * group_size :] / scale[:, -1].unsqueeze(1)
- if zp is not None:
- int_weight_tmp += zp[:, -1].unsqueeze(1)
- int_weight[:, leng * group_size :] = torch.round(int_weight_tmp)
- return int_weight
-
-
-def round_ste(x: torch.Tensor):
- """Straight-Through Estimator for rounding.
- This function is adapted from omniquant.
-
- Args:
- x: torch.Tensor
-
- Returns:
- torch.Tensor
- """
- return (x.round() - x).detach() + x
+from .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,
+ get_dataloader, get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,
+ collect_minmax_scale, get_batch_dim)
class SaveInputs:
@@ -360,59 +173,7 @@ class SaveInputs:
m.orig_forward = m.forward
m.forward = partial(self.get_forward_func(n), m)
break
-
-
-def get_module(model, key):
- """Get module from model by key name.
-
- Args:
- model (torch.nn.Module): original model
- key (str): module name to be replaced
- """
- module = model
- name_list = key.split(".")
- for name in name_list:
- if hasattr(module, name):
- module = getattr(module, name)
- module = module
- return module
-
-
-def set_module(model, key, new_module):
- """Set new module into model by key name.
-
- Args:
- model (torch.nn.Module): original model
- key (str): module name to be replaced
- new_module (torch.nn.Module): new module to be inserted
- """
- module = model
- name_list = key.split(".")
- for name in name_list[:-1]:
- if hasattr(module, name):
- module = getattr(module, name)
- else:
- module = module
- setattr(module, name_list[-1], new_module)
-
-
-def get_scale_shape(weight, group_size):
- """Computes the shape of the scale tensor for quantization based on the weight tensor and group size.
-
- Args:
- weight (torch.Tensor): The weight tensor of the layer.
- group_size (int): The size of the groups for quantization.
-
- Returns:
- The shape of the scale tensor to be used for quantization.
- """
- if group_size == -1 or weight.shape[1] < group_size:
- shape = weight.shape[0]
- else:
- shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)
-
- return shape
-
+
class WrapperLinear(torch.nn.Module):
def __init__(self, orig_layer, enable_minmax_tuning=True):
@@ -602,6 +363,26 @@ class WrapperTransformerConv1d(torch.nn.Module):
return x
+class WrapperMultiblock(torch.nn.Module):
+ """A wrapper for a list of modules to be act as a single block.
+
+ Args:
+ module_list: The list of modules to wrap.
+ """
+
+ def __init__(self, module_list):
+ super(WrapperMultiblock, self).__init__()
+ self.layers = torch.nn.ModuleList(module_list)
+
+ def forward(self, x, **kwargs):
+ hidden_states = x
+ for idx, decoder_layer in enumerate(self.layers):
+ layer_outputs = decoder_layer(hidden_states, **kwargs)
+ hidden_states = layer_outputs
+ if isinstance(hidden_states, tuple) or isinstance(hidden_states, list):
+ hidden_states = layer_outputs[0]
+ return hidden_states
+
def wrapper_block(block, enable_minmax_tuning):
"""Wraps the layers in the given block with a custom Wrapper module.
@@ -660,281 +441,8 @@ def unwrapper_block(block, vs, min_scales, max_scales):
max_scale = torch.clamp(max_scale, -1, 0)
orig_layer = m.unwrapper(v, min_scale, max_scale)
set_module(block, n, orig_layer)
-
-
-def sampling_inputs(input_ids, input_others, indices, seqlen):
- """Samples inputs based on the given indices and sequence length.
-
- Args:
- input_ids: The input tensor containing IDs.
- input_others: A dictionary containing other input data.
- indices: The indices to sample from the input.
- seqlen: The sequence length.
-
- Returns:
- current_input_ids: The sampled input IDs.
- current_input_others: The sampled other input data.
- """
- if len(input_ids.shape) == 3:
- if int(len(input_others["positional_inputs"]) > 0):
- current_input_ids = input_ids[:, indices, :]
- else:
- current_input_ids = input_ids[indices, :, :]
- else:
- n_samples = input_ids.shape[0] // seqlen
- current_input_ids = input_ids.view(n_samples, seqlen, -1)
- current_input_ids = current_input_ids[indices, :, :]
- current_input_ids = current_input_ids.reshape(-1, input.shape[-1])
-
- current_input_others = {"positional_inputs": input_others["positional_inputs"]}
- for key in input_others.keys():
- if "attention_mask" in key or "alibi" in key:
- current_input_others[key] = None
- if input_others[key] is not None:
- current_input_others[key] = input_others[key][indices, ...]
- else:
- current_input_others[key] = input_others[key]
-
- return current_input_ids, current_input_others
-
-
-def move_input_to_device(input, device=torch.device("cpu")):
- """Moves input data to the specified device.
-
- Args:
- input: The input data to be moved.
- device: The target device.
-
- Returns:
- The input data on the specified device.
- """
- if isinstance(input, torch.Tensor):
- return input.to(device)
- if isinstance(input, dict) or isinstance(input, UserDict):
- for inp in input.keys():
- input[inp] = move_input_to_device(input[inp], device)
- elif isinstance(input, list) or isinstance(input, tuple):
- input_res = []
- for inp in input:
- input_res.append(move_input_to_device(inp, device))
- input = input_res
- return input
-
-
-def check_is_cpu(device):
- """Check if the device is a CPU.
-
- Args:
- device: The device to be checked.
-
- Returns:
- bool: True if the device is a CPU, False otherwise.
- """
- return device == torch.device("cpu") or device == "cpu"
-
-
-def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):
- """Performs a forward pass through a block with the given inputs.
-
- Args:
- block: The block to perform the forward pass on.
- input_ids: The input IDs.
- input_others: A dictionary containing other input data.
- amp: A boolean indicating whether to use automatic mixed precision.
- amp_dtype: The data type for automatic mixed precision.
- device: The target device.
-
- Returns:
- output: The output of the forward pass.
- """
- if input_ids.device != device:
- # input_ids, input_others = move_to_device(input_ids, input_others, device)
- input_ids = move_input_to_device(input_ids, device)
- input_others = move_input_to_device(input_others, device)
- if "alibi" in input_others.keys():
- attention_mask = input_others["attention_mask"]
- alibi = input_others["alibi"]
- if alibi is not None:
- alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
- if amp and not check_is_cpu(device):
- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
- output = block(
- input_ids, attention_mask=attention_mask, alibi=alibi
- ) ##TODO is this correct for all models with alibi?
- elif amp and check_is_cpu(device):
- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
- else:
- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
- else:
- input_tuple = input_others.pop("positional_inputs", None)
- if amp and not check_is_cpu(device):
- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
- output = block.forward(input_ids, *input_tuple, **input_others)
- elif amp and check_is_cpu(device):
- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
- output = block.forward(input_ids, *input_tuple, **input_others)
- else:
- output = block.forward(input_ids, *input_tuple, **input_others)
- if isinstance(output, list) or isinstance(output, tuple):
- output = output[0]
- return output
-
-
-def collect_round_v(block):
- """Collects the round values for wrapped linear modules in the given block.
-
- Args:
- block: The input block.
-
- Returns:
- vs: A dictionary of round values for the wrapped linear modules.
- """
- vs = {}
- for n, m in block.named_modules():
- if hasattr(m, "orig_layer"):
- v = m.value.data
- vs[n] = copy.deepcopy(v)
- return vs
-
-
-def collect_minmax_scale(block):
- """Collects the min-max scaling values for wrapped linear modules in the given block.
-
- Args:
- block: The input block.
-
- Returns:
- min_scales: A dictionary of minimum scaling values.
- max_scales: A dictionary of maximum scaling values.
- """
- min_scales = {}
- max_scales = {}
- for n, m in block.named_modules():
- if hasattr(m, "orig_layer"):
- min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, -1, 0))
- max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, -1, 0))
- return min_scales, max_scales
-
-
-def get_batch_dim(input_others):
- """Gets the batch dimension based on the input positional inputs.
-
- Args:
- input_others: A dictionary containing input data.
-
- Returns:
- dim: The batch dimension.
- """
- dim = int(len(input_others["positional_inputs"]) > 0)
- return dim
-
-
-class WrapperMultiblock(torch.nn.Module):
- """A wrapper for a list of modules to be act as a single block.
-
- Args:
- module_list: The list of modules to wrap.
- """
-
- def __init__(self, module_list):
- super(WrapperMultiblock, self).__init__()
- self.layers = torch.nn.ModuleList(module_list)
-
- def forward(self, x, **kwargs):
- hidden_states = x
- for idx, decoder_layer in enumerate(self.layers):
- layer_outputs = decoder_layer(hidden_states, **kwargs)
- hidden_states = layer_outputs
- if isinstance(hidden_states, tuple) or isinstance(hidden_states, list):
- hidden_states = layer_outputs[0]
- return hidden_states
-
-
-def get_block_names(model):
- """Get the block names for transformers-like networks.
-
- Args:
- model: The model.
-
- Returns:
- block_names: A list of block names.
- """
- block_names = []
- target_m = None
- for n, m in model.named_modules():
- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
- target_m = (n, m)
- break ## only find the first modulelist, may be not robust
- for n, m in target_m[1].named_children():
- block_names.append(target_m[0] + "." + n)
- return block_names
-
-
-def get_tokenizer_function(tokenizer, seqlen):
- """Returns a default tokenizer function.
-
- Args:
- tokenizer: The tokenizer to be used for tokenization.
- seqlen: The maximum sequence length.
-
- Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of
- seqlen to the "text" field of examples.
- """
-
- def default_tokenizer_function(examples):
- example = tokenizer(examples["text"], truncation=True, max_length=seqlen)
- return example
-
- return default_tokenizer_function
-
-
-def get_dataloader(tokenizer, seqlen, data_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):
- """Returns a dataloader for the specified dataset and split.
-
- Args:
- tokenizer: The tokenizer to be used for tokenization.
- seqlen: The maximum sequence length.
- data_name: The name of the dataset.
- split: The data split to be used (e.g., "train", "test").
- seed: The random seed for shuffling the dataset.
- bs: The batch size for the dataloader.
-
- Returns:
- A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
- """
- from datasets import load_dataset
- from torch.utils.data import DataLoader
-
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
-
- @torch.no_grad()
- def collate_batch(batch):
- input_ids_new = []
- for text in batch:
- input_ids = text["input_ids"]
- if input_ids.shape[0] < seqlen:
- continue
- input_ids = input_ids[:seqlen]
- input_ids_list = input_ids.tolist()
- if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
- continue
- input_ids_new.append(input_ids)
- if len(input_ids_new) == 0:
- return None
- tmp = torch.vstack(input_ids_new)
- res = {"input_ids": tmp}
- return res
-
- calib_dataset = load_dataset(data_name, split=split)
- calib_dataset = calib_dataset.shuffle(seed=seed)
- calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
- calib_dataset.set_format(type="torch", columns=["input_ids"])
- calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)
- return calib_dataloader
-
-
+
+
class AutoRound(object):
"""This is Signround+ which is an advanced version of Signround. For more information,
please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent
@@ -977,7 +485,7 @@ class AutoRound(object):
seed (int): The random seed (default is 42).
n_blocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_mse (bool): Whether to use mean squared error (default is False).
+ not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
**kwargs: Additional keyword arguments.
@@ -1014,7 +522,7 @@ class AutoRound(object):
seed: int = 42,
n_blocks: int = 1,
gradient_accumulate_steps: int = 1,
- not_use_mse: bool = False,
+ not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int", ##only support data_type
scale_dtype = "fp16",
@@ -1090,7 +598,7 @@ class AutoRound(object):
self.sampler = sampler
self.gradient_accumulate_steps = gradient_accumulate_steps
- self.not_use_mse = not_use_mse
+ self.not_use_best_mse = not_use_best_mse
self.dynamic_max_gap = dynamic_max_gap
self.enable_full_range = enable_full_range
assert self.enable_full_range is False, "only support enable_full_range=False currently"
@@ -1339,16 +847,16 @@ class AutoRound(object):
if total_loss < best_loss:
best_loss = total_loss
- if not self.not_use_mse:
+ if not self.not_use_best_mse:
# print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)
best_v = collect_round_v(block)
best_min_scale, best_max_scale = collect_minmax_scale(block)
last_best_iter = i
- if self.not_use_mse and i == self.iters - 1:
+ if self.not_use_best_mse and i == self.iters - 1:
best_v = collect_round_v(block)
best_min_scale, best_max_scale = collect_minmax_scale(block)
- if not self.not_use_mse:
+ if not self.not_use_best_mse:
if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:
break
self.step(scaler, optimizer, lr_schedule)
@@ -1452,7 +960,7 @@ class AutoRound(object):
pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
force_layer_back_to_cpu=True, use_triton=True)
- from auto_round.export import save_quantized_to_autogptq
+ from auto_round import save_quantized_to_autogptq
sym = self.scheme == "sym"
save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,
iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
@@ -1549,7 +1057,7 @@ class AutoOPTRound(AutoRound):
seed (int): The random seed (default is 42).
n_blocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_mse (bool): Whether to use mean squared error (default is False).
+ not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
optimizer: string or object
@@ -1587,7 +1095,7 @@ class AutoOPTRound(AutoRound):
seed: int = 42,
n_blocks: int = 1,
gradient_accumulate_steps: int = 1,
- not_use_mse: bool = False,
+ not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
optimizer="AdamW",
@@ -1620,7 +1128,7 @@ class AutoOPTRound(AutoRound):
seed,
n_blocks,
gradient_accumulate_steps,
- not_use_mse,
+ not_use_best_mse,
dynamic_max_gap,
data_type,
**kwargs,
@@ -1697,7 +1205,7 @@ class AutoAdamRound(AutoOPTRound):
seed (int): The random seed (default is 42).
n_blocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_mse (bool): Whether to use mean squared error (default is False).
+ not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
optimizer: string or object
@@ -1735,7 +1243,7 @@ class AutoAdamRound(AutoOPTRound):
seed: int = 42,
n_blocks: int = 1,
gradient_accumulate_steps: int = 1,
- not_use_mse: bool = False,
+ not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
optimizer="AdamW",
@@ -1768,10 +1276,9 @@ class AutoAdamRound(AutoOPTRound):
seed,
n_blocks,
gradient_accumulate_steps,
- not_use_mse,
+ not_use_best_mse,
dynamic_max_gap,
data_type,
optimizer,
**kwargs,
)
-
diff --git a/auto_round/export.py b/auto_round/export_to_autogptq.py
similarity index 98%
rename from auto_round/export.py
rename to auto_round/export_to_autogptq.py
index 84188f4db..db17a6422 100644
--- a/auto_round/export.py
+++ b/auto_round/export_to_autogptq.py
@@ -44,6 +44,7 @@ from safetensors.torch import save_file as safe_save
from os.path import join, isfile, isdir
import copy
import json
+from .utils import quant_weight_w_scale
def compress_model(
@@ -95,7 +96,8 @@ def compress_model(
scale_dtype = v["scale_dtype"]
m = get_module(compressed_model, k)
fp_weight = m.weight.data
- scale = torch.tensor(v["scale"], dtype=scale_dtype)
+ # scale = torch.tensor(v["scale"], dtype=scale_dtype)
+ scale = torch.tensor(v["scale"], dtype=torch.float32)
zp = None if scheme == "sym" else torch.tensor(v["zp"], dtype=torch.int32)
int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size)
int_weight = int_weight.type(torch.int32)
@@ -200,4 +202,3 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
quantization_config.model_file_base_name = model_base_name
quantization_config.save_pretrained(save_dir)
-
diff --git a/auto_round/config.py b/auto_round/export_to_speed/config.py
similarity index 97%
rename from auto_round/config.py
rename to auto_round/export_to_speed/config.py
index caf0bae9d..62e87ef6c 100644
--- a/auto_round/config.py
+++ b/auto_round/export_to_speed/config.py
@@ -24,11 +24,12 @@ import json
import os
from dataclasses import dataclass, field
from typing import Any, Dict, Tuple, Union
-from transformers import BitsAndBytesConfig, PretrainedConfig
+from transformers import PretrainedConfig
QUANT_CONFIG = "quantize_config.json"
-class AutoroundQuantConfig(PretrainedConfig):
+class QuantConfig(PretrainedConfig):
+ """A brief quantization configuration for reference when performing model dequantization."""
def __init__(
self,
bits=4,
@@ -221,4 +222,4 @@ class AutoroundQuantConfig(PretrainedConfig):
def get_config_dict(
cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
- return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)
+ return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)
\ No newline at end of file
diff --git a/auto_round/export_to_speed/export.py b/auto_round/export_to_speed/export.py
new file mode 100644
index 000000000..71c6bce9e
--- /dev/null
+++ b/auto_round/export_to_speed/export.py
@@ -0,0 +1,119 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from logging import getLogger
+
+logger = getLogger(__name__)
+import os
+from typing import Dict, List, Optional, Union
+from safetensors.torch import save_file as safe_save
+from os.path import join, isfile, isdir
+import copy
+import json
+from .model_wrapper import WeightOnlyLinear
+from ..utils import quant_weight_w_scale, get_module, set_module
+
+
+def compress_model(
+ model,
+ weight_config:Union[str, dict],
+ enable_full_range=False,
+ compression_dtype=torch.int32,
+ compression_dim=1,
+ scale_dtype=torch.float32,
+ device="cpu",
+ use_optimum_format=True,
+ ):
+ """Convert Linear to WeightOnlyLinear for low memory inference.
+
+ Args:
+ weight_config (str|dict): qconfig dict or Path of qconfig.json.
+ enable_full_range (bool, optional): Whether to leverage the full compression range
+ under symmetric quantization. Defaults to False.
+ compression_dtype (torch.Tensor, optional): The target dtype after comoression.
+ Defaults to torch.int32.
+ compression_dim (int, optional): Select from [0, 1], 0 is output channel,
+ 1 is input channel. Defaults to 1.
+ scale_dtype (torch.Tensor, optional): Use float32 or float16.
+ Defaults to torch.float32.
+ device (str, optional): choose device for compression. Defaults to cpu.
+ use_optimum_format (bool, optional): use the popular huggingface compression format.
+ 1: compression_dim: weight = 1, zeros = 0 and both are transposed.
+ 2: zeros -= 1 before compression. Why we need it?
+ 3: g_idx: use same number for one group instead of recording the channel order.
+ 4. parameter name changed, such as 'packed_weight' -> 'qweight'.
+ 5. zeros is always needed even for sym.
+ """
+ compressed_model = copy.deepcopy(model)
+ if isinstance(weight_config, str):
+ with open(weight_config, "r") as f:
+ q_config = json.load(f)
+ else:
+ q_config = weight_config
+ for k, v in q_config.items():
+ logger.info(f"Compressing {k} on device {device}")
+ if "float" in v["data_type"]:
+ continue
+ dtype = v["data_type"]
+ num_bits = v["bits"]
+ group_size = v["group_size"]
+ scheme = v["scheme"]
+ scale_dtype = v["scale_dtype"]
+ m = get_module(compressed_model, k)
+ fp_weight = m.weight.data
+ # scale = torch.tensor(v["scale"], dtype=scale_dtype)
+ scale, zp = v["scale"], v["zp"]
+ convert_dtype=torch.float32 if fp_weight.device.type == "cpu" else scale_dtype
+ if not isinstance(scale, torch.Tensor):
+ scale = torch.tensor(scale, dtype=convert_dtype)
+ zp = None if scheme == "sym" else torch.tensor(zp, dtype=torch.int32)
+ else:
+ scale = scale.to(dtype=convert_dtype)
+ zp = None if scheme == "sym" else zp.to(dtype=torch.int32)
+ int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)
+ int_weight = int_weight.type(torch.int32)
+ new_module = WeightOnlyLinear(
+ m.in_features,
+ m.out_features,
+ num_bits,
+ group_size,
+ dtype=dtype,
+ scale_dtype=scale_dtype,
+ zp=zp is not None,
+ bias=m.bias is not None,
+ device=device,
+ use_optimum_format=True,
+ )
+ new_module.pack(int_weight, scale, zp, m.bias)
+ set_module(compressed_model, k, new_module)
+ return compressed_model
+
+
+def save_compressed_model(model, output_dir, quantize_config=None, tokenizer=None):
+ """Save configure file and weights for CPU backend inference."""
+
+ if quantize_config is not None:
+ config = model.config
+ setattr(config, "quantization_config", quantize_config.to_dict())
+ config.save_pretrained(output_dir)
+ quantize_config.save_pretrained(output_dir)
+
+ try:
+ model.save_pretrained(output_dir, safe_serialization=True)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
+ except IOError as e: # pragma: no cover
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
\ No newline at end of file
diff --git a/auto_round/model_wrapper.py b/auto_round/export_to_speed/model_wrapper.py
similarity index 99%
rename from auto_round/model_wrapper.py
rename to auto_round/export_to_speed/model_wrapper.py
index bc52c3ec0..256903091 100644
--- a/auto_round/model_wrapper.py
+++ b/auto_round/export_to_speed/model_wrapper.py
@@ -351,4 +351,3 @@ class WeightOnlyLinear(torch.nn.Module):
tmp_str += ", use_optimum_format=True"
return tmp_str
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
new file mode 100644
index 000000000..2e84b82fc
--- /dev/null
+++ b/auto_round/utils.py
@@ -0,0 +1,510 @@
+
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import copy
+from torch.amp import autocast
+from collections import UserDict
+
+
+def round_ste(x: torch.Tensor):
+ """Straight-Through Estimator for rounding.
+ This function is adapted from omniquant.
+
+ Args:
+ x: torch.Tensor
+
+ Returns:
+ torch.Tensor
+ """
+ return (x.round() - x).detach() + x
+
+
+def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
+ """Quantizes and dequantizes weight asymmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2 ** num_bits - 1)
+ zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
+ # zeros = torch.zeros(weight.shape[0], device=weight.device)
+ if isinstance(min_scale, torch.Tensor):
+ wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
+ wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
+ wmin_tmp *= min_scale + 1.0
+ wmax_tmp *= max_scale + 1.0
+ wmax = torch.maximum(wmax_tmp, wmin_tmp)
+ wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ else:
+ wmin = torch.minimum(weight.min(1)[0], zeros)
+ wmax = torch.maximum(weight.max(1)[0], zeros)
+
+ tmp = (wmin == 0) & (wmax == 0)
+ wmin[tmp] = -1
+ wmax[tmp] = +1
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ zp = round_ste(-wmin / scale)
+ scale = scale.unsqueeze(dim=-1)
+ zp = zp.unsqueeze(dim=-1)
+ int_w = round_ste(weight / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ return scale * (q - zp), scale, zp
+
+
+def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
+ """Quantizes and dequantizes weight symmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2 ** (num_bits - 1) - 1).to(weight.device)
+ minq = torch.tensor(-(2 ** (num_bits - 1))).to(weight.device)
+ if num_bits == 1:
+ maxq = torch.tensor(2 ** (num_bits - 1))
+ minq = torch.tensor(2 ** (num_bits - 1) - 1)
+
+ wmax = torch.abs(weight).max(1)[0]
+ wmax *= 1 + max_scale
+ tmp = wmax == 0
+ wmax[tmp] = +1
+ scale = (wmax / ((maxq - minq) / 2)).to(scale_dtype)
+ scale.unsqueeze_(dim=-1)
+ q = torch.clamp(round_ste(weight / scale + v), minq, maxq)
+ return scale * q, scale, None
+
+
+def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):
+ """Quantizes and dequantizes weight symmetrically or asymmetrically .
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ scheme: Sym or asym
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ assert num_bits > 0, "num_bits should be larger than 0"
+ if scheme == "sym":
+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
+ else:
+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
+
+
+def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
+ """Quantizes and dequantizes weight, handing the group size issue .
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: The number of elements shares scale and zero point
+ scheme: Sym or asym
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ if group_size == -1 or weight.shape[1] < group_size:
+ return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,
+ min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)
+ orig_shape = weight.shape
+ if weight.shape[1] % group_size == 0:
+ weight = weight.reshape(-1, group_size)
+ if isinstance(v, torch.Tensor):
+ v = v.reshape(-1, group_size)
+
+ weight, scale, zp = quant_weight_actor(
+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ )
+ weight = weight.reshape(orig_shape)
+ scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d
+ if zp is not None:
+ zp = zp.reshape(weight.shape[0], -1)
+ return weight, scale, zp
+
+ else:
+ pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]
+ weight_new = torch.nn.functional.pad(weight, (0, pad_len))
+ v = torch.nn.functional.pad(v, (0, pad_len))
+ weight_new = weight_new.reshape(-1, group_size)
+ if isinstance(v, torch.Tensor):
+ v = v.reshape(-1, group_size)
+ weight_new, scale, zp = quant_weight_actor(
+ weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ )
+ weight_new = weight_new.reshape(orig_shape[0], -1)
+
+ weight_new = weight_new[:, :-pad_len]
+ scale = scale.reshape(weight_new.shape[0], -1) ##only for linear, conv1d
+ if zp is not None:
+ zp = zp.reshape(weight_new.shape[0], -1)
+ return weight_new, scale, zp
+
+
+def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):
+ """Quant and dequant tensor with group size.
+
+ Args:
+ weight: input weight
+ scale: scale
+ zp: zero point
+ group_size (int, optional): how many elements share one scale/zp. Defaults to -1.
+
+ Returns:
+ output: int weight.
+ """
+ scale = scale.to(device)
+ if zp is not None:
+ zp = zp.to(device)
+ if group_size == -1:
+ return torch.round(weight / scale) if zp is None else torch.round(weight / scale + zp)
+ int_weight = torch.zeros(weight.shape).to(device)
+ leng = weight.shape[1] // group_size
+ tail_flag = False if weight.shape[1] % group_size == 0 else True
+ for i in range(leng):
+ int_weight_tmp = weight[:, i * group_size : (i + 1) * group_size] / scale[:, i].unsqueeze(1)
+ if zp is not None:
+ int_weight_tmp += zp[:, i].unsqueeze(1)
+ int_weight[:, i * group_size : (i + 1) * group_size] = torch.round(int_weight_tmp)
+ if tail_flag:
+ int_weight_tmp = weight[:, leng * group_size :] / scale[:, -1].unsqueeze(1)
+ if zp is not None:
+ int_weight_tmp += zp[:, -1].unsqueeze(1)
+ int_weight[:, leng * group_size :] = torch.round(int_weight_tmp)
+ return int_weight
+
+
+def get_module(model, key):
+ """Get module from model by key name.
+
+ Args:
+ model (torch.nn.Module): original model
+ key (str): module name to be replaced
+ """
+ module = model
+ name_list = key.split(".")
+ for name in name_list:
+ if hasattr(module, name):
+ module = getattr(module, name)
+ module = module
+ return module
+
+
+def set_module(model, key, new_module):
+ """Set new module into model by key name.
+
+ Args:
+ model (torch.nn.Module): original model
+ key (str): module name to be replaced
+ new_module (torch.nn.Module): new module to be inserted
+ """
+ module = model
+ name_list = key.split(".")
+ for name in name_list[:-1]:
+ if hasattr(module, name):
+ module = getattr(module, name)
+ else:
+ module = module
+ setattr(module, name_list[-1], new_module)
+
+
+def get_scale_shape(weight, group_size):
+ """Computes the shape of the scale tensor for quantization based on the weight tensor and group size.
+
+ Args:
+ weight (torch.Tensor): The weight tensor of the layer.
+ group_size (int): The size of the groups for quantization.
+
+ Returns:
+ The shape of the scale tensor to be used for quantization.
+ """
+ if group_size == -1 or weight.shape[1] < group_size:
+ shape = weight.shape[0]
+ else:
+ shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)
+
+ return shape
+
+
+def move_input_to_device(input, device=torch.device("cpu")):
+ """Moves input data to the specified device.
+
+ Args:
+ input: The input data to be moved.
+ device: The target device.
+
+ Returns:
+ The input data on the specified device.
+ """
+ if isinstance(input, torch.Tensor):
+ return input.to(device)
+ if isinstance(input, dict) or isinstance(input, UserDict):
+ for inp in input.keys():
+ input[inp] = move_input_to_device(input[inp], device)
+ elif isinstance(input, list) or isinstance(input, tuple):
+ input_res = []
+ for inp in input:
+ input_res.append(move_input_to_device(inp, device))
+ input = input_res
+ return input
+
+
+def check_is_cpu(device):
+ """Check if the device is a CPU.
+
+ Args:
+ device: The device to be checked.
+
+ Returns:
+ bool: True if the device is a CPU, False otherwise.
+ """
+ return device == torch.device("cpu") or device == "cpu"
+
+
+def get_block_names(model):
+ """Get the block names for transformers-like networks.
+
+ Args:
+ model: The model.
+
+ Returns:
+ block_names: A list of block names.
+ """
+ block_names = []
+ target_m = None
+ for n, m in model.named_modules():
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
+ target_m = (n, m)
+ break ## only find the first modulelist, may be not robust
+ for n, m in target_m[1].named_children():
+ block_names.append(target_m[0] + "." + n)
+ return block_names
+
+
+def get_tokenizer_function(tokenizer, seqlen):
+ """Returns a default tokenizer function.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum sequence length.
+
+ Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of
+ seqlen to the "text" field of examples.
+ """
+
+ def default_tokenizer_function(examples):
+ example = tokenizer(examples["text"], truncation=True, max_length=seqlen)
+ return example
+
+ return default_tokenizer_function
+
+
+def get_dataloader(tokenizer, seqlen, data_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):
+ """Returns a dataloader for the specified dataset and split.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum sequence length.
+ data_name: The name of the dataset.
+ split: The data split to be used (e.g., "train", "test").
+ seed: The random seed for shuffling the dataset.
+ bs: The batch size for the dataloader.
+
+ Returns:
+ A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
+ """
+ from datasets import load_dataset
+ from torch.utils.data import DataLoader
+
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+
+ @torch.no_grad()
+ def collate_batch(batch):
+ input_ids_new = []
+ for text in batch:
+ input_ids = text["input_ids"]
+ if input_ids.shape[0] < seqlen:
+ continue
+ input_ids = input_ids[:seqlen]
+ input_ids_list = input_ids.tolist()
+ if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
+ continue
+ input_ids_new.append(input_ids)
+ if len(input_ids_new) == 0:
+ return None
+ tmp = torch.vstack(input_ids_new)
+ res = {"input_ids": tmp}
+ return res
+
+ calib_dataset = load_dataset(data_name, split=split)
+ calib_dataset = calib_dataset.shuffle(seed=seed)
+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
+ calib_dataset.set_format(type="torch", columns=["input_ids"])
+ calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)
+ return calib_dataloader
+
+
+def collect_round_v(block):
+ """Collects the round values for wrapped linear modules in the given block.
+
+ Args:
+ block: The input block.
+
+ Returns:
+ vs: A dictionary of round values for the wrapped linear modules.
+ """
+ vs = {}
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ v = m.value.data
+ vs[n] = copy.deepcopy(v)
+ return vs
+
+
+def collect_minmax_scale(block):
+ """Collects the min-max scaling values for wrapped linear modules in the given block.
+
+ Args:
+ block: The input block.
+
+ Returns:
+ min_scales: A dictionary of minimum scaling values.
+ max_scales: A dictionary of maximum scaling values.
+ """
+ min_scales = {}
+ max_scales = {}
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, -1, 0))
+ max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, -1, 0))
+ return min_scales, max_scales
+
+
+def get_batch_dim(input_others):
+ """Gets the batch dimension based on the input positional inputs.
+
+ Args:
+ input_others: A dictionary containing input data.
+
+ Returns:
+ dim: The batch dimension.
+ """
+ dim = int(len(input_others["positional_inputs"]) > 0)
+ return dim
+
+
+def sampling_inputs(input_ids, input_others, indices, seqlen):
+ """Samples inputs based on the given indices and sequence length.
+
+ Args:
+ input_ids: The input tensor containing IDs.
+ input_others: A dictionary containing other input data.
+ indices: The indices to sample from the input.
+ seqlen: The sequence length.
+
+ Returns:
+ current_input_ids: The sampled input IDs.
+ current_input_others: The sampled other input data.
+ """
+ if len(input_ids.shape) == 3:
+ if int(len(input_others["positional_inputs"]) > 0):
+ current_input_ids = input_ids[:, indices, :]
+ else:
+ current_input_ids = input_ids[indices, :, :]
+ else:
+ n_samples = input_ids.shape[0] // seqlen
+ current_input_ids = input_ids.view(n_samples, seqlen, -1)
+ current_input_ids = current_input_ids[indices, :, :]
+ current_input_ids = current_input_ids.reshape(-1, input.shape[-1])
+
+ current_input_others = {"positional_inputs": input_others["positional_inputs"]}
+ for key in input_others.keys():
+ if "attention_mask" in key or "alibi" in key:
+ current_input_others[key] = None
+ if input_others[key] is not None:
+ current_input_others[key] = input_others[key][indices, ...]
+ else:
+ current_input_others[key] = input_others[key]
+
+ return current_input_ids, current_input_others
+
+
+def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):
+ """Performs a forward pass through a block with the given inputs.
+
+ Args:
+ block: The block to perform the forward pass on.
+ input_ids: The input IDs.
+ input_others: A dictionary containing other input data.
+ amp: A boolean indicating whether to use automatic mixed precision.
+ amp_dtype: The data type for automatic mixed precision.
+ device: The target device.
+
+ Returns:
+ output: The output of the forward pass.
+ """
+ if input_ids.device != device:
+ # input_ids, input_others = move_to_device(input_ids, input_others, device)
+ input_ids = move_input_to_device(input_ids, device)
+ input_others = move_input_to_device(input_others, device)
+ if "alibi" in input_others.keys():
+ attention_mask = input_others["attention_mask"]
+ alibi = input_others["alibi"]
+ if alibi is not None:
+ alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
+ if amp and not check_is_cpu(device):
+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
+ output = block(
+ input_ids, attention_mask=attention_mask, alibi=alibi
+ ) ##TODO is this correct for all models with alibi?
+ elif amp and check_is_cpu(device):
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
+ else:
+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
+ else:
+ input_tuple = input_others.pop("positional_inputs", None)
+ if amp and not check_is_cpu(device):
+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ elif amp and check_is_cpu(device):
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ else:
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ if isinstance(output, list) or isinstance(output, tuple):
+ output = output[0]
+ return output
\ No newline at end of file
diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.py
index e1d00ea3d..19430def7 100644
--- a/examples/eval/evaluation.py
+++ b/examples/eval/evaluation.py
@@ -220,14 +220,20 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
if os.path.exists(output_dir) and not eval_orig_float:
shutil.rmtree(output_dir)
+
+ if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):
+ dtype = 'bfloat16'
+ pt_dtype = torch.bfloat16
+ else:
+ pt_dtype = torch.float16
+
if not eval_orig_float:
- model = model.to(torch.float16)
+ model = model.to(pt_dtype)
model = model.to("cpu")
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
- if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):
- dtype = 'bfloat16'
+
external_tasks = []
for each in EXT_TASKS:
@@ -366,4 +372,3 @@ if __name__ == "__main__":
print("cost time: ", time.time() - s)
-
diff --git a/examples/eval/utils.py b/examples/eval/utils.py
index 18730a334..2a3a2e4c5 100644
--- a/examples/eval/utils.py
+++ b/examples/eval/utils.py
@@ -213,4 +213,3 @@ def eval_ppl_same_with_gptq(model, testenc, dev):
if hasattr(model.config, "use_cache"):
model.config.use_cache = use_cache
return ppl.item()
-
diff --git a/examples/main.py b/examples/main.py
index f876e2842..95dab31f2 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -5,7 +5,7 @@ from auto_round import (AutoRound,
AutoAdamRound,
compress_model,
save_compressed_model,
- AutoroundQuantConfig)
+ QuantConfig)
parser = argparse.ArgumentParser()
import torch
@@ -90,7 +90,8 @@ if __name__ == '__main__':
help="whether enable weight minmax tuning")
parser.add_argument("--deployment_device", default='fake', type=str,
- help="targeted inference acceleration platform, 'fake', 'cpu' or 'gpu'")
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',"
+ "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
parser.add_argument("--scale_dtype", default='fp32',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
@@ -194,13 +195,15 @@ if __name__ == '__main__':
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args pass
model, q_config = autoround.quantize()
+ export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
if args.deployment_device == 'cpu':
- output_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
compressed_model = compress_model(model, q_config)
- quantize_config = AutoroundQuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)
- save_compressed_model(compressed_model, output_dir=output_dir, quantize_config=quantize_config, tokenizer=tokenizer)
+ quantize_config = QuantConfig(bits=args.bits, sym=args.sym, group_size=args.group_size)
+ save_compressed_model(compressed_model, output_dir=export_dir, quantize_config=quantize_config, tokenizer=tokenizer)
del q_config
del compressed_model
+ elif args.deployment_device == 'gpu':
+ autoround.export_to_autogptq(export_dir, use_triton=True)
torch.cuda.empty_cache()
model.eval()
@@ -213,4 +216,4 @@ if __name__ == '__main__':
print(excel_name, flush=True)
eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,
- limit=None)
+ limit=None)
\ No newline at end of file
diff --git a/examples/requirements.txt b/examples/requirements.txt
index a971eef6d..3692e365b 100644
--- a/examples/requirements.txt
+++ b/examples/requirements.txt
@@ -10,4 +10,3 @@ accelerate
datasets
protobuf
-
diff --git a/examples/run_autoround.sh b/examples/run_autoround.sh
new file mode 100644
index 000000000..2c8b13a9a
--- /dev/null
+++ b/examples/run_autoround.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+set -x
+
+# export CUBLAS_WORKSPACE_CONFIG=':4096:8'
+export HF_HOME="/models/huggingface/"
+export https_proxy=http://child-jf.intel.com:912
+export http_proxy=http://child-jf.intel.com:912
+
+# model="neural-chat-7b-v3-3"
+# for iter in 200 400 1000
+# do
+# CUDA_VISIBLE_DEVICES=2 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
+# examples/main.py \
+# --model_name /models/${model} \
+# --bits 4 --group_size 128 \
+# --enable_minmax_tuning \
+# --use_quant_input \
+# --amp \
+# --iters ${iter} \
+# --deployment_device 'cpu' \
+# --scale_dtype 'fp32' \
+# --eval_bs 16 \
+# --output_dir /data1/zww/autoround/int_llama_${iter} \
+# 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt
+# done &
+
+model="neural-chat-7b-v3-3"
+for iter in 2000
+do
+ CUDA_VISIBLE_DEVICES=4 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
+ examples/main.py \
+ --model_name /models/${model} \
+ --bits 4 --group_size 128 \
+ --enable_minmax_tuning \
+ --use_quant_input \
+ --amp \
+ --iters ${iter} \
+ --deployment_device 'fake' \
+ --scale_dtype 'fp32' \
+ --eval_bs 16 \
+ --output_dir /data1/zww/autoround/int_llama_${iter} \
+ 2>&1 | tee -a /data1/zww/optround_logs/${model}_${iter}_autoround_W4G128.txt
+done
+# model_dir="/data2/zww/gptq_actorder/W4G128_gptq"
+# for model in "static_neural-chat-7b-v3-3" "nonstatic_neural-chat-7b-v3-3"
+# do
+# CUDA_VISIBLE_DEVICES=5 /home/weiweiz1/anaconda3/envs/llm/bin/python3 \
+# examples/main.py \
+# --model_name ${model_dir}/${model} \
+# --bits 4 --group_size 128 \
+# --eval_fp16_baseline \
+# --enable_minmax_tuning \
+# --use_quant_input \
+# --amp \
+# --scale_dtype 'fp32' \
+# --eval_bs 16 \
+# 2>&1 | tee -a /data1/zww/optround_logs/${model}_gptq_W4G128.txt
+# done
+
diff --git a/test/test_autoround.py b/test/test_autoround.py
new file mode 100644
index 000000000..9d87ba35b
--- /dev/null
+++ b/test/test_autoround.py
@@ -0,0 +1,107 @@
+import copy
+import os
+import shutil
+import unittest
+import sys
+sys.path.insert(0,".")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_round import (AutoRound,
+ AutoAdamRound,
+ compress_model,
+ save_compressed_model,
+ AutoroundQuantConfig)
+
+
+class SimpleDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.randn([1, 30])
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
+ approach = "weight_only"
+
+ @classmethod
+ def setUpClass(self):
+ self.dataloader = SimpleDataLoader()
+ self.gptj = transformers.AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM",
+ torchscript=True,
+ )
+ self.tokenizer = transformers.AutoTokenizer.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM",
+ trust_remote_code=True
+ )
+ self.gptj_no_jit = transformers.AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM",
+ )
+ self.llm_dataloader = LLMDataLoader()
+ self.lm_input = torch.ones([1, 10], dtype=torch.long)
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ def test_autoround_int_quant(self):
+ model = copy.deepcopy(self.gptj)
+ # device = "cuda:0"
+ device = "cpu"
+ model=model.to(device)
+ out1 = model(self.lm_input.to(device))
+ round = AutoRound
+ optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=True, seqlen=10, iters=10, scale_dtype='fp16')
+ q_model, weight_config1 = optq_1.quantize()
+ q_model = q_model.to(device)
+ compressed_model = compress_model(q_model, weight_config1, device=device)
+ compressed_model = compressed_model.to(device)
+ q_model = q_model.to(device)
+ model = model.to(device)
+ out2 = model(self.lm_input.to(device))
+ out3 = q_model(self.lm_input.to(q_model.device))
+ out4 = compressed_model(self.lm_input.to(compressed_model.device))
+ self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))
+ self.assertFalse(torch.all(out1[0] == out2[0]))
+ self.assertTrue(torch.all(out2[0] == out3[0]))
+ self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))
+ self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())
+
+ # model = copy.deepcopy(self.gptj)
+ # out6 = model(self.lm_input)
+ # optq_2 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10)
+ # q_model, weight_config2 = optq_2.quantize()
+ # out4 = q_model(self.lm_input)
+ # out5 = model(self.lm_input)
+
+ # self.assertTrue(torch.all(out1[0] == out6[0]))
+ # self.assertTrue(torch.all(out4[0] == out5[0]))
+ # self.assertTrue(torch.all(torch.isclose(out6[0], out5[0], atol=1e-1)))
+
+
+ def test_config(self):
+ config = AutoroundQuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
+ config.save_pretrained("quantization_config_dir")
+ loaded_config = AutoroundQuantConfig.from_pretrained("quantization_config_dir")
+ self.assertEqual(config.group_size, loaded_config.group_size)
+ self.assertEqual(config.true_sequential, loaded_config.true_sequential)
+
+
+
+if __name__ == "__main__":
+ unittest.main()
+