Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rename hypeparameters (#29)

wenhuach21d02f94d4b085
diff --git a/README.md b/README.md
index 4378d30d0..3c0b7ebb5 100644
--- a/README.md
+++ b/README.md
@@ -37,8 +37,8 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
from auto_round import AutoRound
-bits, group_size, scheme = 4, 128, "asym"
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device=tuning_device)
+bits, group_size, sym = 4, 128, False
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, device=tuning_device)
autoround.quantize()
output_dir = "./tmp_autoround"
autoround.save_quantized(output_dir)
@@ -55,7 +55,7 @@ autoround.save_quantized(output_dir)
- `group_size (int)`: Size of the quantization group (default is 128).
-- `scheme (str)`: The quantization scheme (sym/asym) to be used (default is "asym").
+- `sym (bool)`: Whether to use symmetric quantization.
- `use_quant_input (bool)`: Whether to use the output of the previous quantized block as the input for the current block (default is True).
@@ -83,7 +83,7 @@ autoround.save_quantized(output_dir)
- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of a little tuning time (default is True).
-- `dataset_name (str)`: The default dataset name for tuning (default is "NeelNanda/pile-10k").
+- `dataset (str)`: The default dataset name for tuning (default is "NeelNanda/pile-10k").
- `dataset_split (str)`: The split of the dataset to be used for tuning (default is "train").
@@ -98,17 +98,18 @@ autoround.save_quantized(output_dir)
## Support List
-| Model | Supported |
-|-------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |
-| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |
-| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) under review, [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) under review, [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) | |
+| Model | Supported |
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |
+| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |
+| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |
+| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) under review, [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
+| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) under review, [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) | |
mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |
-| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
-| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
-| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
+| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
+| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
+| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |
+| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |
| huggyllama/llama-7b | [example](./examples/language-modeling/) |
| meta-llama/Llama-2-7b-hf | [example](./examples/language-modeling/) |
@@ -139,6 +140,7 @@ from intel_extension_for_transformers.transformers import AutoModelForCausalLM,
from transformers import AutoTokenizer
quantized_model_path = "./tmp_autoround"
+scheme = "sym" if sym else "asym"
woq_config = WeightOnlyQuantConfig(
group_size=group_size, scheme=scheme, use_autoround=True
) ##only supports 4 bits currently
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b8f5f0733..79ea084f5 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -56,7 +56,7 @@ class WrapperLinear(torch.nn.Module):
- orig_layer (torch.nn.Module): The original linear layer being wrapped.
- num_bits (int): The number of bits for quantization.
- group_size (int): The size of the groups for quantization.
- - scheme (str): The quantization scheme to use.
+ - sym (bool): Whether the symmetric quantization is to be used.
- value (torch.nn.Parameter): The learnable parameter for quantization.
- enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.
- min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.
@@ -67,7 +67,7 @@ class WrapperLinear(torch.nn.Module):
self.num_bits = self.orig_layer.bits
self.group_size = self.orig_layer.group_size
self.scale_dtype = self.orig_layer.scale_dtype
- self.scheme = self.orig_layer.scheme
+ self.sym = self.orig_layer.sym
self.value = torch.nn.Parameter(
torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True
)
@@ -103,7 +103,7 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.weight,
self.num_bits,
self.group_size,
- self.scheme,
+ self.sym,
v,
min_scale,
max_scale,
@@ -133,7 +133,7 @@ class WrapperLinear(torch.nn.Module):
weight,
self.num_bits,
self.group_size,
- self.scheme,
+ self.sym,
self.value,
self.min_scale,
self.max_scale,
@@ -153,14 +153,14 @@ class WrapperTransformerConv1d(torch.nn.Module):
- orig_layer (torch.nn.Module): The original 1D convolutional layer to be wrapped.
- num_bits (int): The number of bits for quantization.
- group_size (int): The size of the groups for quantization.
- - scheme (str): The quantization scheme to use.
+ - sym (bool): Whether symmetric quantization is to be used.
- enable_minmax_tuning (bool): Whether to enable min-max scaling tuning. Default is True.
Attributes:
- orig_layer (torch.nn.Module): The original 1D convolutional layer being wrapped.
- num_bits (int): The number of bits for quantization.
- group_size (int): The size of the groups for quantization.
- - scheme (str): The quantization scheme to use.
+ - sym (bool): Whether symmetric quantization is to be used.
- weight_t (torch.Tensor): Transposed weight tensor of the original layer.
- value (torch.nn.Parameter): The learnable parameter for quantization.
- enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.
@@ -171,7 +171,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.orig_layer = orig_layer
self.num_bits = self.orig_layer.bits
self.group_size = self.orig_layer.group_size
- self.scheme = self.orig_layer.scheme
+ self.sym = self.orig_layer.sym
device = self.orig_layer.weight.device
self.weight_t = self.orig_layer.weight.t()
self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)
@@ -198,7 +198,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
min_scale.clamp_(-1, 0)
max_scale.clamp_(-1, 0)
weight_q, scale, zp = quant_weight(
- self.weight_t, self.num_bits, self.group_size, self.scheme, v, min_scale, max_scale, self.scale_dtype
+ self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale, max_scale, self.scale_dtype
)
self.orig_layer.weight.data.copy_(weight_q.t())
self.orig_layer.weight.grad = None
@@ -222,7 +222,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.weight_t,
self.num_bits,
self.group_size,
- self.scheme,
+ self.sym,
self.value,
self.min_scale,
self.max_scale,
@@ -264,29 +264,32 @@ def wrapper_block(block, enable_minmax_tuning):
enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.
Returns:
- list: A list of names of the wrapped layers.
+ list: A list of names of the wrapped layers and unwrapped layers.
"""
- names = []
+ quantized_layers = []
+ unquantized_layers = []
for n, m in block.named_modules():
if isinstance(m, torch.nn.Linear):
if not check_to_quantized(m):
+ unquantized_layers.append(n)
continue
new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning)
set_module(block, n, new_m)
- names.append(n)
+ quantized_layers.append(n)
try:
import transformers
if isinstance(m, transformers.modeling_utils.Conv1D):
if not check_to_quantized(m):
+ unquantized_layers.append(n)
continue
new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)
set_module(block, n, new_m)
- names.append(n)
+ quantized_layers.append(n)
except:
pass
- return names
+ return quantized_layers, unquantized_layers
@torch.no_grad()
@@ -326,7 +329,7 @@ class AutoRound(object):
tokenizer: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
- scheme (str): The quantization scheme (sym/asym) to be used (default is "asym").
+ sym (bool): Whether symmetric quantization is to be used (default is False).
weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
weight_config={
'layer1':##layer_name
@@ -334,14 +337,14 @@ class AutoRound(object):
'data_type': 'int',
'bits': 4,
'group_size': 32,
- 'scheme': "asym", ## or sym
+ 'sym': False
}
...
}
enable_full_range (bool): Whether to enable full range quantization (default is False).
- bs (int): Batch size for training (default is 8).
+ batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for tuning (default is "cuda:0").
+ device: The device to be used for tuning (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
@@ -376,15 +379,15 @@ class AutoRound(object):
tokenizer,
bits: int = 4,
group_size: int = 128,
- scheme: str = "asym",
+ sym: bool = False,
weight_config: dict = {},
enable_full_range: bool = False, ##for symmetric, TODO support later
- bs: int = 8,
+ batch_size: int = 8,
amp: bool = True,
device=None,
lr_scheduler=None,
dataloader=None, ## to support later
- dataset_name: str = "NeelNanda/pile-10k",
+ dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
@@ -413,7 +416,7 @@ class AutoRound(object):
self.n_blocks = n_blocks
self.bits = bits
self.group_size = group_size
- self.scheme = scheme
+ self.sym = sym
self.low_gpu_mem_usage = low_gpu_mem_usage
self.data_type = data_type
self.supported_types = [torch.nn.Linear]
@@ -428,7 +431,7 @@ class AutoRound(object):
self.seed = seed
self.tokenizer = tokenizer
self.seqlen = seqlen
- self.train_bs = bs
+ self.train_bs = batch_size
self.n_blocks = n_blocks
self.device = detect_device(device)
@@ -454,7 +457,7 @@ class AutoRound(object):
else:
self.model = self.model.to(torch.float32)
logger.info(f"using {self.model.dtype} for quantization tuning")
- self.dataset_name = dataset_name
+ self.dataset_name = dataset
self.dataloader = dataloader
self.iters = iters
@@ -478,6 +481,7 @@ class AutoRound(object):
self.set_layerwise_config(self.weight_config)
self.optimizer = self.get_optimizer(None)
self.check_configs()
+ torch.set_printoptions(precision=5)
def get_optimizer(self, optimizer):
"""Returns the specified optimizer. In SignRound, we fix the optimizer.
@@ -568,7 +572,7 @@ class AutoRound(object):
weight_config[n]["data_type"] = self.data_type
weight_config[n]["bits"] = self.bits
weight_config[n]["group_size"] = self.group_size
- weight_config[n]["scheme"] = self.scheme
+ weight_config[n]["sym"] = self.sym
else:
if "data_type" not in weight_config[n].keys():
weight_config[n]["data_type"] = self.data_type
@@ -576,14 +580,14 @@ class AutoRound(object):
weight_config[n]["bits"] = self.bits
if "group_size" not in weight_config[n].keys():
weight_config[n]["group_size"] = self.group_size
- if "scheme" not in weight_config[n].keys():
- weight_config[n]["scheme"] = self.scheme
+ if "sym" not in weight_config[n].keys():
+ weight_config[n]["sym"] = self.sym
weight_config[n]["scale_dtype"] = self.scale_dtype
m.data_type = weight_config[n]["data_type"]
m.bits = weight_config[n]["bits"]
m.group_size = weight_config[n]["group_size"]
- m.scheme = weight_config[n]["scheme"]
+ m.sym = weight_config[n]["sym"]
m.scale_dtype = weight_config[n]["scale_dtype"]
@torch.no_grad()
@@ -801,8 +805,7 @@ class AutoRound(object):
if q_input is not None:
input_ids = q_input.to(cache_device)
- names = wrapper_block(block, self.enable_minmax_tuning)
- logger.info(names)
+ quantized_layer_names, unquantized_layer_names = wrapper_block(block, self.enable_minmax_tuning)
round_params = []
minmax_params = []
@@ -837,6 +840,7 @@ class AutoRound(object):
best_loss = torch.finfo(torch.float).max
mse_loss = torch.nn.MSELoss().to(device)
scaler = self.get_scaler() # pylint: disable=assignment-from-none
+ init_loss = None
for i in range(self.iters):
if self.sampler == "rand":
indices = torch.randperm(n_samples)[:pick_samples]
@@ -871,6 +875,9 @@ class AutoRound(object):
)
total_loss += loss.item() / self.gradient_accumulate_steps
+ if i == 0:
+ init_loss = total_loss
+
self.scale_loss_and_backward(scaler, loss)
if total_loss < best_loss:
@@ -888,6 +895,20 @@ class AutoRound(object):
if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:
break
self.step(scaler, optimizer, lr_schedule)
+
+ last_loss = total_loss
+ best_iter = self.iters
+ if not self.not_use_best_mse:
+ last_loss = best_loss
+ best_iter = last_best_iter
+ dump_info = (
+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))}"
+ f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ )
+ logger.info(dump_info)
+ if len(unquantized_layer_names) != 0:
+ logger.info(f"{unquantized_layer_names} have not been quantized")
+
unwrapper_block(block, best_v, best_min_scale, best_max_scale)
if self.use_quant_input:
q_outputs = self.get_block_outputs(
@@ -974,7 +995,7 @@ class AutoRound(object):
inplace=inplace,
bits=self.bits,
group_size=self.group_size,
- scheme=self.scheme,
+ sym=self.sym,
iters=self.iters,
lr=self.lr,
minmax_lr=self.minmax_lr,
@@ -1049,6 +1070,21 @@ class AutoRound(object):
end_time = time.time()
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
+ ## dump a summary
+ quantized_layers = []
+ unquantized_layers = []
+ for n, m in self.model.named_modules():
+ if isinstance(m, tuple(self.supported_types)):
+ if self.weight_config[n]["bits"] == 16:
+ unquantized_layers.append(n)
+ else:
+ quantized_layers.append(n)
+ logger.info(
+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
+ )
+ if len(unquantized_layers) > 0:
+ logger.info(f"Summary: {unquantized_layers} have not been quantized")
+
self.quantized = True
self.model = self.model.to(self.model_orig_dtype)
return self.model, self.weight_config
@@ -1062,12 +1098,12 @@ class AutoOPTRound(AutoRound):
tokenizer: An optional tokenizer for processing input data.
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
- scheme (str): The quantization scheme to be used (default is "asym").
+ sym (bool): Whether sym to be used (default is False).
weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
enable_full_range (bool): Whether to enable full range quantization (default is False).
- bs (int): Batch size for training (default is 8).
+ batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "cuda:0").
+ device: The device to be used for training (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
@@ -1087,6 +1123,8 @@ class AutoOPTRound(AutoRound):
not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
+ scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels
+ have different choices.
optimizer: string or object
**kwargs: Additional keyword arguments.
@@ -1100,15 +1138,15 @@ class AutoOPTRound(AutoRound):
tokenizer=None,
bits: int = 4,
group_size: int = 128,
- scheme: str = "asym",
+ sym: bool = False,
weight_config: dict = {},
enable_full_range: bool = False,
- bs: int = 8,
+ batch_size: int = 8,
amp: bool = True,
- device="cuda:0",
+ device="auto",
lr_scheduler=None,
dataloader=None,
- dataset_name: str = "NeelNanda/pile-10k",
+ dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
@@ -1125,6 +1163,7 @@ class AutoOPTRound(AutoRound):
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
+ scale_dtype: str = "fp32",
optimizer="AdamW",
**kwargs,
):
@@ -1133,15 +1172,15 @@ class AutoOPTRound(AutoRound):
tokenizer,
bits,
group_size,
- scheme,
+ sym,
weight_config,
enable_full_range,
- bs,
+ batch_size,
amp,
device,
lr_scheduler,
dataloader,
- dataset_name,
+ dataset,
dataset_split,
use_quant_input,
enable_minmax_tuning,
@@ -1158,6 +1197,7 @@ class AutoOPTRound(AutoRound):
not_use_best_mse,
dynamic_max_gap,
data_type,
+ scale_dtype,
**kwargs,
)
@@ -1212,12 +1252,12 @@ class AutoAdamRound(AutoOPTRound):
tokenizer: An optional tokenizer for processing input data.
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
- scheme (str): The quantization scheme to be used (default is "asym").
+ sym (str): Whether symmetric quantization to be used (default is False).
weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
enable_full_range (bool): Whether to enable full range quantization (default is False).
- bs (int): Batch size for training (default is 8).
+ batch_size (int): Batch size for training (default is 8).
amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "cuda:0").
+ device: The device to be used for training (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
@@ -1238,6 +1278,8 @@ class AutoAdamRound(AutoOPTRound):
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
optimizer: string or object
+ scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels
+ have different choices.
**kwargs: Additional keyword arguments.
Returns:
@@ -1250,15 +1292,15 @@ class AutoAdamRound(AutoOPTRound):
tokenizer=None,
bits: int = 4,
group_size: int = 128,
- scheme: str = "asym",
+ sym: bool = False,
weight_config: dict = {},
enable_full_range: bool = False,
- bs: int = 8,
+ batch_size: int = 8,
amp: bool = True,
- device="cuda:0",
+ device="auto",
lr_scheduler=None,
dataloader=None,
- dataset_name: str = "NeelNanda/pile-10k",
+ dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
use_quant_input: bool = True,
enable_minmax_tuning: bool = True,
@@ -1275,6 +1317,7 @@ class AutoAdamRound(AutoOPTRound):
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
+ scale_dtype: str = "fp32",
optimizer="AdamW",
**kwargs,
):
@@ -1283,15 +1326,15 @@ class AutoAdamRound(AutoOPTRound):
tokenizer,
bits,
group_size,
- scheme,
+ sym,
weight_config,
enable_full_range,
- bs,
+ batch_size,
amp,
device,
lr_scheduler,
dataloader,
- dataset_name,
+ dataset,
dataset_split,
use_quant_input,
enable_minmax_tuning,
@@ -1308,6 +1351,7 @@ class AutoAdamRound(AutoOPTRound):
not_use_best_mse,
dynamic_max_gap,
data_type,
+ scale_dtype,
optimizer,
**kwargs,
)
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index d69d8404c..17f6be1d1 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -52,7 +52,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
"""Export the model to autogptq format to easily leverage cuda kernel."""
model = kwargs["model"]
weight_config = kwargs["weight_config"]
- sym = kwargs["scheme"] == "sym"
+ sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
iters = kwargs["iters"]
diff --git a/auto_round/export/export_to_itrex/__init__.py b/auto_round/export/export_to_itrex/__init__.py
index 75fecd3b9..a6c225a0c 100644
--- a/auto_round/export/export_to_itrex/__init__.py
+++ b/auto_round/export/export_to_itrex/__init__.py
@@ -11,5 +11,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from .export import save_quantized_as_itrex
+from .export import save_quantized_as_itrex, pack_model
from .config import QuantConfig
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index a0f0ff024..6f58b6517 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -33,7 +33,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
"""Save configure file and weights for CPU backend inference."""
model = kwargs["model"]
weight_config = kwargs["weight_config"]
- sym = kwargs["scheme"] == "sym"
+ sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
iters = kwargs["iters"]
@@ -44,7 +44,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
- compressed_model = _pack_model(model, weight_config, inplace=inplace)
+ compressed_model = pack_model(model, weight_config, inplace=inplace)
if output_dir is None:
return compressed_model
quantize_config = QuantConfig(
@@ -73,7 +73,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
return compressed_model
-def _pack_model(
+def pack_model(
model,
weight_config: Union[str, dict],
enable_full_range=False,
@@ -118,7 +118,7 @@ def _pack_model(
dtype = v["data_type"]
num_bits = v["bits"]
group_size = v["group_size"]
- scheme = v["scheme"]
+ sym = v["sym"]
scale_dtype = v["scale_dtype"]
m = get_module(compressed_model, k)
fp_weight = m.weight.data
@@ -126,13 +126,13 @@ def _pack_model(
convert_dtype = torch.float32 if fp_weight.device.type == "cpu" else scale_dtype
if not isinstance(scale, torch.Tensor):
scale = torch.tensor(scale, dtype=convert_dtype)
- zp = None if scheme == "sym" else torch.tensor(zp, dtype=torch.int32)
+ zp = torch.tensor(zp, dtype=torch.int32)
else:
if not inplace:
scale = scale.clone()
- zp = None if scheme == "sym" else zp.clone()
+ zp = zp.clone()
scale = scale.to(dtype=convert_dtype)
- zp = None if scheme == "sym" else zp.to(dtype=torch.int32)
+ zp = zp.to(dtype=torch.int32)
int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)
int_weight = int_weight.type(torch.int32)
new_module = WeightOnlyLinear(
diff --git a/auto_round/utils.py b/auto_round/utils.py
index fec585402..1ccc154a3 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -149,13 +149,13 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dt
return scale * (q - zp), scale, zp
-def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):
+def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16):
"""Quantizes and dequantizes weight symmetrically or asymmetrically .
Args:
weight: Tensor containing the weight to be quantized
num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- scheme: Sym or asym
+ sym: Sym or asym
v: Rounding value perturbation
min_scale: Minimum scale coefficient for weight
max_scale: Maximum scale coefficient for weight
@@ -164,14 +164,14 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_
Quantized and dequantized weight, scale, zero-point
"""
assert num_bits > 0, "num_bits should be larger than 0"
- if scheme == "sym":
+ if sym:
return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
else:
return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
def quant_weight(
- weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16
+ weight, num_bits=4, group_size=-1, sym=False, v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16
):
"""Quantizes and dequantizes weight, handing the group size issue .
@@ -179,7 +179,7 @@ def quant_weight(
weight: Tensor containing the weight to be quantized
num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
group_size: The number of elements shares scale and zero point
- scheme: Sym or asym
+ sym: Sym or asym
v: Rounding value perturbation
min_scale: Minimum scale coefficient for weight
max_scale: Maximum scale coefficient for weight
@@ -189,7 +189,7 @@ def quant_weight(
"""
if group_size == -1 or weight.shape[1] < group_size:
return quant_weight_actor(
- weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
)
orig_shape = weight.shape
if weight.shape[1] % group_size == 0:
@@ -198,7 +198,7 @@ def quant_weight(
v = v.reshape(-1, group_size)
weight, scale, zp = quant_weight_actor(
- weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
)
weight = weight.reshape(orig_shape)
scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d
@@ -214,7 +214,7 @@ def quant_weight(
if isinstance(v, torch.Tensor):
v = v.reshape(-1, group_size)
weight_new, scale, zp = quant_weight_actor(
- weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ weight_new, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
)
weight_new = weight_new.reshape(orig_shape[0], -1)
@@ -510,9 +510,20 @@ def check_to_quantized(config):
def detect_device(device=None):
- if device is None:
+ def is_valid_digit(s):
+ try:
+ num = int(s)
+ return 0 <= num
+ except:
+ return False
+
+ dev_idx = None
+ if is_valid_digit(device):
+ dev_idx = int(device)
+ device = "auto"
+ if device is None or device == "auto":
if torch.cuda.is_available():
- device = torch.device("cuda:0")
+ device = torch.device("cuda")
logger.info("Using GPU device")
elif is_hpu_available:
device = torch.device("hpu")
@@ -521,6 +532,8 @@ def detect_device(device=None):
else:
device = torch.device("cpu")
logger.info("Using CPU device")
+ if dev_idx is not None:
+ device = str(device) + f":{dev_idx}"
return str(device)
elif isinstance(device, torch.device):
device = str(device)
diff --git a/docs/Llama-2-7b-chat-hf-acc.md b/docs/Llama-2-7b-chat-hf-acc.md
new file mode 100644
index 000000000..4b8740661
--- /dev/null
+++ b/docs/Llama-2-7b-chat-hf-acc.md
@@ -0,0 +1,21 @@
+Due to licensing restrictions, we are unable to release the model.
+
+Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d.
+
+Since we encountered an issue evaluating this model with lm-eval, we opted to evaluate the qdq model instead. In our assessment, we found that its accuracy closely matches that of the real quantized model in most cases except for some small models like opt-125m.
+
+
+| Metric | FP16 | int4 qdq |
+| -------------- | ------ | -------- |
+| Avg. | 0.5901 | 0.5897 |
+| mmlu | 0.4640 | 0.4545 |
+| lambada_openai | 0.7105 | 0.7037 |
+| hellaswag | 0.5780 | 0.5706 |
+| winogrande | 0.6638 | 0.6614 |
+| piqa | 0.7639 | 0.7633 |
+| truthfulqa_mc1 | 0.3023 | 0.3035 |
+| openbookqa | 0.3340 | 0.3260 |
+| boolq | 0.7976 | 0.8064 |
+| rte | 0.6968 | 0.7292 |
+| arc_easy | 0.7382 | 0.7336 |
+| arc_challenge | 0.4420 | 0.4352 |
diff --git a/docs/Mixtral-8x7B-v0.1-acc.md b/docs/Mixtral-8x7B-v0.1-acc.md
index c3ea904b8..78f26e6d7 100644
--- a/docs/Mixtral-8x7B-v0.1-acc.md
+++ b/docs/Mixtral-8x7B-v0.1-acc.md
@@ -7,17 +7,17 @@ Download the model from hf(coming soon) or follow examples/language-modeling/scr
lm_eval --model hf --model_args pretrained="Intel/Mixtral-8x7B-v0.1-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32
~~~
-| Metric | FP16 | INT4 | 2e-3 |
-| -------------- | ------ | ------ | ------ |
-| Avg. | 0.6698 | 0.6633 | |
-| mmlu | 0.6802 | 0.6693 | 0.6713 |
-| lambada_openai | 0.7827 | 0.7825 | 0.7819 |
-| hellaswag | 0.6490 | 0.6459 | 0.6447 |
-| winogrande | 0.7648 | 0.7514 | 0.7569 |
-| piqa | 0.8248 | 0.8210 | 0.8210 |
-| truthfulqa_mc1 | 0.3427 | 0.3219 | 0.3121 |
-| openbookqa | 0.3540 | 0.3560 | 0.3380 |
-| boolq | 0.8523 | 0.8474 | 0.8465 |
-| rte | 0.7076 | 0.6931 | 0.7112 |
-| arc_easy | 0.8430 | 0.8430 | 0.8333 |
-| arc_challenge | 0.5666 | 0.5648 | 0.5469 |
+| Metric | FP16 | INT4 |
+| -------------- | ------ | ------ |
+| Avg. | 0.6698 | 0.6633 |
+| mmlu | 0.6802 | 0.6693 |
+| lambada_openai | 0.7827 | 0.7825 |
+| hellaswag | 0.6490 | 0.6459 |
+| winogrande | 0.7648 | 0.7514 |
+| piqa | 0.8248 | 0.8210 |
+| truthfulqa_mc1 | 0.3427 | 0.3219 |
+| openbookqa | 0.3540 | 0.3560 |
+| boolq | 0.8523 | 0.8474 |
+| rte | 0.7076 | 0.6931 |
+| arc_easy | 0.8430 | 0.8430 |
+| arc_challenge | 0.5666 | 0.5648 |
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
index c721abfbd..64d0a253d 100644
--- a/examples/code-generation/main.py
+++ b/examples/code-generation/main.py
@@ -156,13 +156,10 @@ if __name__ == '__main__':
if not args.low_gpu_mem_usage:
model = model.to(torch_device)
- scheme = "asym"
- if args.sym:
- scheme = "sym"
round = AutoRound
if args.adam:
round = AutoAdamRound
- autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,
+ autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, bs=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 41421c77d..1020b08bc 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -34,8 +34,8 @@ if __name__ == '__main__':
parser.add_argument("--eval_bs", default=4, type=int,
help="eval batch size")
- parser.add_argument("--device", default=None, type=str,
- help="The device to be used for tuning. The default is set to None,"
+ parser.add_argument("--device", default="auto", type=str,
+ help="The device to be used for tuning. The default is set to auto/None,"
"allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
parser.add_argument("--sym", action='store_true',
@@ -97,16 +97,11 @@ if __name__ == '__main__':
parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
help="Where to store the final model.")
- parser.add_argument("--eval_legacy", action='store_true',
- help="Whether to evaluate with a old lm_eval version(e.g. 0.3.0).")
-
parser.add_argument("--disable_eval", action='store_true',
help="Whether to do lmeval evaluation.")
args = parser.parse_args()
set_seed(args.seed)
-
-
tasks = args.tasks
use_eval_legacy = False
import subprocess
@@ -116,13 +111,9 @@ if __name__ == '__main__':
return version
except subprocess.CalledProcessError:
return "Library not found"
-
-
res = get_library_version("lm-eval")
if res == "0.3.0":
use_eval_legacy = True
-
-
if not use_eval_legacy:
from eval import eval_model
else:
@@ -214,15 +205,12 @@ if __name__ == '__main__':
if not args.low_gpu_mem_usage:
model = model.to(torch_device)
- scheme = "asym"
- if args.sym:
- scheme = "sym"
round = AutoRound
if args.adam:
round = AutoAdamRound
weight_config = {}
- if args.deployment_device == 'gpu':
+ if 'gpu' in args.deployment_device:
for n, m in model.named_modules():
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
@@ -230,7 +218,7 @@ if __name__ == '__main__':
print(
f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
- autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,
+ autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
@@ -264,4 +252,3 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name)
-
diff --git a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
new file mode 100644
index 000000000..59b075efd
--- /dev/null
+++ b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
@@ -0,0 +1,15 @@
+## Due to licensing restrictions, we are unable to release the model. To run the int4 model on GPU, please install transformers==4.32.0, currently the 4.38.1 version has some issues.
+python3 main.py \
+--model_name meta-llama/Llama-2-7b-chat-hf \
+--device 0 \
+--group_size 128 \
+--bits 4 \
+--iters 1000 \
+--enable_minmax_tuning \
+--use_quant_input \
+--low_gpu_mem_usage \
+--deployment_device 'gpu' \
+--scale_dtype 'fp32' \
+--eval_bs 32 \
+--output_dir "./tmp_autoround" \
+--amp
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 58992fd05..ba02e6f82 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -6,19 +6,11 @@ import unittest
sys.path.insert(0, ".")
import torch
import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
-class SimpleDataLoader:
- def __init__(self):
- self.batch_size = 1
-
- def __iter__(self):
- for i in range(2):
- yield torch.randn([1, 30])
-
-
class LLMDataLoader:
def __init__(self):
self.batch_size = 1
@@ -28,74 +20,129 @@ class LLMDataLoader:
yield torch.ones([1, 10], dtype=torch.long)
-class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
- approach = "weight_only"
-
+class TestAutoRound(unittest.TestCase):
@classmethod
def setUpClass(self):
- self.dataloader = SimpleDataLoader()
- self.gptj = transformers.AutoModelForCausalLM.from_pretrained(
- "hf-internal-testing/tiny-random-GPTJForCausalLM",
- torchscript=True,
- )
- self.tokenizer = transformers.AutoTokenizer.from_pretrained(
- "hf-internal-testing/tiny-random-GPTJForCausalLM", trust_remote_code=True
- )
- self.gptj_no_jit = transformers.AutoModelForCausalLM.from_pretrained(
- "hf-internal-testing/tiny-random-GPTJForCausalLM",
- )
+ model_name = "facebook/opt-125m"
+ self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
- self.lm_input = torch.ones([1, 10], dtype=torch.long)
@classmethod
def tearDownClass(self):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- def test_autoround_int_quant(self):
- model = copy.deepcopy(self.gptj)
- device = "cpu"
- model = model
- out1 = model(self.lm_input)
- round = AutoRound
- optq_1 = round(
- model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype="fp32"
+ def test_default(self):
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+ if torch.cuda.is_available():
+ autoround.save_quantized(output_dir="./saved", inplace=False)
+ autoround.save_quantized(output_dir="./saved", inplace=False, format="itrex")
+
+ def test_sym(self):
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_w4g1(self):
+ bits, group_size, sym = 4, -1, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_w3g128(self):
+ bits, group_size, sym = 3, 128, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_w2g128(self):
+ bits, group_size, sym = 2, 128, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_disable_use_quant_input(self):
+ bits, group_size, sym = 4, -1, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ use_quant_input=False,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_disable_minmax_tuning(self):
+ bits, group_size, sym = 4, -1, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ enable_minmax_tuning=False,
+ dataloader=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_signround(self):
+ bits, group_size, sym = 4, -1, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ enable_minmax_tuning=False,
+ use_quant_input=False,
+ dataloader=self.llm_dataloader,
)
- q_model, weight_config1 = optq_1.quantize()
- q_model = q_model
- from auto_round.export.export_to_itrex import compress_model
-
- compressed_model, _ = compress_model(q_model, weight_config1)
- q_model = q_model
- model = model
- out2 = model(self.lm_input)
- out3 = q_model(self.lm_input)
- out4 = compressed_model(self.lm_input)
- self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))
- self.assertFalse(torch.all(out1[0] == out2[0]))
- self.assertTrue(torch.all(out2[0] == out3[0]))
- self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))
- self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())
-
- # model = copy.deepcopy(self.gptj)
- # out6 = model(self.lm_input)
- # optq_2 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10)
- # q_model, weight_config2 = optq_2.quantize()
- # out4 = q_model(self.lm_input)
- # out5 = model(self.lm_input)
-
- # self.assertTrue(torch.all(out1[0] == out6[0]))
- # self.assertTrue(torch.all(out4[0] == out5[0]))
- # self.assertTrue(torch.all(torch.isclose(out6[0], out5[0], atol=1e-1)))
-
- def test_config(self):
- from auto_round import QuantConfig
-
- config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
- config.save_pretrained("quantization_config_dir")
- loaded_config = QuantConfig.from_pretrained("quantization_config_dir")
- self.assertEqual(config.group_size, loaded_config.group_size)
- self.assertEqual(config.true_sequential, loaded_config.true_sequential)
+ autoround.quantize()
if __name__ == "__main__":
diff --git a/test/test_autoround_export.py b/test/test_autoround_export.py
new file mode 100644
index 000000000..0f4a13233
--- /dev/null
+++ b/test/test_autoround_export.py
@@ -0,0 +1,96 @@
+import copy
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, ".")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+
+class SimpleDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.randn([1, 30])
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoroundExport(unittest.TestCase):
+ approach = "weight_only"
+
+ @classmethod
+ def setUpClass(self):
+ self.gptj = transformers.AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM",
+ torchscript=True,
+ )
+ self.tokenizer = transformers.AutoTokenizer.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM", trust_remote_code=True
+ )
+ self.gptj_no_jit = transformers.AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-GPTJForCausalLM",
+ )
+ self.llm_dataloader = LLMDataLoader()
+ self.lm_input = torch.ones([1, 10], dtype=torch.long)
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_autoround_int_quant(self):
+ model = copy.deepcopy(self.gptj)
+ out1 = model(self.lm_input)
+ round = AutoRound
+ optq_1 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10, iters=10)
+ q_model, weight_config1 = optq_1.quantize()
+ from auto_round.export.export_to_itrex import pack_model
+
+ compressed_model = pack_model(model=q_model, weight_config=weight_config1)
+ out2 = model(self.lm_input)
+ out3 = q_model(self.lm_input)
+ out4 = compressed_model(self.lm_input)
+ self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))
+ self.assertFalse(torch.all(out1[0] == out2[0]))
+ self.assertTrue(torch.all(out2[0] == out3[0]))
+ self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))
+ self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())
+
+ model = copy.deepcopy(self.gptj)
+ out6 = model(self.lm_input)
+ optq_2 = round(model, self.tokenizer, device="cpu", n_samples=20, seqlen=10)
+ q_model, weight_config2 = optq_2.quantize()
+ compressed_model = pack_model(model=q_model, weight_config=weight_config2, inplace=False)
+ out4 = q_model(self.lm_input)
+ out5 = compressed_model(self.lm_input)
+ self.assertTrue(torch.all(out1[0] == out6[0]))
+ self.assertTrue(torch.all(torch.isclose(out4[0], out5[0], atol=1e-5)))
+
+ def test_config(self):
+ from auto_round.export import QuantConfig
+
+ config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
+ config.save_pretrained("quantization_config_dir")
+ loaded_config = QuantConfig.from_pretrained("quantization_config_dir")
+ self.assertEqual(config.group_size, loaded_config.group_size)
+ self.assertEqual(config.desc_act, loaded_config.desc_act)
+ self.assertEqual(config.bits, loaded_config.bits)
+ self.assertEqual(config.sym, loaded_config.sym)
+
+
+if __name__ == "__main__":
+ unittest.main()