Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

1 fix a bug in autoround format with the latest transformers 2 rename n_samples n_blocks to nsamples nblocks (#163)

wenhuach21f9e7d79e9b64
diff --git a/README.md b/README.md
index d9027c20d..377560986 100644
--- a/README.md
+++ b/README.md
@@ -11,12 +11,9 @@ AutoRound
<div align="left">
AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference. It's tailored for a wide range
-of models and consistently delivers noticeable improvements, often significantly outperforming SignRound with the cost
-of more tuning time for quantization.
-
-Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,
+of models. Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,
which competes impressively against recent methods without introducing any additional inference overhead. The below
-image presents an overview of AutoRound.
+image presents an overview of AutoRound. Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)
<div align="center">
@@ -26,7 +23,6 @@ image presents an overview of AutoRound.
## What's New
* [2024/06] AutoRound format supports mixed bit-widths and group sizes for inference, resolving the significant performance drop issue with the asymmetric kernel
-* [2024/05] Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)
* [2024/05] AutoRound supports lm-head quantization, saving 0.7G for LLaMA3-8B at W4G128.
* [2024/05] AutoRound performs well
in [low_bit_open_llm_leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)
@@ -102,7 +98,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).
-- `n_samples (int)`: Number of samples for tuning (default is 512).
+- `nsamples (int)`: Number of samples for tuning (default is 512).
- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).
@@ -113,7 +109,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r
- `amp (bool)`: Whether to use automatic mixed precision (default is True).
-- `n_blocks (int)`: Packing several blocks as one for tuning together (default is 1).
+- `nblocks (int)`: Packing several blocks as one for tuning together (default is 1).
- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).
@@ -137,13 +133,13 @@ Please run the quantization code first.
### CPU
```python
-##Install the latest https://github.com/intel/intel-extension-for-transformers from source first.
+##pip install intel-extension-for-transformers
from intel_extension_for_transformers.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
@@ -156,8 +152,8 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
##from auto_round.auto_quantizer import AutoHfQuantizer ## uncomment it for models with auto_round format
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
@@ -174,10 +170,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)
| tiiuae/falcon-7b | [HF-int4-model-G64](https://huggingface.co/Intel/falcon-7b-int4-inc), [accuracy](./docs/falcon-7b-acc.md), [recipe](./examples/language-modeling/scripts/falcon-7b.sh), [example](./examples/language-modeling/) |
| mistralai/Mistral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
| mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-Instruct-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |
| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
| meta-llama/Meta-Llama-3-8B-Instruct | [accuracy](./docs/Meta-Llama-3-8B-Instruct-acc.md), [recipe](./examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh), [example](./examples/language-modeling/) |
+| google/gemma-7b | [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
| meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) |
| Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [sym recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh), [asym recipe ](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh), [example](./examples/language-modeling/) |
| baichuan-inc/Baichuan2-7B-Chat | [accuracy](./docs/baichuan2-7b-chat-acc.md), [recipe](./examples/language-modeling/scripts/baichuan2-7b-chat.sh), [example](./examples/language-modeling/) |
@@ -205,9 +201,7 @@ average accuracies of 11 zero-shot tasks.
1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time.
-2 Setting 'enable_quanted_input' to False has been observed to occasionally yield improved results.
-
-3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
+2 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
## Reference
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index a0e08db41..045a64572 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -43,7 +43,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quanti
from auto_round.utils import get_module, set_module, dynamic_import_inference_linear
import auto_round_extension.qbits.qlinear_qbits as qlinear_qbits
-
+from enum import Enum
logger = getLogger(__name__)
import sys
@@ -194,6 +194,9 @@ class AutoHfQuantizer:
return quantization_config
+class AutoRoundQuantizationMethod(str, Enum):
+ AutoRound = "intel/auto-round"
+
@dataclass
class AutoRoundConfig(QuantizationConfigMixin):
@@ -222,6 +225,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
weight_config: dict = None,
**kwargs,
):
+
self.bits = bits
self.tokenizer = tokenizer
self.dataset = dataset
@@ -232,7 +236,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
if kwargs is not None:
for key in kwargs.keys():
setattr(self, key, kwargs[key])
-
+ self.quant_method = AutoRoundQuantizationMethod.AutoRound
self.post_init()
def get_loading_attributes(self):
@@ -378,11 +382,6 @@ class AutoRoundQuantizer(HfQuantizer):
The input model
"""
- #
- # if self.bits == 4: if get_device(model) == torch.device("cpu") or ( hasattr(model, "hf_device_map") and
- # any(d in model.hf_device_map for d in ["cpu", "disk"]) ): raise ValueError( "Found modules on cpu/disk.
- # Using Exllamav2 backend requires all the modules to be on GPU." "You can deactivate exllama backend by
- # setting `disable_exllama=True` in the quantization config object" )
class StoreAttr(object):
pass
@@ -406,11 +405,7 @@ class AutoRoundQuantizer(HfQuantizer):
model = self.post_init_model(model)
else:
raise NotImplementedError
- # if self.quantization_config.tokenizer is None:
- # self.quantization_config.tokenizer = model.name_or_path
- #
- # self.optimum_quantizer.quantize_model(model, self.quantization_config.tokenizer)
- # model.config.quantization_config = GPTQConfig.from_dict(self.optimum_quantizer.to_dict())
+
@property
def is_trainable(self, model: Optional["PreTrainedModel"] = None):
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c5df1ad1a..ff3889110 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -79,15 +79,15 @@ class AutoRound(object):
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
iters (int): Number of iterations (default is 200).
seqlen (int): Data length of the sequence for tuning (default is 2048).
- n_samples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 512).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
- n_blocks (int): Number of blocks (default is 1).
+ nblocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels
+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
Returns:
@@ -115,10 +115,10 @@ class AutoRound(object):
low_gpu_mem_usage: bool = True,
iters: int = 200,
seqlen: int = 2048,
- n_samples: int = 512,
+ nsamples: int = 512,
sampler: str = "rand",
seed: int = 42,
- n_blocks: int = 1,
+ nblocks: int = 1,
gradient_accumulate_steps: int = 1,
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
@@ -132,8 +132,8 @@ class AutoRound(object):
self.amp = amp
self.enable_quanted_input = enable_quanted_input
self.enable_minmax_tuning = enable_minmax_tuning
- self.n_samples = n_samples
- self.n_blocks = n_blocks
+ self.nsamples = nsamples
+ self.nblocks = nblocks
self.bits = bits
self.group_size = group_size
self.sym = sym
@@ -145,7 +145,7 @@ class AutoRound(object):
self.tokenizer = tokenizer
self.seqlen = seqlen
self.train_bs = batch_size
- self.n_blocks = n_blocks
+ self.nblocks = nblocks
self.device = detect_device(device)
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
self.set_amp_dtype()
@@ -156,12 +156,9 @@ class AutoRound(object):
if self.iters <= 0:
logger.warning("iters must be positive, reset it to 200")
self.iters = 200
- self.lr = lr
- if self.lr is None:
- self.lr = 1.0 / self.iters
- self.minmax_lr = minmax_lr
- if self.minmax_lr is None:
- self.minmax_lr = self.lr
+
+ self.lr = lr or (1.0 / self.iters)
+ self.minmax_lr = minmax_lr or self.lr
self.sampler = sampler
self.gradient_accumulate_steps = gradient_accumulate_steps
@@ -176,35 +173,6 @@ class AutoRound(object):
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
- serialization_keys = [
- "bits",
- "group_size",
- "sym",
- "data_type",
- "enable_quanted_input",
- "enable_minmax_tuning",
- "data_type",
- "seqlen",
- "train_bs",
- "scale_dtype",
- "lr",
- "minmax_lr",
- "gradient_accumulate_steps",
- "iters",
- "amp",
- "n_samples",
- "low_gpu_mem_usage",
- ]
- if isinstance(dataset, str):
- serialization_keys.append("dataset")
- self.serialization_dict = {}
- for key in serialization_keys:
- self.serialization_dict[key] = getattr(self, key)
- from .version import __version__
-
- self.serialization_dict["autoround_version"] = __version__
- if "scale_dtype" in self.serialization_dict.keys():
- self.serialization_dict["scale_dtype"] = str(self.serialization_dict["scale_dtype"])
if is_optimum_habana_available():
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
@@ -222,7 +190,7 @@ class AutoRound(object):
assert self.train_bs > 0, "batch size must be positive"
assert self.iters > 0, "iters must be positive"
assert self.seqlen > 0, "seqlen must be positive"
- assert self.n_blocks > 0, "n_blocks must be positive"
+ assert self.nblocks > 0, "nblocks must be positive"
assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
assert self.enable_full_range is False, "only support enable_full_range=False currently"
# assert self.tokenizer != None or self.dataloader != None
@@ -245,7 +213,7 @@ class AutoRound(object):
layer_names = self.get_quantized_layer_names_outside_blocks()
self.start_time = time.time()
- all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.n_samples, layer_names=layer_names)
+ all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.nsamples, layer_names=layer_names)
del self.inputs
inputs = all_inputs[block_names[0]]
@@ -254,7 +222,7 @@ class AutoRound(object):
del self.inputs
if "input_ids" in inputs.keys():
total_samples = len(inputs["input_ids"])
- self.n_samples = total_samples
+ self.nsamples = total_samples
if total_samples < self.train_bs:
self.train_bs = total_samples
logger.warning(f"force the train batch size to {total_samples} ")
@@ -265,7 +233,7 @@ class AutoRound(object):
self.model,
inputs,
block_names,
- n_blocks=self.n_blocks,
+ nblocks=self.nblocks,
device=self.device,
)
@@ -344,7 +312,7 @@ class AutoRound(object):
return
q_layer_inputs = None
if self.enable_quanted_input:
- q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.n_samples, layer_names=layer_names)
+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)
self.model = self.model.to("cpu")
torch.cuda.empty_cache()
@@ -370,11 +338,11 @@ class AutoRound(object):
Returns:
None
"""
- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types)
+ layers_inblocks = get_layer_names_in_block(self.model, self.supported_types)
for n, m in self.model.named_modules():
if not isinstance(m, tuple(self.supported_types)):
continue
- if n not in weight_config.keys() and n in layers_in_blocks:
+ if n not in weight_config.keys() and n in layers_inblocks:
weight_config[n] = {}
weight_config[n]["data_type"] = self.data_type
weight_config[n]["bits"] = self.bits
@@ -424,8 +392,8 @@ class AutoRound(object):
"""
output = []
- for i in range(0, self.n_samples, bs):
- end_index = min(self.n_samples, i + bs)
+ for i in range(0, self.nsamples, bs):
+ end_index = min(self.nsamples, i + bs)
indices = torch.arange(i, end_index).to(torch.long)
tmp_input_ids, tmp_input_others = sampling_inputs(
input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim
@@ -439,7 +407,7 @@ class AutoRound(object):
return output
@torch.no_grad()
- def calib(self, n_samples, bs):
+ def calib(self, nsamples, bs):
"""Perform calibration for quantization.
This method calibrates the model for quantization by processing a specified
@@ -448,7 +416,7 @@ class AutoRound(object):
is less than the specified number, it logs a warning. If no samples are processed,
it logs an error and exits.
Args:
- n_samples (int): The number of samples to use for calibration.
+ nsamples (int): The number of samples to use for calibration.
bs (int): The number of samples to use for calibration
"""
@@ -460,7 +428,7 @@ class AutoRound(object):
dataset,
self.seed,
bs,
- self.n_samples,
+ self.nsamples,
)
else:
self.dataloader = self.dataset
@@ -499,7 +467,7 @@ class AutoRound(object):
except Exception as error:
logger.error(error)
total_cnt += input_ids.shape[0]
- if total_cnt >= n_samples:
+ if total_cnt >= nsamples:
break
if total_cnt == 0:
logger.error(
@@ -507,19 +475,19 @@ class AutoRound(object):
f"dataset or decease the sequence length"
)
exit()
- elif total_cnt < n_samples:
+ elif total_cnt < nsamples:
logger.warning(
f"Insufficient number of samples collected may affect the quantification. "
- f"Valid samples size:{total_cnt}, Target sample size:{n_samples}"
+ f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"
)
@torch.no_grad()
- def try_cache_inter_data_gpucpu(self, block_names, n_samples, layer_names=[], last_cache_name=None):
+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=[], last_cache_name=None):
"""Attempts to cache intermediate data on GPU,if failed, then using CPU.
Args:
block_names (list): List of block names to cache data for.
- n_samples (int): Number of samples to use for caching.
+ nsamples (int): Number of samples to use for caching.
layer_names (list, optional): List of layer names to cache data for. Defaults to [].
last_cache_name (str, optional): Name of the last cache. Defaults to None.
@@ -532,7 +500,7 @@ class AutoRound(object):
try:
self.model = self.model.to(self.device)
all_inputs = self.cache_inter_data(
- block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name
+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
self.model = self.model.to("cpu")
torch.cuda.empty_cache()
@@ -541,12 +509,12 @@ class AutoRound(object):
self.model = self.model.to("cpu")
torch.cuda.empty_cache()
all_inputs = self.cache_inter_data(
- block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name
+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
return all_inputs
@torch.no_grad()
- def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_name=None):
+ def cache_inter_data(self, block_names, nsamples, layer_names=[], last_cache_name=None):
"""Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.
This method temporarily replaces the forward method of the model to capture
@@ -556,7 +524,7 @@ class AutoRound(object):
Args:
block_names (list): The names of the blocks for which inputs are to be saved.
layer_names (list):The names of the layers for which inputs are to be saved.
- n_samples (int): The number of samples to use for calibration.
+ nsamples (int): The number of samples to use for calibration.
last_cache_name (str, optional): The name of the last layer to be cached,
we could break the forward in this layer to save time
@@ -577,7 +545,7 @@ class AutoRound(object):
calib_bs = self.train_bs
self.hook_handles = []
self._replace_forward()
- self.calib(n_samples, calib_bs)
+ self.calib(nsamples, calib_bs)
self._recover_forward()
res = self.inputs
del self.last_cache_name
@@ -732,7 +700,7 @@ class AutoRound(object):
)
else:
lr_schedule = copy.deepcopy(self.lr_scheduler)
- n_samples = len(inputs)
+ nsamples = len(inputs)
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
mse_loss = torch.nn.MSELoss().to(device)
@@ -744,13 +712,12 @@ class AutoRound(object):
pick_samples = train_bs * gradient_accumulate_steps
if self.sampler != "rand":
- whole_indices = torch.randperm(n_samples)[:pick_samples]
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
- whole_indices = torch.randperm(n_samples)[:pick_samples]
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
for tmp_step in range(gradient_accumulate_steps):
- org_input = None
indices = whole_indices[tmp_step * train_bs: (tmp_step + 1) * train_bs]
if q_inputs is not None:
current_input = [q_inputs[i] for i in indices]
@@ -859,9 +826,9 @@ class AutoRound(object):
lr_schedule = copy.deepcopy(self.lr_scheduler)
pick_samples = self.train_bs * self.gradient_accumulate_steps
- n_samples = len(input_ids)
+ nsamples = len(input_ids)
if self.sampler != "rand":
- whole_indices = torch.randperm(n_samples)[:pick_samples]
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
mse_loss = torch.nn.MSELoss().to(device)
@@ -871,7 +838,7 @@ class AutoRound(object):
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
- whole_indices = torch.randperm(n_samples)[:pick_samples]
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
for tmp_step in range(self.gradient_accumulate_steps):
indices = whole_indices[tmp_step * self.train_bs: (tmp_step + 1) * self.train_bs]
current_input_ids, current_input_others = sampling_inputs(
@@ -956,7 +923,7 @@ class AutoRound(object):
model: torch.nn.Module,
inputs,
block_names,
- n_blocks=1,
+ nblocks=1,
device=torch.device("cpu"),
):
"""Quantize and dequantize the weights of the specified blocks in the model.
@@ -965,7 +932,7 @@ class AutoRound(object):
model: The PyTorch model to be quantized.
inputs: The input data for quantization.
block_names: The names of the blocks to be quantized and dequantized.
- n_blocks: The number of blocks to quantize and dequantize.
+ nblocks: The number of blocks to quantize and dequantize.
device: The device for quantization and dequantization.
Returns:
@@ -995,13 +962,13 @@ class AutoRound(object):
for i in range(len(input_others[key])):
input_others[key][i].to(tmp_dtype)
- for i in range(0, len(block_names), n_blocks):
- if n_blocks == 1:
+ for i in range(0, len(block_names), nblocks):
+ if nblocks == 1:
n = block_names[i]
logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")
m = get_module(model, n)
else:
- names = block_names[i: i + n_blocks]
+ names = block_names[i: i + nblocks]
logger.info(names)
modules = [get_module(model, n) for n in names]
m = WrapperMultiblock(modules)
@@ -1046,6 +1013,36 @@ class AutoRound(object):
logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
exit()
save_quantized_as_format = EXPORT_FORMAT.get(format)
+ serialization_keys = [
+ "bits",
+ "group_size",
+ "sym",
+ "data_type",
+ "enable_quanted_input",
+ "enable_minmax_tuning",
+ "data_type",
+ "seqlen",
+ "train_bs",
+ "scale_dtype",
+ "lr",
+ "minmax_lr",
+ "gradient_accumulate_steps",
+ "iters",
+ "amp",
+ "nsamples",
+ "low_gpu_mem_usage",
+ ]
+ if isinstance(self.dataset, str):
+ serialization_keys.append("dataset")
+ serialization_dict = {}
+ for key in serialization_keys:
+ serialization_dict[key] = getattr(self, key)
+ from .version import __version__
+
+ serialization_dict["autoround_version"] = __version__
+ if "scale_dtype" in serialization_dict.keys():
+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])
+
compressed_model = save_quantized_as_format( ##TODO refine the code
output_dir,
model=self.model,
@@ -1063,7 +1060,7 @@ class AutoRound(object):
tokenizer=self.tokenizer,
supported_types=self.supported_types,
data_type=self.data_type,
- serialization_dict=self.serialization_dict,
+ serialization_dict=serialization_dict,
**kwargs,
)
return compressed_model
@@ -1187,15 +1184,15 @@ class AutoOPTRound(AutoRound):
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
- n_samples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 512).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
- n_blocks (int): Number of blocks (default is 1).
+ nblocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels
+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
**kwargs: Additional keyword arguments.
@@ -1224,10 +1221,10 @@ class AutoOPTRound(AutoRound):
low_gpu_mem_usage: bool = True,
iters: int = 200,
seqlen: int = 2048,
- n_samples: int = 512,
+ nsamples: int = 512,
sampler: str = "rand",
seed: int = 42,
- n_blocks: int = 1,
+ nblocks: int = 1,
gradient_accumulate_steps: int = 1,
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
@@ -1256,10 +1253,10 @@ class AutoOPTRound(AutoRound):
low_gpu_mem_usage,
iters,
seqlen,
- n_samples,
+ nsamples,
sampler,
seed,
- n_blocks,
+ nblocks,
gradient_accumulate_steps,
not_use_best_mse,
dynamic_max_gap,
@@ -1335,10 +1332,10 @@ class AutoAdamRound(AutoOPTRound):
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
- n_samples (int): Number of samples (default is 512).
+ nsamples (int): Number of samples (default is 512).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
- n_blocks (int): Number of blocks (default is 1).
+ nblocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
@@ -1372,10 +1369,10 @@ class AutoAdamRound(AutoOPTRound):
low_gpu_mem_usage: bool = True,
iters: int = 200,
seqlen: int = 2048,
- n_samples: int = 512,
+ nsamples: int = 512,
sampler: str = "rand",
seed: int = 42,
- n_blocks: int = 1,
+ nblocks: int = 1,
gradient_accumulate_steps: int = 1,
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
@@ -1404,10 +1401,10 @@ class AutoAdamRound(AutoOPTRound):
low_gpu_mem_usage,
iters,
seqlen,
- n_samples,
+ nsamples,
sampler,
seed,
- n_blocks,
+ nblocks,
gradient_accumulate_steps,
not_use_best_mse,
dynamic_max_gap,
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index d4b4d649a..fe8a1be6e 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -227,7 +227,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
return calib_dataset
-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, n_samples=512):
+def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):
"""Generate a DataLoader for calibration using specified parameters.
Args:
@@ -239,7 +239,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
split (str, optional): The data split to use. Defaults to None.
seed (int, optional): The random seed for reproducibility. Defaults to 42.
bs (int, optional): The batch size. Defaults to 4.
- n_samples (int, optional): The total number of samples to include. Defaults to 512.
+ nsamples (int, optional): The total number of samples to include. Defaults to 512.
Returns:
DataLoader: The DataLoader for the calibrated dataset.
@@ -340,14 +340,14 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
dataset_names = [dataset_names[index] for index in indices]
cnt = 0 if not data_lens else sum(data_lens.values())
dataset_cnt_info = {}
- if cnt > n_samples:
+ if cnt > nsamples:
cnt = 0
for i in range(len(datasets)):
name = dataset_names[i].split(':')[0]
if name not in data_lens:
- target_cnt = (n_samples - cnt) // (len(datasets) - len(data_lens)) if data_lens \
- else (n_samples - cnt) // (len(datasets) - i)
+ target_cnt = (nsamples - cnt) // (len(datasets) - len(data_lens)) if data_lens \
+ else (nsamples - cnt) // (len(datasets) - i)
target_cnt = min(target_cnt, len(datasets[i]))
cnt += target_cnt
else:
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index e83324592..67f7b6765 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -50,7 +50,8 @@ from ..utils import convert_dtype_torch2str_hf
@register_format("auto_gptq")
-def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwargs):##TODO align with autoround format
+def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True,
+ **kwargs):
"""Export the model to autogptq format to easily leverage cuda kernel."""
try:
import auto_gptq
@@ -58,15 +59,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
raise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")
model = kwargs["model"]
weight_config = kwargs["weight_config"]
- sym = kwargs["sym"]
bits = kwargs["bits"]
group_size = kwargs["group_size"]
- iters = kwargs["iters"]
- lr = kwargs["lr"]
- minmax_lr = kwargs["minmax_lr"]
- enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- enable_quanted_input = kwargs["enable_quanted_input"]
- scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
supported_types = kwargs["supported_types"]
@@ -102,172 +96,61 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
from auto_gptq.modeling._utils import pack_model # pylint: disable=E0401
if bits == 3 or use_triton is False:
- if bits == 3 and use_triton is True:
- logger.warning("triton does not support 3 bits, reset it to False")
- quantizers = {}
- for key in weight_config:
- if key == "lm_head": ##TODO remove this after pr 87 is merged
- continue
- info = weight_config[key]
- if not check_to_quantized(info):
- continue
- ##force to float32 to be compatible with torch 2.0
- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])
- pack_model(
- compressed_model,
- quantizers,
- bits,
- group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=False,
- )
- else:
- quantizers = {}
- for key in weight_config:
- if key == "lm_head": ##TODO remove this after pr 87 is merged
- continue
- info = weight_config[key]
- if not check_to_quantized(info):
- continue
- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])
- pack_model(
- compressed_model,
- quantizers,
- bits,
- group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=True,
- )
- if output_dir is None:
- return compressed_model
-
- _save_quantized_to_autogptq(
+ use_triton = False
+ quantizers = {}
+ for key in weight_config:
+ if key == "lm_head": ##TODO remove this after pr 87 is merged
+ continue
+ info = weight_config[key]
+ if not check_to_quantized(info):
+ continue
+ quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])
+ pack_model(
compressed_model,
- output_dir,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=iters,
- lr=lr,
- minmax_lr=minmax_lr,
- enable_minmax_tuning=enable_minmax_tuning,
- enable_quanted_input=enable_quanted_input,
- scale_dtype=scale_dtype,
- use_safetensors=True,
- modules_in_block_to_quantize=modules_in_block_to_quantize,
- )
-
-
-def _save_quantized_to_autogptq(
- model,
- save_dir: str,
- bits=4,
- group_size=128,
- sym=False,
- iters=200,
- lr=5e-3,
- minmax_lr=5e-3,
- enable_minmax_tuning=True,
- enable_quanted_input=True,
- use_safetensors: bool = True,
- scale_dtype=torch.float32,
- safetensors_metadata: Optional[Dict[str, str]] = None,
- modules_in_block_to_quantize=None,
-):
- """Save quantized model and configs to local disk for cuda."""
- os.makedirs(save_dir, exist_ok=True)
- model.to("cpu")
-
- model_base_name = "model"
- if use_safetensors:
- model_save_name = model_base_name + ".safetensors"
- state_dict = model.state_dict()
- state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}
- if safetensors_metadata is None:
- safetensors_metadata = {}
- elif not isinstance(safetensors_metadata, dict):
- raise TypeError("safetensors_metadata must be a dictionary.")
- else:
- logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")
- new_safetensors_metadata = {}
- converted_keys = False
- for key, value in safetensors_metadata.items():
- if not isinstance(key, str) or not isinstance(value, str):
- converted_keys = True
- try:
- new_key = str(key)
- new_value = str(value)
- except Exception as e:
- raise TypeError(
- "safetensors_metadata: both keys and values must be strings"
- / f" and an error occurred when trying to convert them: {e}"
- )
- if new_key in new_safetensors_metadata:
- logger.warning(
- f"After converting safetensors_metadata keys to strings, the key '{new_key}' "
- / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."
- )
- new_safetensors_metadata[new_key] = new_value
- safetensors_metadata = new_safetensors_metadata
- if converted_keys:
- logger.debug(
- "One or more safetensors_metadata keys or values had to be converted to str()."
- / f" Final safetensors_metadata: {safetensors_metadata}"
- )
-
- # Format is required to enable Accelerate to load the metadata
- # otherwise it raises an OSError
- safetensors_metadata["format"] = "pt"
-
- # Store the quantization configuration as safetensors metadata
- from auto_round import __version__
-
- safetensors_metadata["autoround_version"] = str(__version__)
- safetensors_metadata["bits"] = str(bits)
- safetensors_metadata["group_size"] = str(group_size)
- safetensors_metadata["iters"] = str(iters)
- safetensors_metadata["lr"] = str(lr)
- safetensors_metadata["minmax_lr"] = str(minmax_lr)
- safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)
- safetensors_metadata["enable_quanted_input"] = str(enable_quanted_input)
- safetensors_metadata["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)
- safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
- else:
- model_save_name = model_base_name + ".bin"
- torch.save(model.state_dict(), join(save_dir, model_save_name))
-
- from auto_gptq.modeling._base import BaseQuantizeConfig # pylint: disable=E0401
-
- quantization_config = BaseQuantizeConfig(
- bits=bits,
- group_size=group_size,
+ quantizers,
+ bits,
+ group_size,
+ use_cuda_fp16=True,
desc_act=False,
- sym=sym,
- true_sequential=False,
- static_groups=False,
- model_file_base_name=model_base_name,
+ force_layer_back_to_cpu=True,
+ use_triton=use_triton
)
- quantization_config.model_file_base_name = model_base_name
-
- config_dict = quantization_config.to_dict()
- config_dict["quant_method"] = "intel/auto-round"
- config_dict["autoround_version"] = __version__
- config_dict["iters"] = iters
- config_dict["lr"] = lr
- config_dict["minmax_lr"] = minmax_lr
- config_dict["enable_minmax_tuning"] = enable_minmax_tuning
- config_dict["enable_quanted_input"] = enable_quanted_input
- config_dict["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)
+ if output_dir is None:
+ return compressed_model
+ quantization_config = kwargs["serialization_dict"]
+ quantization_config["quant_method"] = "gptq"
+ quantization_config.pop("dataset", None) ## pile-10k is not supported in gptq
if modules_in_block_to_quantize is not None:
- config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize
-
- with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:
- json.dump(config_dict, f, indent=2)
-
- config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value
- model.config.quantization_config = config_dict
- model.config.save_pretrained(save_dir)
+ quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize
+ if hasattr(model, "config"):
+ model.config.quantization_config = quantization_config
+ save(compressed_model, output_dir)
+
+
+def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ os.makedirs(save_dir, exist_ok=True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_file = "quantization_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ model.config.quantization_config["quant_method"] = "intel/auto-round"
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(model.config.quantization_config, f, indent=2)
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index da08449af..6a633d51a 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -56,6 +56,11 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4
set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)
reduce the train bs to 4(potential accuracy drop)
+
+reduce the nsamples to 128 (potential accuracy drop)
+
+or combine them
+
- **Speedup the tuning:**
disable_low_gpu_mem_usage(more gpu memory)
@@ -64,11 +69,13 @@ reduce the train bs to 4(little accuracy drop)
reduce the seqlen to 512(potential large accuracy drop)
+reduce the nsamples to 128 (potential accuracy drop)
+
or combine them
- **Enable quantized lm-head:**
-Currently only support in Intel xpu,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.
+Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.
```bash
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage
```
@@ -77,10 +84,6 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4
Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.
-- **Running the Original SignRound:**
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input
-```
- **Code generation LLM:**
@@ -117,7 +120,7 @@ CUDA_VISIBLE_DEVICES=1,2 python3 eval/evaluation.py --model_name /save_model_pat
You can also utilize the official lm_eval [link](https://github.com/EleutherAI/lm-evaluation-harness/tree/main?tab=readme-ov-file#basic-usage).
## 4. Known Issues
-* Random issues in tuning Qwen models
+* Random issues in tuning some models
* ChatGlm-V1 is not supported
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 3569ce671..690f5bdc2 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -74,9 +74,9 @@ if __name__ == '__main__':
parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")
+ parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
- parser.add_argument("--n_samples", default=512, type=int,
+ parser.add_argument("--nsamples", default=512, type=int,
help="number of samples")
parser.add_argument("--low_gpu_mem_usage", action='store_true',
@@ -298,9 +298,9 @@ if __name__ == '__main__':
gpu_format = "auto_round"
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
- dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
+ dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
- amp=not args.disable_amp, n_samples=args.n_samples,
+ amp=not args.disable_amp, nsamples=args.nsamples,
low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, weight_config=weight_config,
@@ -344,8 +344,10 @@ if __name__ == '__main__':
if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:
model_args = f"pretrained={export_dir}-gpu"
- else:
+ elif "fake" in deployment_device:
model_args = f"pretrained={output_dir}"
+ else:
+ exit() ## does not support cpu,xpu model eval
res = simple_evaluate(model="hf", model_args=model_args,
tasks=tasks,
diff --git a/test/test_autoround_export.py b/test/test_autoround_export.py
index c49395e8d..591064594 100644
--- a/test/test_autoround_export.py
+++ b/test/test_autoround_export.py
@@ -56,7 +56,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10, iters=10)
+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)
q_model, weight_config1 = optq_1.quantize()
from auto_round.export.export_to_itrex import pack_model
@@ -72,7 +72,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out6 = model(self.lm_input)
- optq_2 = round(model, self.tokenizer, device="cpu", n_samples=20, seqlen=10)
+ optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)
q_model, weight_config2 = optq_2.quantize()
compressed_model = pack_model(model=q_model, weight_config=weight_config2, inplace=False)
out4 = q_model(self.lm_input)