Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

1.8X speedup by disable_low_gpu_mem_usage and reduce memory usage by avoid using torch.cat (#106)

wenhuach2123b60c3a67c6
diff --git a/README.md b/README.md
index 59fc584b7..fc75033b1 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,9 @@ image presents an overview of AutoRound.
<div align="left">
+## What's New
+* [2024/05] AutoRound performs well in [low_bit_open_llm_leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)
+
## Prerequisites
- Python 3.9 or higher
@@ -69,16 +72,16 @@ autoround.save_quantized(output_dir)
- `model`: The PyTorch model to be quantized.
-- `tokenizer`: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.
+- `tokenizer`: An optional tokenizer for processing input data. If none, a dataset must be provided.
- `bits (int)`: Number of bits for quantization (default is 4).
- `group_size (int)`: Size of the quantization group (default is 128).
-- `sym (bool)`: Whether to use symmetric quantization.
+- `sym (bool)`: Whether to use symmetric quantization (default is False).
- `enable_quanted_input (bool)`: Whether to use the output of the previous quantized block as the input for the current
- block (default is True).
+ block for tuning (default is True).
- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).
@@ -94,7 +97,7 @@ autoround.save_quantized(output_dir)
- `batch_size (int)`: Batch size for training (default is 8).
-- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float32"), different kernels have
+- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float16"), different kernels have
different choices.
- `amp (bool)`: Whether to use automatic mixed precision (default is True).
@@ -103,7 +106,7 @@ autoround.save_quantized(output_dir)
- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).
-- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of a little tuning time (default is True).
+- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is True).
- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 59e076d66..ec277077d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -30,6 +30,7 @@ from .utils import (
check_to_quantized,
collect_minmax_scale,
collect_round_v,
+ convert_dtype_str2torch,
detect_device,
get_block_names,
get_module,
@@ -37,10 +38,10 @@ from .utils import (
htcore,
is_optimum_habana_available,
logger,
- move_input_to_device,
quant_weight,
sampling_inputs,
set_module,
+ to_device,
)
@@ -172,6 +173,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.num_bits = self.orig_layer.bits
self.group_size = self.orig_layer.group_size
self.sym = self.orig_layer.sym
+ self.scale_dtype = self.orig_layer.scale_dtype
device = self.orig_layer.weight.device
self.weight_t = self.orig_layer.weight.t()
self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)
@@ -420,7 +422,7 @@ class AutoRound(object):
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int", ##only support int for now
- scale_dtype: str = "fp32",
+ scale_dtype: str = "fp16",
**kwargs,
):
self.quantized = False
@@ -444,30 +446,9 @@ class AutoRound(object):
self.train_bs = batch_size
self.n_blocks = n_blocks
self.device = detect_device(device)
-
- if scale_dtype == "fp16" or scale_dtype == "float16":
- self.scale_dtype = torch.float16
- elif scale_dtype == "bf16" or scale_dtype == "bfloat16":
- self.scale_dtype = torch.bfloat16
- else:
- self.scale_dtype = torch.float32
-
- self.amp_dtype = torch.float16
- if self.model.dtype != torch.float32:
- self.amp_dtype = self.model.dtype
- if self.device == "cpu" or "hpu" in self.device:
- self.amp_dtype = torch.bfloat16
- if self.amp:
- if self.device == "cpu" and not CpuInfo().bf16:
- self.amp = False
- self.model = self.model.to(torch.float32)
- logger.warning(
- f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
- )
- else:
- self.model = self.model.to(self.amp_dtype)
- else:
- self.model = self.model.to(torch.float32)
+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)
+ self.set_amp_dtype()
+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else device
logger.info(f"using {self.model.dtype} for quantization tuning")
self.dataset = dataset
self.iters = iters
@@ -486,11 +467,9 @@ class AutoRound(object):
self.not_use_best_mse = not_use_best_mse
self.dynamic_max_gap = dynamic_max_gap
self.enable_full_range = enable_full_range
- assert self.enable_full_range is False, "only support enable_full_range=False currently"
self.lr_scheduler = lr_scheduler
self.set_layerwise_config(self.weight_config)
self.optimizer = self.get_optimizer(None)
- self.check_configs()
self.share_attention_mask_flag = None
self.hidden_dim_flag = None
torch.set_printoptions(precision=3, sci_mode=True)
@@ -498,73 +477,156 @@ class AutoRound(object):
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401
+ self.check_configs()
- def get_optimizer(self, optimizer):
- """Returns the specified optimizer. In SignRound, we fix the optimizer.
+ def check_configs(self):
+ """Checks if the configurations are valid.
- Args:
- optimizer: The optimizer to be used.
+ Raises:
+ AssertionError: If any of the configurations are invalid.
+ """
+ assert isinstance(self.model, torch.nn.Module)
+ assert self.bits > 0, "bits must be positive"
+ assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
+ assert self.train_bs > 0, "batch size must be positive"
+ assert self.iters > 0, "iters must be positive"
+ assert self.seqlen > 0, "seqlen must be positive"
+ assert self.n_blocks > 0, "n_blocks must be positive"
+ assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
+ assert self.enable_full_range is False, "only support enable_full_range=False currently"
+ # assert self.tokenizer != None or self.dataloader != None
+
+ def quantize(self):
+ """Quantize the model and return the quantized model along with weight configurations.
+ the entry of AutoRound.
Returns:
- The specified optimizer.
+ The quantized model and weight configurations.
"""
- from auto_round.sign_sgd import SGD
+ # logger.info("cache block input")
+ block_names = get_block_names(self.model)
+ if len(block_names) == 0:
+ logger.warning("could not find blocks, exit with original model")
+ return self.model, self.weight_config
- return SGD
+ if self.amp:
+ self.model = self.model.to(self.amp_dtype)
- def get_scaler(self):
- """Returns scaler, in SignRound, no need to use scaler."""
- return None
+ layer_names = self.get_quantized_layer_names_outside_blocks()
+ self.start_time = time.time()
+ all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.n_samples, layer_names=layer_names)
+ del self.inputs
+ inputs = all_inputs[block_names[0]]
- def scale_loss_and_backward(self, scaler, loss):
- """Scales the loss and performs backward pass.
+ all_inputs.pop(block_names[0])
+ self.inputs = None
+ del self.inputs
+ if "input_ids" in inputs.keys():
+ total_samples = len(inputs["input_ids"])
+ self.n_samples = total_samples
+ if total_samples < self.train_bs:
+ self.train_bs = total_samples
+ logger.warning(f"force the train batch size to {total_samples} ")
+ self.model = self.model.to("cpu")
+ torch.cuda.empty_cache()
+ self.quant_blocks(
+ self.model,
+ inputs,
+ block_names,
+ n_blocks=self.n_blocks,
+ device=self.device,
+ )
+
+ self.quant_layers(layer_names, all_inputs)
+
+ self.dump_data_to_weight_config()
+
+ end_time = time.time()
+ cost_time = end_time - self.start_time
+ logger.info(f"quantization tuning time {cost_time}")
+
+ ## dump a summary
+ quantized_layers = []
+ unquantized_layers = []
+ for n, m in self.model.named_modules():
+ if isinstance(m, tuple(self.supported_types)):
+ if self.weight_config[n]["bits"] == 16:
+ unquantized_layers.append(n)
+ else:
+ quantized_layers.append(n)
+ summary_info = (
+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
+ )
+ if len(unquantized_layers) > 0:
+ summary_info += f", {unquantized_layers} have not been quantized"
+ logger.info(summary_info)
+
+ self.quantized = True
+ self.model = self.model.to(self.model_orig_dtype)
+ return self.model, self.weight_config
+
+ def dump_data_to_weight_config(self):
+ """
+ dump quantization scale and zp to weight configuration
Args:
- scaler: The scaler to be used.
- loss: The loss to be scaled.
Returns:
- The scaled loss.
+ None
"""
- scale_loss = loss * 1000
- scale_loss.backward()
- if is_optimum_habana_available():
- htcore.mark_step()
- return scale_loss
+ for n, m in self.model.named_modules():
+ if n not in self.weight_config.keys():
+ continue
+ if hasattr(m, "scale"):
+ self.weight_config[n]["scale"] = m.scale
+ self.weight_config[n]["zp"] = m.zp
+ if self.group_size <= 0:
+ self.weight_config[n]["g_idx"] = torch.tensor(
+ [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+ )
+ else:
+ self.weight_config[n]["g_idx"] = torch.tensor(
+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+ )
+ delattr(m, "scale")
+ delattr(m, "zp")
+ else:
+ self.weight_config[n]["data_type"] = "float"
+ if self.amp_dtype == torch.bfloat16:
+ self.weight_config[n]["data_type"] = "bfloat"
+ self.weight_config[n]["bits"] = 16
+ self.weight_config[n]["group_size"] = None
+ self.weight_config[n]["sym"] = None
- def step(self, scaler, optimizer, lr_schedule):
- """Performs a step in the optimization process.
+ def quant_layers(self, layer_names, layer_inputs):
+ """Quantizes specified layers based on inputs and configuration.
Args:
- scaler: The scaler to be used.
- optimizer: The optimizer for the step.
- lr_schedule: The learning rate schedule.
+ layer_names (list): List of layer names to quantize.
+ layer_inputs (dict): Dictionary mapping layer names to input data.
Returns:
- None
+ None
"""
- optimizer.step()
- # for hpu
- if is_optimum_habana_available():
- htcore.mark_step()
- optimizer.zero_grad()
- lr_schedule.step()
-
- def check_configs(self):
- """Checks if the configurations are valid.
+ ##TODO currently we take all the layers outside blocks as post block layers which is not optimal
+ if len(layer_names) == 0:
+ return
+ q_layer_inputs = None
+ if self.enable_quanted_input:
+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.n_samples, layer_names=layer_names)
- Raises:
- AssertionError: If any of the configurations are invalid.
- """
- assert isinstance(self.model, torch.nn.Module)
- assert self.bits > 0, "bits must be positive"
- assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
- assert self.train_bs > 0, "batch size must be positive"
- assert self.iters > 0, "iters must be positive"
- assert self.seqlen > 0, "seqlen must be positive"
- assert self.n_blocks > 0, "n_blocks must be positive"
- assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
- # assert self.tokenizer != None or self.dataloader != None
+ self.model = self.model.to("cpu")
+ torch.cuda.empty_cache()
+ for layer_name in layer_names:
+ layer_input = layer_inputs[layer_name]
+ layer_input = to_device(layer_input, self.cache_device)
+ q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None
+ q_layer_input = to_device(q_layer_input, self.cache_device)
+ self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)
+ for i in range(len(layer_inputs)):
+ layer_input[i] = None
+ q_layer_input[i] = None
+ torch.cuda.empty_cache()
def set_layerwise_config(self, weight_config):
"""Sets the layer-wise configuration based on the provided weight_config.
@@ -639,9 +701,9 @@ class AutoRound(object):
tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
cache_device
)
- output.append(tmp_output)
- output = torch.cat(output, dim=self.input_dim)
+ output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))
torch.cuda.empty_cache()
+
return output
@torch.no_grad()
@@ -719,6 +781,37 @@ class AutoRound(object):
f"Valid samples size:{total_cnt}, Target sample size:{n_samples}"
)
+ @torch.no_grad()
+ def try_cache_inter_data_gpucpu(self, block_names, n_samples, layer_names=[], last_cache_name=None):
+ """Attempts to cache intermediate data on GPU,if failed, then using CPU.
+
+ Args:
+ block_names (list): List of block names to cache data for.
+ n_samples (int): Number of samples to use for caching.
+ layer_names (list, optional): List of layer names to cache data for. Defaults to [].
+ last_cache_name (str, optional): Name of the last cache. Defaults to None.
+
+ Returns:
+ all_inputs: Cached intermediate data.
+
+ Raises:
+ Exception: If caching on GPU fails, switches to CPU and caches there.
+ """
+ try:
+ self.model = self.model.to(self.device)
+ all_inputs = self.cache_inter_data(
+ block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name
+ )
+ self.model = self.model.to("cpu")
+ except:
+ logger.info("switch to cpu to cache inputs")
+ self.model = self.model.to("cpu")
+ torch.cuda.empty_cache()
+ all_inputs = self.cache_inter_data(
+ block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name
+ )
+ return all_inputs
+
@torch.no_grad()
def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_name=None):
"""Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.
@@ -740,13 +833,10 @@ class AutoRound(object):
self.inputs = {}
self.to_cached_layers = block_names + layer_names
tmp_dtype = None
- if (
- len(block_names) > 1 or len(layer_names) > 0
- ) and self.low_gpu_mem_usage: ## have bug if block name is not the first block
+ ## have bug if block name is not the first block
+ if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:
tmp_dtype = self.model.dtype
- self.model = (
- self.model.to(torch.bfloat16) if self.amp else self.model.to(torch.float32)
- ) ##force to dtype supported on cpu
+ self.model = self.model.to(torch.bfloat16) if self.amp else self.model.to(torch.float32)
self.last_cache_name = last_cache_name
if last_cache_name is None and len(block_names) + len(layer_names) == 1:
@@ -792,16 +882,15 @@ class AutoRound(object):
self.input_dim = check_hidden_state_dim(self.model, positional_args)
self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs)
if name in self.inputs:
- data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=self.input_dim)
- self.inputs[name]["input_ids"] = data
+ self.inputs[name]["input_ids"].extend(list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))
else:
self.inputs[name] = {}
- self.inputs[name]["input_ids"] = hidden_states.to("cpu")
+ self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))
if "positional_inputs" not in self.inputs[name]:
self.inputs[name]["positional_inputs"] = []
for idx, item in enumerate(positional_args):
- self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)
+ self.inputs[name]["positional_inputs"] = to_device(positional_args)
for key in kwargs.keys():
if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):
@@ -810,11 +899,10 @@ class AutoRound(object):
self.inputs[name][key] = None
if kwargs[key] is not None:
if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:
- self.inputs[name][key] = torch.cat(
- [self.inputs[name][key], kwargs[key].to("cpu")], dim=0
- )
+ self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
+
else:
- self.inputs[name][key] = kwargs[key].to("cpu")
+ self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0))
elif "alibi" in key:
if key not in self.inputs[name].keys():
self.inputs[name][key] = None
@@ -823,11 +911,13 @@ class AutoRound(object):
batch = kwargs["attention_mask"].shape[0]
alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])
if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:
- self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)
+ self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))
else:
- self.inputs[name][key] = alibi.to("cpu")
+
+ self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))
+
elif key not in self.inputs[name].keys():
- self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))
+ self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))
if name == self.last_cache_name:
raise NotImplementedError
else:
@@ -846,9 +936,9 @@ class AutoRound(object):
if isinstance(inputs, tuple) or isinstance(input, list):
input = inputs[0]
if name in self.inputs:
- self.inputs[name] = torch.cat([self.inputs[name], input.to("cpu")], dim=0)
+ self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
else:
- self.inputs[name] = input.to("cpu")
+ self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
return cache_input_hook
@@ -890,9 +980,10 @@ class AutoRound(object):
logger.info(f"quantizing layer {layer_name}")
layer = get_module(self.model, layer_name)
layer = layer.to(device)
- inputs = inputs.to(layer.weight.dtype)
- if q_inputs is not None:
- q_inputs = q_inputs.to(layer.weight.dtype)
+ for i in range(len(inputs)):
+ inputs[i] = inputs[i].to(layer.weight.dtype)
+ if q_inputs is not None:
+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
wrapper_linear = WrapperLinear(layer, self.enable_minmax_tuning).to(device)
round_params = []
@@ -915,7 +1006,7 @@ class AutoRound(object):
train_bs = self.train_bs
pick_samples = train_bs
- n_samples = inputs.shape[0]
+ n_samples = len(inputs)
if self.sampler != "rand":
indices = torch.randperm(n_samples)[:pick_samples]
last_best_iter = 0
@@ -932,10 +1023,13 @@ class AutoRound(object):
if self.sampler == "rand":
indices = torch.randperm(n_samples)[:pick_samples]
if q_inputs is not None:
- current_input = q_inputs[indices, ...].to(device)
- org_input = inputs[indices, ...].to(device)
+ current_input = [q_inputs[i] for i in indices]
+ current_input = torch.cat(current_input, dim=0).to(device)
+ org_input = [inputs[i] for i in indices]
+ org_input = torch.cat(org_input, dim=0).to(device)
else:
- current_input = inputs[indices, ...].to(device)
+ current_input = [q_inputs[i] for i in indices]
+ current_input = torch.cat(current_input, dim=0).to(device)
org_input = current_input
with torch.no_grad():
current_output = layer(org_input)
@@ -978,10 +1072,10 @@ class AutoRound(object):
if not self.not_use_best_mse:
last_loss = best_loss
best_iter = last_best_iter
- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- logger.info(dump_info)
with torch.no_grad():
unwrapper_layer(self.model, wrapper_linear, layer_name, best_v, best_min_scale, best_max_scale)
+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ logger.info(dump_info)
def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
"""Quantize the weights of a given block of the model.
@@ -996,23 +1090,14 @@ class AutoRound(object):
Returns:
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
- cache_device = "cpu" ## force cache device to "cpu"
- ##change to block dtype:
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
- for (
- key
- ) in (
- input_others.keys()
- ): ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
- if isinstance(input_others[key], torch.Tensor) and (
- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
- ):
- input_others[key] = input_others[key].to(tmp_dtype)
- output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)
+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, self.cache_device)
if q_input is not None:
- input_ids = q_input.to(cache_device)
+ for i in range(len(input_ids)):
+ input_ids[i] = None
+ torch.cuda.empty_cache()
+ input_ids = q_input
quantized_layer_names, unquantized_layer_names = wrapper_block(block, self.enable_minmax_tuning)
@@ -1039,10 +1124,7 @@ class AutoRound(object):
lr_schedule = copy.deepcopy(self.lr_scheduler)
pick_samples = self.train_bs
- if len(input_ids.shape) == 3:
- n_samples = input_ids.shape[self.input_dim]
- else:
- n_samples = input_ids.shape[0] // self.seqlen
+ n_samples = len(input_ids)
if self.sampler != "rand":
indices = torch.randperm(n_samples)[:pick_samples]
last_best_iter = 0
@@ -1064,18 +1146,11 @@ class AutoRound(object):
share_attention_mask_flag=self.share_attention_mask_flag,
input_dim=self.input_dim,
)
- if len(input_ids.shape) == 3:
- if self.input_dim == 0:
- current_output = output[indices, :, :]
- elif self.input_dim == 1:
- current_output = output[:, indices, :]
- else:
- current_output = output[:, :, indices]
- else:
- current_output = output.view(n_samples, self.seqlen, -1)
- current_output = current_output[indices, :, :]
- current_output = current_output.reshape(-1, current_output.shape[-1])
- current_output = move_input_to_device(current_output, device)
+
+ current_output = [output[i] for i in indices]
+ current_output = torch.cat(current_output, dim=self.input_dim)
+
+ current_output = to_device(current_output, device)
output_q = block_forward(
block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
@@ -1125,14 +1200,23 @@ class AutoRound(object):
with torch.no_grad():
unwrapper_block(block, best_v, best_min_scale, best_max_scale)
if self.enable_quanted_input:
- q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)
+
+ q_outputs = self.get_block_outputs(
+ block, input_ids, input_others, self.train_bs, device, cache_device=self.cache_device
+ )
+ for i in range(len(input_ids)):
+ input_ids[i] = None
+ torch.cuda.empty_cache()
return q_outputs, output
else:
+ for i in range(len(input_ids)):
+ input_ids[i] = None
+ torch.cuda.empty_cache()
return None, output
- def qdq_weight_round(
+ def quant_blocks(
self,
model: torch.nn.Module,
inputs,
@@ -1160,6 +1244,22 @@ class AutoRound(object):
inputs.pop("input_ids", None)
input_others = inputs
torch.cuda.empty_cache()
+ input_ids = to_device(input_ids, self.cache_device)
+ input_others = to_device(input_others, self.cache_device)
+ ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
+ tmp_dtype = self.amp_dtype if self.amp else torch.float32
+ for i in range(len(input_ids)):
+ input_ids[i] = input_ids[i].to(tmp_dtype)
+
+ for key in input_others.keys():
+ if isinstance(input_others[key], torch.Tensor) and (
+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
+ ):
+ input_others[key] = input_others[key].to(tmp_dtype)
+ elif isinstance(input_others[key], list):
+ for i in range(len(input_others[key])):
+ input_others[key][i].to(tmp_dtype)
+
for i in range(0, len(block_names), n_blocks):
if n_blocks == 1:
n = block_names[i]
@@ -1277,109 +1377,76 @@ class AutoRound(object):
return layer_names
- def quantize(self):
- """Quantize the model and return the quantized model along with weight configurations.
+ def set_amp_dtype(self):
+ self.amp_dtype = torch.float16
+ if self.model.dtype != torch.float32:
+ self.amp_dtype = self.model.dtype
+ if self.device == "cpu" or "hpu" in self.device:
+ self.amp_dtype = torch.bfloat16
+ if self.amp:
+ if self.device == "cpu" and not CpuInfo().bf16:
+ self.amp = False
+ self.amp_dtype = torch.float32
+ self.model = self.model.to(torch.float32)
+ logger.warning(
+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
+ )
+ else:
+ self.model = self.model.to(self.amp_dtype)
+ else:
+ self.amp_dtype = torch.float32
+ self.model = self.model.to(torch.float32)
+
+ def get_optimizer(self, optimizer):
+ """Returns the specified optimizer. In SignRound, we fix the optimizer.
+
+ Args:
+ optimizer: The optimizer to be used.
Returns:
- The quantized model and weight configurations.
+ The specified optimizer.
"""
- # logger.info("cache block input")
- block_names = get_block_names(self.model)
- if len(block_names) == 0:
- logger.warning("could not find blocks, exit with original model")
- return
- if self.amp:
- self.model = self.model.to(self.amp_dtype)
- if not self.low_gpu_mem_usage:
- self.model = self.model.to(self.device)
+ from auto_round.sign_sgd import SGD
- layer_names = self.get_quantized_layer_names_outside_blocks()
- self.start_time = time.time()
- all_inputs = self.cache_inter_data([block_names[0]], self.n_samples, layer_names=layer_names)
- del self.inputs
- inputs = all_inputs[block_names[0]]
+ return SGD
- all_inputs.pop(block_names[0])
- self.inputs = None
- del self.inputs
- if "input_ids" in inputs.keys():
- dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) ##polish the code
- total_samples = inputs["input_ids"].shape[dim]
- self.n_samples = total_samples
- if total_samples < self.train_bs:
- self.train_bs = total_samples
- logger.warning(f"force the train batch size to {total_samples} ")
- self.model = self.model.to("cpu")
- torch.cuda.empty_cache()
- self.qdq_weight_round(
- self.model,
- inputs,
- block_names,
- n_blocks=self.n_blocks,
- device=self.device,
- )
- ##TODO currently we take all the layers outside blocks as post block layers which is not optimal
- if len(layer_names) > 0:
- torch.cuda.empty_cache()
- layer_inputs = all_inputs
- q_layer_inputs = None
- if self.enable_quanted_input:
- if not self.low_gpu_mem_usage:
- self.model = self.model.to(self.device)
- q_layer_inputs = self.cache_inter_data([], self.n_samples, layer_names=layer_names)
- self.model = self.model.to("cpu")
- torch.cuda.empty_cache()
- for layer_name in layer_names:
- q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None
- self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)
- torch.cuda.empty_cache()
+ def get_scaler(self):
+ """Returns scaler, in SignRound, no need to use scaler."""
+ return None
- for n, m in self.model.named_modules():
- if n in self.weight_config.keys():
- if hasattr(m, "scale"):
- self.weight_config[n]["scale"] = m.scale
- self.weight_config[n]["zp"] = m.zp
- if self.group_size <= 0:
- self.weight_config[n]["g_idx"] = torch.tensor(
- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
- )
- else:
- self.weight_config[n]["g_idx"] = torch.tensor(
- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
- )
- delattr(m, "scale")
- delattr(m, "zp")
- else:
- self.weight_config[n]["data_type"] = "float"
- if self.amp_dtype == torch.bfloat16:
- self.weight_config[n]["data_type"] = "bfloat"
- self.weight_config[n]["bits"] = 16
- self.weight_config[n]["group_size"] = None
- self.weight_config[n]["sym"] = None
+ def scale_loss_and_backward(self, scaler, loss):
+ """Scales the loss and performs backward pass.
- end_time = time.time()
- cost_time = end_time - self.start_time
- logger.info(f"quantization tuning time {cost_time}")
- ## dump a summary
- quantized_layers = []
- unquantized_layers = []
- for n, m in self.model.named_modules():
- if isinstance(m, tuple(self.supported_types)):
- if self.weight_config[n]["bits"] == 16:
- unquantized_layers.append(n)
- else:
- quantized_layers.append(n)
- summary_info = (
- f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
- )
- if len(unquantized_layers) > 0:
- summary_info += f", {unquantized_layers} have not been quantized"
+ Args:
+ scaler: The scaler to be used.
+ loss: The loss to be scaled.
- logger.info(summary_info)
+ Returns:
+ The scaled loss.
+ """
+ scale_loss = loss * 1000
+ scale_loss.backward()
+ if is_optimum_habana_available():
+ htcore.mark_step()
+ return scale_loss
- self.quantized = True
- self.model = self.model.to(self.model_orig_dtype)
- return self.model, self.weight_config
+ def step(self, scaler, optimizer, lr_schedule):
+ """Performs a step in the optimization process.
+
+ Args:
+ scaler: The scaler to be used.
+ optimizer: The optimizer for the step.
+ lr_schedule: The learning rate schedule.
+
+ Returns:
+ None
+ """
+ optimizer.step()
+ # for hpu
+ if is_optimum_habana_available():
+ htcore.mark_step()
+ optimizer.zero_grad()
+ lr_schedule.step()
class AutoOPTRound(AutoRound):
@@ -1450,7 +1517,7 @@ class AutoOPTRound(AutoRound):
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
- scale_dtype: str = "fp32",
+ scale_dtype: str = "fp16",
optimizer="AdamW",
**kwargs,
):
@@ -1598,7 +1665,7 @@ class AutoAdamRound(AutoOPTRound):
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
data_type: str = "int",
- scale_dtype: str = "fp32",
+ scale_dtype: str = "fp16",
optimizer="AdamW",
**kwargs,
):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 84073d447..0e2b832f6 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -330,7 +330,7 @@ def get_scale_shape(weight, group_size):
return shape
-def move_input_to_device(input, device=torch.device("cpu")):
+def to_device(input, device=torch.device("cpu")):
"""Moves input data to the specified device.
Args:
@@ -340,16 +340,24 @@ def move_input_to_device(input, device=torch.device("cpu")):
Returns:
The input data on the specified device.
"""
+ if input is None:
+ return None
if isinstance(input, torch.Tensor):
return input.to(device)
if isinstance(input, dict) or isinstance(input, UserDict):
for inp in input.keys():
- input[inp] = move_input_to_device(input[inp], device)
+ input[inp] = to_device(input[inp], device)
+
elif isinstance(input, list) or isinstance(input, tuple):
+ if len(input) == 0:
+ return input
input_res = []
for inp in input:
- input_res.append(move_input_to_device(inp, device))
+ input_res.append(to_device(inp, device))
+ if isinstance(input, tuple):
+ input_res = tuple(input_res)
input = input_res
+
return input
@@ -421,11 +429,12 @@ def collect_minmax_scale(block):
return min_scales, max_scales
def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_mask_flag=False, input_dim=0):
"""Samples inputs based on the given indices and sequence length.
Args:
- input_ids: The input tensor containing IDs.
+ input_ids: The list of input tensor containing input_ids.
input_others: A dictionary containing other input data.
indices: The indices to sample from the input.
seqlen: The sequence length.
@@ -434,22 +443,17 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_ma
current_input_ids: The sampled input IDs.
current_input_others: The sampled other input data.
"""
- if len(input_ids.shape) == 3:
- if bool(input_dim):
- current_input_ids = input_ids[:, indices, :]
- else:
- current_input_ids = input_ids[indices, :, :]
- else:
- n_samples = input_ids.shape[0] // seqlen
- current_input_ids = input_ids.view(n_samples, seqlen, -1)
- current_input_ids = current_input_ids[indices, :, :]
- current_input_ids = current_input_ids.reshape(-1, input.shape[-1])
+ current_input_ids = [input_ids[i] for i in indices]
+ current_input_ids = torch.cat(current_input_ids, dim=input_dim)
+
current_input_others = {"positional_inputs": input_others["positional_inputs"]}
for key in input_others.keys():
if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key):
current_input_others[key] = None
if input_others[key] is not None:
- current_input_others[key] = input_others[key][indices, ...]
+ current_input_others[key] = [input_others[key][i] for i in indices]
+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)
+
else:
current_input_others[key] = input_others[key]
@@ -472,8 +476,8 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo
"""
if input_ids.device != device:
# input_ids, input_others = move_to_device(input_ids, input_others, device)
- input_ids = move_input_to_device(input_ids, device)
- input_others = move_input_to_device(input_others, device)
+ input_ids = to_device(input_ids, device)
+ input_others = to_device(input_others, device)
if "alibi" in input_others.keys():
attention_mask = input_others["attention_mask"]
alibi = input_others["alibi"]
@@ -634,11 +638,11 @@ def convert_dtype_str2torch(str_dtype):
return str_dtype
if str_dtype == "int8":
return torch.int8
- elif str_dtype == "fp32" or str_dtype == "auto":
+ elif str_dtype == "fp32" or str_dtype == "float32" or str_dtype == "auto":
return torch.float
- elif str_dtype == "fp16":
+ elif str_dtype == "fp16" or str_dtype == "float16":
return torch.float16
- elif str_dtype == "bf16":
+ elif str_dtype == "bf16" or str_dtype == "bfloat16":
return torch.bfloat16
else:
assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 92faa77c4..6ff8c0c04 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -19,7 +19,6 @@ The transformers version required varies across different types of models. Here,
| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36 |
| mosaicml/mpt-7b | 4.28/4.30/4.34/4.36 |
| mosaicml/mpt-7b-chat | 4.34 |
-| bigscience/bloom-3b | 4.28/4.30/4.34/4.36 |
| bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36 |
| baichuan-inc/Baichuan2-7B-Chat | 4.36 |
| Qwen/Qwen-7B | 4.28/4.30/4.34/4.36 |
@@ -34,6 +33,7 @@ The transformers version required varies across different types of models. Here,
| rinna/bilingual-gpt-neox-4b | 4.36 |
| microsoft/phi-2 | 4.36 |
| google/gemma-7b | 4.38/4.40 |
+| Salesforce/codegen25-7b-multi | 4.33.2|
## 2. Prepare Calibration Dataset
@@ -84,9 +84,13 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8
```
+- **Speedup the tuning:**
+
+disable_low_gpu_mem_usage(more gpu memory) or set the n_sample to 128( little accuracy drop) or change the train bs to 4(little accuracy drop) or combine them
+
- **Enable quantized lm-head:**
---disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.
+Currently only support in Intel xpu,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.
```bash
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage
```
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index d1858b684..90a83d93a 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -253,9 +253,6 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name)
exit()
- if args.disable_low_gpu_mem_usage:
- model = model.to(torch_device)
-
round = AutoRound
if args.adam:
round = AutoAdamRound
@@ -267,22 +264,22 @@ if __name__ == '__main__':
weight_config[n] = {"data_type": "fp"}
print(
f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
+ lm_head_layer_name = "lm_head"
if args.quant_lm_head:
from transformers import AutoConfig
- config = AutoConfig.from_pretrained(model_name)
- if config.tie_word_embeddings:
- if hasattr(model, "_tied_weights_keys"):
- tied_keys = model._tied_weights_keys
- for item in tied_keys:
- if "lm_head" in item:##TODO extend to encoder-decoder layer, seq classification model
- args.quant_lm_head = False
- print(
- f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "
- f"supported currently")
- break
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):
+ tied_keys = model._tied_weights_keys
+ for item in tied_keys:
+ if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model
+ args.quant_lm_head = False
+ print(
+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "
+ f"supported currently")
+ break
if args.quant_lm_head:
- weight_config['lm_head'] = {"data_type": "int"}
+ weight_config[lm_head_layer_name] = {"data_type": "int"}
if args.quant_lm_head and not args.disable_low_gpu_mem_usage:
print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "
diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.sh
index df702a886..79abcbc6a 100644
--- a/examples/language-modeling/scripts/phi-2.sh
+++ b/examples/language-modeling/scripts/phi-2.sh
@@ -7,4 +7,4 @@ python3 main.py \
--deployment_device 'gpu' \
--disable_trust_remote_code \
--output_dir "./tmp_autoround" \
---disable_quanted_input \
\ No newline at end of file
+--disable_quanted_input
\ No newline at end of file