PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
1.8X speedup by disable_low_gpu_mem_usage and reduce memory usage by avoid using torch.cat (#106)
wenhuach2123b60c3a67c6
diff --git a/README.md b/README.mdindex 59fc584b7..fc75033b1 100644--- a/README.md+++ b/README.md@@ -24,6 +24,9 @@ image presents an overview of AutoRound.<div align="left">+## What's New+* [2024/05] AutoRound performs well in [low_bit_open_llm_leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)+## Prerequisites- Python 3.9 or higher@@ -69,16 +72,16 @@ autoround.save_quantized(output_dir)- `model`: The PyTorch model to be quantized.-- `tokenizer`: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.+- `tokenizer`: An optional tokenizer for processing input data. If none, a dataset must be provided.- `bits (int)`: Number of bits for quantization (default is 4).- `group_size (int)`: Size of the quantization group (default is 128).-- `sym (bool)`: Whether to use symmetric quantization.+- `sym (bool)`: Whether to use symmetric quantization (default is False).- `enable_quanted_input (bool)`: Whether to use the output of the previous quantized block as the input for the current- block (default is True).+ block for tuning (default is True).- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).@@ -94,7 +97,7 @@ autoround.save_quantized(output_dir)- `batch_size (int)`: Batch size for training (default is 8).-- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float32"), different kernels have+- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float16"), different kernels havedifferent choices.- `amp (bool)`: Whether to use automatic mixed precision (default is True).@@ -103,7 +106,7 @@ autoround.save_quantized(output_dir)- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).-- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of a little tuning time (default is True).+- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of ~20% more tuning time (default is True).- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 59e076d66..ec277077d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -30,6 +30,7 @@ from .utils import (check_to_quantized,collect_minmax_scale,collect_round_v,+ convert_dtype_str2torch,detect_device,get_block_names,get_module,@@ -37,10 +38,10 @@ from .utils import (htcore,is_optimum_habana_available,logger,- move_input_to_device,quant_weight,sampling_inputs,set_module,+ to_device,)@@ -172,6 +173,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.num_bits = self.orig_layer.bitsself.group_size = self.orig_layer.group_sizeself.sym = self.orig_layer.sym+ self.scale_dtype = self.orig_layer.scale_dtypedevice = self.orig_layer.weight.deviceself.weight_t = self.orig_layer.weight.t()self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)@@ -420,7 +422,7 @@ class AutoRound(object):not_use_best_mse: bool = False,dynamic_max_gap: int = -1,data_type: str = "int", ##only support int for now- scale_dtype: str = "fp32",+ scale_dtype: str = "fp16",**kwargs,):self.quantized = False@@ -444,30 +446,9 @@ class AutoRound(object):self.train_bs = batch_sizeself.n_blocks = n_blocksself.device = detect_device(device)-- if scale_dtype == "fp16" or scale_dtype == "float16":- self.scale_dtype = torch.float16- elif scale_dtype == "bf16" or scale_dtype == "bfloat16":- self.scale_dtype = torch.bfloat16- else:- self.scale_dtype = torch.float32-- self.amp_dtype = torch.float16- if self.model.dtype != torch.float32:- self.amp_dtype = self.model.dtype- if self.device == "cpu" or "hpu" in self.device:- self.amp_dtype = torch.bfloat16- if self.amp:- if self.device == "cpu" and not CpuInfo().bf16:- self.amp = False- self.model = self.model.to(torch.float32)- logger.warning(- f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."- )- else:- self.model = self.model.to(self.amp_dtype)- else:- self.model = self.model.to(torch.float32)+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)+ self.set_amp_dtype()+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else devicelogger.info(f"using {self.model.dtype} for quantization tuning")self.dataset = datasetself.iters = iters@@ -486,11 +467,9 @@ class AutoRound(object):self.not_use_best_mse = not_use_best_mseself.dynamic_max_gap = dynamic_max_gapself.enable_full_range = enable_full_range- assert self.enable_full_range is False, "only support enable_full_range=False currently"self.lr_scheduler = lr_schedulerself.set_layerwise_config(self.weight_config)self.optimizer = self.get_optimizer(None)- self.check_configs()self.share_attention_mask_flag = Noneself.hidden_dim_flag = Nonetorch.set_printoptions(precision=3, sci_mode=True)@@ -498,73 +477,156 @@ class AutoRound(object):logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401+ self.check_configs()- def get_optimizer(self, optimizer):- """Returns the specified optimizer. In SignRound, we fix the optimizer.+ def check_configs(self):+ """Checks if the configurations are valid.- Args:- optimizer: The optimizer to be used.+ Raises:+ AssertionError: If any of the configurations are invalid.+ """+ assert isinstance(self.model, torch.nn.Module)+ assert self.bits > 0, "bits must be positive"+ assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"+ assert self.train_bs > 0, "batch size must be positive"+ assert self.iters > 0, "iters must be positive"+ assert self.seqlen > 0, "seqlen must be positive"+ assert self.n_blocks > 0, "n_blocks must be positive"+ assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"+ assert self.enable_full_range is False, "only support enable_full_range=False currently"+ # assert self.tokenizer != None or self.dataloader != None++ def quantize(self):+ """Quantize the model and return the quantized model along with weight configurations.+ the entry of AutoRound.Returns:- The specified optimizer.+ The quantized model and weight configurations."""- from auto_round.sign_sgd import SGD+ # logger.info("cache block input")+ block_names = get_block_names(self.model)+ if len(block_names) == 0:+ logger.warning("could not find blocks, exit with original model")+ return self.model, self.weight_config- return SGD+ if self.amp:+ self.model = self.model.to(self.amp_dtype)- def get_scaler(self):- """Returns scaler, in SignRound, no need to use scaler."""- return None+ layer_names = self.get_quantized_layer_names_outside_blocks()+ self.start_time = time.time()+ all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.n_samples, layer_names=layer_names)+ del self.inputs+ inputs = all_inputs[block_names[0]]- def scale_loss_and_backward(self, scaler, loss):- """Scales the loss and performs backward pass.+ all_inputs.pop(block_names[0])+ self.inputs = None+ del self.inputs+ if "input_ids" in inputs.keys():+ total_samples = len(inputs["input_ids"])+ self.n_samples = total_samples+ if total_samples < self.train_bs:+ self.train_bs = total_samples+ logger.warning(f"force the train batch size to {total_samples} ")+ self.model = self.model.to("cpu")+ torch.cuda.empty_cache()+ self.quant_blocks(+ self.model,+ inputs,+ block_names,+ n_blocks=self.n_blocks,+ device=self.device,+ )++ self.quant_layers(layer_names, all_inputs)++ self.dump_data_to_weight_config()++ end_time = time.time()+ cost_time = end_time - self.start_time+ logger.info(f"quantization tuning time {cost_time}")++ ## dump a summary+ quantized_layers = []+ unquantized_layers = []+ for n, m in self.model.named_modules():+ if isinstance(m, tuple(self.supported_types)):+ if self.weight_config[n]["bits"] == 16:+ unquantized_layers.append(n)+ else:+ quantized_layers.append(n)+ summary_info = (+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"+ )+ if len(unquantized_layers) > 0:+ summary_info += f", {unquantized_layers} have not been quantized"+ logger.info(summary_info)++ self.quantized = True+ self.model = self.model.to(self.model_orig_dtype)+ return self.model, self.weight_config++ def dump_data_to_weight_config(self):+ """+ dump quantization scale and zp to weight configurationArgs:- scaler: The scaler to be used.- loss: The loss to be scaled.Returns:- The scaled loss.+ None"""- scale_loss = loss * 1000- scale_loss.backward()- if is_optimum_habana_available():- htcore.mark_step()- return scale_loss+ for n, m in self.model.named_modules():+ if n not in self.weight_config.keys():+ continue+ if hasattr(m, "scale"):+ self.weight_config[n]["scale"] = m.scale+ self.weight_config[n]["zp"] = m.zp+ if self.group_size <= 0:+ self.weight_config[n]["g_idx"] = torch.tensor(+ [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"+ )+ else:+ self.weight_config[n]["g_idx"] = torch.tensor(+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"+ )+ delattr(m, "scale")+ delattr(m, "zp")+ else:+ self.weight_config[n]["data_type"] = "float"+ if self.amp_dtype == torch.bfloat16:+ self.weight_config[n]["data_type"] = "bfloat"+ self.weight_config[n]["bits"] = 16+ self.weight_config[n]["group_size"] = None+ self.weight_config[n]["sym"] = None- def step(self, scaler, optimizer, lr_schedule):- """Performs a step in the optimization process.+ def quant_layers(self, layer_names, layer_inputs):+ """Quantizes specified layers based on inputs and configuration.Args:- scaler: The scaler to be used.- optimizer: The optimizer for the step.- lr_schedule: The learning rate schedule.+ layer_names (list): List of layer names to quantize.+ layer_inputs (dict): Dictionary mapping layer names to input data.Returns:- None+ None"""- optimizer.step()- # for hpu- if is_optimum_habana_available():- htcore.mark_step()- optimizer.zero_grad()- lr_schedule.step()-- def check_configs(self):- """Checks if the configurations are valid.+ ##TODO currently we take all the layers outside blocks as post block layers which is not optimal+ if len(layer_names) == 0:+ return+ q_layer_inputs = None+ if self.enable_quanted_input:+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.n_samples, layer_names=layer_names)- Raises:- AssertionError: If any of the configurations are invalid.- """- assert isinstance(self.model, torch.nn.Module)- assert self.bits > 0, "bits must be positive"- assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"- assert self.train_bs > 0, "batch size must be positive"- assert self.iters > 0, "iters must be positive"- assert self.seqlen > 0, "seqlen must be positive"- assert self.n_blocks > 0, "n_blocks must be positive"- assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"- # assert self.tokenizer != None or self.dataloader != None+ self.model = self.model.to("cpu")+ torch.cuda.empty_cache()+ for layer_name in layer_names:+ layer_input = layer_inputs[layer_name]+ layer_input = to_device(layer_input, self.cache_device)+ q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None+ q_layer_input = to_device(q_layer_input, self.cache_device)+ self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)+ for i in range(len(layer_inputs)):+ layer_input[i] = None+ q_layer_input[i] = None+ torch.cuda.empty_cache()def set_layerwise_config(self, weight_config):"""Sets the layer-wise configuration based on the provided weight_config.@@ -639,9 +701,9 @@ class AutoRound(object):tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(cache_device)- output.append(tmp_output)- output = torch.cat(output, dim=self.input_dim)+ output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))torch.cuda.empty_cache()+return output@torch.no_grad()@@ -719,6 +781,37 @@ class AutoRound(object):f"Valid samples size:{total_cnt}, Target sample size:{n_samples}")+ @torch.no_grad()+ def try_cache_inter_data_gpucpu(self, block_names, n_samples, layer_names=[], last_cache_name=None):+ """Attempts to cache intermediate data on GPU,if failed, then using CPU.++ Args:+ block_names (list): List of block names to cache data for.+ n_samples (int): Number of samples to use for caching.+ layer_names (list, optional): List of layer names to cache data for. Defaults to [].+ last_cache_name (str, optional): Name of the last cache. Defaults to None.++ Returns:+ all_inputs: Cached intermediate data.++ Raises:+ Exception: If caching on GPU fails, switches to CPU and caches there.+ """+ try:+ self.model = self.model.to(self.device)+ all_inputs = self.cache_inter_data(+ block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name+ )+ self.model = self.model.to("cpu")+ except:+ logger.info("switch to cpu to cache inputs")+ self.model = self.model.to("cpu")+ torch.cuda.empty_cache()+ all_inputs = self.cache_inter_data(+ block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name+ )+ return all_inputs+@torch.no_grad()def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_name=None):"""Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.@@ -740,13 +833,10 @@ class AutoRound(object):self.inputs = {}self.to_cached_layers = block_names + layer_namestmp_dtype = None- if (- len(block_names) > 1 or len(layer_names) > 0- ) and self.low_gpu_mem_usage: ## have bug if block name is not the first block+ ## have bug if block name is not the first block+ if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:tmp_dtype = self.model.dtype- self.model = (- self.model.to(torch.bfloat16) if self.amp else self.model.to(torch.float32)- ) ##force to dtype supported on cpu+ self.model = self.model.to(torch.bfloat16) if self.amp else self.model.to(torch.float32)self.last_cache_name = last_cache_nameif last_cache_name is None and len(block_names) + len(layer_names) == 1:@@ -792,16 +882,15 @@ class AutoRound(object):self.input_dim = check_hidden_state_dim(self.model, positional_args)self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs)if name in self.inputs:- data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=self.input_dim)- self.inputs[name]["input_ids"] = data+ self.inputs[name]["input_ids"].extend(list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))else:self.inputs[name] = {}- self.inputs[name]["input_ids"] = hidden_states.to("cpu")+ self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))if "positional_inputs" not in self.inputs[name]:self.inputs[name]["positional_inputs"] = []for idx, item in enumerate(positional_args):- self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)+ self.inputs[name]["positional_inputs"] = to_device(positional_args)for key in kwargs.keys():if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):@@ -810,11 +899,10 @@ class AutoRound(object):self.inputs[name][key] = Noneif kwargs[key] is not None:if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:- self.inputs[name][key] = torch.cat(- [self.inputs[name][key], kwargs[key].to("cpu")], dim=0- )+ self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))+else:- self.inputs[name][key] = kwargs[key].to("cpu")+ self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0))elif "alibi" in key:if key not in self.inputs[name].keys():self.inputs[name][key] = None@@ -823,11 +911,13 @@ class AutoRound(object):batch = kwargs["attention_mask"].shape[0]alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:- self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)+ self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))else:- self.inputs[name][key] = alibi.to("cpu")++ self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))+elif key not in self.inputs[name].keys():- self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))+ self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))if name == self.last_cache_name:raise NotImplementedErrorelse:@@ -846,9 +936,9 @@ class AutoRound(object):if isinstance(inputs, tuple) or isinstance(input, list):input = inputs[0]if name in self.inputs:- self.inputs[name] = torch.cat([self.inputs[name], input.to("cpu")], dim=0)+ self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))else:- self.inputs[name] = input.to("cpu")+ self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))return cache_input_hook@@ -890,9 +980,10 @@ class AutoRound(object):logger.info(f"quantizing layer {layer_name}")layer = get_module(self.model, layer_name)layer = layer.to(device)- inputs = inputs.to(layer.weight.dtype)- if q_inputs is not None:- q_inputs = q_inputs.to(layer.weight.dtype)+ for i in range(len(inputs)):+ inputs[i] = inputs[i].to(layer.weight.dtype)+ if q_inputs is not None:+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)wrapper_linear = WrapperLinear(layer, self.enable_minmax_tuning).to(device)round_params = []@@ -915,7 +1006,7 @@ class AutoRound(object):train_bs = self.train_bspick_samples = train_bs- n_samples = inputs.shape[0]+ n_samples = len(inputs)if self.sampler != "rand":indices = torch.randperm(n_samples)[:pick_samples]last_best_iter = 0@@ -932,10 +1023,13 @@ class AutoRound(object):if self.sampler == "rand":indices = torch.randperm(n_samples)[:pick_samples]if q_inputs is not None:- current_input = q_inputs[indices, ...].to(device)- org_input = inputs[indices, ...].to(device)+ current_input = [q_inputs[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)+ org_input = [inputs[i] for i in indices]+ org_input = torch.cat(org_input, dim=0).to(device)else:- current_input = inputs[indices, ...].to(device)+ current_input = [q_inputs[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)org_input = current_inputwith torch.no_grad():current_output = layer(org_input)@@ -978,10 +1072,10 @@ class AutoRound(object):if not self.not_use_best_mse:last_loss = best_lossbest_iter = last_best_iter- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"- logger.info(dump_info)with torch.no_grad():unwrapper_layer(self.model, wrapper_linear, layer_name, best_v, best_min_scale, best_max_scale)+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ logger.info(dump_info)def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):"""Quantize the weights of a given block of the model.@@ -996,23 +1090,14 @@ class AutoRound(object):Returns:Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)"""- cache_device = "cpu" ## force cache device to "cpu"- ##change to block dtype:- tmp_dtype = self.amp_dtype if self.amp else torch.float32- for (- key- ) in (- input_others.keys()- ): ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage- if isinstance(input_others[key], torch.Tensor) and (- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16- ):- input_others[key] = input_others[key].to(tmp_dtype)- output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, self.cache_device)if q_input is not None:- input_ids = q_input.to(cache_device)+ for i in range(len(input_ids)):+ input_ids[i] = None+ torch.cuda.empty_cache()+ input_ids = q_inputquantized_layer_names, unquantized_layer_names = wrapper_block(block, self.enable_minmax_tuning)@@ -1039,10 +1124,7 @@ class AutoRound(object):lr_schedule = copy.deepcopy(self.lr_scheduler)pick_samples = self.train_bs- if len(input_ids.shape) == 3:- n_samples = input_ids.shape[self.input_dim]- else:- n_samples = input_ids.shape[0] // self.seqlen+ n_samples = len(input_ids)if self.sampler != "rand":indices = torch.randperm(n_samples)[:pick_samples]last_best_iter = 0@@ -1064,18 +1146,11 @@ class AutoRound(object):share_attention_mask_flag=self.share_attention_mask_flag,input_dim=self.input_dim,)- if len(input_ids.shape) == 3:- if self.input_dim == 0:- current_output = output[indices, :, :]- elif self.input_dim == 1:- current_output = output[:, indices, :]- else:- current_output = output[:, :, indices]- else:- current_output = output.view(n_samples, self.seqlen, -1)- current_output = current_output[indices, :, :]- current_output = current_output.reshape(-1, current_output.shape[-1])- current_output = move_input_to_device(current_output, device)++ current_output = [output[i] for i in indices]+ current_output = torch.cat(current_output, dim=self.input_dim)++ current_output = to_device(current_output, device)output_q = block_forward(block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device@@ -1125,14 +1200,23 @@ class AutoRound(object):with torch.no_grad():unwrapper_block(block, best_v, best_min_scale, best_max_scale)if self.enable_quanted_input:- q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)++ q_outputs = self.get_block_outputs(+ block, input_ids, input_others, self.train_bs, device, cache_device=self.cache_device+ )+ for i in range(len(input_ids)):+ input_ids[i] = None+ torch.cuda.empty_cache()return q_outputs, outputelse:+ for i in range(len(input_ids)):+ input_ids[i] = None+ torch.cuda.empty_cache()return None, output- def qdq_weight_round(+ def quant_blocks(self,model: torch.nn.Module,inputs,@@ -1160,6 +1244,22 @@ class AutoRound(object):inputs.pop("input_ids", None)input_others = inputstorch.cuda.empty_cache()+ input_ids = to_device(input_ids, self.cache_device)+ input_others = to_device(input_others, self.cache_device)+ ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ for i in range(len(input_ids)):+ input_ids[i] = input_ids[i].to(tmp_dtype)++ for key in input_others.keys():+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)+ elif isinstance(input_others[key], list):+ for i in range(len(input_others[key])):+ input_others[key][i].to(tmp_dtype)+for i in range(0, len(block_names), n_blocks):if n_blocks == 1:n = block_names[i]@@ -1277,109 +1377,76 @@ class AutoRound(object):return layer_names- def quantize(self):- """Quantize the model and return the quantized model along with weight configurations.+ def set_amp_dtype(self):+ self.amp_dtype = torch.float16+ if self.model.dtype != torch.float32:+ self.amp_dtype = self.model.dtype+ if self.device == "cpu" or "hpu" in self.device:+ self.amp_dtype = torch.bfloat16+ if self.amp:+ if self.device == "cpu" and not CpuInfo().bf16:+ self.amp = False+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)+ logger.warning(+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."+ )+ else:+ self.model = self.model.to(self.amp_dtype)+ else:+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)++ def get_optimizer(self, optimizer):+ """Returns the specified optimizer. In SignRound, we fix the optimizer.++ Args:+ optimizer: The optimizer to be used.Returns:- The quantized model and weight configurations.+ The specified optimizer."""- # logger.info("cache block input")- block_names = get_block_names(self.model)- if len(block_names) == 0:- logger.warning("could not find blocks, exit with original model")- return- if self.amp:- self.model = self.model.to(self.amp_dtype)- if not self.low_gpu_mem_usage:- self.model = self.model.to(self.device)+ from auto_round.sign_sgd import SGD- layer_names = self.get_quantized_layer_names_outside_blocks()- self.start_time = time.time()- all_inputs = self.cache_inter_data([block_names[0]], self.n_samples, layer_names=layer_names)- del self.inputs- inputs = all_inputs[block_names[0]]+ return SGD- all_inputs.pop(block_names[0])- self.inputs = None- del self.inputs- if "input_ids" in inputs.keys():- dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) ##polish the code- total_samples = inputs["input_ids"].shape[dim]- self.n_samples = total_samples- if total_samples < self.train_bs:- self.train_bs = total_samples- logger.warning(f"force the train batch size to {total_samples} ")- self.model = self.model.to("cpu")- torch.cuda.empty_cache()- self.qdq_weight_round(- self.model,- inputs,- block_names,- n_blocks=self.n_blocks,- device=self.device,- )- ##TODO currently we take all the layers outside blocks as post block layers which is not optimal- if len(layer_names) > 0:- torch.cuda.empty_cache()- layer_inputs = all_inputs- q_layer_inputs = None- if self.enable_quanted_input:- if not self.low_gpu_mem_usage:- self.model = self.model.to(self.device)- q_layer_inputs = self.cache_inter_data([], self.n_samples, layer_names=layer_names)- self.model = self.model.to("cpu")- torch.cuda.empty_cache()- for layer_name in layer_names:- q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None- self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)- torch.cuda.empty_cache()+ def get_scaler(self):+ """Returns scaler, in SignRound, no need to use scaler."""+ return None- for n, m in self.model.named_modules():- if n in self.weight_config.keys():- if hasattr(m, "scale"):- self.weight_config[n]["scale"] = m.scale- self.weight_config[n]["zp"] = m.zp- if self.group_size <= 0:- self.weight_config[n]["g_idx"] = torch.tensor(- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"- )- else:- self.weight_config[n]["g_idx"] = torch.tensor(- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"- )- delattr(m, "scale")- delattr(m, "zp")- else:- self.weight_config[n]["data_type"] = "float"- if self.amp_dtype == torch.bfloat16:- self.weight_config[n]["data_type"] = "bfloat"- self.weight_config[n]["bits"] = 16- self.weight_config[n]["group_size"] = None- self.weight_config[n]["sym"] = None+ def scale_loss_and_backward(self, scaler, loss):+ """Scales the loss and performs backward pass.- end_time = time.time()- cost_time = end_time - self.start_time- logger.info(f"quantization tuning time {cost_time}")- ## dump a summary- quantized_layers = []- unquantized_layers = []- for n, m in self.model.named_modules():- if isinstance(m, tuple(self.supported_types)):- if self.weight_config[n]["bits"] == 16:- unquantized_layers.append(n)- else:- quantized_layers.append(n)- summary_info = (- f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"- )- if len(unquantized_layers) > 0:- summary_info += f", {unquantized_layers} have not been quantized"+ Args:+ scaler: The scaler to be used.+ loss: The loss to be scaled.- logger.info(summary_info)+ Returns:+ The scaled loss.+ """+ scale_loss = loss * 1000+ scale_loss.backward()+ if is_optimum_habana_available():+ htcore.mark_step()+ return scale_loss- self.quantized = True- self.model = self.model.to(self.model_orig_dtype)- return self.model, self.weight_config+ def step(self, scaler, optimizer, lr_schedule):+ """Performs a step in the optimization process.++ Args:+ scaler: The scaler to be used.+ optimizer: The optimizer for the step.+ lr_schedule: The learning rate schedule.++ Returns:+ None+ """+ optimizer.step()+ # for hpu+ if is_optimum_habana_available():+ htcore.mark_step()+ optimizer.zero_grad()+ lr_schedule.step()class AutoOPTRound(AutoRound):@@ -1450,7 +1517,7 @@ class AutoOPTRound(AutoRound):not_use_best_mse: bool = False,dynamic_max_gap: int = -1,data_type: str = "int",- scale_dtype: str = "fp32",+ scale_dtype: str = "fp16",optimizer="AdamW",**kwargs,):@@ -1598,7 +1665,7 @@ class AutoAdamRound(AutoOPTRound):not_use_best_mse: bool = False,dynamic_max_gap: int = -1,data_type: str = "int",- scale_dtype: str = "fp32",+ scale_dtype: str = "fp16",optimizer="AdamW",**kwargs,):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 84073d447..0e2b832f6 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -330,7 +330,7 @@ def get_scale_shape(weight, group_size):return shape-def move_input_to_device(input, device=torch.device("cpu")):+def to_device(input, device=torch.device("cpu")):"""Moves input data to the specified device.Args:@@ -340,16 +340,24 @@ def move_input_to_device(input, device=torch.device("cpu")):Returns:The input data on the specified device."""+ if input is None:+ return Noneif isinstance(input, torch.Tensor):return input.to(device)if isinstance(input, dict) or isinstance(input, UserDict):for inp in input.keys():- input[inp] = move_input_to_device(input[inp], device)+ input[inp] = to_device(input[inp], device)+elif isinstance(input, list) or isinstance(input, tuple):+ if len(input) == 0:+ return inputinput_res = []for inp in input:- input_res.append(move_input_to_device(inp, device))+ input_res.append(to_device(inp, device))+ if isinstance(input, tuple):+ input_res = tuple(input_res)input = input_res+return input@@ -421,11 +429,12 @@ def collect_minmax_scale(block):return min_scales, max_scales[email protected]_grad()def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_mask_flag=False, input_dim=0):"""Samples inputs based on the given indices and sequence length.Args:- input_ids: The input tensor containing IDs.+ input_ids: The list of input tensor containing input_ids.input_others: A dictionary containing other input data.indices: The indices to sample from the input.seqlen: The sequence length.@@ -434,22 +443,17 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_macurrent_input_ids: The sampled input IDs.current_input_others: The sampled other input data."""- if len(input_ids.shape) == 3:- if bool(input_dim):- current_input_ids = input_ids[:, indices, :]- else:- current_input_ids = input_ids[indices, :, :]- else:- n_samples = input_ids.shape[0] // seqlen- current_input_ids = input_ids.view(n_samples, seqlen, -1)- current_input_ids = current_input_ids[indices, :, :]- current_input_ids = current_input_ids.reshape(-1, input.shape[-1])+ current_input_ids = [input_ids[i] for i in indices]+ current_input_ids = torch.cat(current_input_ids, dim=input_dim)+current_input_others = {"positional_inputs": input_others["positional_inputs"]}for key in input_others.keys():if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key):current_input_others[key] = Noneif input_others[key] is not None:- current_input_others[key] = input_others[key][indices, ...]+ current_input_others[key] = [input_others[key][i] for i in indices]+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+else:current_input_others[key] = input_others[key]@@ -472,8 +476,8 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo"""if input_ids.device != device:# input_ids, input_others = move_to_device(input_ids, input_others, device)- input_ids = move_input_to_device(input_ids, device)- input_others = move_input_to_device(input_others, device)+ input_ids = to_device(input_ids, device)+ input_others = to_device(input_others, device)if "alibi" in input_others.keys():attention_mask = input_others["attention_mask"]alibi = input_others["alibi"]@@ -634,11 +638,11 @@ def convert_dtype_str2torch(str_dtype):return str_dtypeif str_dtype == "int8":return torch.int8- elif str_dtype == "fp32" or str_dtype == "auto":+ elif str_dtype == "fp32" or str_dtype == "float32" or str_dtype == "auto":return torch.float- elif str_dtype == "fp16":+ elif str_dtype == "fp16" or str_dtype == "float16":return torch.float16- elif str_dtype == "bf16":+ elif str_dtype == "bf16" or str_dtype == "bfloat16":return torch.bfloat16else:assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex 92faa77c4..6ff8c0c04 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -19,7 +19,6 @@ The transformers version required varies across different types of models. Here,| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36 || mosaicml/mpt-7b | 4.28/4.30/4.34/4.36 || mosaicml/mpt-7b-chat | 4.34 |-| bigscience/bloom-3b | 4.28/4.30/4.34/4.36 || bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36 || baichuan-inc/Baichuan2-7B-Chat | 4.36 || Qwen/Qwen-7B | 4.28/4.30/4.34/4.36 |@@ -34,6 +33,7 @@ The transformers version required varies across different types of models. Here,| rinna/bilingual-gpt-neox-4b | 4.36 || microsoft/phi-2 | 4.36 || google/gemma-7b | 4.38/4.40 |+| Salesforce/codegen25-7b-multi | 4.33.2|## 2. Prepare Calibration Dataset@@ -84,9 +84,13 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8```+- **Speedup the tuning:**++disable_low_gpu_mem_usage(more gpu memory) or set the n_sample to 128( little accuracy drop) or change the train bs to 4(little accuracy drop) or combine them+- **Enable quantized lm-head:**---disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.+Currently only support in Intel xpu,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.```bashCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage```diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex d1858b684..90a83d93a 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -253,9 +253,6 @@ if __name__ == '__main__':device=torch_device, excel_file=excel_name)exit()- if args.disable_low_gpu_mem_usage:- model = model.to(torch_device)-round = AutoRoundif args.adam:round = AutoAdamRound@@ -267,22 +264,22 @@ if __name__ == '__main__':weight_config[n] = {"data_type": "fp"}print(f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")+ lm_head_layer_name = "lm_head"if args.quant_lm_head:from transformers import AutoConfig- config = AutoConfig.from_pretrained(model_name)- if config.tie_word_embeddings:- if hasattr(model, "_tied_weights_keys"):- tied_keys = model._tied_weights_keys- for item in tied_keys:- if "lm_head" in item:##TODO extend to encoder-decoder layer, seq classification model- args.quant_lm_head = False- print(- f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "- f"supported currently")- break+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):+ tied_keys = model._tied_weights_keys+ for item in tied_keys:+ if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model+ args.quant_lm_head = False+ print(+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "+ f"supported currently")+ breakif args.quant_lm_head:- weight_config['lm_head'] = {"data_type": "int"}+ weight_config[lm_head_layer_name] = {"data_type": "int"}if args.quant_lm_head and not args.disable_low_gpu_mem_usage:print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.shindex df702a886..79abcbc6a 100644--- a/examples/language-modeling/scripts/phi-2.sh+++ b/examples/language-modeling/scripts/phi-2.sh@@ -7,4 +7,4 @@ python3 main.py \--deployment_device 'gpu' \--disable_trust_remote_code \--output_dir "./tmp_autoround" \---disable_quanted_input \\ No newline at end of file+--disable_quanted_input\ No newline at end of file