PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support lm head quantizaiton and export to Intel cpu (#76)
wenhuach2116f9b7bda5e2
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex e433275d1..54abf173c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -17,6 +17,7 @@ import copyimport timeimport torch+from torch import autocastfrom .calib_dataset import get_dataloaderfrom .special_model_handler import check_hidden_state_dim, check_share_attention_mask@@ -24,6 +25,7 @@ from .utils import (CpuInfo,block_forward,check_is_cpu,+ check_memory_availability,check_to_quantized,collect_minmax_scale,collect_round_v,@@ -186,7 +188,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.min_scale = torch.tensor(0, device=device)self.max_scale = torch.tensor(0, device=device)- def unwrapper(self, v, min_scale, max_scale):+ def unwrapper(self, v=0, min_scale=0, max_scale=0):"""Unwrapper the layer to the original conv1d layer.Args:@@ -294,6 +296,31 @@ def wrapper_block(block, enable_minmax_tuning):return quantized_layers, unquantized_layers[email protected]_grad()+def unwrapper_layer(model, layer, layer_name, v=0, min_scale=0, max_scale=0):+ """Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.++ Args:+ block: The input block containing wrapped modules to be unwrapped.+ vs: A dictionary of scaling parameters for the wrapped modules.+ min_scales: A dictionary of minimum scaling values for the wrapped modules.+ max_scales: A dictionary of maximum scaling values for the wrapped modules.+ """++ if hasattr(layer, "orig_layer"):++ if isinstance(min_scale, torch.Tensor):+ min_scale = torch.clamp(min_scale, -1, 0)+ max_scale = torch.clamp(max_scale, -1, 0)++ else:+ min_scale = torch.tensor(0)+ max_scale = torch.tensor(0)+ orig_layer = layer.unwrapper(v, min_scale, max_scale)+ orig_layer = orig_layer.to("cpu")+ set_module(model, layer_name, orig_layer)++@torch.no_grad()def unwrapper_block(block, vs, min_scales, max_scales):"""Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.@@ -369,7 +396,6 @@ class AutoRound(object):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernelshave different choices.- **kwargs: Additional keyword arguments.Returns:The quantized model.@@ -409,6 +435,7 @@ class AutoRound(object):scale_dtype: str = "fp32",**kwargs,):+ self.quantized = Falseself.model_orig_dtype = model.dtypeself.model = model.eval().to("cpu")self.amp = amp@@ -483,6 +510,8 @@ class AutoRound(object):self.set_layerwise_config(self.weight_config)self.optimizer = self.get_optimizer(None)self.check_configs()+ self.share_attention_mask_flag = None+ self.hidden_dim_flag = Nonetorch.set_printoptions(precision=3, sci_mode=True)def get_optimizer(self, optimizer):@@ -554,6 +583,7 @@ class AutoRound(object):def set_layerwise_config(self, weight_config):"""Sets the layer-wise configuration based on the provided weight_config.+ By default, only quantize layers in blocks.Args:weight_config: The weight configuration.@@ -561,21 +591,18 @@ class AutoRound(object):Returns:None"""+ layers_in_blocks = self.get_layer_names_in_block()for n, m in self.model.named_modules():- is_supported_type = False- for supported_type in self.supported_types:- if isinstance(m, supported_type):- is_supported_type = True- break- if not is_supported_type:+ if not isinstance(m, tuple(self.supported_types)):continue- if n not in weight_config.keys():+ if n not in weight_config.keys() and n in layers_in_blocks:weight_config[n] = {}weight_config[n]["data_type"] = self.data_typeweight_config[n]["bits"] = self.bitsweight_config[n]["group_size"] = self.group_sizeweight_config[n]["sym"] = self.sym- else:+ weight_config[n]["scale_dtype"] = self.scale_dtype+ elif n in weight_config.keys():if "data_type" not in weight_config[n].keys():weight_config[n]["data_type"] = self.data_typeif "bits" not in weight_config[n].keys():@@ -584,7 +611,15 @@ class AutoRound(object):weight_config[n]["group_size"] = self.group_sizeif "sym" not in weight_config[n].keys():weight_config[n]["sym"] = self.sym- weight_config[n]["scale_dtype"] = self.scale_dtype+ if "scale_dtype" not in weight_config[n].keys():+ weight_config[n]["scale_dtype"] = self.scale_dtype+ else:+ weight_config[n] = {}+ weight_config[n]["data_type"] = "float"+ weight_config[n]["bits"] = 16+ weight_config[n]["group_size"] = self.group_size+ weight_config[n]["sym"] = self.sym+ weight_config[n]["scale_dtype"] = self.scale_dtypem.data_type = weight_config[n]["data_type"]m.bits = weight_config[n]["bits"]@@ -625,7 +660,7 @@ class AutoRound(object):return output@torch.no_grad()- def calib(self, n_samples):+ def calib(self, n_samples, bs):"""Perform calibration for quantization.This method calibrates the model for quantization by processing a specified@@ -635,6 +670,7 @@ class AutoRound(object):it logs an error and exits.Args:n_samples (int): The number of samples to use for calibration.+ bs (int): The number of samples to use for calibration"""if self.dataloader is None:@@ -644,11 +680,9 @@ class AutoRound(object):self.dataset_name,self.dataset_split,self.seed,- self.train_bs,+ bs,self.n_samples,)-- self.start_time = time.time()total_cnt = 0for data in self.dataloader:if data is None:@@ -673,8 +707,7 @@ class AutoRound(object):input_ids = data_new["input_ids"]if input_ids.shape[-1] < self.seqlen:continue- # if total_cnt + input_ids.shape[0] > n_samples:- # input_ids = input_ids[: n_samples - total_cnt, ...]+try:if isinstance(data_new, torch.Tensor):self.model(data_new)@@ -700,32 +733,55 @@ class AutoRound(object):)@torch.no_grad()- def cache_block_input(self, block_name, n_samples):- """Save the inputs of the first block for calibration.+ def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_name=None):+ """Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.This method temporarily replaces the forward method of the model to capturethe inputs passing through the specified block. It then calibrates the modelusing a specified number of samples. Finally, it restores the original forwardmethod and returns the inputs for the specified block.Args:- block_name (str): The name of the block for which inputs are to be saved.+ block_names (list): The names of the blocks for which inputs are to be saved.+ layer_names (list):The names of the layers for which inputs are to be saved.n_samples (int): The number of samples to use for calibration.+ last_cache_name (str, optional): The name of the last layer to be cached,+ we could break the forward in this layer to save time+Returns:dict: A dictionary containing the inputs for the specified block."""self.inputs = {}- self.tmp_block_name = block_name- self.share_attention_mask_flag = None- self.hidden_dim_flag = None+ self.to_cached_layers = block_names + layer_names+ tmp_dtype = None+ if (+ len(block_names) > 1 or len(layer_names) > 0+ ) and self.low_gpu_mem_usage: ## have bug if block name is not the first block+ tmp_dtype = self.model.dtype+ self.model = (+ self.model.to(torch.bfloat16) if self.amp else self.model.to(torch.float32)+ ) ##force to dtype supported on cpu++ self.last_cache_name = last_cache_name+ if last_cache_name is None and len(block_names) + len(layer_names) == 1:+ self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0]+ calib_bs = self.train_bs+ if not self.low_gpu_mem_usage and len(layer_names) > 1: ## persume has lm-head+ calib_bs = 1++ self.hook_handles = []self._replace_forward()- self.calib(n_samples)+ self.calib(n_samples, calib_bs)self._recover_forward()- res = self.inputs[self.tmp_block_name]- del self.tmp_block_name+ res = self.inputs+ del self.last_cache_name+ del self.to_cached_layers+ if tmp_dtype is not None:+ self.model = self.model.to(tmp_dtype)+return res@torch.no_grad()- def get_forward_func(self, name):+ def get_block_forward_func(self, name):"""Gets the forward function.Args:@@ -734,7 +790,7 @@ class AutoRound(object):function: The forward function."""- def forward(_, hidden_states, *positional_args, **kwargs):+ def forward(m, hidden_states, *positional_args, **kwargs):"""Rewrite forward function, process and collect input data.Args:@@ -785,27 +841,173 @@ class AutoRound(object):self.inputs[name][key] = alibi.to("cpu")elif key not in self.inputs[name].keys():self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))- raise NotImplementedError+ if name == self.last_cache_name:+ raise NotImplementedError+ else:+ return m.orig_forward(hidden_states, *positional_args, **kwargs)return forward+ @torch.no_grad()+ def _get_cache_data_hook_for_layer(self, name):+ """A forward hook to save input max of a module+ :param name: the module name+ :return: A hook function."""++ def cache_input_hook(module, inputs, outputs):+ input = inputs+ if isinstance(inputs, tuple) or isinstance(input, list):+ input = inputs[0]+ if name in self.inputs:+ self.inputs[name] = torch.cat([self.inputs[name], input.to("cpu")], dim=0)+ else:+ self.inputs[name] = input.to("cpu")++ return cache_input_hook+def _recover_forward(self):"""Recovers the forward function."""for n, m in self.model.named_modules():- if n == self.tmp_block_name:+ if hasattr(m, "orig_forward"):m.forward = m.orig_forwarddelattr(m, "orig_forward")- break+ for hook_handle in self.hook_handles:+ hook_handle.remove()+ self.hook_handles = []def _replace_forward(self):"""Replaces the forward function."""from functools import partialfor n, m in self.model.named_modules():- if n == self.tmp_block_name:+ if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##blockm.orig_forward = m.forward- m.forward = partial(self.get_forward_func(n), m)- break+ m.forward = partial(self.get_block_forward_func(n), m)+ elif n in self.to_cached_layers: ##linear layer or conv1d layer+ hook_func = self._get_cache_data_hook_for_layer(n)+ hook_handle = m.register_forward_hook(hook_func)+ self.hook_handles.append(hook_handle)++ def quant_layer(self, layer_name, inputs, q_inputs=None, device=torch.device("cpu")):+ """Quantize a specific layer of the model using the provided inputs.++ Args:+ layer_name (str): The name of the layer to quantize.+ inputs (torch.Tensor): Input data for quantization.+ q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.+ device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").++ Returns:+ None+ """+ logger.info(f"quantizing layer {layer_name}")+ with torch.no_grad():+ layer = get_module(self.model, layer_name)+ cache_device = "cpu"+ layer = layer.to(device)++ inputs = inputs.to(layer.weight.dtype)+ if q_inputs is not None:+ q_inputs = q_inputs.to(layer.weight.dtype)++ output = []+ train_bs = self.train_bs+ for i in range(0, self.n_samples, train_bs):+ end_index = min(self.n_samples, i + train_bs)+ tmp_inputs = inputs[i:end_index, ...].to(device)+ tmp_output = layer.forward(tmp_inputs).to(cache_device)+ output.append(tmp_output)+ torch.cuda.empty_cache() ##too large for lm head, maybe need to decrease n_sample++ output = torch.cat(output, dim=0)+ torch.cuda.empty_cache()++ wrapper_linear = WrapperLinear(layer, self.enable_minmax_tuning).to(device)+ round_params = []+ minmax_params = []+ round_params.append(wrapper_linear.value)+ minmax_params.append(wrapper_linear.min_scale)+ minmax_params.append(wrapper_linear.max_scale)+ if self.enable_minmax_tuning:+ optimizer = self.optimizer(+ [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0+ )+ else:+ optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)+ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)++ pick_samples = train_bs++ n_samples = inputs.shape[0]+ if self.sampler != "rand":+ indices = torch.randperm(n_samples)[:pick_samples]+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ mse_loss = torch.nn.MSELoss().to(device)+ scaler = self.get_scaler() # pylint: disable=assignment-from-none+ init_loss = None+ best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(0), torch.tensor(0)+ gradient_accumulate_steps = self.train_bs // train_bs+ for i in range(self.iters):+ if self.sampler == "rand":+ indices = torch.randperm(n_samples)[:pick_samples]+ total_loss = 0+ for _ in range(gradient_accumulate_steps):+ if q_inputs is not None:+ current_input = q_inputs[indices, ...].to(device)+ else:+ current_input = inputs[indices, ...].to(device)++ current_output = output[indices, ...].to(device)+ if self.amp:+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable+ else:+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32), current_output.to(torch.float32)+ )++ total_loss += loss.item() / gradient_accumulate_steps+ if i == 0:+ init_loss = total_loss++ self.scale_loss_and_backward(scaler, loss)+ torch.cuda.empty_cache()++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_v = copy.deepcopy(wrapper_linear.value.data)+ best_min_scale = copy.deepcopy(torch.clamp(wrapper_linear.min_scale.data, -1, 0))+ best_max_scale = copy.deepcopy(torch.clamp(wrapper_linear.max_scale.data, -1, 0))++ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_v = copy.deepcopy(wrapper_linear.value.data)+ best_min_scale = copy.deepcopy(torch.clamp(wrapper_linear.min_scale.data, -1, 0))+ best_max_scale = copy.deepcopy(torch.clamp(wrapper_linear.max_scale.data, -1, 0))++ if not self.not_use_best_mse:+ if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:+ break+ self.step(scaler, optimizer, lr_schedule)++ last_loss = total_loss+ best_iter = self.iters+ if not self.not_use_best_mse:+ last_loss = best_loss+ best_iter = last_best_iter+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ logger.info(dump_info)+ with torch.no_grad():+ unwrapper_layer(self.model, wrapper_linear, layer_name, best_v, best_min_scale, best_max_scale)def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):"""Quantize the weights of a given block of the model.@@ -820,14 +1022,19 @@ class AutoRound(object):Returns:Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)"""- from torch.amp import autocast+ cache_device = "cpu" ## force cache device to "cpu"+ ##change to block dtype:+ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ for (+ key+ ) in (+ input_others.keys()+ ): ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)- if not self.low_gpu_mem_usage and input_ids.device != device:- input_ids = move_input_to_device(input_ids, device)- input_others = move_input_to_device(input_others, device)- cache_device = device- if self.low_gpu_mem_usage:- cache_device = "cpu"output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)if q_input is not None:@@ -909,6 +1116,7 @@ class AutoRound(object):)total_loss += loss.item() / self.gradient_accumulate_steps+ torch.cuda.empty_cache()if i == 0:init_loss = total_loss@@ -942,8 +1150,8 @@ class AutoRound(object):logger.info(dump_info)if len(unquantized_layer_names) != 0:logger.info(f"{unquantized_layer_names} have not been quantized")-- unwrapper_block(block, best_v, best_min_scale, best_max_scale)+ with torch.no_grad():+ unwrapper_block(block, best_v, best_min_scale, best_max_scale)if self.use_quant_input:q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)@@ -1000,7 +1208,7 @@ class AutoRound(object):q_input=q_input,device=device,)- m.to("cpu")+ m = m.to("cpu")torch.cuda.empty_cache()del q_input@@ -1011,6 +1219,17 @@ class AutoRound(object):torch.cuda.empty_cache()def save_quantized(self, output_dir=None, format="auto_gptq", inplace=True, **kwargs):+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_gptq".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.++ Returns:+ object: The compressed model object.+ """if not self.quantized:logger.warning("please run autoround.quantize first")return@@ -1040,6 +1259,74 @@ class AutoRound(object):)return compressed_model+ #+ # @torch.no_grad()+ # def gets_layer_names_outside_blocks(self):+ # """Gets the names of layers outside blocks in the model.+ #+ # Returns:+ # list: List of layer names outside blocks.+ # """+ # all_layer_names = set()+ # for n, m in self.model.named_modules():+ # if isinstance(m, tuple(self.supported_types)):+ # m.tmp_name = n+ # all_layer_names.add(n)+ # block_names = get_block_names(self.model)+ # all_layer_names_in_block = set()+ # for block_name in block_names:+ # block = get_module(self.model, block_name)+ # for n, m in block.named_modules():+ # if isinstance(m, tuple(self.supported_types)):+ # all_layer_names_in_block.add(m.tmp_name)+ #+ # res = all_layer_names - all_layer_names_in_block+ # for n, m in self.model.named_modules():+ # if isinstance(m, tuple(self.supported_types)):+ # delattr(m, "tmp_name")+ # return list(res)++ def get_layer_names_in_block(self):+ for n, m in self.model.named_modules():+ if isinstance(m, tuple(self.supported_types)):+ m.tmp_name = n+ layers_in_block = []+ block_names = get_block_names(self.model)+ for block_name in block_names:+ block = get_module(self.model, block_name)+ for n, m in block.named_modules():+ if hasattr(m, "tmp_name"):+ layers_in_block.append(m.tmp_name)+ for n, m in self.model.named_modules():+ if hasattr(m, "tmp_name"):+ delattr(m, "tmp_name")+ return layers_in_block++ def get_quantized_layer_names_outside_blocks(self):+ """Gets the names of quantized layers outside blocks in the model.++ Returns:+ list: List of layer names outside blocks.+ """+ if self.weight_config is None or len(self.weight_config) == 0:+ return []++ layer_names = []+ all_layers_in_block = self.get_layer_names_in_block()++ for key in self.weight_config.keys():+ if key in all_layers_in_block:+ continue+ try:+ layer = get_module(self.model, key)+ if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.weight_config[key]):+ layer_names.append(key)+ except:+ logger.error(f"could not find layer {key} in the model, exit...")+ exit()++ return layer_names+def quantize(self):"""Quantize the model and return the quantized model along with weight configurations.@@ -1055,10 +1342,18 @@ class AutoRound(object):self.model = self.model.to(self.amp_dtype)if not self.low_gpu_mem_usage:self.model = self.model.to(self.device)- inputs = self.cache_block_input(block_names[0], self.n_samples)++ layer_names = self.get_quantized_layer_names_outside_blocks()+ self.start_time = time.time()+ all_inputs = self.cache_inter_data([block_names[0]], self.n_samples, layer_names=layer_names)+ del self.inputs+ inputs = all_inputs[block_names[0]]++ all_inputs.pop(block_names[0])+ self.inputs = Nonedel self.inputsif "input_ids" in inputs.keys():- dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) ##polish the codetotal_samples = inputs["input_ids"].shape[dim]self.n_samples = total_samplesif total_samples < self.train_bs:@@ -1073,6 +1368,22 @@ class AutoRound(object):n_blocks=self.n_blocks,device=self.device,)+ ##TODO currently we take all the layers outside blocks as post block layers which is not optimal+ if len(layer_names) > 0:+ torch.cuda.empty_cache()+ layer_inputs = all_inputs+ q_layer_inputs = None+ if self.use_quant_input:+ if not self.low_gpu_mem_usage:+ self.model = self.model.to(self.device)+ q_layer_inputs = self.cache_inter_data([], self.n_samples, layer_names=layer_names)+ self.model = self.model.to("cpu")+ torch.cuda.empty_cache()+ for layer_name in layer_names:+ q_layer_input = q_layer_inputs[layer_name] if self.use_quant_input else None+ self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)+ torch.cuda.empty_cache()+for n, m in self.model.named_modules():if n in self.weight_config.keys():if hasattr(m, "scale"):@@ -1156,7 +1467,6 @@ class AutoOPTRound(AutoRound):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernelshave different choices.- optimizer: string or object**kwargs: Additional keyword arguments.Returns:@@ -1311,7 +1621,6 @@ class AutoAdamRound(AutoOPTRound):optimizer: string or objectscale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernelshave different choices.- **kwargs: Additional keyword arguments.Returns:The quantized model.diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 41197f5a7..242604fbb 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -15,7 +15,6 @@ def register_dataset(name):Decorator function used before a Pattern subclass.Args:- cls (class): The subclass of register.name: A string. Define the dataset type.Returns:diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.pyindex ae5a42384..76023b788 100644--- a/auto_round/export/export_to_itrex/config.py+++ b/auto_round/export/export_to_itrex/config.py@@ -25,18 +25,18 @@ from typing import Any, Dict, Tuple, Unionimport torchfrom transformers import PretrainedConfig-from auto_round.utils import logger+from auto_round.utils import convert_dtype_str2torch, convert_dtype_torch2str, loggerQUANT_CONFIG = "quantize_config.json"class QuantConfig(PretrainedConfig):- """A brief quantization configuration for reference when performing model dequantization."""+ """A brief quantization configuration for reference when performing model de-quantization."""def __init__(self,bits=4,- scale_dtype="torch.float32",+ scale_dtype="fp32",group_size=128,sym=False,quant_method="autoround",@@ -47,11 +47,12 @@ class QuantConfig(PretrainedConfig):lr=0.001,minmax_lr=0.001,use_quant_input=True,+ compute_dtype=None,**kwargs,):self.bits = bitsself.group_size = group_size- self.scale_dtype = scale_dtype+ self.scale_dtype = convert_dtype_torch2str(scale_dtype)self.sym = symself.quant_method = quant_methodself.model_name_or_path = model_name_or_path@@ -61,24 +62,73 @@ class QuantConfig(PretrainedConfig):self.lr = lrself.minmax_lr = minmax_lrself.use_quant_input = use_quant_input-- ### Redundant parameters, will be removed later. ###- self.damp_percent = 0.01- self.desc_act = False- self.true_sequential = False- self.quant_method = "gptq"+ self.compute_dtype = convert_dtype_torch2str(compute_dtype)++ if "export_to_xpu" not in kwargs or not kwargs["export_to_xpu"]:+ ### Redundant parameters, will be removed later. ###+ self.damp_percent = 0.01+ self.desc_act = False+ self.true_sequential = False+ self.quant_method = "gptq"+ else:+ ### XPU special parameters. ###+ self.weight_dtype = "int4_fullrange" # Due to ipex format limitations. Actually, it's int4_clip.def post_init(self):- r"""- Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.- """+ r"""Safety checker for CPU that arguments are correct+ also replaces some NoneType arguments with their default values."""- if self.scale_dtype not in ["torch.float32", "torch.float16", "torch.bfloat16"]:+ if self.scale_dtype not in ["fp32", "fp16", "bf16"]:raise ValueError("scale_dtype must be 'fp32', 'fp16' or 'bf16'.")if self.group_size not in [-1, 32, 128]:raise ValueError("group_size must be an integer in [-1, 32, 128]")+ def post_init_xpu(self):+ r"""+ Safety checker for XPU that arguments are correct+ - also replaces some NoneType arguments with their default values.+ """++ if self.compute_dtype is not None and self.compute_dtype not in ["fp16"]:+ raise ValueError("compute_dtype must be 'fp16'.")+ elif self.compute_dtype is None:+ self.compute_dtype = "fp16"++ if self.bits is None:+ self.bits = 4+ elif self.bits not in [4]:+ raise ValueError(f"Only support quantization to [4] bits but found {self.bits}")++ if self.weight_dtype is None:+ self.weight_dtype = "int4_fullrange"++ elif self.weight_dtype not in [+ "int4_fullrange",+ ]:+ raise ValueError(f"weight_dtype must be a string in 'int4_fullrange', but get {self.weight_dtype}.")++ if self.scale_dtype is not None and self.scale_dtype not in ["fp16"]:+ raise ValueError("scale_dtype must be a string in 'fp16'")+ elif self.scale_dtype is None:+ self.scale_dtype = "fp16"++ # if not isinstance(self.use_double_quant, bool):+ # raise ValueError("use_double_quant must be a boolean")++ # if self.use_double_quant and not isinstance(self.double_quant_dtype, str):+ # raise ValueError("double_quant_dtype must be a string")++ # if self.use_double_quant and not isinstance(self.scale_dtype, str):+ # raise ValueError("scale_dtype must be a string")++ if not isinstance(self.group_size, int):+ raise ValueError("group_size must be a int")++ if self.sym is not True:+ raise ValueError("asym is not support, only support 'sym' now!")+ self.use_neural_speed = False+def quantization_method(self):r"""This method returns the quantization method used for the model."""return self.quant_method@@ -229,3 +279,25 @@ class QuantConfig(PretrainedConfig):cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]:return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)++ def remove_redundant_parameters(self):+ remove_parameters = [+ "calib_dataloader",+ "dataset",+ "scheme",+ "tokenizer",+ "use_neural_speed",+ "use_quant_input",+ "layer_wise",+ "nsamples",+ "lr",+ "minmax_lr",+ "iters",+ "use_quant_input",+ "model_file_base_name",+ "enable_minmax_tuning",+ "model_name_or_path",+ ]+ for parameter in remove_parameters:+ if hasattr(self, parameter):+ delattr(self, parameter)diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 578f50ec8..f56797a41 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -56,9 +56,58 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):minmax_lr=minmax_lr,enable_minmax_tuning=enable_minmax_tuning,use_quant_input=use_quant_input,- scale_dtype=str(scale_dtype),+ scale_dtype=scale_dtype,)if quantize_config is not None:+ quantize_config.post_init()+ config = compressed_model.config+ setattr(config, "quantization_config", quantize_config.to_dict())+ config.save_pretrained(output_dir)+ quantize_config.save_pretrained(output_dir)+ try:+ compressed_model.save_pretrained(output_dir, safe_serialization=True)+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))+ except IOError as e: # pragma: no cover+ logger.error("Fail to save configure file and weights due to {}.".format(e))+ return compressed_model+++@register_format("itrex_xpu")+def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):+ """Save configure file and weights for CPU backend inference."""+ model = kwargs["model"]+ weight_config = kwargs["weight_config"]+ sym = kwargs["sym"]+ bits = kwargs["bits"]+ group_size = kwargs["group_size"]+ iters = kwargs["iters"]+ lr = kwargs["lr"]+ minmax_lr = kwargs["minmax_lr"]+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]+ use_quant_input = kwargs["use_quant_input"]+ scale_dtype = kwargs["scale_dtype"]+ tokenizer = kwargs["tokenizer"]++ compressed_model = pack_model(inplace=inplace, **kwargs)+ if output_dir is None:+ return compressed_model+ quantize_config = QuantConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=iters,+ lr=lr,+ minmax_lr=minmax_lr,+ enable_minmax_tuning=enable_minmax_tuning,+ use_quant_input=use_quant_input,+ scale_dtype=scale_dtype,+ export_to_xpu=True,+ )+ if quantize_config is not None:+ quantize_config.post_init_xpu()+ quantize_config.remove_redundant_parameters()config = compressed_model.configsetattr(config, "quantization_config", quantize_config.to_dict())config.save_pretrained(output_dir)@@ -82,6 +131,7 @@ def pack_model(device="cpu",use_optimum_format=True,inplace=False,+ **kwargs,):"""Convert Linear to WeightOnlyLinear for low memory inference.@@ -101,6 +151,13 @@ def pack_model(4. parameter name changed, such as 'packed_weight' -> 'qweight'.5. zeros is always needed even for sym.inplace (bool, optional): Compress the model in place, or copy the model and compress it.++ xpu args:+ compression_dtype=torch.int8,+ compression_dim=0,+ use_optimum_format=False,+ scale_dtype=convert_dtype_str2torch(config.scale_dtype),+ device="xpu","""if inplace:compressed_model = model@@ -127,12 +184,20 @@ def pack_model(if not isinstance(scale, torch.Tensor):scale = torch.tensor(scale, dtype=convert_dtype)zp = torch.tensor(zp, dtype=torch.int32)+ if device == "xpu":+ scale = torch.tensor(v["scale"], dtype=torch.float32)+ zp = None if sym else torch.tensor(v["zp"], dtype=torch.int32)else:if not inplace:scale = scale.clone()zp = zp.clone()- scale = scale.to(dtype=convert_dtype)- zp = zp.to(dtype=torch.int32)+ if device == "xpu":+ # Please note that for XPU, the scale data type is forcibly set to fp32+ scale = scale.to(dtype=torch.float32)+ zp = None if sym else zp.to(dtype=torch.int32)+ else:+ scale = scale.to(dtype=convert_dtype)+ zp = zp.to(dtype=torch.int32)int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)int_weight = int_weight.type(torch.int32)@@ -145,10 +210,11 @@ def pack_model(scale_dtype=scale_dtype,zp=zp is not None,bias=m.bias is not None,- device=device,- use_optimum_format=True,+ device="cuda" if device == "xpu" else device,+ compression_dtype=compression_dtype,+ compression_dim=compression_dim,+ use_optimum_format=use_optimum_format, # xpu is False)new_module.pack(int_weight, scale, zp, m.bias)set_module(compressed_model, k, new_module)-return compressed_modeldiff --git a/auto_round/utils.py b/auto_round/utils.pyindex d5524c1e9..f53939daa 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -20,6 +20,7 @@ from collections import UserDict# for cpu usageimport cpuinfo+import numpy as npimport psutilimport torchfrom torch.amp import autocast@@ -588,4 +589,109 @@ class CpuInfo(object):def is_local_path(path):+ """Checks if a given path exists locally.++ Args:+ path (str): The path to check.++ Returns:+ bool: True if the path exists locally, False otherwise.+ """return os.path.exists(path)+++def convert_dtype_str2torch(str_dtype):+ """Converts a string dtype to its corresponding PyTorch dtype.++ Args:+ str_dtype (str): The string representation of the dtype.++ Returns:+ torch.dtype: The PyTorch dtype.++ Raises:+ AssertionError: If the input str_dtype is unsupported.+ """+ if isinstance(str_dtype, torch.dtype) or str_dtype is None:+ return str_dtype+ if str_dtype == "int8":+ return torch.int8+ elif str_dtype == "fp32" or str_dtype == "auto":+ return torch.float+ elif str_dtype == "fp16":+ return torch.float16+ elif str_dtype == "bf16":+ return torch.bfloat16+ else:+ assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)+++def convert_dtype_torch2str(dtype):+ """Converts a PyTorch dtype to its corresponding string representation.++ Args:+ dtype: PyTorch dtype or str. The dtype to convert.++ Returns:+ str: The string representation of the dtype.++ Raises:+ AssertionError: If the input dtype is unsupported.+ """+ if isinstance(dtype, str) or dtype is None:+ return dtype+ if dtype == torch.int8:+ return "int8"+ elif dtype == torch.float:+ return "fp32"+ elif dtype == torch.float16:+ return "fp16"+ elif dtype == torch.bfloat16:+ return "bf16"+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:+ return dtype+ else:+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+++def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):+ """Checks the availability of memory on the specified device for processing inputs using a given weight tensor.++ Args:+ device (str): The device type ('cuda' for GPU or 'hpu' for HPU).+ inputs (torch.Tensor): Input tensor.+ weight (torch.Tensor): Weight tensor.+ org_seqlen (int): Original sequence length.+ org_bs (int): Original batch size.++ Returns:+ tuple: A tuple containing availability status (bool), modified sequence length (int),+ and modified batch size (int).+ """+ weight_memory = weight.numel() * weight.element_size()+ if "cuda" in device:+ current_gpu_index = torch.cuda.current_device()+ total_memory = torch.cuda.get_device_properties(current_gpu_index).total_memory+ used_memory = torch.cuda.memory_allocated(current_gpu_index)+ free_space = total_memory - used_memory+ elif "hpu" in device:+ current_hpu_index = torch.hpu.current_device()+ free_space = torch.hpu.memory_reserved(current_hpu_index)+ else:+ return True, org_seqlen, org_bs++ free_space = free_space - weight_memory * 10 # for min_max_scale & grad usage+ seqlen = org_seqlen+ bs = org_bs+ in_feature = weight.shape[1]+ out_feature = weight.shape[0]+ while seqlen >= 128:+ input_size = bs * seqlen * in_feature+ output_size = bs * seqlen * out_feature+ input_output_memory = 2 * (input_size * inputs.element_size() + output_size * inputs.element_size())+ if input_output_memory < free_space:+ return True, seqlen, bs+ seqlen = seqlen // 2+ bs = 1++ return False, seqlen, bsdiff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 31540bd1e..3a65084c7 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -44,7 +44,7 @@ if __name__ == '__main__':parser.add_argument("--iters", default=200, type=int,help=" iters")-+parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,help="The dataset for quantization training. It can be a custom one.")@@ -89,7 +89,7 @@ if __name__ == '__main__':help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")- parser.add_argument("--scale_dtype", default='fp32',+ parser.add_argument("--scale_dtype", default='fp16',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")parser.add_argument("--tasks",@@ -116,6 +116,8 @@ if __name__ == '__main__':parser.add_argument("--disable_trust_remote_code", action='store_true',help="Whether to disable trust_remote_code")+ parser.add_argument("--quant_lm_head", action='store_true',+ help="quant_lm_head")args = parser.parse_args()if args.low_gpu_mem_usage:@@ -152,6 +154,7 @@ if __name__ == '__main__':from eval import eval_modelelse:from eval_legacy import eval_model+if isinstance(tasks, list):if "mmlu" in tasks:tmp_tasks = tasks@@ -167,7 +170,7 @@ if __name__ == '__main__':if use_eval_legacy:print("Using the legacy lm_eval(0.3.0)")else:- print("Using the latest lm_eval(0.4.1)")+ print(f"Using the latest {res}")model_name = args.model_nameif model_name[-1] == "/":@@ -187,7 +190,8 @@ if __name__ == '__main__':model = AutoModel.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)else:model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=not args.disable_trust_remote_code+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ trust_remote_code=not args.disable_trust_remote_code)from auto_round import (AutoRound,@@ -203,14 +207,14 @@ if __name__ == '__main__':model.seqlen = seqlenseqlen = args.seqlen- if "llama" in model_name:- from transformers import LlamaTokenizer-- tokenizer = LlamaTokenizer.from_pretrained(model_name)- if tokenizer.pad_token is None:- tokenizer.add_special_tokens({'pad_token': '[PAD]'})- else:- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ # if "llama" in model_name:+ # from transformers import LlamaTokenizer+ #+ # tokenizer = LlamaTokenizer.from_pretrained(model_name)+ # if tokenizer.pad_token is None:+ # tokenizer.add_special_tokens({'pad_token': '[PAD]'})+ # else:+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)if hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:@@ -219,6 +223,7 @@ if __name__ == '__main__':seqlen = min(seqlen, tokenizer.model_max_length)args.seqlen = seqlen+ excel_name = f"{model_name}_{args.bits}_{args.group_size}"pt_dtype = torch.float16if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):dtype = 'bfloat16'@@ -233,7 +238,7 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"if "bloom" in model_name:- args.disable_low_gpu_mem_usage=True+ args.disable_low_gpu_mem_usage = Trueif args.eval_fp16_baseline:if args.disable_low_gpu_mem_usage:model = model.to(torch_device)@@ -257,7 +262,8 @@ if __name__ == '__main__':weight_config[n] = {"data_type": "fp"}print(f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")-+ if args.quant_lm_head:+ weight_config['lm_head'] = {"data_type": "int"}autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,@@ -265,7 +271,7 @@ if __name__ == '__main__':low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, weight_config=weight_config,- enable_minmax_tuning=not args.disable_minmax_tuning) ##TODO args pass+ enable_minmax_tuning=not args.disable_minmax_tuning)model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")@@ -278,6 +284,10 @@ if __name__ == '__main__':deployment_device = args.deployment_device.split(',')if 'gpu' in deployment_device:autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)+ if 'xpu' in deployment_device:+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=False,+ compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,+ device="xpu")if "cpu" in deployment_device:autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)if "fake" in deployment_device:diff --git a/examples/language-modeling/run_xpu.sh b/examples/language-modeling/run_xpu.shnew file mode 100644index 000000000..12126ef72--- /dev/null+++ b/examples/language-modeling/run_xpu.sh@@ -0,0 +1,23 @@+#!/bin/bash++device=1+eval_bs=32+model_name="/facebook/opt-125m"+++CUDA_VISIBLE_DEVICES=$device \+python3 main.py \+--model_name $model_name \+--device 0 \+--group_size 128 \+--bits 4 \+--iters 10 \+--use_quant_input \+--deployment_device 'xpu,fake' \+--quantize_layers_outside_blocks \+--disable_low_gpu_mem_usage \+--disable_eval \+--output_dir "./test-export" \+--scale_dtype "fp16" \+--group_size 32 \+2>&1 | tee -a 4.16_xpu.logdiff --git a/test/test_autoround.py b/test/test_autoround.pyindex 2235fd5e0..cb03e7430 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -152,6 +152,24 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_lm_head(self):+ bits, group_size, sym = 4, -1, False+ weight_config = {"lm_head": {"data_type": "int"}}+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=10,+ enable_minmax_tuning=False,+ use_quant_input=False,+ dataloader=self.llm_dataloader,+ weight_config=weight_config,+ )+ autoround.quantize()+if __name__ == "__main__":unittest.main()