PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add initial support for activation quantization (#176)
wenhuach215f67048c878b
diff --git a/README.md b/README.mdindex 72afb77b1..ea6b26e8c 100644--- a/README.md+++ b/README.md@@ -2,7 +2,7 @@AutoRound===========================-<h3> Advanced Weight-Only Quantization Algorithm for LLMs</h3>+<h3> Advanced Quantization Algorithm for LLMs</h3>[](https://github.com/intel/auto-round)[](https://github.com/intel/auto-round)@@ -10,7 +10,7 @@ AutoRound---<div align="left">-AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference. It's tailored for a wide range+AutoRound is an advanced quantization algorithm for low-bits LLM inference. It's tailored for a wide rangeof models. Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,which competes impressively against recent methods without introducing any additional inference overhead. The belowimage presents an overview of AutoRound. Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)@@ -120,7 +120,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_rNeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"-- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits+- `layer_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bitsor mixed precision.- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex dab5c7f21..2ddd1f1f0 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -224,7 +224,7 @@ class AutoRoundConfig(QuantizationConfigMixin):group_size: int = 128,sym: bool = False,backend="autoround:exllamav2",- weight_config: dict = None,+ layer_config: dict = None,**kwargs,):@@ -234,7 +234,7 @@ class AutoRoundConfig(QuantizationConfigMixin):self.group_size = group_sizeself.sym = symself.backend = backend- self.weight_config = weight_config+ self.layer_config = layer_configif kwargs is not None:for key in kwargs.keys():setattr(self, key, kwargs[key])diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 51159c14c..85421f2ec 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -54,14 +54,19 @@ class AutoRound(object):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (bool): Whether symmetric quantization is to be used (default is False).- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).- weight_config={+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config={'layer1':##layer_name{'data_type': 'int','bits': 4,- 'group_size': 32,+ 'group_size': 128,'sym': False+ 'act_data_type': None,+ 'act_bits': 32,+ 'group_size': None,+ 'sym': None,+}...}@@ -89,6 +94,10 @@ class AutoRound(object):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.+ act_bits (int): Number of bits for activation quantization. Default is 32.+ act_group_size (int): Group size for activation quantization. Default is None.+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.Returns:The quantized model.@@ -101,11 +110,11 @@ class AutoRound(object):bits: int = 4,group_size: int = 128,sym: bool = False,- weight_config: dict = {},+ layer_config: dict = {},enable_full_range: bool = False, ##for symmetric, TODO support laterbatch_size: int = 8,amp: bool = True,- device=None,+ device: str = None,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,@@ -122,8 +131,12 @@ class AutoRound(object):gradient_accumulate_steps: int = 1,not_use_best_mse: bool = False,dynamic_max_gap: int = -1,- data_type: str = "int", ##only support int for now+ data_type: str = "int",scale_dtype: str = "fp16",+ act_bits: int = 32,+ act_group_size: int = None,+ act_sym: bool = None,+ act_dynamic: bool = True,**kwargs,):self.quantized = False@@ -140,7 +153,7 @@ class AutoRound(object):self.low_gpu_mem_usage = low_gpu_mem_usageself.data_type = data_typeself.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]- self.weight_config = weight_config+ self.layer_config = layer_configself.seed = seedself.tokenizer = tokenizerself.seqlen = seqlen@@ -165,11 +178,15 @@ class AutoRound(object):self.dynamic_max_gap = dynamic_max_gapself.enable_full_range = enable_full_rangeself.lr_scheduler = lr_scheduler- self.set_layerwise_config(self.weight_config)self.optimizer = self.get_optimizer(None)self.share_attention_mask_flag = Noneself.hidden_dim_flag = Noneself.infer_bs_coeff = 1+ self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size+ self.act_bits = act_bits if not (act_bits is None) else self.bits+ self.act_sym = act_sym if not (act_sym is None) else self.sym+ self.act_dynamic = act_dynamic+ self.set_layerwise_config(self.layer_config)torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()logger.info(f"using {self.model.dtype} for quantization tuning")@@ -186,27 +203,31 @@ class AutoRound(object):"""assert isinstance(self.model, torch.nn.Module)assert self.bits > 0, "bits must be positive"+ assert self.act_bits > 0, "bits must be positive"assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"+ assert self.act_group_size == -1 or self.act_group_size >= 1,\+ "only supports positive group_size or -1(per channel)"assert self.train_bs > 0, "batch size must be positive"assert self.iters > 0, "iters must be positive"assert self.seqlen > 0, "seqlen must be positive"assert self.nblocks > 0, "nblocks must be positive"assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"assert self.enable_full_range is False, "only support enable_full_range=False currently"+ assert self.act_dynamic is True, "only support dynamic quantization for activation currently"# assert self.tokenizer != None or self.dataloader != Nonedef quantize(self):- """Quantize the model and return the quantized model along with weight configurations.+ """Quantize the model and return the quantized model along with layer configurations.the entry of AutoRound.Returns:- The quantized model and weight configurations.+ The quantized model and layer configurations."""# logger.info("cache block input")block_names = get_block_names(self.model)if len(block_names) == 0:logger.warning("could not find blocks, exit with original model")- return self.model, self.weight_config+ return self.model, self.layer_configif self.amp:self.model = self.model.to(self.amp_dtype)@@ -239,7 +260,7 @@ class AutoRound(object):self.quant_layers(layer_names, all_inputs)- self.dump_data_to_weight_config()+ self.dump_qinfo_to_layer_config()end_time = time.time()cost_time = end_time - self.start_time@@ -250,7 +271,7 @@ class AutoRound(object):unquantized_layers = []for n, m in self.model.named_modules():if isinstance(m, tuple(self.supported_types)):- if self.weight_config[n]["bits"] == 16:+ if m.bits > 8:unquantized_layers.append(n)else:quantized_layers.append(n)@@ -263,39 +284,45 @@ class AutoRound(object):self.quantized = True##self.model = self.model.to(self.model_orig_dtype)##keep it as amp dtype- return self.model, self.weight_config+ return self.model, self.layer_config- def dump_data_to_weight_config(self):+ def dump_qinfo_to_layer_config(self):"""- dump quantization scale and zp to weight configuration+ dump quantization scale and zp to layer configurationArgs:Returns:None"""for n, m in self.model.named_modules():- if n not in self.weight_config.keys():+ if n not in self.layer_config.keys():continueif hasattr(m, "scale"):- self.weight_config[n]["scale"] = m.scale- self.weight_config[n]["zp"] = m.zp+ self.layer_config[n]["scale"] = m.scale+ self.layer_config[n]["zp"] = m.zp+ if isinstance(m, transformers.modeling_utils.Conv1D):+ weight = m.weight.t()+ else:+ weight = m.weight+if self.group_size <= 0:- self.weight_config[n]["g_idx"] = torch.tensor(- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"++ self.layer_config[n]["g_idx"] = torch.tensor(+ [0 for i in range(weight.shape[1])], dtype=torch.int32, device="cpu")else:- self.weight_config[n]["g_idx"] = torch.tensor(- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"+ self.layer_config[n]["g_idx"] = torch.tensor(+ [i // self.group_size for i in range(weight.shape[1])], dtype=torch.int32, device="cpu")delattr(m, "scale")delattr(m, "zp")else:- self.weight_config[n]["data_type"] = "float"+ self.layer_config[n]["data_type"] = "float"if self.amp_dtype == torch.bfloat16:- self.weight_config[n]["data_type"] = "bfloat"- self.weight_config[n]["bits"] = 16- self.weight_config[n]["group_size"] = None- self.weight_config[n]["sym"] = None+ self.layer_config[n]["data_type"] = "bfloat"+ self.layer_config[n]["bits"] = 16+ self.layer_config[n]["group_size"] = None+ self.layer_config[n]["sym"] = Nonedef quant_layers(self, layer_names, layer_inputs):"""Quantizes specified layers based on inputs and configuration.@@ -328,51 +355,41 @@ class AutoRound(object):q_layer_input[i] = Nonetorch.cuda.empty_cache()- def set_layerwise_config(self, weight_config):- """Sets the layer-wise configuration based on the provided weight_config.+ def set_layerwise_config(self, layer_config):+ """Sets the layer-wise configuration based on the provided layer_config.By default, only quantize layers in blocks.Args:- weight_config: The weight configuration.+ layer_config: The layer configuration.Returns:None"""layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types)+ keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",+ "act_dynamic"]for n, m in self.model.named_modules():if not isinstance(m, tuple(self.supported_types)):continue- if n not in weight_config.keys() and n in layers_in_blocks:- weight_config[n] = {}- weight_config[n]["data_type"] = self.data_type- weight_config[n]["bits"] = self.bits- weight_config[n]["group_size"] = self.group_size- weight_config[n]["sym"] = self.sym- weight_config[n]["scale_dtype"] = self.scale_dtype- elif n in weight_config.keys():- if "data_type" not in weight_config[n].keys():- weight_config[n]["data_type"] = self.data_type- if "bits" not in weight_config[n].keys():- weight_config[n]["bits"] = self.bits- if "group_size" not in weight_config[n].keys():- weight_config[n]["group_size"] = self.group_size- if "sym" not in weight_config[n].keys():- weight_config[n]["sym"] = self.sym- if "scale_dtype" not in weight_config[n].keys():- weight_config[n]["scale_dtype"] = self.scale_dtype- else:- weight_config[n] = {}- weight_config[n]["data_type"] = "float"- weight_config[n]["bits"] = 16- weight_config[n]["group_size"] = self.group_size- weight_config[n]["sym"] = self.sym- weight_config[n]["scale_dtype"] = self.scale_dtype-- m.data_type = weight_config[n]["data_type"]- m.bits = weight_config[n]["bits"]- m.group_size = weight_config[n]["group_size"]- m.sym = weight_config[n]["sym"]- m.scale_dtype = weight_config[n]["scale_dtype"]+ ##not set in layer config, so use the default values+ if n not in layer_config.keys() and n in layers_in_blocks:+ layer_config[n] = {}+ for key in keys:+ layer_config[n][key] = getattr(self, key)+ elif n in layer_config.keys(): ## partly set+ for key in keys:+ if key not in layer_config[n].keys():+ layer_config[n][key] = getattr(self, key)+ else: ##not in layer_config and layers in block,+ layer_config[n] = {}+ for key in keys:+ layer_config[n][key] = getattr(self, key)+ layer_config[n]["bits"] = 32+ layer_config[n]["act_bits"] = 32++ for key in keys:+ setattr(m, key, layer_config[n][key])+ tmp=1@torch.no_grad()def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device):@@ -1009,6 +1026,13 @@ class AutoRound(object):if not self.quantized:logger.warning("please run autoround.quantize first")return+ if format == "fake" or format == "qdq" or self.act_bits <= 8: ##TODO fix act quantizaiton later+ self.model = self.model.to("cpu")+ self.model.save_pretrained(output_dir)+ if self.tokenizer is not None:+ self.tokenizer.save_pretrained(output_dir)+ return+from auto_round.export import EXPORT_FORMATbackend = formatformat = format.split(":")[0]@@ -1049,7 +1073,7 @@ class AutoRound(object):compressed_model = save_quantized_as_format( ##TODO refine the codeoutput_dir,model=self.model,- weight_config=self.weight_config,+ layer_config=self.layer_config,inplace=inplace,bits=self.bits,group_size=self.group_size,@@ -1075,20 +1099,20 @@ class AutoRound(object):Returns:list: List of layer names outside blocks."""- if self.weight_config is None or len(self.weight_config) == 0:+ if self.layer_config is None or len(self.layer_config) == 0:return []layer_names = []all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types)- for key in self.weight_config.keys():+ for key in self.layer_config.keys():if key in all_layers_in_block:continuelayer = get_module(self.model, key)if layer is None:logger.error(f"could not find layer {key} in the model, exit...")exit()- if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.weight_config[key]):+ if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):layer_names.append(key)return layer_names@@ -1174,7 +1198,7 @@ class AutoOPTRound(AutoRound):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (bool): Whether sym to be used (default is False).- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).enable_full_range (bool): Whether to enable full range quantization (default is False).batch_size (int): Batch size for training (default is 8).amp (bool): Whether to use automatic mixed precision (default is True).@@ -1198,6 +1222,11 @@ class AutoOPTRound(AutoRound):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.+ act_bits (int): Number of bits for activation quantization. Default is 32.+ act_group_size (int): Group size for activation quantization. Default is None.+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.+**kwargs: Additional keyword arguments.Returns:@@ -1211,7 +1240,7 @@ class AutoOPTRound(AutoRound):bits: int = 4,group_size: int = 128,sym: bool = False,- weight_config: dict = {},+ layer_config: dict = {},enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,@@ -1234,6 +1263,10 @@ class AutoOPTRound(AutoRound):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",+ act_bits: int = 32,+ act_group_size: int = None,+ act_sym: bool = None,+ act_dynamic: bool = True,optimizer="AdamW",**kwargs,):@@ -1243,7 +1276,7 @@ class AutoOPTRound(AutoRound):bits,group_size,sym,- weight_config,+ layer_config,enable_full_range,batch_size,amp,@@ -1266,6 +1299,10 @@ class AutoOPTRound(AutoRound):dynamic_max_gap,data_type,scale_dtype,+ act_bits,+ act_group_size,+ act_sym,+ act_dynamic,**kwargs,)@@ -1321,7 +1358,7 @@ class AutoAdamRound(AutoOPTRound):bits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (str): Whether symmetric quantization to be used (default is False).- weight_config (dict): Configuration for weight quantization (default is an empty dictionary).+ layer_config (dict): Configuration for weight quantization (default is an empty dictionary).enable_full_range (bool): Whether to enable full range quantization (default is False).batch_size (int): Batch size for training (default is 8).amp (bool): Whether to use automatic mixed precision (default is True).@@ -1347,6 +1384,10 @@ class AutoAdamRound(AutoOPTRound):optimizer: string or objectscale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.+ act_bits (int): Number of bits for activation quantization. Default is 32.+ act_group_size (int): Group size for activation quantization. Default is None.+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.Returns:The quantized model.@@ -1359,7 +1400,7 @@ class AutoAdamRound(AutoOPTRound):bits: int = 4,group_size: int = 128,sym: bool = False,- weight_config: dict = {},+ layer_config: dict = {},enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,@@ -1382,6 +1423,10 @@ class AutoAdamRound(AutoOPTRound):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",+ act_bits: int = 32,+ act_group_size: int = None,+ act_sym: bool = None,+ act_dynamic: bool = True,optimizer="AdamW",**kwargs,):@@ -1391,7 +1436,7 @@ class AutoAdamRound(AutoOPTRound):bits,group_size,sym,- weight_config,+ layer_config,enable_full_range,batch_size,amp,@@ -1414,6 +1459,10 @@ class AutoAdamRound(AutoOPTRound):dynamic_max_gap,data_type,scale_dtype,+ act_bits,+ act_group_size,+ act_sym,+ act_dynamic,optimizer,**kwargs,)diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex acff4fe4a..a80015823 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -57,7 +57,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarexcept ImportError:raise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")model = kwargs["model"]- weight_config = kwargs["weight_config"]+ layer_config = kwargs["layer_config"]sym = kwargs["sym"]bits = kwargs["bits"]group_size = kwargs["group_size"]@@ -105,10 +105,10 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarif bits == 3 and use_triton is True:logger.warning("triton does not support 3 bits, reset it to False")quantizers = {}- for key in weight_config:+ for key in layer_config:if key == "lm_head": ##TODO remove this after pr 87 is mergedcontinue- info = weight_config[key]+ info = layer_config[key]if not check_to_quantized(info):continue##force to float32 to be compatible with torch 2.0@@ -125,10 +125,10 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar)else:quantizers = {}- for key in weight_config:+ for key in layer_config:if key == "lm_head": ##TODO remove this after pr 87 is mergedcontinue- info = weight_config[key]+ info = layer_config[key]if not check_to_quantized(info):continuequantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 68aa4585f..f7c5e535d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -147,7 +147,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exlDefault is "autoround:exllamav2".**kwargs: Additional keyword arguments including:- model (nn.Module): The model to be quantized.- - weight_config (dict): The weight configuration for each layer.+ - layer_config (dict): The layer configuration for each layer.- serialization_dict (dict): The serialization configuration.- tokenizer (Tokenizer, optional): The tokenizer to be saved.@@ -171,12 +171,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exlmodel = copy.deepcopy(model.to("cpu"))layer_names_in_block = get_layer_names_in_block(model)- weight_config = kwargs["weight_config"]+ layer_config = kwargs["layer_config"]with tctl.threadpool_limits(limits=1):- for name in weight_config.keys():+ for name in layer_config.keys():- config = kwargs["weight_config"][name]+ config = kwargs["layer_config"][name]if config["data_type"] != "int" and config["bits"] >= 16:continuelogger.info(f"packing {name}")@@ -208,8 +208,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exlnew_layer.device = deviceset_module(model, name, new_layer)qlayer = new_layer- scale = weight_config[name]["scale"]- zero = weight_config[name]["zp"]+ scale = layer_config[name]["scale"]+ zero = layer_config[name]["zp"]# so far can only pack layer on CPUqlayer.to("cpu")##force to float32 to be compatible with torch 2.0@@ -225,18 +225,18 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exlquantization_config["quant_method"] = "intel/auto-round"quantization_config["backend"] = backendextra_config = {}- for layer_name in weight_config:- if weight_config[layer_name]["bits"] >= 16:+ for layer_name in layer_config:+ if layer_config[layer_name]["bits"] >= 16:continueif layer_name not in layer_names_in_block:extra_config[layer_name] = {}- extra_config[layer_name]["bits"] = weight_config[layer_name]["bits"]- extra_config[layer_name]["data_type"] = weight_config[layer_name]["data_type"]- extra_config[layer_name]["group_size"] = weight_config[layer_name]["group_size"]- extra_config[layer_name]["sym"] = weight_config[layer_name]["sym"]+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]else:neq_keys = check_neq_config(- weight_config[layer_name],+ layer_config[layer_name],data_type=quantization_config["data_type"],bits=quantization_config["bits"],group_size=quantization_config["group_size"],@@ -245,7 +245,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exlif len(neq_keys) > 0:extra_config[layer_name] = {}for key in neq_keys:- extra_config[layer_name][key] = weight_config[layer_name][key]+ extra_config[layer_name][key] = layer_config[layer_name][key]if len(extra_config) > 0:quantization_config["extra_config"] = extra_configif hasattr(model, "config"):diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex d6d6fe679..d7220a78d 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -64,7 +64,7 @@ def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):"""Save configure file and weights for CPU backend inference."""model = kwargs["model"]- weight_config = kwargs["weight_config"]+ layer_config = kwargs["layer_config"]sym = kwargs["sym"]bits = kwargs["bits"]group_size = kwargs["group_size"]@@ -76,7 +76,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):scale_dtype = kwargs["scale_dtype"]tokenizer = kwargs["tokenizer"]- compressed_model = pack_model(model, weight_config, inplace=inplace)+ compressed_model = pack_model(model, layer_config, inplace=inplace)if output_dir is None:return compressed_modelquantize_config = QuantConfig(@@ -110,7 +110,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):"""Save configure file and weights for CPU backend inference."""model = kwargs["model"]- weight_config = kwargs["weight_config"]+ layer_config = kwargs["layer_config"]sym = kwargs["sym"]bits = kwargs["bits"]group_size = kwargs["group_size"]@@ -156,7 +156,7 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):def pack_model(model,- weight_config: Union[str, dict],+ layer_config: Union[str, dict],enable_full_range=False,compression_dtype=torch.int32,compression_dim=1,@@ -168,7 +168,7 @@ def pack_model("""Convert Linear to WeightOnlyLinear for low memory inference.Args:- weight_config (str|dict): qconfig dict or Path of qconfig.json.+ layer_config (str|dict): qconfig dict or Path of qconfig.json.enable_full_range (bool, optional): Whether to leverage the full compression rangeunder symmetric quantization. Defaults to False.compression_dtype (torch.Tensor, optional): The target dtype after comoression.@@ -195,11 +195,11 @@ def pack_model(compressed_model = modelelse:compressed_model = copy.deepcopy(model)- if isinstance(weight_config, str):- with open(weight_config, "r") as f:+ if isinstance(layer_config, str):+ with open(layer_config, "r") as f:q_config = json.load(f)else:- q_config = weight_config+ q_config = layer_configfor k, v in q_config.items():logger.info(f"Packing {k}")if "float" in v["data_type"]:diff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 9d8738331..151d14266 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -35,8 +35,8 @@ def round_ste(x: torch.Tensor):return (x.round() - x).detach() + x-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- weight_min=None, weight_max=None):+def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None, q_scale_thresh=0.0):"""Quantizes and dequantizes weight asymmetrically.Args:@@ -75,6 +75,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scawmin[tmp] = -1wmax[tmp] = +1scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)zp = round_ste(-wmin / scale) # pylint: disable=E1130scale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)@@ -83,8 +84,8 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scareturn scale * (q - zp), scale, zp-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,- weight_max=None):+def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,+ weight_max=None, q_scale_thresh=0.0):"""Quantizes and dequantizes weight symmetrically.Args:@@ -129,7 +130,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scalwmin_new[tmp] = -1wmax_new[tmp] = +1scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)-+ scale = torch.clamp(scale, min=q_scale_thresh)scale = scale.unsqueeze(dim=-1)zp = torch.full_like(scale, (maxq + 1) / 2)@@ -138,8 +139,8 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scalreturn scale * (q - zp), scale, zp-def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,- weight_max=None):+def quant_tensor_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,+ weight_max=None, q_scale_thresh=0.0):"""Quantizes and dequantizes weight symmetrically or asymmetrically .Args:@@ -157,9 +158,11 @@ def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dty"""assert num_bits > 0, "num_bits should be larger than 0"if sym:- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)+ return quant_tensor_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,+ q_scale_thresh)else:- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)+ return quant_tensor_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,+ q_scale_thresh)def reshape_tensor(v, group_size=-1):@@ -182,14 +185,14 @@ def reshape_tensor(v, group_size=-1):return v-def quant_weight(- weight, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- weight_min=None, weight_max=None+def quant_tensor(+ data, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None, q_scale_thresh=0.0):"""Quantizes and dequantizes weight, handing the group size issue .Args:- weight: Tensor containing the weight to be quantized+ data: Tensor containing the weight to be quantizednum_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)group_size: The number of elements shares scale and zero pointsym: Sym or asym@@ -202,32 +205,47 @@ def quant_weight(Returns:Quantized and dequantized weight, scale, zero-point"""- if group_size == -1 or weight.shape[1] < group_size:- return quant_weight_actor(- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,- weight_min=weight_min, weight_max=weight_max- )- orig_shape = weight.shape- if weight.shape[1] % group_size == 0:- weight = weight.reshape(-1, group_size)- weight, scale, zp = quant_weight_actor(- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,- weight_min=weight_min, weight_max=weight_max- )- weight = weight.reshape(orig_shape)- return weight, scale, zp+ orig_shape = data.shape+ if len(data.shape) > 2:+ data = data.reshape(-1, orig_shape[-1])+ if group_size == -1 or data.shape[1] < group_size:+ data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,+ q_scale_thresh=q_scale_thresh)+ data = data.reshape(orig_shape)+ return data, scale, zp++ if data.shape[1] % group_size == 0:+ data = data.reshape(-1, group_size)+ data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,+ q_scale_thresh=q_scale_thresh)+ data = data.reshape(orig_shape)+ return data, scale, zpelse:- pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]- weight_new = torch.nn.functional.pad(weight, (0, pad_len))- weight_new = weight_new.reshape(-1, group_size)- weight_new, scale, zp = quant_weight_actor(- weight_new, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,- weight_min=weight_min, weight_max=weight_max- )- weight_new = weight_new.reshape(orig_shape[0], -1)- weight_new = weight_new[:, :-pad_len]- return weight_new, scale, zp+ tmp_shape = data.shape+ pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]+ data_new = torch.nn.functional.pad(data, (0, pad_len))+ data_new = data_new.reshape(-1, group_size)+ data_new, scale, zp = quant_tensor_actor(data_new, num_bits, sym=sym, v=v, min_scale=min_scale,+ max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,+ weight_max=weight_max, q_scale_thresh=q_scale_thresh)+ data_new = data_new.reshape(tmp_shape[0], -1)+ data_new = data_new[:, :-pad_len]+ data_new = data_new.reshape(orig_shape)+ return data_new, scale, zp+++class WrapperWALayer(torch.nn.Module):+ def __init__(self, orig_layer):+ super(WrapperWALayer, self).__init__()+ self.orig_layer = orig_layer++ def forward(self, x):+ x, _, _ = quant_tensor(x, self.orig_layer.act_bits, self.orig_layer.group_size, self.orig_layer.sym,+ scale_dtype=self.orig_layer.scale_dtype, q_scale_thresh=self.orig_layer.q_scale_thresh)+ return self.orig_layer.forward(x)class WrapperLinear(torch.nn.Module):@@ -254,6 +272,13 @@ class WrapperLinear(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.scale_dtype = self.orig_layer.scale_dtypeself.sym = self.orig_layer.sym+ self.act_bits = self.orig_layer.act_bits+ self.act_group_size = self.orig_layer.act_group_size+ self.act_sym = self.orig_layer.act_sym+ self.act_dynamic = self.orig_layer.act_dynamic+ self.act_quant = self.act_bits <= 8+ self.q_scale_thresh = 1e-5+weight_dtype = torch.float32self.value = torch.nn.Parameter(reshape_tensor(@@ -291,25 +316,19 @@ class WrapperLinear(torch.nn.Module):min_scale.clamp_(0, 1.0)max_scale.clamp_(0, 1.0)- qdq_weight, scale, zp = quant_weight(- self.orig_layer.weight,- self.num_bits,- self.group_size,- self.sym,- v,- min_scale,- max_scale,- self.scale_dtype,- self.weight_min,- self.weight_max,- )+ qdq_weight, scale, zp = quant_tensor(self.orig_layer.weight, self.num_bits, self.group_size, self.sym, v,+ min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)self.orig_layer.weight.data.copy_(qdq_weight)- self.orig_layer.weight.grad = None ##clear grad+ self.orig_layer.weight.grad = Noneself.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu") if zp is not None else None+ self.orig_layer.q_scale_thresh = self.q_scale_thresh+ if self.act_quant:+ wrapper_layer = WrapperWALayer(self.orig_layer)+ return wrapper_layerreturn self.orig_layerdef forward(self, x):@@ -326,19 +345,12 @@ class WrapperLinear(torch.nn.Module):weight = self.orig_layer.weightself.min_scale.data.copy_(torch.clamp(self.min_scale.data, 0, 1.0))self.max_scale.data.copy_(torch.clamp(self.max_scale.data, 0, 1.0))- weight_q, _, _ = quant_weight(- weight,- self.num_bits,- self.group_size,- self.sym,- self.value,- self.min_scale,- self.max_scale,- self.scale_dtype,- self.weight_min,- self.weight_max- )+ weight_q, _, _ = quant_tensor(weight, self.num_bits, self.group_size, self.sym, self.value, self.min_scale,+ self.max_scale, self.scale_dtype, self.weight_min, self.weight_max)weight_q = weight_q.to(weight.dtype)+ if self.act_quant:+ x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)# pylint: disable=not-callablereturn F.linear(x, weight_q, self.orig_layer.bias)@@ -372,6 +384,12 @@ class WrapperTransformerConv1d(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.sym = self.orig_layer.symself.scale_dtype = self.orig_layer.scale_dtype+ self.act_bits = self.orig_layer.act_bits+ self.act_group_size = self.orig_layer.act_group_size+ self.act_sym = self.orig_layer.act_sym+ self.act_dynamic = self.orig_layer.act_dynamic+ self.act_quant = self.act_bits <= 8+ self.q_scale_thresh = 1e-5weight_dtype = torch.float32device = self.orig_layer.weight.deviceself.weight_t = self.orig_layer.weight.t()@@ -410,10 +428,8 @@ class WrapperTransformerConv1d(torch.nn.Module):"""min_scale.clamp_(0, 1.0)max_scale.clamp_(0, 1.0)- qdq_weight, scale, zp = quant_weight(- self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale, max_scale, self.scale_dtype,- self.weight_min, self.weight_max- )+ qdq_weight, scale, zp = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale,+ max_scale, self.scale_dtype, self.weight_min, self.weight_max)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)@@ -421,6 +437,10 @@ class WrapperTransformerConv1d(torch.nn.Module):self.orig_layer.weight.grad = Noneself.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu")+ self.orig_layer.q_scale_thresh = self.q_scale_thresh+ if self.act_quant:+ wrapper_layer = WrapperWALayer(self.orig_layer)+ return wrapper_layerreturn self.orig_layerdef forward(self, x):@@ -435,20 +455,14 @@ class WrapperTransformerConv1d(torch.nn.Module):with torch.no_grad():self.min_scale.clamp_(0, 1.0)self.max_scale.clamp_(0, 1.0)- weight_q, _, _ = quant_weight(- self.weight_t,- self.num_bits,- self.group_size,- self.sym,- self.value,- self.min_scale,- self.max_scale,- self.scale_dtype,- self.weight_min,- self.weight_max- )+ weight_q, _, _ = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, self.value,+ self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,+ self.weight_max)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)+ if self.act_quant:+ x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())x = x.view(*size_out)return xdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 8b98f97d5..7dab5830e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -307,11 +307,11 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flodef check_to_quantized(config):if isinstance(config, dict):- if config["bits"] > 8 or "fp" in config["data_type"] or "float" in config["data_type"]:+ if config["bits"] > 8:return Falsereturn Trueelse:- if config.bits > 8 or "fp" in config.data_type or "float" in config.data_type:+ if config.bits > 8:return Falsereturn Truediff --git a/examples/language-modeling/eval_042/evaluation.py b/examples/language-modeling/eval_042/evaluation.pyindex b14a67c82..fa1a6f4f0 100644--- a/examples/language-modeling/eval_042/evaluation.py+++ b/examples/language-modeling/eval_042/evaluation.py@@ -54,6 +54,7 @@ def simple_evaluate(random_seed: int = 0,numpy_random_seed: int = 1234,torch_random_seed: int = 1234,+ user_model = None, ##user model does not support tensor parallelism):"""Instantiate and evaluate a model on a list of tasks.@@ -185,6 +186,8 @@ def simple_evaluate(+ str(lm.rank)+ ".db",)+ if user_model is not None:+ lm._model = user_modelif task_manager is None:task_manager = TaskManager(verbosity)diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 14d9b5c60..5ec9dcb23 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -119,6 +119,9 @@ if __name__ == '__main__':parser.add_argument("--model_dtype", default=None, type=str,help="force to convert the dtype, some backends supports fp16 dtype better")+ parser.add_argument("--act_bits", default=32, type=int,+ help="activation bits")+args = parser.parse_args()if args.enable_minmax_tuning:@@ -131,6 +134,10 @@ if __name__ == '__main__':print("enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")+ if args.act_bits <= 8:+ print(+ "Warning, activation quantization is an experiment feature")+set_seed(args.seed)tasks = args.tasksuse_eval_legacy = False@@ -138,6 +145,9 @@ if __name__ == '__main__':if "marlin" in args.deployment_device and args.sym == False:assert False, "marlin backend only supports sym quantization, please set --sym"+ if args.act_bits <= 8 and args.deployment_device != "fake":+ assert False, "only support fake mode for activation quantization currently"+model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]@@ -205,11 +215,11 @@ if __name__ == '__main__':if args.adam:round = AutoAdamRound- weight_config = {}+ layer_config = {}for n, m in model.named_modules():if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- weight_config[n] = {"data_type": "fp"}+ layer_config[n] = {"bits": 32}print(f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")lm_head_layer_name = "lm_head"@@ -229,7 +239,7 @@ if __name__ == '__main__':f"supported currently")breakif args.quant_lm_head:- weight_config[lm_head_layer_name] = {"data_type": "int"}+ layer_config[lm_head_layer_name] = {"bits": args.bits}transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]if transformers_version[0] == 4 and transformers_version[1] < 38:error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."@@ -245,8 +255,8 @@ if __name__ == '__main__':amp=not args.disable_amp, nsamples=args.nsamples,low_gpu_mem_usage=args.low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, weight_config=weight_config,- enable_minmax_tuning=not args.disable_minmax_tuning)+ scale_dtype=args.scale_dtype, layer_config=layer_config,+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits)model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")@@ -264,7 +274,7 @@ if __name__ == '__main__':gpu_formats.append(item)if 'gpu' in deployment_device:- if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":+ if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":gpu_formats.append("auto_round")else:gpu_formats.append("auto_gptq")@@ -349,10 +359,13 @@ if __name__ == '__main__':model_args = f"pretrained={eval_folder}"else:exit() ## does not support cpu,xpu model eval+ user_model = None+ if args.act_bits <= 8:+ user_model = model.to(device_str)res = simple_evaluate(model="hf", model_args=model_args,tasks=tasks,- batch_size=args.eval_bs)+ batch_size=args.eval_bs, user_model=user_model)from lm_eval.utils import make_tableprint(make_table(res))diff --git a/test/test_autoround.py b/test/test_autoround.pyindex aff2f53f0..279f5dee9 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -34,7 +34,7 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("runs", ignore_errors=True)def test_remove_whole_block(self):- weight_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},+ layer_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},"model.decoder.layers.0.self_attn.v_proj": {"data_type": "float"},"model.decoder.layers.0.self_attn.q_proj": {"data_type": "float"},"model.decoder.layers.0.self_attn.out_proj": {"data_type": "float"},@@ -51,7 +51,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- weight_config=weight_config+ layer_config=layer_config)autoround.quantize()@@ -177,7 +177,7 @@ class TestAutoRound(unittest.TestCase):def test_lm_head(self):bits, group_size, sym = 4, -1, False- weight_config = {"lm_head": {"data_type": "int"}}+ layer_config = {"lm_head": {"data_type": "int"}}autoround = AutoRound(self.model,self.tokenizer,@@ -189,9 +189,24 @@ class TestAutoRound(unittest.TestCase):enable_minmax_tuning=False,enable_quanted_input=False,dataset=self.llm_dataloader,- weight_config=weight_config,+ layer_config=layer_config,)autoround.quantize()+ def test_wa_quant(self):+ bits, group_size, sym, act_bits = 4, 128, False, 4+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ act_bits=4,+ )+ autoround.quantize()+if __name__ == "__main__":diff --git a/test/test_autoround_export.py b/test/test_autoround_export.pyindex d90ca15f1..3c005d224 100644--- a/test/test_autoround_export.py+++ b/test/test_autoround_export.py@@ -57,10 +57,10 @@ class TestAutoroundExport(unittest.TestCase):out1 = model(self.lm_input)round = AutoRoundoptq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)- q_model, weight_config1 = optq_1.quantize()+ q_model, layer_config1 = optq_1.quantize()from auto_round.export.export_to_itrex import pack_model- compressed_model = pack_model(model=q_model, weight_config=weight_config1)+ compressed_model = pack_model(model=q_model, layer_config=layer_config1)out2 = model(self.lm_input)out3 = q_model(self.lm_input)out4 = compressed_model(self.lm_input)@@ -73,8 +73,8 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out6 = model(self.lm_input)optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)- q_model, weight_config2 = optq_2.quantize()- compressed_model = pack_model(model=q_model, weight_config=weight_config2, inplace=False)+ q_model, layer_config2 = optq_2.quantize()+ compressed_model = pack_model(model=q_model, layer_config=layer_config2, inplace=False)out4 = q_model(self.lm_input)out5 = compressed_model(self.lm_input)self.assertTrue(torch.all(out1[0] == out6[0]))diff --git a/test/test_conv1d.py b/test/test_conv1d.pynew file mode 100644index 000000000..b2e51a24e--- /dev/null+++ b/test/test_conv1d.py@@ -0,0 +1,52 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "..")+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestQuantizationConv1d(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "MBZUAI/LaMini-GPT-124M"+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+++ def test_wa_quant(self):+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,++ )+ autoround.quantize()+ autoround.save_quantized("./saved")+++if __name__ == "__main__":+ unittest.main()