PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refine quntization config (#17)
WeiweiZhang12b821bcae998
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 1167d7512..cb46acfde 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -26,14 +26,9 @@ logger.addHandler(fh)import copyimport time-from torch.amp import autocast-from functools import partial-from torch.functional import Ffrom .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,- collect_minmax_scale, get_batch_dim, is_hpu_available, check_to_quantized)-from .calib_dataset import CALIB_DATASETS-+ collect_minmax_scale, get_batch_dim, htcore, is_hpu_available, check_to_quantized)class SaveInputs:"""Cache the inputs of the first block."""@@ -171,6 +166,7 @@ class SaveInputs:def _replace_forward(self):"""Replaces the forward function."""+ from functools import partialfor n, m in self.model.named_modules():if n == self.block_name:m.orig_forward = m.forward@@ -258,6 +254,7 @@ class WrapperLinear(torch.nn.Module):Returns:- torch.Tensor: The output tensor after applying the linear transformation with quantized weights."""+ from torch.functional import Fweight = self.orig_layer.weightself.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))@@ -266,6 +263,7 @@ class WrapperLinear(torch.nn.Module):self.scale_dtype)weight_q = weight_q.to(weight.dtype)+ # pylint: disable=not-callablereturn F.linear(x, weight_q, self.orig_layer.bias)@@ -533,6 +531,7 @@ class AutoRound(object):scale_dtype="fp16",**kwargs,):+ from .calib_dataset import CALIB_DATASETSself.model = modelself.model = self.model.to("cpu")self.amp = amp@@ -778,6 +777,7 @@ class AutoRound(object):Returns:Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)"""+ from torch.amp import autocastbatch_dim = get_batch_dim(input_others)if not self.low_gpu_mem_usage and input_ids.device != device:input_ids = move_input_to_device(input_ids, device)@@ -853,8 +853,10 @@ class AutoRound(object):)if self.amp and not check_is_cpu(device):with autocast(device_type="cuda", dtype=self.amp_dtype):+ # pylint: disable=not-callableloss = mse_loss(output_q, current_output)else:+ # pylint: disable=not-callableloss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))total_loss += loss.item() / self.gradient_accumulate_steps@@ -945,14 +947,16 @@ class AutoRound(object):torch.cuda.empty_cache()def save_quantized(self, output_dir, format="itrex", **kwargs):+ compress_model = Noneif format == "itrex":- self.save_quantized_as_itrex(output_dir)+ compress_model = self.save_quantized_as_itrex(output_dir, **kwargs)elif format == "auto_gptq":self.save_quantized_as_autogptq(output_dir, **kwargs)else:logger.error("export only supports itrex and auto_gptq now")+ return compress_model- def save_quantized_as_autogptq(self, output_dir, use_triton=False):+ def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):"""Export the model to autogptq format to easily leverage cuda kernel"""@@ -982,10 +986,14 @@ class AutoRound(object):modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptqif all_to_quantized:modules_in_block_to_quantize = None-- model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy++ if inplace:+ model = self.model.to("cpu")+ else:+ model = copy.deepcopy(self.model.to("cpu"))from auto_gptq.modeling._utils import pack_model+ sym = self.scheme == 'sym'if self.bits == 3 or use_triton is False:if self.bits == 3 and use_triton is True:logger.warning("triton does not support 3 bits, reset it to False")@@ -1003,22 +1011,26 @@ class AutoRound(object):info = self.weight_config[key]if not check_to_quantized(info):continue- quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'].to(torch.float32), info['g_idx'])-+ info['zp'] = None if sym else info['zp'].to(torch.float32)+ quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'], info['g_idx'])pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,force_layer_back_to_cpu=True, use_triton=True)from auto_round import save_quantized_to_autogptq- sym = self.scheme == "sym"save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,scale_dtype=self.scale_dtype,use_safetensors=True, modules_in_block_to_quantize=modules_in_block_to_quantize)- def save_quantized_as_itrex(self, output_dir):+ def save_quantized_as_itrex(self, output_dir, inplace=True):"""Save configure file and weights for CPU backend inference."""- from auto_round.export.export_to_itrex import compress_model- compressed_model, quantize_config = compress_model(self.model, self.weight_config)+ from auto_round.export.export_to_itrex import compress_model, QuantConfig+ compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)+ sym = self.scheme == "sym"+ quantize_config = QuantConfig(bits=self.bits, group_size=self.group_size, sym=sym,+ iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,+ scale_dtype=str(self.scale_dtype))if quantize_config is not None:config = compressed_model.configsetattr(config, "quantization_config", quantize_config.to_dict())@@ -1032,6 +1044,7 @@ class AutoRound(object):logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))except IOError as e: # pragma: no coverlogger.error("Fail to save configure file and weights due to {}.".format(e))+ return compressed_modeldef quantize(self):"""Quantize the model and return the quantized model along with weight configurations.@@ -1354,3 +1367,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 617494b69..7935112db 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -113,8 +113,8 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',tokenizer: The tokenizer to be used for tokenization.seqlen: The maximum sequence length.- Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of- seqlen to the "text" field of examples.+ Returns: A default tokenizer function that applies the provided tokenizer with truncation and+ a maximum length of seqlen to the "text" field of examples."""def default_tokenizer_function(examples):@@ -159,3 +159,4 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',calib_dataloader = DataLoader(samples, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader+diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex a81c8b2b8..aa393fc3a 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -76,15 +76,18 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symnew_value = str(value)except Exception as e:raise TypeError(- f"safetensors_metadata: both keys and values must be strings and an error occured when trying to convert them: {e}")+ f"safetensors_metadata: both keys and values must be strings" /+ f" and an error occurred when trying to convert them: {e}")if new_key in new_safetensors_metadata:logger.warning(- f"After converting safetensors_metadata keys to strings, the key '{new_key}' is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' " /+ f"is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")new_safetensors_metadata[new_key] = new_valuesafetensors_metadata = new_safetensors_metadataif converted_keys:logger.debug(- f"One or more safetensors_metadata keys or values had to be converted to str(). Final safetensors_metadata: {safetensors_metadata}")+ f"One or more safetensors_metadata keys or values had to be converted to str()." /+ f" Final safetensors_metadata: {safetensors_metadata}")# Format is required to enable Accelerate to load the metadata# otherwise it raises an OSError@@ -131,3 +134,4 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symconfig_dict["quant_method"] = "gptq" ##hf transformers could only recognize this valuemodel.config.quantization_config = config_dictmodel.config.save_pretrained(save_dir)+diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.pyindex 62e87ef6c..25ac08e6a 100644--- a/auto_round/export/export_to_itrex/config.py+++ b/auto_round/export/export_to_itrex/config.py@@ -33,21 +33,31 @@ class QuantConfig(PretrainedConfig):def __init__(self,bits=4,- scale_dtype="fp32",+ scale_dtype='torch.float32',group_size=128,sym=False,- quant_method="Autoround",+ quant_method="autoround",model_name_or_path=None,model_file_base_name='model',+ enable_minmax_tuning=True,+ iters=1000,+ lr=0.001,+ minmax_lr=0.001,+ use_quant_input=True,**kwargs,):self.bits = bitsself.group_size = group_size- self.scale_dtype = scale_dtype ## reserved parameter+ self.scale_dtype = scale_dtypeself.sym = symself.quant_method = quant_methodself.model_name_or_path = model_name_or_pathself.model_file_base_name = model_file_base_name+ self.enable_minmax_tuning=enable_minmax_tuning+ self.iters=iters+ self.lr=lr+ self.minmax_lr=minmax_lr+ self.use_quant_input=use_quant_input### Redundant parameters, will be removed later. ###self.damp_percent = 0.01@@ -61,8 +71,8 @@ class QuantConfig(PretrainedConfig):Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."""- if self.scale_dtype not in ["fp32", "fp16"]:- raise ValueError("scale_dtype must be 'fp32', 'fp16'.")+ if self.scale_dtype not in ['torch.float32', 'torch.float16', 'torch.bfloat16']:+ raise ValueError("scale_dtype must be 'fp32', 'fp16' or 'bf16'.")if self.group_size not in [-1, 32, 128]:raise ValueError("group_size must be an integer in [-1, 32, 128]")@@ -78,7 +88,7 @@ class QuantConfig(PretrainedConfig):@classmethoddef from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):"""- Instantiates a [`AutoroundQuantConfig`] from a Python dictionary of parameters.+ Instantiates a [`QuantConfig`] from a Python dictionary of parameters.Args:config_dict (`Dict[str, Any]`):Dictionary that will be used to instantiate the configuration object.@@ -88,7 +98,7 @@ class QuantConfig(PretrainedConfig):kwargs (`Dict[str, Any]`):Additional parameters from which to initialize the configuration object.Returns:- [`AutoroundQuantConfig`]: The configuration object instantiated from those parameters.+ [`QuantConfig`]: The configuration object instantiated from those parameters."""config = cls(**config_dict)@@ -143,7 +153,7 @@ class QuantConfig(PretrainedConfig):Args:use_diff (`bool`, *optional*, defaults to `True`):If set to `True`, only the difference between the config instance and the default- `AutoroundQuantConfig()`+ `QuantConfig()`is serialized to JSON string.Returns:`str`: String containing all the attributes that make up this configuration instance in JSON format.@@ -165,7 +175,7 @@ class QuantConfig(PretrainedConfig):config_dict = self.to_dict()# get the default config dict- default_config_dict = AutoroundQuantConfig().to_dict()+ default_config_dict = QuantConfig().to_dict()serializable_config_dict = {}@@ -222,4 +232,4 @@ class QuantConfig(PretrainedConfig):def get_config_dict(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]:- return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)\ No newline at end of file+ return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 60d529a01..9db61f692 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -35,6 +35,7 @@ def compress_model(compression_dim=1,device="cpu",use_optimum_format=True,+ inplace=False):"""Convert Linear to WeightOnlyLinear for low memory inference.@@ -53,8 +54,12 @@ def compress_model(3: g_idx: use same number for one group instead of recording the channel order.4. parameter name changed, such as 'packed_weight' -> 'qweight'.5. zeros is always needed even for sym.+ inplace (bool, optional): Compress the model in place, or copy the model and compress it."""- compressed_model = copy.deepcopy(model)+ if inplace:+ compressed_model = model+ else:+ compressed_model = copy.deepcopy(model)if isinstance(weight_config, str):with open(weight_config, "r") as f:q_config = json.load(f)@@ -71,13 +76,15 @@ def compress_model(scale_dtype = v["scale_dtype"]m = get_module(compressed_model, k)fp_weight = m.weight.data- # scale = torch.tensor(v["scale"], dtype=scale_dtype)scale, zp = v["scale"], v["zp"]convert_dtype = torch.float32 if fp_weight.device.type == "cpu" else scale_dtypeif not isinstance(scale, torch.Tensor):scale = torch.tensor(scale, dtype=convert_dtype)zp = None if scheme == "sym" else torch.tensor(zp, dtype=torch.int32)else:+ if not inplace:+ scale = scale.clone()+ zp = None if scheme == "sym" else zp.clone()scale = scale.to(dtype=convert_dtype)zp = None if scheme == "sym" else zp.to(dtype=torch.int32)int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)@@ -97,5 +104,5 @@ def compress_model(new_module.pack(int_weight, scale, zp, m.bias)set_module(compressed_model, k, new_module)- quantize_config = QuantConfig(bits=num_bits, sym=(scheme == "sym"), group_size=group_size)- return compressed_model, quantize_config+ return compressed_model+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex c3347842b..45ecc1dc3 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -35,6 +35,7 @@ try:is_hpu_available = Falseexcept ImportError:is_hpu_available = False+ htcore = Nonedef round_ste(x: torch.Tensor):"""Straight-Through Estimator for rounding.@@ -138,7 +139,15 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)-def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):+def quant_weight(+ weight,+ num_bits=4,+ group_size=-1,+ scheme="asym",+ v=0,+ min_scale=0,+ max_scale=0,+ scale_dtype=torch.float16):"""Quantizes and dequantizes weight, handing the group size issue .Args:@@ -223,14 +232,13 @@ def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):return int_weight-def get_module(model, key):+def get_module(module, key):"""Get module from model by key name.Args:- model (torch.nn.Module): original model+ module (torch.nn.Module): original modelkey (str): module name to be replaced"""- module = modelname_list = key.split(".")for name in name_list:if hasattr(module, name):@@ -474,4 +482,4 @@ def check_to_quantized(config):else:if config.bits > 8 or "fp" in config.data_type or "float" in config.data_type:return False- return True\ No newline at end of file+ return Truediff --git a/auto_round/version.py b/auto_round/version.pyindex 1a1828052..952b12d4e 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -11,5 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-"""Intel® auto-round: An open-source Python library supporting popular model weight only compression based on signround."""-__version__ = "0.1"\ No newline at end of file+"""Intel® auto-round: An open-source Python library+supporting popular model weight only compression based on signround."""++__version__ = "0.1"diff --git a/docs/acc.md b/docs/acc.mdindex a3e286235..a7c3925b1 100644--- a/docs/acc.md+++ b/docs/acc.md@@ -1,5 +1,5 @@## Comparison with other methods-To ensure a fair comparison as much as possible and alleviate overfitting in perplexity evaluation on Wikitext or C4, we utilized 512 samples from NeelNanda/pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enalbed act-order and true-seqential in GPTQ, and also activated static group in scenarios where group_size!=-1. The notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues. With these configurations, the tuning costs for GPTQ, AWQ, and ours are comparable.+To ensure a fair comparison as much as possible and alleviate overfitting in perplexity evaluation on Wikitext or C4, we utilized 512 samples from NeelNanda/pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enabled act-order and true-seqential in GPTQ, and also activated static group in scenarios where group_size!=-1. The notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues. With these configurations, the tuning costs for GPTQ, AWQ, and ours are comparable.For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequence length to 2048. For lm-eval wikitext ppl, we adopt lm-eval. The lm-eval-harness git id we used in the following is 008fc2a23245c40384f2312718433eeb1e0f87a9 and we evaluated on qdq fake model.@@ -2737,3 +2737,4 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc<td>11.35</td> <! lm-eval wikitext ppl --></tr></table>+diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pyindex e11520cc5..c721abfbd 100644--- a/examples/code-generation/main.py+++ b/examples/code-generation/main.py@@ -53,7 +53,7 @@ if __name__ == '__main__':help="whether to use the output of quantized block to tune the next block")parser.add_argument("--lr", default=None, type=float,- help="learning rate, if None, it will be set to 1.0/iters automatially")+ help="learning rate, if None, it will be set to 1.0/iters automatically")parser.add_argument("--minmax_lr", default=None, type=float,help="minmax learning rate, if None,it will beset to be the same with lr")@@ -196,3 +196,4 @@ if __name__ == '__main__':model = model.to("cpu")model.save_pretrained(output_dir)tokenizer.save_pretrained(output_dir)+diff --git a/examples/language-modeling/eval/__init__.py b/examples/language-modeling/eval/__init__.pyindex e69de29bb..07f1ae76d 100644--- a/examples/language-modeling/eval/__init__.py+++ b/examples/language-modeling/eval/__init__.py@@ -0,0 +1,5 @@+if __name__ == "__main__":+ import sys+ sys.path.insert(0, './')+from .evaluation import eval_model+diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex 3fef08105..86c68fadd 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -3,15 +3,22 @@ import osos.environ["TOKENIZERS_PARALLELISM"] = "false"import os.pathimport torch-+import loggingimport pprintimport reimport shutilimport transformers+from typing import TYPE_CHECKING, Optional, Union+import time+try:+ from lm_eval.tasks import TaskManager+except ImportError:+ TaskManager = None+if __name__ == "__main__":import syssys.path.insert(0, './')-import time+EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']fewshots_dict = {}fewshots_dict['paper'] = {@@ -60,30 +67,39 @@ fewshots_dict['all'] = {def simple_evaluate(model,- model_args=None,- tasks=[],- num_fewshot=None,- batch_size=None,- max_batch_size=None,- device=None,- use_cache=None,- limit=None,+ model_args: Optional[Union[str, dict, None]] = None,+ tasks=None,+ num_fewshot: Optional[int] = None,+ batch_size: Optional[int] = None,+ max_batch_size: Optional[int] = None,+ device: Optional[str] = None,+ use_cache: Optional[str] = None,+ cache_requests: bool = False,+ rewrite_requests_cache: bool = False,+ delete_requests_cache: bool = False,+ limit: Optional[Union[int, float]] = None,bootstrap_iters: int = 100000,check_integrity: bool = False,decontamination_ngrams_path=None,write_out: bool = False,log_samples: bool = True,gen_kwargs: str = None,+ task_manager: TaskManager = None,+ verbosity: str = "INFO",+ predict_only: bool = False,+ random_seed: int = 1234,+ numpy_random_seed: int = 1234,+ torch_random_seed: int = 1234,lm=None):"""Instantiate and evaluate a model on a list of tasks.:param model: Union[str, LM]Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model- :param model_args: Optional[str]- String arguments for each model class, see LM.create_from_arg_string.+ :param model_args: Optional[str, dict]+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.Ignored if `model` argument is a LM object.- :param tasks: list[Union[str, Task]]+ :param tasks: list[Union[str, dict, Task]]List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.:param num_fewshot: intNumber of examples in few-shot context@@ -95,6 +111,12 @@ def simple_evaluate(PyTorch device (e.g. "cpu" or "cuda:0") for running models:param use_cache: str, optionalA path to a sqlite db file for caching model responses. `None` if not caching.+ :param cache_requests: bool, optional+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.+ :param rewrite_requests_cache: bool, optional+ Rewrites all of the request cache if set to `True`. `None` if not desired.+ :param delete_requests_cache: bool, optional+ Deletes all of the request cache if set to `True`. `None` if not desired.:param limit: int or float, optionalLimit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.:param bootstrap_iters:@@ -108,6 +130,15 @@ def simple_evaluate(:param gen_kwargs: strString arguments for model generationIgnored for all tasks with loglikelihood output_type+ :param predict_only: bool+ If true only model outputs will be generated and returned. Metrics will not be evaluated+ :param random_seed: int+ Random seed for python's random module. If set to None, the seed will not be set.+ :param numpy_random_seed: int+ Random seed for numpy. If set to None, the seed will not be set.+ :param torch_random_seed: int+ Random seed for torch. If set to None, the seed will not be set.+:returnDictionary of results"""@@ -120,32 +151,46 @@ def simple_evaluate(import lm_eval.api.metricsimport lm_eval.api.registryfrom lm_eval.evaluator import evaluate- lm_eval.tasks.initialize_tasks()+ from lm_eval.logging_utils import add_env_info, get_git_commit_hash+ from lm_eval.tasks import get_task_dictfrom lm_eval.utils import (- positional_deprecated,- run_task_tests,- make_table,- get_git_commit_hash,- simple_parse_args_string,- eval_logger,+ eval_logger,+ positional_deprecated,+ run_task_tests,+ simple_parse_args_string,)- random.seed(1234)- np.random.seed(1234)- torch.manual_seed(- 1234- ) # TODO: this may affect training runs that are run with evaluation mid-run.+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))++ if delete_requests_cache:+ eval_logger.info("Deleting requests cache...")+ delete_cache()++ if random_seed is not None:+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412+ eval_logger.info(f"Setting random seed to {random_seed}")+ random.seed(random_seed)++ if numpy_random_seed is not None:+ eval_logger.info(f"Setting numpy seed to {numpy_random_seed}")+ np.random.seed(numpy_random_seed)++ if torch_random_seed is not None:+ eval_logger.info(f"Setting torch manual seed to {torch_random_seed}")+ torch.manual_seed(torch_random_seed)- assert (- tasks != []- ), "No tasks specified, or no tasks found. Please verify the task names."+ if tasks is None:+ tasks = []+ assert (+ tasks != []+ ), "No tasks specified, or no tasks found. Please verify the task names."if lm == None:if gen_kwargs is not None:gen_kwargs = simple_parse_args_string(gen_kwargs)eval_logger.warning(- f"generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks."+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!")if gen_kwargs == "":gen_kwargs = None@@ -153,16 +198,28 @@ def simple_evaluate(if isinstance(model, str):if model_args is None:model_args = ""- lm = lm_eval.api.registry.get_model(model).create_from_arg_string(- model_args,- {- "batch_size": batch_size,- "max_batch_size": max_batch_size,- "device": device,- },- )++ elif isinstance(model_args, dict):+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(+ model_args,+ {+ "batch_size": batch_size,+ "max_batch_size": max_batch_size,+ "device": device,+ },+ )++ else:+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(+ model_args,+ {+ "batch_size": batch_size,+ "max_batch_size": max_batch_size,+ "device": device,+ },+ )elif isinstance(model, transformers.PreTrainedModel):- lm = lm_eval.api.registry.get_model("hf-causal")(+ lm = lm_eval.api.registry.get_model("hf")(pretrained=model,batch_size=batch_size,max_batch_size=max_batch_size,@@ -182,33 +239,46 @@ def simple_evaluate(+ model_args.replace("=", "-").replace(",", "_").replace("/", "-")+ ".db",)++ if task_manager is None:+ task_manager = TaskManager(verbosity)- task_dict = lm_eval.tasks.get_task_dict(tasks)+ eval_logger.info(+ "get_task_dict has been updated to accept an optional argument, `task_manager`"+ "Read more here:https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage"+ )+ task_dict = get_task_dict(tasks, task_manager)for task_name in task_dict.keys():task_obj = task_dict[task_name]- if type(task_obj) == tuple:- group, task_obj = task_obj+ if isinstance(task_obj, tuple):+ _, task_obj = task_objif task_obj is None:continue- config = task_obj._config- if config["output_type"] == "generate_until" and gen_kwargs is not None:- config["generation_kwargs"].update(gen_kwargs)+ if task_obj.get_config("output_type") == "generate_until":+ if gen_kwargs is not None:+ task_obj.set_config(+ key="generation_kwargs", value=gen_kwargs, update=True+ )++ if predict_only:+ log_samples = True+ eval_logger.info(+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"+ )+ # we have to change the class properties post-hoc. This is pretty hacky.+ task_obj.override_metric(metric_name="bypass")if num_fewshot is not None:- if config["num_fewshot"] == 0:+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:eval_logger.info(f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored.")else:- # default_num_fewshot = config["num_fewshot"]- # eval_logger.warning(- # f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"- # )-- task_obj._config["num_fewshot"] = num_fewshot- else:- num_fewshot = config["num_fewshot"]+ eval_logger.warning(+ f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"+ )+ task_obj.set_config(key="num_fewshot", value=num_fewshot)if check_integrity:run_task_tests(task_list=tasks)@@ -217,40 +287,47 @@ def simple_evaluate(lm=lm,task_dict=task_dict,limit=limit,+ cache_requests=cache_requests,+ rewrite_requests_cache=rewrite_requests_cache,bootstrap_iters=bootstrap_iters,decontamination_ngrams_path=decontamination_ngrams_path,write_out=write_out,log_samples=log_samples,+ verbosity=verbosity,)if lm.rank == 0:+ if isinstance(model, str):+ model_name = model+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):+ model_name = model.config._name_or_path+ else:+ model_name = type(model).__name__+# add info about the model and few shot configresults["config"] = {- "model": model- if isinstance(model, str)- else model.model.config._name_or_path,+ "model": model_name,"model_args": model_args,"batch_size": batch_size,- "batch_sizes": list(lm.batch_sizes.values())- if hasattr(lm, "batch_sizes")- else [],+ "batch_sizes": (+ list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []+ ),"device": device,"use_cache": use_cache,"limit": limit,- "num_fewshot": num_fewshot,"bootstrap_iters": bootstrap_iters,"gen_kwargs": gen_kwargs,}results["git_hash"] = get_git_commit_hash()+ add_env_info(results) # additional environment info to resultsreturn results, lmelse:return None-def eval_model(output_dir=None, model=None, tokenizer=None,- tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],+def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],eval_bs=32, use_accelerate=True, dtype="float16", limit=None,- device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):+ device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):print("evaluation with official lm-eval", flush=True)try:import lm_eval.api@@ -264,29 +341,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,raise ImportError("""follow requirements to install dependencies.""")import collectionsorg_s = time.time()- ##save model- if output_dir is None:- output_dir = "./tmp_signround"-- if os.path.exists(output_dir) and not eval_orig_float:- shutil.rmtree(output_dir)-- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):- dtype = 'bfloat16'- pt_dtype = torch.bfloat16- else:- if str(device) != "cpu":- pt_dtype = torch.float16- dtype = 'float16'- else:- pt_dtype = torch.float32- dtype = 'float32'-- if not eval_orig_float:- model = model.to(pt_dtype)- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)external_tasks = []for each in EXT_TASKS:@@ -304,8 +358,8 @@ def eval_model(output_dir=None, model=None, tokenizer=None,task_s = time.time()for shot in num_fewshot:model_type = "hf"- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'- if use_accelerate: # bool(re.search("chatglm", output_dir.lower()))+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'+ if use_accelerate: # bool(re.search("chatglm", model_path.lower()))model_args += f',parallelize=True'if "wikitext" in tmp_tasks:@@ -331,7 +385,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,print(str(e))continue- tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)model = lm.model# for external tasks# maybe adjust for specific model@@ -420,10 +474,11 @@ if __name__ == "__main__":"arc_easy", "arc_challenge"]model_name = args.model_name.rstrip('/')excel_name = model_name.split('/')[-1] + ".xlsx"- eval_model(output_dir=args.model_name,+ eval_model(model_path=args.model_name,tasks=test_tasks,- eval_bs=args.eval_bs, eval_orig_float=True, limit=None, excel_file=excel_name)+ eval_bs=args.eval_bs, limit=None, excel_file=excel_name)print("cost time: ", time.time() - s)+diff --git a/examples/language-modeling/eval_legacy/__init__.py b/examples/language-modeling/eval_legacy/__init__.pynew file mode 100644index 000000000..07f1ae76d--- /dev/null+++ b/examples/language-modeling/eval_legacy/__init__.py@@ -0,0 +1,5 @@+if __name__ == "__main__":+ import sys+ sys.path.insert(0, './')+from .evaluation import eval_model+diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.pyindex b0a423993..2ba6a29f9 100644--- a/examples/language-modeling/eval_legacy/evaluation.py+++ b/examples/language-modeling/eval_legacy/evaluation.py@@ -11,7 +11,7 @@ import transformersif __name__ == "__main__":import syssys.path.insert(0, './')-from eval.parse_results import result_parser+from .parse_results import result_parserimport timeEXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']fewshots_dict = {}@@ -201,10 +201,9 @@ def simple_evaluate(return results, lm-def eval_model(output_dir=None, model=None, tokenizer=None,- tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],+def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],eval_bs=32, use_accelerate=True, dtype="float16", limit=None,- device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):+ device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):print("evaluation with official lm-eval", flush=True)try:import lm_eval@@ -214,31 +213,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,raise ImportError("""follow requirements to install dependencies.""")org_s = time.time()- ##save model- if output_dir is None:- output_dir = "./tmp_signround"-- if os.path.exists(output_dir) and not eval_orig_float:- shutil.rmtree(output_dir)-- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):- dtype = 'bfloat16'- pt_dtype = torch.bfloat16- else:- if str(device) != "cpu":- pt_dtype = torch.float16- dtype = 'float16'- else:- pt_dtype = torch.float32- dtype = 'float32'-- if not eval_orig_float:- model = model.to(pt_dtype)- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)--external_tasks = []for each in EXT_TASKS:@@ -256,11 +230,11 @@ def eval_model(output_dir=None, model=None, tokenizer=None,print(f'********* {tmp_tasks} evaluate ************')task_s = time.time()for shot in num_fewshot:- if bool(re.search("chatglm", output_dir.lower())):- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'+ if bool(re.search("chatglm", model_path.lower())):+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'model_type = "hf-causal"else:- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'model_type = "hf-causal-experimental"if "wikitext" in task_names:@@ -280,7 +254,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,print(str(e))continue- tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)model = lm.model# for external tasks# maybe adjust for specific model@@ -371,10 +345,11 @@ if __name__ == "__main__":"arc_challenge"]model_name = args.model_name.rstrip('/')excel_name = model_name.split('/')[-1] + ".xlsx"- eval_model(output_dir=args.model_name,+ eval_model(model_path=args.model_name,tasks=test_tasks,- eval_bs=args.eval_bs, eval_orig_float=True, limit=None, excel_file=excel_name)+ eval_bs=args.eval_bs, limit=None, excel_file=excel_name)print("cost time: ", time.time() - s)+diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 47b51ffe6..365ddfeaf 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -1,6 +1,7 @@import argparseimport syssys.path.insert(0, '../..')+from auto_round import (AutoRound,AutoAdamRound)@@ -85,7 +86,7 @@ if __name__ == '__main__':help="whether enable weight minmax tuning")parser.add_argument("--deployment_device", default='fake', type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',"+ help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")parser.add_argument("--scale_dtype", default='fp32',@@ -96,30 +97,43 @@ if __name__ == '__main__':"mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte","arc_easy", "arc_challenge"],help="lm-eval tasks for lm_eval version 0.4")-- # parser.add_argument("--tasks",- # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',- # "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",- # "arc_easy", "arc_challenge"],- # help="lm-eval tasks for lm_eval version 0.3")parser.add_argument("--output_dir", default="./tmp_autoround", type=str,help="Where to store the final model.")parser.add_argument("--eval_legacy", action='store_true',help="Whether to evaluate with a old lm_eval version(e.g. 0.3.0).")++ parser.add_argument("--disable_lmeval", action='store_true',+ help="Whether to do lmeval evaluation.")args = parser.parse_args()set_seed(args.seed)++ tasks = args.tasks+ if not args.eval_legacy:+ from eval import eval_model+ else:+ from eval_legacy import eval_model+ if isinstance(tasks, str):+ tasks = tasks.split(',')+ if isinstance(tasks, list):+ if "mmlu" in tasks:+ tmp_tasks = tasks+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:+ tmp_tasks = tasks+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]+ tasks = list(set(tasks))+ if isinstance(args.tasks, str):+ tasks = ','.join(tasks)model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]print(model_name, flush=True)- tasks = args.tasks-if args.device == "cpu":device_str = "cpu"else:@@ -154,6 +168,7 @@ if __name__ == '__main__':tokenizer.add_special_tokens({'pad_token': '[PAD]'})else:tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+if hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",@@ -161,14 +176,26 @@ if __name__ == '__main__':seqlen = min(seqlen, tokenizer.model_max_length)args.seqlen = seqlen+ pt_dtype = torch.float16+ if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):+ dtype = 'bfloat16'+ pt_dtype = torch.bfloat16+ else:+ if str(args.device) != "cpu":+ pt_dtype = torch.float16+ dtype = 'float16'+ else:+ pt_dtype = torch.float32+ dtype = 'float32'+excel_name = f"{model_name}_{args.bits}_{args.group_size}"if args.eval_fp16_baseline:if not args.low_gpu_mem_usage:model = model.to(torch_device)excel_name += "_fp16.xlsx"- eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device,- eval_orig_float=True, excel_file=excel_name)+ eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,+ device=torch_device, excel_file=excel_name)exit()if not args.low_gpu_mem_usage:@@ -196,44 +223,34 @@ if __name__ == '__main__':amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass- model, q_config = autoround.quantize()+ model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"-- if args.deployment_device == 'cpu':- export_dir += "-cpu"- autoround.save_quantized(output_dir=export_dir)- del q_config- elif args.deployment_device == 'gpu':- export_dir += "-gpu"- autoround.save_quantized(export_dir, format="auto_gptq", use_triton=True)- model = model.eval()-++ model.eval()if args.device != "cpu":torch.cuda.empty_cache()- model.eval()++ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"-- pt_dtype = torch.float16- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):- dtype = 'bfloat16'- pt_dtype = torch.bfloat16- else:- if str(args.device) != "cpu":- pt_dtype = torch.float16- dtype = 'float16'+ deployment_device = args.deployment_device.split(',')+ for dev in deployment_device:+ if dev == 'gpu':+ autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)+ elif dev == 'cpu':+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', inplace=False)+ elif dev == 'fake':+ model = model.to("cpu")+ model.save_pretrained(output_dir)+ tokenizer.save_pretrained(output_dir)else:- pt_dtype = torch.float32- dtype = 'float32'+ print(f"The '{dev}' deployment type is not enabled yet.")++ if not args.disable_lmeval:+ excel_name = f"{output_dir}_result.xlsx"+ output_dir += "/"+ print(excel_name, flush=True)+ eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,+ device=torch_device, excel_file=excel_name)- excel_name = f"{output_dir}_result.xlsx"- output_dir += "/"- print(excel_name, flush=True)- if not args.eval_legacy:- from eval.evaluation import eval_model- else:- from eval_legacy.evaluation import eval_model- eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device, excel_file=excel_name,- limit=None)diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txtindex 61b731f0a..4cc53c9ce 100644--- a/examples/language-modeling/requirements.txt+++ b/examples/language-modeling/requirements.txt@@ -1,6 +1,8 @@transformerstorch-git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9+git+https://github.com/EleutherAI/lm-evaluation-harness.git@96d185fa6232a5ab685ba7c43e45d1dbb3bb906d+# For the paper results use the old lm_eval (0.3.0)+# git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9stiktokentransformers_stream_generatorpeft@@ -11,3 +13,5 @@ datasetsprotobufauto-gptqopenpyxl+wandb+diff --git a/examples/language-modeling/run_autoround.sh b/examples/language-modeling/run_autoround.shindex c6574049a..60c47f4ab 100644--- a/examples/language-modeling/run_autoround.sh+++ b/examples/language-modeling/run_autoround.sh@@ -12,7 +12,7 @@ python3 main.py \--use_quant_input \--amp \--iters 200 \---deployment_device 'cpu' \+--deployment_device 'fake,cpu' \--scale_dtype 'fp32' \--eval_bs 32 \--output_dir "./tmp_autoround"