PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support marlin in auto_round format (#172)
wenhuach212b1448d42036
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 045a64572..dab5c7f21 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -44,6 +44,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quantifrom auto_round.utils import get_module, set_module, dynamic_import_inference_linearimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbitsfrom enum import Enum+logger = getLogger(__name__)import sys@@ -194,6 +195,7 @@ class AutoHfQuantizer:return quantization_config+class AutoRoundQuantizationMethod(str, Enum):AutoRound = "intel/auto-round"@@ -336,12 +338,13 @@ class AutoRoundQuantizer(HfQuantizer):bits = config["bits"]group_size = config["group_size"]data_type = config["data_type"]- if not (bits <= 8 and data_type == "int"):+ sym = config["sym"]+ if not (bits <= 8):continuelayer = get_module(module, layer_name)device = get_device(layer)- QuantLinear = dynamic_import_inference_linear(bits, group_size, backend)+ QuantLinear = dynamic_import_inference_linear(backend, bits, group_size, sym)if isinstance(layer, nn.Linear):in_features = layer.in_featuresout_features = layer.out_features@@ -382,7 +385,6 @@ class AutoRoundQuantizer(HfQuantizer):The input model"""-class StoreAttr(object):pass@@ -406,7 +408,6 @@ class AutoRoundQuantizer(HfQuantizer):else:raise NotImplementedError-@propertydef is_trainable(self, model: Optional["PreTrainedModel"] = None):return Truediff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 361c820f7..da040752c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1008,7 +1008,8 @@ class AutoRound(object):logger.warning("please run autoround.quantize first")returnfrom auto_round.export import EXPORT_FORMAT-+ backend = format+ format = format.split(":")[0]if format not in EXPORT_FORMAT:logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")exit()@@ -1061,7 +1062,8 @@ class AutoRound(object):supported_types=self.supported_types,data_type=self.data_type,serialization_dict=serialization_dict,- **kwargs,+ backend=backend,+ **kwargs)return compressed_modeldiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 9d3ded310..68aa4585f 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -23,9 +23,24 @@ import transformersfrom auto_round.export.register import register_formatfrom auto_round.utils import get_layer_names_in_block, get_module, logger, set_module+import threadpoolctl as tctl+import inspectdef check_neq_config(config, data_type, bits, group_size, sym):+ """+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.++ Args:+ config (dict): A dictionary containing the configuration parameters.+ data_type (str): The expected data type.+ bits (int): The expected number of bits.+ group_size (int): The expected group size.+ sym (bool): The expected symmetry flag.++ Returns:+ list: A list of strings indicating which configuration parameters do not match.+ """res = []if data_type != config["data_type"]:res.append("data_type")@@ -39,58 +54,81 @@ def check_neq_config(config, data_type, bits, group_size, sym):return res-def get_autogptq_backend_config(backend, bits=4):- use_triton = False- disable_exllamav2 = False+def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):+ """+ Configures and returns a QuantLinear class based on the specified backend and parameters.++ Args:+ backend (str): The backend to be used for quantization. Supported values include "qigen", "triton", "marlin",+ "exllama", and "cuda".+ bits (int, optional): The number of bits for quantization. Default is 4.+ group_size (int, optional): The group size for quantization. Default is 128.+ sym (bool, optional): Flag indicating whether to use symmetric quantization. Default is False.++ Returns:+ class: The dynamically imported QuantLinear class configured according to the specified parameters.+ """+ use_triton = True+ disable_exllamav2 = Truedisable_exllamav1 = Falsedisable_marlin = Trueuse_qigen = False- if backend == "gptq:qigen":+ if "qigen" in "backend":+ use_triton = Falseuse_qigen = True- if backend == "gptq:triton": ##TODO refine the code+ elif "triton" in backend:use_triton = True- if backend == "gptq:marlin":+ elif "marlin" in backend and sym:use_triton = Falsedisable_marlin = False- if backend == "gptq:exllamav2": ##need v1 code to export- use_triton = False- disable_marlin = True- if backend == "gptq:exllamav1":- use_triton = False+ elif "exllama" in backend: ##need v1 code to export+ use_triton = True ##same with tritondisable_marlin = True- if backend == "gptq:cuda":+ elif "cuda" in backend:use_triton = Falsedisable_marlin = Truedisable_exllamav2 = Truedisable_exllamav1 = True- if bits not in [2, 4, 8]:- use_qigen = False- if bits not in [2, 4]:- use_triton = False- return use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin---def dynamic_QuantLienar_for_packing(backend, bits, group_size):- if "gptq" in backend:- use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(- backend, bits- )- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401- QuantLinear = dynamically_import_QuantLinear(- use_triton=use_triton,- desc_act=False,- group_size=group_size,- bits=bits,- disable_exllama=disable_exllamav1,- disable_exllamav2=disable_exllamav2,- use_qigen=use_qigen,- disable_marlin=disable_marlin,- )- return QuantLinear- ##export all use trition, inference use exllamav2- elif "autoround" in backend or "auto-round" in backend or "auto_round" in backend:++ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401+ QuantLinear = dynamically_import_QuantLinear(+ use_triton=use_triton,+ desc_act=False,+ group_size=group_size,+ bits=bits,+ disable_exllama=disable_exllamav1,+ disable_exllamav2=disable_exllamav2,+ use_qigen=use_qigen,+ disable_marlin=disable_marlin,+ )+ return QuantLinear+++def dynamic_import_quantLienar_for_packing(backend, bits, group_size, sym):+ """+ Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.++ Args:+ backend (str): The backend to be used for quantization. Supported values include "auto_round" and "gptq".+ bits (int): The number of bits for quantization.+ group_size (int): The group size for quantization.+ sym (bool): Flag indicating whether to use symmetric quantization.++ Returns:+ class: The dynamically imported QuantLinear class configured according to the specified parameters.++ Raises:+ AssertionError: If the backend is not supported.+ """+ if "auto_round" in backend:+ ##only support triton and exllamav2+ if not ("triton" in backend or "exllamav2" in backend):+ logger.warning_once(f"autoround format does not support {backend}, try to packing with autogptq")+ return get_autogptq_packing_qlinear(backend, bits, group_size, sym)from auto_round_extension.cuda.qliner_triton import QuantLinearreturn QuantLinear+ elif "gptq" in backend:+ return get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:assert False, f"only support gptq and autoround backend"@@ -98,53 +136,91 @@ def dynamic_QuantLienar_for_packing(backend, bits, group_size):@register_format("auto_round")def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exllamav2", **kwargs):+ """+ Saves a quantized model in the auto-round format.++ Args:+ output_dir (str): The directory where the quantized model will be saved.+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.+ Default is True.+ backend (str, optional): The backend to be used for quantization.+ Default is "autoround:exllamav2".+ **kwargs: Additional keyword arguments including:+ - model (nn.Module): The model to be quantized.+ - weight_config (dict): The weight configuration for each layer.+ - serialization_dict (dict): The serialization configuration.+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.++ Returns:+ None++ Raises:+ AssertionError: If the backend is not supported.+ """+ if ":" not in backend:+ backend = "autoround:exllamav2"+ if not ("triton" in backend or "exllamav2" in backend):+ logger.info(f"autoround format does not support {backend}, try to packing with autogptq")++ backend = backend.replace("autoround", "auto_round")+ backend = backend.replace("auto-round", "auto_round")+ backend = backend.replace("auto_round", "auto_gptq")model = kwargs["model"]+ model = model.to(torch.float16) ##force to fp16if not inplace:model = copy.deepcopy(model.to("cpu"))layer_names_in_block = get_layer_names_in_block(model)weight_config = kwargs["weight_config"]- for name in weight_config.keys():- config = kwargs["weight_config"][name]- if config["data_type"] != "int" and config["bits"] >= 16:- continue- logger.info(f"packing {name}")-- bits = config["bits"]- group_size = config["group_size"]-- layer = get_module(model, name)- device = layer.weight.device-- QuantLinear = dynamic_QuantLienar_for_packing(backend, bits, group_size)-- if isinstance(layer, nn.Linear):- in_features = layer.in_features- out_features = layer.out_features- elif isinstance(layer, nn.Conv2d):- in_features = layer.in_channels- out_features = layer.out_channels- elif isinstance(layer, transformers.pytorch_utils.Conv1D):- in_features = layer.weight.shape[0]- out_features = layer.weight.shape[1]- bias = layer.bias is not None and torch.any(layer.bias)-- new_layer = QuantLinear( ##pylint: disable=E1123- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype- )-- new_layer.device = device- set_module(model, name, new_layer)- qlayer = new_layer- scale = weight_config[name]["scale"]- zero = weight_config[name]["zp"]- # so far can only pack layer on CPU- qlayer.to("cpu")- ##force to float32 to be compatible with torch 2.0- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)- qlayer.pack(layer, scale, zero, None)- qlayer.to(device)+ with tctl.threadpool_limits(limits=1):+ for name in weight_config.keys():++ config = kwargs["weight_config"][name]+ if config["data_type"] != "int" and config["bits"] >= 16:+ continue+ logger.info(f"packing {name}")++ bits = config["bits"]+ group_size = config["group_size"]+ sym = config["sym"]++ layer = get_module(model, name)+ device = layer.weight.device++ QuantLinear = dynamic_import_quantLienar_for_packing(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]+ bias = layer.bias is not None and torch.any(layer.bias)++ new_layer = QuantLinear( ##pylint: disable=E1123+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype+ )++ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = weight_config[name]["scale"]+ zero = weight_config[name]["zp"]+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ sig = inspect.signature(qlayer.pack)+ param_count = len(sig.parameters)+ if param_count == 2:+ qlayer.pack(layer, scale)+ else:+ qlayer.pack(layer, scale, zero, None)+ qlayer.to(device)quantization_config = kwargs["serialization_dict"]quantization_config["quant_method"] = "intel/auto-round"quantization_config["backend"] = backenddiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 2b8d817bf..8b98f97d5 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -26,6 +26,8 @@ import torchfrom torch.amp import autocastfrom functools import lru_cache++@lru_cache(None)def warning_once(self, msg: str):self.warning(msg)@@ -43,6 +45,7 @@ logger.addHandler(fh)import importlibimport transformers+class LazyImport(object):"""Lazy import python module till use."""@@ -583,37 +586,46 @@ def is_autoround_exllamav2_available():return res-def get_autogptq_backend_config(backend, bits=4):+def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):use_triton = Falsedisable_exllamav2 = Falsedisable_exllamav1 = Falsedisable_marlin = Trueuse_qigen = False- if backend == "gptq:qigen":+ if "qigen" in backend:use_qigen = True- if backend == "gptq:triton": ##TODO refine the code+ elif "triton" in backend:use_triton = True- if backend == "gptq:marlin":+ elif "marlin" in backend:use_triton = False- disable_marlin = True- if backend == "gptq:exllamav2": ##need v1 code to export+ disable_marlin = False+ elif "exllamav2" in backend:use_triton = False+ disable_exllamav2 = Falsedisable_marlin = True- if backend == "gptq:exllamav1":+ elif "exllamav1" in backend:use_triton = Falsedisable_marlin = True- if backend == "gptq:cuda":+ elif "cuda" in backend:use_triton = Falsedisable_marlin = Truedisable_exllamav2 = Truedisable_exllamav1 = True- if bits not in [2, 4, 8]:- use_qigen = False- if bits not in [2, 4]:- use_triton = False- return use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin+ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401+ QuantLinear = dynamically_import_QuantLinear(+ use_triton=use_triton,+ desc_act=False,+ group_size=group_size,+ bits=bits,+ disable_exllama=disable_exllamav1,+ disable_exllamav2=disable_exllamav2,+ use_qigen=use_qigen,+ disable_marlin=disable_marlin,+ )+ return QuantLinear-def dynamic_import_inference_linear(bits, group_size, backend):++def dynamic_import_inference_linear(backend, bits, group_size, sym):"""Dynamically imports and returns the appropriate QuantLinear class based on the given bits and backend.Args:@@ -627,10 +639,11 @@ def dynamic_import_inference_linear(bits, group_size, backend):The appropriate QuantLinear class for the given configuration."""exllama2_available = is_autoround_exllamav2_available()-- if (not torch.cuda.is_available()) or "qbits" in backend or "cpu" in backend:+ ##TODO may have bug for marlin backend+ if (+ not torch.cuda.is_available()) or "qbits" in backend or "cpu" in backend:try:- from intel_extension_for_transformers import qbits # pylint: disable=E0401+ from intel_extension_for_transformers import qbits # pylint: disable=E0401except Exception as e:raise ImportError("Please install Intel Extension for Transformers via 'pip install ""intel-extension-for-transformers' to inference on X86 CPU")@@ -638,29 +651,15 @@ def dynamic_import_inference_linear(bits, group_size, backend):return qlinear_qbits.QuantLinearif "gptq" in backend:try:- import auto_gptq # pylint: disable=E0401+ import auto_gptq # pylint: disable=E0401except Exception as e:raise ImportError("Please install auto-gptq via 'pip install auto-gptq' to support GPTQ backend ")- use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(- backend, bits- )- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401- QuantLinear = dynamically_import_QuantLinear(- use_triton=use_triton,- desc_act=False,- group_size=group_size,- bits=bits,- disable_exllama=disable_exllamav1,- disable_exllamav2=disable_exllamav2,- use_qigen=use_qigen,- disable_marlin=disable_marlin,- )- return QuantLinear+ return get_autogptq_infer_linear(backend, bits, group_size, sym)if bits == 4 and exllama2_available and "exllamav2" in backend:from auto_round_extension.cuda.qliner_exllamav2 import QuantLinearelif bits == 4 and "exllamav2" in backend:logger.warning_once("Please install auto-round from source to enable exllamav2 kernels, switch to triton "- "kernels for now")+ "kernels for now")from auto_round_extension.cuda.qliner_triton import QuantLinearelse:from auto_round_extension.cuda.qliner_triton import QuantLineardiff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex 6a633d51a..bfb5a9fd5 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -80,6 +80,14 @@ Currently only support in Intel xpu and AutoRound format,however, we found the fCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage```+- **Enable marlin kernel:**++```bash+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --sym --bits 4 --group_size 128 --disable_low_gpu_mem_usage --deployment_device "auto_round:marlin"+```++for AutoGPTQ format, enable `--sym` in the quantization and set the deployment_device to "gpu" and then follow the API in AutoGPTQ+- **Utilizing the AdamW Optimizer:**Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 690f5bdc2..9145b6cc3 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -6,6 +6,7 @@ parser = argparse.ArgumentParser()import torchimport osimport transformers+import subprocessos.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"torch.use_deterministic_algorithms(True, warn_only=True)@@ -60,9 +61,6 @@ if __name__ == '__main__':parser.add_argument("--seed", default=42, type=int,help="seed")- parser.add_argument("--eval_fp16_baseline", action='store_true',- help="whether to eval FP16 baseline")-parser.add_argument("--amp", action='store_true',help="amp is deprecated ")@@ -141,44 +139,9 @@ if __name__ == '__main__':set_seed(args.seed)tasks = args.tasksuse_eval_legacy = False- import subprocess--- def get_library_version(library_name):- try:- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]- return version- except subprocess.CalledProcessError:- return "Library not found"--- lm_eval_version = get_library_version("lm-eval")- if lm_eval_version == "0.3.0":- use_eval_legacy = True-- if isinstance(tasks, str):- tasks = tasks.split(',')- if not use_eval_legacy:- from eval import eval_model- else:- from eval_legacy import eval_model-- if isinstance(tasks, list):- if "mmlu" in tasks:- tmp_tasks = tasks- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:- tmp_tasks = tasks- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]- seen = set()- tmp_tasks = tasks- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]- if 'fake' in args.deployment_device and not args.disable_eval:- if use_eval_legacy:- print("Using the legacy lm_eval(0.3.0)")- else:- print(f"Using the latest {lm_eval_version}")+ if "marlin" in args.deployment_device and args.sym == False:+ assert False, "marlin backend only supports sym quantization, please set --sym"model_name = args.model_nameif model_name[-1] == "/":@@ -242,14 +205,6 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"if "bloom" in model_name:args.disable_low_gpu_mem_usage = True- if args.eval_fp16_baseline:- if args.disable_low_gpu_mem_usage:- model = model.to(torch_device)- excel_name += "_fp16.xlsx"- eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \- eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,- device=torch_device, excel_file=excel_name)- exit()round = AutoRoundif args.adam:@@ -288,14 +243,6 @@ if __name__ == '__main__':if args.quant_lm_head and not args.disable_low_gpu_mem_usage:print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "f"gpu")- deployment_device = args.deployment_device.split(',')- gpu_format = "auto_gptq"- if 'gpu' in deployment_device:- if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":- gpu_format = "auto_round"-- if "autoround" in deployment_device or "auto-round" in deployment_device or "auto_round" in deployment_device:- gpu_format = "auto_round"autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,@@ -315,9 +262,29 @@ if __name__ == '__main__':export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"+ deployment_device = args.deployment_device.split(',')+ gpu_formats = []+ for item in deployment_device:+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item:+ gpu_formats.append(item)++ if 'gpu' in deployment_device:+ if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":+ gpu_formats.append("auto_round")+ else:+ gpu_formats.append("auto_gptq")+ gpu_formats = list(set(gpu_formats))+inplace = True if len(deployment_device) < 2 else False- if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:- autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)+ eval_folder = None+ for gpu_format in gpu_formats:+ if "round" in gpu_format:+ eval_folder = f'{export_dir}-round'+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ elif "gptq" in gpu_format:+ eval_folder = f'{export_dir}-gpu'+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+if 'xpu' in deployment_device:autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,@@ -328,6 +295,45 @@ if __name__ == '__main__':model = model.to("cpu")model.save_pretrained(output_dir)tokenizer.save_pretrained(output_dir)+ if eval_folder is None:+ eval_folder = output_dir+++ def get_library_version(library_name):+ try:+ version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]+ return version+ except subprocess.CalledProcessError:+ return "Library not found"+++ lm_eval_version = get_library_version("lm-eval")+ if lm_eval_version == "0.3.0":+ use_eval_legacy = True++ if isinstance(tasks, str):+ tasks = tasks.split(',')+ if not use_eval_legacy:+ from eval import eval_model+ else:+ from eval_legacy import eval_model++ if isinstance(tasks, list):+ if "mmlu" in tasks:+ tmp_tasks = tasks+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:+ tmp_tasks = tasks+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]+ seen = set()+ tmp_tasks = tasks+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]++ if 'fake' in args.deployment_device and not args.disable_eval:+ if use_eval_legacy:+ print("Using the legacy lm_eval(0.3.0)")+ else:+ print(f"Using the latest {lm_eval_version}")if not args.disable_eval and "fake" in deployment_device and lm_eval_version != "0.4.2":excel_name = f"{output_dir}_result.xlsx"@@ -338,16 +344,16 @@ if __name__ == '__main__':device=torch_device, excel_file=excel_name)if not args.disable_eval and lm_eval_version == "0.4.2":- if "round" in deployment_device:+ if "round" in eval_folder:from auto_round.auto_quantizer import AutoHfQuantizerfrom eval_042.evaluation import simple_evaluate- if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:- model_args = f"pretrained={export_dir}-gpu"+ if 'gpu' in deployment_device or len(gpu_formats) > 0:+ model_args = f"pretrained={eval_folder}"elif "fake" in deployment_device:- model_args = f"pretrained={output_dir}"+ model_args = f"pretrained={eval_folder}"else:- exit() ## does not support cpu,xpu model eval+ exit() ## does not support cpu,xpu model evalres = simple_evaluate(model="hf", model_args=model_args,tasks=tasks,@@ -355,4 +361,3 @@ if __name__ == '__main__':from lm_eval.utils import make_tableprint(make_table(res))-diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txtindex 13d67e77c..dab669f97 100644--- a/examples/language-modeling/requirements.txt+++ b/examples/language-modeling/requirements.txt@@ -18,4 +18,4 @@ openpyxlwandbpy-cpuinfonumpy < 2.0-+threadpoolctldiff --git a/requirements.txt b/requirements.txtindex 0f5c4f7af..83ec1fe4e 100644--- a/requirements.txt+++ b/requirements.txt@@ -6,3 +6,4 @@ torchtransformerstritonnumpy < 2.0+threadpoolctl\ No newline at end of file