Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support marlin in auto_round format (#172)

wenhuach212b1448d42036
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 045a64572..dab5c7f21 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -44,6 +44,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quanti
from auto_round.utils import get_module, set_module, dynamic_import_inference_linear
import auto_round_extension.qbits.qlinear_qbits as qlinear_qbits
from enum import Enum
+
logger = getLogger(__name__)
import sys
@@ -194,6 +195,7 @@ class AutoHfQuantizer:
return quantization_config
+
class AutoRoundQuantizationMethod(str, Enum):
AutoRound = "intel/auto-round"
@@ -336,12 +338,13 @@ class AutoRoundQuantizer(HfQuantizer):
bits = config["bits"]
group_size = config["group_size"]
data_type = config["data_type"]
- if not (bits <= 8 and data_type == "int"):
+ sym = config["sym"]
+ if not (bits <= 8):
continue
layer = get_module(module, layer_name)
device = get_device(layer)
- QuantLinear = dynamic_import_inference_linear(bits, group_size, backend)
+ QuantLinear = dynamic_import_inference_linear(backend, bits, group_size, sym)
if isinstance(layer, nn.Linear):
in_features = layer.in_features
out_features = layer.out_features
@@ -382,7 +385,6 @@ class AutoRoundQuantizer(HfQuantizer):
The input model
"""
-
class StoreAttr(object):
pass
@@ -406,7 +408,6 @@ class AutoRoundQuantizer(HfQuantizer):
else:
raise NotImplementedError
-
@property
def is_trainable(self, model: Optional["PreTrainedModel"] = None):
return True
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 361c820f7..da040752c 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1008,7 +1008,8 @@ class AutoRound(object):
logger.warning("please run autoround.quantize first")
return
from auto_round.export import EXPORT_FORMAT
-
+ backend = format
+ format = format.split(":")[0]
if format not in EXPORT_FORMAT:
logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
exit()
@@ -1061,7 +1062,8 @@ class AutoRound(object):
supported_types=self.supported_types,
data_type=self.data_type,
serialization_dict=serialization_dict,
- **kwargs,
+ backend=backend,
+ **kwargs
)
return compressed_model
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 9d3ded310..68aa4585f 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -23,9 +23,24 @@ import transformers
from auto_round.export.register import register_format
from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module
+import threadpoolctl as tctl
+import inspect
def check_neq_config(config, data_type, bits, group_size, sym):
+ """
+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.
+
+ Args:
+ config (dict): A dictionary containing the configuration parameters.
+ data_type (str): The expected data type.
+ bits (int): The expected number of bits.
+ group_size (int): The expected group size.
+ sym (bool): The expected symmetry flag.
+
+ Returns:
+ list: A list of strings indicating which configuration parameters do not match.
+ """
res = []
if data_type != config["data_type"]:
res.append("data_type")
@@ -39,58 +54,81 @@ def check_neq_config(config, data_type, bits, group_size, sym):
return res
-def get_autogptq_backend_config(backend, bits=4):
- use_triton = False
- disable_exllamav2 = False
+def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
+ """
+ Configures and returns a QuantLinear class based on the specified backend and parameters.
+
+ Args:
+ backend (str): The backend to be used for quantization. Supported values include "qigen", "triton", "marlin",
+ "exllama", and "cuda".
+ bits (int, optional): The number of bits for quantization. Default is 4.
+ group_size (int, optional): The group size for quantization. Default is 128.
+ sym (bool, optional): Flag indicating whether to use symmetric quantization. Default is False.
+
+ Returns:
+ class: The dynamically imported QuantLinear class configured according to the specified parameters.
+ """
+ use_triton = True
+ disable_exllamav2 = True
disable_exllamav1 = False
disable_marlin = True
use_qigen = False
- if backend == "gptq:qigen":
+ if "qigen" in "backend":
+ use_triton = False
use_qigen = True
- if backend == "gptq:triton": ##TODO refine the code
+ elif "triton" in backend:
use_triton = True
- if backend == "gptq:marlin":
+ elif "marlin" in backend and sym:
use_triton = False
disable_marlin = False
- if backend == "gptq:exllamav2": ##need v1 code to export
- use_triton = False
- disable_marlin = True
- if backend == "gptq:exllamav1":
- use_triton = False
+ elif "exllama" in backend: ##need v1 code to export
+ use_triton = True ##same with triton
disable_marlin = True
- if backend == "gptq:cuda":
+ elif "cuda" in backend:
use_triton = False
disable_marlin = True
disable_exllamav2 = True
disable_exllamav1 = True
- if bits not in [2, 4, 8]:
- use_qigen = False
- if bits not in [2, 4]:
- use_triton = False
- return use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin
-
-
-def dynamic_QuantLienar_for_packing(backend, bits, group_size):
- if "gptq" in backend:
- use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(
- backend, bits
- )
- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
- QuantLinear = dynamically_import_QuantLinear(
- use_triton=use_triton,
- desc_act=False,
- group_size=group_size,
- bits=bits,
- disable_exllama=disable_exllamav1,
- disable_exllamav2=disable_exllamav2,
- use_qigen=use_qigen,
- disable_marlin=disable_marlin,
- )
- return QuantLinear
- ##export all use trition, inference use exllamav2
- elif "autoround" in backend or "auto-round" in backend or "auto_round" in backend:
+
+ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
+ QuantLinear = dynamically_import_QuantLinear(
+ use_triton=use_triton,
+ desc_act=False,
+ group_size=group_size,
+ bits=bits,
+ disable_exllama=disable_exllamav1,
+ disable_exllamav2=disable_exllamav2,
+ use_qigen=use_qigen,
+ disable_marlin=disable_marlin,
+ )
+ return QuantLinear
+
+
+def dynamic_import_quantLienar_for_packing(backend, bits, group_size, sym):
+ """
+ Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.
+
+ Args:
+ backend (str): The backend to be used for quantization. Supported values include "auto_round" and "gptq".
+ bits (int): The number of bits for quantization.
+ group_size (int): The group size for quantization.
+ sym (bool): Flag indicating whether to use symmetric quantization.
+
+ Returns:
+ class: The dynamically imported QuantLinear class configured according to the specified parameters.
+
+ Raises:
+ AssertionError: If the backend is not supported.
+ """
+ if "auto_round" in backend:
+ ##only support triton and exllamav2
+ if not ("triton" in backend or "exllamav2" in backend):
+ logger.warning_once(f"autoround format does not support {backend}, try to packing with autogptq")
+ return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
from auto_round_extension.cuda.qliner_triton import QuantLinear
return QuantLinear
+ elif "gptq" in backend:
+ return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
assert False, f"only support gptq and autoround backend"
@@ -98,53 +136,91 @@ def dynamic_QuantLienar_for_packing(backend, bits, group_size):
@register_format("auto_round")
def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exllamav2", **kwargs):
+ """
+ Saves a quantized model in the auto-round format.
+
+ Args:
+ output_dir (str): The directory where the quantized model will be saved.
+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.
+ Default is True.
+ backend (str, optional): The backend to be used for quantization.
+ Default is "autoround:exllamav2".
+ **kwargs: Additional keyword arguments including:
+ - model (nn.Module): The model to be quantized.
+ - weight_config (dict): The weight configuration for each layer.
+ - serialization_dict (dict): The serialization configuration.
+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.
+
+ Returns:
+ None
+
+ Raises:
+ AssertionError: If the backend is not supported.
+ """
+ if ":" not in backend:
+ backend = "autoround:exllamav2"
+ if not ("triton" in backend or "exllamav2" in backend):
+ logger.info(f"autoround format does not support {backend}, try to packing with autogptq")
+
+ backend = backend.replace("autoround", "auto_round")
+ backend = backend.replace("auto-round", "auto_round")
+ backend = backend.replace("auto_round", "auto_gptq")
model = kwargs["model"]
+ model = model.to(torch.float16) ##force to fp16
if not inplace:
model = copy.deepcopy(model.to("cpu"))
layer_names_in_block = get_layer_names_in_block(model)
weight_config = kwargs["weight_config"]
- for name in weight_config.keys():
- config = kwargs["weight_config"][name]
- if config["data_type"] != "int" and config["bits"] >= 16:
- continue
- logger.info(f"packing {name}")
-
- bits = config["bits"]
- group_size = config["group_size"]
-
- layer = get_module(model, name)
- device = layer.weight.device
-
- QuantLinear = dynamic_QuantLienar_for_packing(backend, bits, group_size)
-
- if isinstance(layer, nn.Linear):
- in_features = layer.in_features
- out_features = layer.out_features
- elif isinstance(layer, nn.Conv2d):
- in_features = layer.in_channels
- out_features = layer.out_channels
- elif isinstance(layer, transformers.pytorch_utils.Conv1D):
- in_features = layer.weight.shape[0]
- out_features = layer.weight.shape[1]
- bias = layer.bias is not None and torch.any(layer.bias)
-
- new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
-
- new_layer.device = device
- set_module(model, name, new_layer)
- qlayer = new_layer
- scale = weight_config[name]["scale"]
- zero = weight_config[name]["zp"]
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- ##force to float32 to be compatible with torch 2.0
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
- qlayer.pack(layer, scale, zero, None)
- qlayer.to(device)
+ with tctl.threadpool_limits(limits=1):
+ for name in weight_config.keys():
+
+ config = kwargs["weight_config"][name]
+ if config["data_type"] != "int" and config["bits"] >= 16:
+ continue
+ logger.info(f"packing {name}")
+
+ bits = config["bits"]
+ group_size = config["group_size"]
+ sym = config["sym"]
+
+ layer = get_module(model, name)
+ device = layer.weight.device
+
+ QuantLinear = dynamic_import_quantLienar_for_packing(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+ bias = layer.bias is not None and torch.any(layer.bias)
+
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
+ )
+
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = weight_config[name]["scale"]
+ zero = weight_config[name]["zp"]
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ ##force to float32 to be compatible with torch 2.0
+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
+ sig = inspect.signature(qlayer.pack)
+ param_count = len(sig.parameters)
+ if param_count == 2:
+ qlayer.pack(layer, scale)
+ else:
+ qlayer.pack(layer, scale, zero, None)
+ qlayer.to(device)
quantization_config = kwargs["serialization_dict"]
quantization_config["quant_method"] = "intel/auto-round"
quantization_config["backend"] = backend
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 2b8d817bf..8b98f97d5 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -26,6 +26,8 @@ import torch
from torch.amp import autocast
from functools import lru_cache
+
+
@lru_cache(None)
def warning_once(self, msg: str):
self.warning(msg)
@@ -43,6 +45,7 @@ logger.addHandler(fh)
import importlib
import transformers
+
class LazyImport(object):
"""Lazy import python module till use."""
@@ -583,37 +586,46 @@ def is_autoround_exllamav2_available():
return res
-def get_autogptq_backend_config(backend, bits=4):
+def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
use_triton = False
disable_exllamav2 = False
disable_exllamav1 = False
disable_marlin = True
use_qigen = False
- if backend == "gptq:qigen":
+ if "qigen" in backend:
use_qigen = True
- if backend == "gptq:triton": ##TODO refine the code
+ elif "triton" in backend:
use_triton = True
- if backend == "gptq:marlin":
+ elif "marlin" in backend:
use_triton = False
- disable_marlin = True
- if backend == "gptq:exllamav2": ##need v1 code to export
+ disable_marlin = False
+ elif "exllamav2" in backend:
use_triton = False
+ disable_exllamav2 = False
disable_marlin = True
- if backend == "gptq:exllamav1":
+ elif "exllamav1" in backend:
use_triton = False
disable_marlin = True
- if backend == "gptq:cuda":
+ elif "cuda" in backend:
use_triton = False
disable_marlin = True
disable_exllamav2 = True
disable_exllamav1 = True
- if bits not in [2, 4, 8]:
- use_qigen = False
- if bits not in [2, 4]:
- use_triton = False
- return use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin
+ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
+ QuantLinear = dynamically_import_QuantLinear(
+ use_triton=use_triton,
+ desc_act=False,
+ group_size=group_size,
+ bits=bits,
+ disable_exllama=disable_exllamav1,
+ disable_exllamav2=disable_exllamav2,
+ use_qigen=use_qigen,
+ disable_marlin=disable_marlin,
+ )
+ return QuantLinear
-def dynamic_import_inference_linear(bits, group_size, backend):
+
+def dynamic_import_inference_linear(backend, bits, group_size, sym):
"""Dynamically imports and returns the appropriate QuantLinear class based on the given bits and backend.
Args:
@@ -627,10 +639,11 @@ def dynamic_import_inference_linear(bits, group_size, backend):
The appropriate QuantLinear class for the given configuration.
"""
exllama2_available = is_autoround_exllamav2_available()
-
- if (not torch.cuda.is_available()) or "qbits" in backend or "cpu" in backend:
+ ##TODO may have bug for marlin backend
+ if (
+ not torch.cuda.is_available()) or "qbits" in backend or "cpu" in backend:
try:
- from intel_extension_for_transformers import qbits # pylint: disable=E0401
+ from intel_extension_for_transformers import qbits # pylint: disable=E0401
except Exception as e:
raise ImportError("Please install Intel Extension for Transformers via 'pip install "
"intel-extension-for-transformers' to inference on X86 CPU")
@@ -638,29 +651,15 @@ def dynamic_import_inference_linear(bits, group_size, backend):
return qlinear_qbits.QuantLinear
if "gptq" in backend:
try:
- import auto_gptq # pylint: disable=E0401
+ import auto_gptq # pylint: disable=E0401
except Exception as e:
raise ImportError("Please install auto-gptq via 'pip install auto-gptq' to support GPTQ backend ")
- use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(
- backend, bits
- )
- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
- QuantLinear = dynamically_import_QuantLinear(
- use_triton=use_triton,
- desc_act=False,
- group_size=group_size,
- bits=bits,
- disable_exllama=disable_exllamav1,
- disable_exllamav2=disable_exllamav2,
- use_qigen=use_qigen,
- disable_marlin=disable_marlin,
- )
- return QuantLinear
+ return get_autogptq_infer_linear(backend, bits, group_size, sym)
if bits == 4 and exllama2_available and "exllamav2" in backend:
from auto_round_extension.cuda.qliner_exllamav2 import QuantLinear
elif bits == 4 and "exllamav2" in backend:
logger.warning_once("Please install auto-round from source to enable exllamav2 kernels, switch to triton "
- "kernels for now")
+ "kernels for now")
from auto_round_extension.cuda.qliner_triton import QuantLinear
else:
from auto_round_extension.cuda.qliner_triton import QuantLinear
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 6a633d51a..bfb5a9fd5 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -80,6 +80,14 @@ Currently only support in Intel xpu and AutoRound format,however, we found the f
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage
```
+- **Enable marlin kernel:**
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --sym --bits 4 --group_size 128 --disable_low_gpu_mem_usage --deployment_device "auto_round:marlin"
+```
+
+for AutoGPTQ format, enable `--sym` in the quantization and set the deployment_device to "gpu" and then follow the API in AutoGPTQ
+
- **Utilizing the AdamW Optimizer:**
Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 690f5bdc2..9145b6cc3 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -6,6 +6,7 @@ parser = argparse.ArgumentParser()
import torch
import os
import transformers
+import subprocess
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=True)
@@ -60,9 +61,6 @@ if __name__ == '__main__':
parser.add_argument("--seed", default=42, type=int,
help="seed")
- parser.add_argument("--eval_fp16_baseline", action='store_true',
- help="whether to eval FP16 baseline")
-
parser.add_argument("--amp", action='store_true',
help="amp is deprecated ")
@@ -141,44 +139,9 @@ if __name__ == '__main__':
set_seed(args.seed)
tasks = args.tasks
use_eval_legacy = False
- import subprocess
-
-
- def get_library_version(library_name):
- try:
- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
- return version
- except subprocess.CalledProcessError:
- return "Library not found"
-
-
- lm_eval_version = get_library_version("lm-eval")
- if lm_eval_version == "0.3.0":
- use_eval_legacy = True
-
- if isinstance(tasks, str):
- tasks = tasks.split(',')
- if not use_eval_legacy:
- from eval import eval_model
- else:
- from eval_legacy import eval_model
-
- if isinstance(tasks, list):
- if "mmlu" in tasks:
- tmp_tasks = tasks
- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
- tmp_tasks = tasks
- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
- seen = set()
- tmp_tasks = tasks
- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
- if 'fake' in args.deployment_device and not args.disable_eval:
- if use_eval_legacy:
- print("Using the legacy lm_eval(0.3.0)")
- else:
- print(f"Using the latest {lm_eval_version}")
+ if "marlin" in args.deployment_device and args.sym == False:
+ assert False, "marlin backend only supports sym quantization, please set --sym"
model_name = args.model_name
if model_name[-1] == "/":
@@ -242,14 +205,6 @@ if __name__ == '__main__':
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
if "bloom" in model_name:
args.disable_low_gpu_mem_usage = True
- if args.eval_fp16_baseline:
- if args.disable_low_gpu_mem_usage:
- model = model.to(torch_device)
- excel_name += "_fp16.xlsx"
- eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \
- eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,
- device=torch_device, excel_file=excel_name)
- exit()
round = AutoRound
if args.adam:
@@ -288,14 +243,6 @@ if __name__ == '__main__':
if args.quant_lm_head and not args.disable_low_gpu_mem_usage:
print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "
f"gpu")
- deployment_device = args.deployment_device.split(',')
- gpu_format = "auto_gptq"
- if 'gpu' in deployment_device:
- if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":
- gpu_format = "auto_round"
-
- if "autoround" in deployment_device or "auto-round" in deployment_device or "auto_round" in deployment_device:
- gpu_format = "auto_round"
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
@@ -315,9 +262,29 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
+ deployment_device = args.deployment_device.split(',')
+ gpu_formats = []
+ for item in deployment_device:
+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item:
+ gpu_formats.append(item)
+
+ if 'gpu' in deployment_device:
+ if lm_head_layer_name in weight_config.keys() and weight_config[lm_head_layer_name]["data_type"] == "int":
+ gpu_formats.append("auto_round")
+ else:
+ gpu_formats.append("auto_gptq")
+ gpu_formats = list(set(gpu_formats))
+
inplace = True if len(deployment_device) < 2 else False
- if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:
- autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)
+ eval_folder = None
+ for gpu_format in gpu_formats:
+ if "round" in gpu_format:
+ eval_folder = f'{export_dir}-round'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ elif "gptq" in gpu_format:
+ eval_folder = f'{export_dir}-gpu'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+
if 'xpu' in deployment_device:
autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,
@@ -328,6 +295,45 @@ if __name__ == '__main__':
model = model.to("cpu")
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
+ if eval_folder is None:
+ eval_folder = output_dir
+
+
+ def get_library_version(library_name):
+ try:
+ version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
+ return version
+ except subprocess.CalledProcessError:
+ return "Library not found"
+
+
+ lm_eval_version = get_library_version("lm-eval")
+ if lm_eval_version == "0.3.0":
+ use_eval_legacy = True
+
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+ if not use_eval_legacy:
+ from eval import eval_model
+ else:
+ from eval_legacy import eval_model
+
+ if isinstance(tasks, list):
+ if "mmlu" in tasks:
+ tmp_tasks = tasks
+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
+ tmp_tasks = tasks
+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
+ seen = set()
+ tmp_tasks = tasks
+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
+
+ if 'fake' in args.deployment_device and not args.disable_eval:
+ if use_eval_legacy:
+ print("Using the legacy lm_eval(0.3.0)")
+ else:
+ print(f"Using the latest {lm_eval_version}")
if not args.disable_eval and "fake" in deployment_device and lm_eval_version != "0.4.2":
excel_name = f"{output_dir}_result.xlsx"
@@ -338,16 +344,16 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name)
if not args.disable_eval and lm_eval_version == "0.4.2":
- if "round" in deployment_device:
+ if "round" in eval_folder:
from auto_round.auto_quantizer import AutoHfQuantizer
from eval_042.evaluation import simple_evaluate
- if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:
- model_args = f"pretrained={export_dir}-gpu"
+ if 'gpu' in deployment_device or len(gpu_formats) > 0:
+ model_args = f"pretrained={eval_folder}"
elif "fake" in deployment_device:
- model_args = f"pretrained={output_dir}"
+ model_args = f"pretrained={eval_folder}"
else:
- exit() ## does not support cpu,xpu model eval
+ exit() ## does not support cpu,xpu model eval
res = simple_evaluate(model="hf", model_args=model_args,
tasks=tasks,
@@ -355,4 +361,3 @@ if __name__ == '__main__':
from lm_eval.utils import make_table
print(make_table(res))
-
diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txt
index 13d67e77c..dab669f97 100644
--- a/examples/language-modeling/requirements.txt
+++ b/examples/language-modeling/requirements.txt
@@ -18,4 +18,4 @@ openpyxl
wandb
py-cpuinfo
numpy < 2.0
-
+threadpoolctl
diff --git a/requirements.txt b/requirements.txt
index 0f5c4f7af..83ec1fe4e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ torch
transformers
triton
numpy < 2.0
+threadpoolctl
\ No newline at end of file