PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix example (#256)
n1ck-guod8ee1c071c12
diff --git a/README.md b/README.mdindex 31364a528..1f24f9006 100644--- a/README.md+++ b/README.md@@ -152,7 +152,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.--group_size 128 \--nsamples 512 \--iters 1000 \- --low_gpu_mem_usage+ --low_gpu_mem_usage \+ --disable_eval``````bash@@ -163,7 +164,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.--nsamples 128 \--iters 200 \--seqlen 512 \- --batch_size 4+ --batch_size 4 \+ --disable_eval```</details>diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex ea24d8b4f..b298ada72 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -169,7 +169,6 @@ def tune(args):torch_dtype = "auto"if "hpu" in device_str:torch_dtype = torch.bfloat16- torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))low_cpu_mem_usage = False@@ -307,7 +306,7 @@ def tune(args):if "cpu" not in device_str:torch.cuda.empty_cache()- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"format_list = args.format.replace(' ', '').split(',')diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 78e68cc60..6f84275c7 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -52,14 +52,6 @@ from .utils import (from .low_cpu_mem.utils import get_layers_before_block-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'-import logging-import numexpr-dataset_logger = logging.getLogger("datasets")-dataset_logger.disabled = True-numexpr_logger = logging.getLogger("numexpr")-numexpr_logger.disabled = True-class AutoRound(object):"""This is Signround+ which is an advanced version of Signround. For more information,diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 6eeb81c8a..c63367ea8 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -15,6 +15,7 @@import copyimport loggingimport os+import sysimport subprocessfrom collections import UserDict@@ -826,15 +827,25 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearreturn QuantLinear-def get_library_version(library_name):- import pkg_resources- try:- version = pkg_resources.get_distribution(library_name).version- return version- except pkg_resources.DistributionNotFound:- return f"{library_name} is not installed"-+ from packaging.version import Version+ python_vesion = Version(sys.version.split()[0])+ if python_vesion < Version("3.8"):+ import warnings+ warnings.filterwarnings('ignore', category=DeprecationWarning)+ import pkg_resources+ try:+ version = pkg_resources.get_distribution(library_name).version+ return version+ except pkg_resources.DistributionNotFound:+ return f"{library_name} is not installed"+ else:+ import importlib_metadata+ try:+ version = importlib_metadata.version(library_name)+ return version+ except importlib_metadata.PackageNotFoundError:+ return f"{library_name} is not installed"def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):"""diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.pyindex 7a0666b70..458986dc8 100644--- a/examples/language-modeling/eval_legacy/evaluation.py+++ b/examples/language-modeling/eval_legacy/evaluation.py@@ -342,7 +342,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "org_s = time.time()if dtype == None:- from eval.utils import convert_dtype_torch2str_hf+ from .utils import convert_dtype_torch2str_hffrom transformers import AutoConfigconfig = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)if hasattr(config, "torch_dtype"):@@ -420,7 +420,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "# model.seqlen = seqlenmodel.seqlen = 2048- from eval.utils import get_loaders, eval_ppl_same_with_gptq+ from .utils import get_loaders, eval_ppl_same_with_gptqfor dataset in external_tasks:try:dataloader, testloader = get_loaders(diff --git a/examples/language-modeling/eval_legacy/evaluation_legacy.py b/examples/language-modeling/eval_legacy/evaluation_legacy.pyindex 68656a7d9..5aecf3c24 100644--- a/examples/language-modeling/eval_legacy/evaluation_legacy.py+++ b/examples/language-modeling/eval_legacy/evaluation_legacy.py@@ -215,7 +215,7 @@ def eval_model_legacy(model_path=None, tasks=["lambada_openai", "hellaswag", "wiorg_s = time.time()if dtype == None:- from eval.utils import convert_dtype_torch2str_hf+ from .utils import convert_dtype_torch2str_hffrom transformers import AutoConfigconfig = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)if hasattr(config, "torch_dtype"):@@ -285,7 +285,7 @@ def eval_model_legacy(model_path=None, tasks=["lambada_openai", "hellaswag", "wi# model.seqlen = seqlenmodel.seqlen = 2048- from eval.utils import get_loaders, eval_ppl_same_with_gptq+ from .utils import get_loaders, eval_ppl_same_with_gptqfor dataset in external_tasks:try:dataloader, testloader = get_loaders(diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 49206aea9..48ed2770f 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -5,6 +5,7 @@ import argparseimport subprocessfrom packaging import version+sys.path.insert(0, '../..')parser = argparse.ArgumentParser()import torch@@ -15,6 +16,17 @@ torch.use_deterministic_algorithms(True, warn_only=True)from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel+import logging+import warnings+import numexpr+warnings.filterwarnings('ignore', category=DeprecationWarning)+warnings.filterwarnings('ignore', category=FutureWarning)+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'+dataset_logger = logging.getLogger("datasets")+dataset_logger.disabled = True+numexpr_logger = logging.getLogger("numexpr")+numexpr_logger.disabled = True+os.environ["TOKENIZERS_PARALLELISM"] = "false"if __name__ == '__main__':@@ -335,8 +347,8 @@ if __name__ == '__main__':if "cpu" not in device_str:torch.cuda.empty_cache()- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}-qdq"eval_folder = Noneif args.format:@@ -379,16 +391,8 @@ if __name__ == '__main__':print('does not support cpu, xpu model evaluation.')exit() ## does not support cpu,xpu model eval- def get_library_version(library_name):- import pkg_resources- try:- version = pkg_resources.get_distribution(library_name).version- return version- except pkg_resources.DistributionNotFound:- return f"{library_name} is not installed"--from packaging.version import Version+ from auto_round.utils import get_library_versionlm_eval_version = get_library_version("lm_eval")lm_eval_version = Version(lm_eval_version)if lm_eval_version == Version("0.3.0"):@@ -398,19 +402,6 @@ if __name__ == '__main__':use_eval_legacy = Falsefrom eval_legacy import eval_model- if isinstance(tasks, str):- tasks = tasks.split(',')- if isinstance(tasks, list):- if "mmlu" in tasks:- tmp_tasks = tasks- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:- tmp_tasks = tasks- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]- seen = set()- tmp_tasks = tasks- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]-use_qdq = Falseif args.deployment_device and 'fake' in args.deployment_device:use_qdq = True@@ -424,7 +415,21 @@ if __name__ == '__main__':else:print(f"Using the latest {lm_eval_version}")+ if isinstance(tasks, str):+ tasks = tasks.split(',')+if use_qdq and lm_eval_version < Version("0.4.2"):+ if use_eval_legacy:+ if "mmlu" in tasks:+ tmp_tasks = tasks+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:+ tmp_tasks = tasks+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]+ seen = set()+ tmp_tasks = tasks+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]+excel_name = f"{output_dir}_result.xlsx"output_dir += "/"print(excel_name, flush=True)