Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix example (#256)

n1ck-guod8ee1c071c12
diff --git a/README.md b/README.md
index 31364a528..1f24f9006 100644
--- a/README.md
+++ b/README.md
@@ -152,7 +152,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.
--group_size 128 \
--nsamples 512 \
--iters 1000 \
- --low_gpu_mem_usage
+ --low_gpu_mem_usage \
+ --disable_eval
```
```bash
@@ -163,7 +164,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.
--nsamples 128 \
--iters 200 \
--seqlen 512 \
- --batch_size 4
+ --batch_size 4 \
+ --disable_eval
```
</details>
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index ea24d8b4f..b298ada72 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -169,7 +169,6 @@ def tune(args):
torch_dtype = "auto"
if "hpu" in device_str:
torch_dtype = torch.bfloat16
- torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
low_cpu_mem_usage = False
@@ -307,7 +306,7 @@ def tune(args):
if "cpu" not in device_str:
torch.cuda.empty_cache()
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"
format_list = args.format.replace(' ', '').split(',')
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 78e68cc60..6f84275c7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -52,14 +52,6 @@ from .utils import (
from .low_cpu_mem.utils import get_layers_before_block
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
-import logging
-import numexpr
-dataset_logger = logging.getLogger("datasets")
-dataset_logger.disabled = True
-numexpr_logger = logging.getLogger("numexpr")
-numexpr_logger.disabled = True
-
class AutoRound(object):
"""This is Signround+ which is an advanced version of Signround. For more information,
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 6eeb81c8a..c63367ea8 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -15,6 +15,7 @@
import copy
import logging
import os
+import sys
import subprocess
from collections import UserDict
@@ -826,15 +827,25 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
return QuantLinear
-
def get_library_version(library_name):
- import pkg_resources
- try:
- version = pkg_resources.get_distribution(library_name).version
- return version
- except pkg_resources.DistributionNotFound:
- return f"{library_name} is not installed"
-
+ from packaging.version import Version
+ python_vesion = Version(sys.version.split()[0])
+ if python_vesion < Version("3.8"):
+ import warnings
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
+ import pkg_resources
+ try:
+ version = pkg_resources.get_distribution(library_name).version
+ return version
+ except pkg_resources.DistributionNotFound:
+ return f"{library_name} is not installed"
+ else:
+ import importlib_metadata
+ try:
+ version = importlib_metadata.version(library_name)
+ return version
+ except importlib_metadata.PackageNotFoundError:
+ return f"{library_name} is not installed"
def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
"""
diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.py
index 7a0666b70..458986dc8 100644
--- a/examples/language-modeling/eval_legacy/evaluation.py
+++ b/examples/language-modeling/eval_legacy/evaluation.py
@@ -342,7 +342,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
org_s = time.time()
if dtype == None:
- from eval.utils import convert_dtype_torch2str_hf
+ from .utils import convert_dtype_torch2str_hf
from transformers import AutoConfig
config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
if hasattr(config, "torch_dtype"):
@@ -420,7 +420,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
# model.seqlen = seqlen
model.seqlen = 2048
- from eval.utils import get_loaders, eval_ppl_same_with_gptq
+ from .utils import get_loaders, eval_ppl_same_with_gptq
for dataset in external_tasks:
try:
dataloader, testloader = get_loaders(
diff --git a/examples/language-modeling/eval_legacy/evaluation_legacy.py b/examples/language-modeling/eval_legacy/evaluation_legacy.py
index 68656a7d9..5aecf3c24 100644
--- a/examples/language-modeling/eval_legacy/evaluation_legacy.py
+++ b/examples/language-modeling/eval_legacy/evaluation_legacy.py
@@ -215,7 +215,7 @@ def eval_model_legacy(model_path=None, tasks=["lambada_openai", "hellaswag", "wi
org_s = time.time()
if dtype == None:
- from eval.utils import convert_dtype_torch2str_hf
+ from .utils import convert_dtype_torch2str_hf
from transformers import AutoConfig
config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
if hasattr(config, "torch_dtype"):
@@ -285,7 +285,7 @@ def eval_model_legacy(model_path=None, tasks=["lambada_openai", "hellaswag", "wi
# model.seqlen = seqlen
model.seqlen = 2048
- from eval.utils import get_loaders, eval_ppl_same_with_gptq
+ from .utils import get_loaders, eval_ppl_same_with_gptq
for dataset in external_tasks:
try:
dataloader, testloader = get_loaders(
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 49206aea9..48ed2770f 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -5,6 +5,7 @@ import argparse
import subprocess
from packaging import version
+
sys.path.insert(0, '../..')
parser = argparse.ArgumentParser()
import torch
@@ -15,6 +16,17 @@ torch.use_deterministic_algorithms(True, warn_only=True)
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+import logging
+import warnings
+import numexpr
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+warnings.filterwarnings('ignore', category=FutureWarning)
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+dataset_logger = logging.getLogger("datasets")
+dataset_logger.disabled = True
+numexpr_logger = logging.getLogger("numexpr")
+numexpr_logger.disabled = True
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
if __name__ == '__main__':
@@ -335,8 +347,8 @@ if __name__ == '__main__':
if "cpu" not in device_str:
torch.cuda.empty_cache()
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"
+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}-qdq"
eval_folder = None
if args.format:
@@ -379,16 +391,8 @@ if __name__ == '__main__':
print('does not support cpu, xpu model evaluation.')
exit() ## does not support cpu,xpu model eval
- def get_library_version(library_name):
- import pkg_resources
- try:
- version = pkg_resources.get_distribution(library_name).version
- return version
- except pkg_resources.DistributionNotFound:
- return f"{library_name} is not installed"
-
-
from packaging.version import Version
+ from auto_round.utils import get_library_version
lm_eval_version = get_library_version("lm_eval")
lm_eval_version = Version(lm_eval_version)
if lm_eval_version == Version("0.3.0"):
@@ -398,19 +402,6 @@ if __name__ == '__main__':
use_eval_legacy = False
from eval_legacy import eval_model
- if isinstance(tasks, str):
- tasks = tasks.split(',')
- if isinstance(tasks, list):
- if "mmlu" in tasks:
- tmp_tasks = tasks
- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
- tmp_tasks = tasks
- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
- seen = set()
- tmp_tasks = tasks
- tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
-
use_qdq = False
if args.deployment_device and 'fake' in args.deployment_device:
use_qdq = True
@@ -424,7 +415,21 @@ if __name__ == '__main__':
else:
print(f"Using the latest {lm_eval_version}")
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+
if use_qdq and lm_eval_version < Version("0.4.2"):
+ if use_eval_legacy:
+ if "mmlu" in tasks:
+ tmp_tasks = tasks
+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
+ tmp_tasks = tasks
+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
+ seen = set()
+ tmp_tasks = tasks
+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
+
excel_name = f"{output_dir}_result.xlsx"
output_dir += "/"
print(excel_name, flush=True)