Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix old eval tasks order (#78)

WeiweiZhang1257c5be2cded
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 54abf173c..ef8dc6cbd 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -34,8 +34,8 @@ from .utils import (
get_module,
get_scale_shape,
htcore,
- is_hpu_available,
is_local_path,
+ is_optimum_habana_available,
logger,
move_input_to_device,
quant_weight,
@@ -43,10 +43,6 @@ from .utils import (
set_module,
)
-if is_hpu_available:
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401
-
class WrapperLinear(torch.nn.Module):
def __init__(self, orig_layer, enable_minmax_tuning=True):
@@ -513,6 +509,10 @@ class AutoRound(object):
self.share_attention_mask_flag = None
self.hidden_dim_flag = None
torch.set_printoptions(precision=3, sci_mode=True)
+ if is_optimum_habana_available():
+ logger.info("Optimum Habana is available, import htcore explicitly.")
+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401
+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401
def get_optimizer(self, optimizer):
"""Returns the specified optimizer. In SignRound, we fix the optimizer.
@@ -543,7 +543,7 @@ class AutoRound(object):
"""
scale_loss = loss * 1000
scale_loss.backward()
- if is_hpu_available:
+ if is_optimum_habana_available():
htcore.mark_step()
return scale_loss
@@ -560,7 +560,7 @@ class AutoRound(object):
"""
optimizer.step()
# for hpu
- if is_hpu_available:
+ if is_optimum_habana_available():
htcore.mark_step()
optimizer.zero_grad()
lr_schedule.step()
@@ -1566,7 +1566,7 @@ class AutoOPTRound(AutoRound):
loss = scaler.scale(loss)
loss.backward()
- if is_hpu_available:
+ if is_optimum_habana_available():
htcore.mark_step()
return loss
@@ -1580,7 +1580,7 @@ class AutoOPTRound(AutoRound):
optimizer.step()
optimizer.zero_grad()
lr_schedule.step()
- if is_hpu_available:
+ if is_optimum_habana_available():
htcore.mark_step()
diff --git a/auto_round/utils.py b/auto_round/utils.py
index f53939daa..d7cf32a42 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -33,27 +33,49 @@ fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(line
fh.setFormatter(fh_formatter)
logger.addHandler(fh)
+import importlib
-def is_optimum_habana_available():
- import importlib
- from transformers.utils.import_utils import is_optimum_available
+class LazyImport(object):
+ """Lazy import python module till use."""
- return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not None
+ def __init__(self, module_name):
+ """Init LazyImport object.
+ Args:
+ module_name (string): The name of module imported later
+ """
+ self.module_name = module_name
+ self.module = None
-try:
- import habana_frameworks.torch.core as htcore
- import habana_frameworks.torch.hpu as hthpu
+ def __getattr__(self, name):
+ """Get the attributes of the module by name."""
+ try:
+ self.module = importlib.import_module(self.module_name)
+ mod = getattr(self.module, name)
+ except:
+ spec = importlib.util.find_spec(str(self.module_name + "." + name))
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
- if is_optimum_habana_available():
- is_hpu_available = True
- else:
- print("Should install optimum-habana when the environment has habana frameworks")
- is_hpu_available = False
-except ImportError:
- is_hpu_available = False
- htcore = None
+ def __call__(self, *args, **kwargs):
+ """Call the function in that module."""
+ function_name = self.module_name.split(".")[-1]
+ module_name = self.module_name.split(f".{function_name}")[0]
+ self.module = importlib.import_module(module_name)
+ function = getattr(self.module, function_name)
+ return function(*args, **kwargs)
+
+
+auto_gptq = LazyImport("auto_gptq")
+htcore = LazyImport("habana_frameworks.torch.core")
+
+
+def is_optimum_habana_available():
+ from transformers.utils.import_utils import is_optimum_available
+
+ return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not None
def round_ste(x: torch.Tensor):
@@ -507,7 +529,7 @@ def detect_device(device=None):
if torch.cuda.is_available():
device = torch.device("cuda")
logger.info("Using GPU device")
- elif is_hpu_available:
+ elif is_optimum_habana_available():
device = torch.device("hpu")
logger.info("Using HPU device")
# Use CPU as a fallback
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index adc6bdef3..9c3b62f31 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -10,9 +10,6 @@ import shutil
import transformers
from typing import TYPE_CHECKING, Optional, Union
import time
-
-
-
if __name__ == "__main__":
import sys
@@ -344,7 +341,7 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
raise ImportError("""follow requirements to install dependencies.""")
import collections
org_s = time.time()
-
+ print(f"Using {dtype} as evaluation data type.")
external_tasks = []
for each in EXT_TASKS:
if each in tasks:
@@ -375,7 +372,10 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,
num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
- sub_name = f'{tmp_tasks} {shot}-shot'
+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
+ sub_name = f'hendrycksTest-* {shot}-shot'
+ else:
+ sub_name = f'{tmp_tasks} {shot}-shot'
print(f'{sub_name}: ')
pprint.pprint(tmp_results["results"])
print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
@@ -438,6 +438,22 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
new_dict["wikitext 0-shot_word_perplexity"] = 0
new_dict["wikitext 0-shot_byte_perplexity"] = 0
new_dict["wikitext 0-shot_bits_per_byte"] = 0
+
+ # Special handling of mmlu for compatibility with the old excel results sequence
+ search_str = 'hendrycksTest'
+ mmlu_matching_keys = [key for key in results.keys() if search_str.lower() in key.lower()]
+ for key in mmlu_matching_keys:
+ data = results.pop(key)
+ for sub_key in data.keys():
+ for sub_sub_key in data[sub_key].keys():
+ new_key = key + "-" + sub_key + "-" + sub_sub_key
+ if "std" in new_key or "alias" in new_key:
+ continue
+ new_key = new_key.split(",")[0]
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+ new_key = new_key + "_norm"
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+
for key in results.keys():
if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
continue
@@ -447,14 +463,12 @@ def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "
continue
for sub_key in data.keys():
for sub_sub_key in data[sub_key].keys():
- if "mmlu" in key:
- new_key = key + "-" + sub_key + "-" + sub_sub_key
- else:
- new_key = key + "_" + sub_sub_key
+ new_key = key + "_" + sub_sub_key
if "std" in new_key or "alias" in new_key:
continue
- new_key = new_key.rstrip(",none")
+ new_key = new_key.split(",")[0]
new_dict[new_key] = data[sub_key][sub_sub_key]
+
import pandas as pd
df = pd.DataFrame(data=new_dict, index=[0])
@@ -497,7 +511,3 @@ if __name__ == "__main__":
eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
print("cost time: ", time.time() - s)
-
-
-
-
diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.py
index 2ba6a29f9..bafff6c59 100644
--- a/examples/language-modeling/eval_legacy/evaluation.py
+++ b/examples/language-modeling/eval_legacy/evaluation.py
@@ -211,7 +211,7 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
from lm_eval.tasks import ALL_TASKS, get_task_dict
except:
raise ImportError("""follow requirements to install dependencies.""")
-
+ print(f"Using {dtype} as evaluation data type.")
org_s = time.time()
external_tasks = []
@@ -353,3 +353,5 @@ if __name__ == "__main__":
+
+
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 3a65084c7..dea8b935f 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -93,9 +93,8 @@ if __name__ == '__main__':
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
parser.add_argument("--tasks",
- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
- "arc_easy", "arc_challenge"],
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge,wikitext2,ptb-new,c4-new",
help="lm-eval tasks for lm_eval version 0.4")
parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
@@ -162,7 +161,9 @@ if __name__ == '__main__':
if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
tmp_tasks = tasks
tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
- tasks = list(set(tasks))
+ seen = set()
+ tmp_tasks = tasks
+ tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
if isinstance(args.tasks, str):
tasks = ','.join(tasks)
@@ -302,3 +303,4 @@ if __name__ == '__main__':
eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,
device=torch_device, excel_file=excel_name)
+