Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update eval and fix example (#260)

n1ck-guo7816eea5dde5
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index b298ada72..d2f098596 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -17,7 +17,6 @@ import re
import argparse
import torch
-import subprocess
import transformers
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=True)
@@ -26,7 +25,7 @@ from lm_eval.utils import make_table # pylint: disable=E0401
from auto_round import AutoRoundConfig
from auto_round.eval.evaluation import simple_evaluate
-from auto_round.utils import detect_device
+from auto_round.utils import detect_device, get_library_version
def setup_parser():
parser = argparse.ArgumentParser()
@@ -36,7 +35,7 @@ def setup_parser():
)
parser.add_argument('--eval', action='store_true',
- help="whether to use eval mode.")
+ help="whether to use eval only mode.")
parser.add_argument("--bits", default=4, type=int,
help="number of bits")
@@ -109,7 +108,7 @@ def setup_parser():
help="Where to store the final model.")
parser.add_argument("--disable_eval", action='store_true',
- help="Whether to do lmeval evaluation.")
+ help="Whether to do lm-eval evaluation after tuning.")
parser.add_argument("--disable_amp", action='store_true',
help="disable amp")
@@ -177,7 +176,6 @@ def tune(args):
args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")
if args.low_cpu_mem_mode == 2:
from auto_round.low_cpu_mem.utils import load_model_with_hooks
-
model = load_model_with_hooks(
model_name,
model_cls,
@@ -189,7 +187,6 @@ def tune(args):
)
elif args.low_cpu_mem_mode == 1:
from auto_round.low_cpu_mem.utils import load_empty_model
-
low_cpu_mem_usage = True
model = load_empty_model(
model_name,
@@ -205,18 +202,16 @@ def tune(args):
trust_remote_code=not args.disable_trust_remote_code
)
- from auto_round import (AutoRound,
- AutoAdamRound)
+ from auto_round import AutoRound, AutoAdamRound
model = model.eval()
# align with GPTQ to eval ppl
+ seqlen = args.seqlen
if "opt" in model_name:
seqlen = model.config.max_position_embeddings
model.seqlen = model.config.max_position_embeddings
else:
seqlen = 2048
- model.seqlen = seqlen
- seqlen = args.seqlen
if args.model_dtype != None:
if args.model_dtype == "float16" or args.model_dtype == "fp16":
@@ -262,7 +257,6 @@ def tune(args):
lm_head_layer_name = n
if args.quant_lm_head:
from transformers import AutoConfig
-
config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):
tied_keys = model._tied_weights_keys
@@ -299,7 +293,6 @@ def tune(args):
model_name = args.model.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
-
shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
model.eval()
@@ -308,22 +301,12 @@ def tune(args):
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}"
-
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
for format_ in format_list:
eval_folder = f'{export_dir}-{format_}'
autoround.save_quantized(eval_folder, format=format_, inplace=inplace)
-
- def get_library_version(library_name):
- try:
- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
- return version
- except subprocess.CalledProcessError:
- return "Library not found"
-
-
lm_eval_version = get_library_version("lm-eval")
if isinstance(tasks, str):
@@ -343,28 +326,22 @@ def tune(args):
tasks=tasks,
batch_size=args.eval_bs,
user_model=user_model)
-
print(make_table(res))
def eval(args):
- quantization_config = AutoRoundConfig(backend=args.device)
device_str = detect_device(args.device)
- user_model = AutoModelForCausalLM.from_pretrained(
- args.model,
- device_map=device_str, quantization_config=quantization_config)
model_args = f"pretrained={args.model},trust_remote_code={not args.disable_trust_remote_code}"
if isinstance(args.tasks, str):
tasks = args.tasks.split(',')
res = simple_evaluate(
model="hf",
model_args=model_args,
- user_model=user_model,
tasks=tasks,
+ device=device_str,
batch_size=args.eval_bs)
from lm_eval.utils import make_table # pylint: disable=E0401
-
print(make_table(res))
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 1b4db5aea..8e50d889c 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -17,269 +17,54 @@ import random
import time
from typing import TYPE_CHECKING, List, Optional, Union
-import numpy as np
-import torch
+import lm_eval
+from lm_eval import simple_evaluate as lm_simple_evaluate
-import lm_eval.api.metrics # pylint: disable=E0401
-import lm_eval.api.registry # pylint: disable=E0401
-import lm_eval.models # pylint: disable=E0401
-from lm_eval.caching.cache import delete_cache # pylint: disable=E0401
-from lm_eval.evaluator import evaluate # pylint: disable=E0401
-from lm_eval.evaluator_utils import run_task_tests # pylint: disable=E0401
-from lm_eval.loggers.utils import add_env_info, get_git_commit_hash # pylint: disable=E0401
-from lm_eval.tasks import TaskManager, get_task_dict # pylint: disable=E0401
-from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string # pylint: disable=E0401
-if TYPE_CHECKING:
- from lm_eval.api.model import LM
- from lm_eval.tasks import Task
-
-
-@positional_deprecated
def simple_evaluate(
model,
model_args: Optional[Union[str, dict]] = None,
- tasks: Optional[List[Union[str, dict, object]]] = None,
- num_fewshot: Optional[int] = None,
+ user_model = None,
batch_size: Optional[int] = None,
max_batch_size: Optional[int] = None,
device: Optional[str] = None,
- use_cache: Optional[str] = None,
- cache_requests: bool = False,
- rewrite_requests_cache: bool = False,
- delete_requests_cache: bool = False,
- limit: Optional[Union[int, float]] = None,
- bootstrap_iters: int = 100000,
- check_integrity: bool = False,
- write_out: bool = False,
- log_samples: bool = True,
- gen_kwargs: Optional[str] = None,
- task_manager: Optional[TaskManager] = None,
- verbosity: str = "INFO",
- predict_only: bool = False,
- random_seed: int = 0,
- numpy_random_seed: int = 1234,
- torch_random_seed: int = 1234,
- user_model = None, ##user model does not support tensor parallelism
-):
- """Instantiate and evaluate a model on a list of tasks.
-
- :param model: Union[str, LM]
- Name of model or LM object, see lm_eval.models.get_model
- :param model_args: Optional[str, dict]
- String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
- Ignored if `model` argument is a LM object.
- :param tasks: list[Union[str, dict, Task]]
- List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME
- if defined and type(task).__name__ otherwise.
- :param num_fewshot: int
- Number of examples in few-shot context
- :param batch_size: int or str, optional
- Batch size for model
- :param max_batch_size: int, optional
- Maximal batch size to try with automatic batch size detection
- :param device: str, optional
- PyTorch device (e.g. "cpu" or "cuda:0") for running models
- :param use_cache: str, optional
- A path to a sqlite db file for caching model responses. `None` if not caching.
- :param cache_requests: bool, optional
- Speed up evaluation by caching the building of dataset requests. `None` if not caching.
- :param rewrite_requests_cache: bool, optional
- Rewrites all of the request cache if set to `True`. `None` if not desired.
- :param delete_requests_cache: bool, optional
- Deletes all of the request cache if set to `True`. `None` if not desired.
- :param limit: int or float, optional
- Limit the number of examples per task (only use this for testing), If <1,
- limit is a percentage of the total number of examples.
- :param bootstrap_iters:
- Number of iterations for bootstrap statistics
- :param check_integrity: bool
- Whether to run the relevant part of the test suite for the tasks
- :param write_out: bool
- If True, write out an example document and model input for checking task integrity
- :param log_samples: bool
- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
- :param gen_kwargs: str
- String arguments for model generation
- Ignored for all tasks with loglikelihood output_type
- :param predict_only: bool
- If true only model outputs will be generated and returned. Metrics will not be evaluated
- :param random_seed: int
- Random seed for python's random module. If set to None, the seed will not be set.
- :param numpy_random_seed: int
- Random seed for numpy. If set to None, the seed will not be set.
- :param torch_random_seed: int
- Random seed for torch. If set to None, the seed will not be set.
-
- :return
- Dictionary of results
- """
- from auto_round.auto_quantizer import AutoHfQuantizer
- eval_logger.setLevel(getattr(logging, f"{verbosity}"))
- start_date = time.time()
-
- if delete_requests_cache:
- eval_logger.info("Deleting requests cache...")
- delete_cache()
-
- seed_message = []
- if random_seed is not None:
- # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
- seed_message.append(f"Setting random seed to {random_seed}")
- random.seed(random_seed)
-
- if numpy_random_seed is not None:
- seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
- np.random.seed(numpy_random_seed)
-
- if torch_random_seed is not None:
- seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
- torch.manual_seed(torch_random_seed)
-
- if seed_message:
- eval_logger.info(" | ".join(seed_message))
-
- if tasks is None:
- tasks = []
- if len(tasks) == 0:
- raise ValueError(
- "No tasks specified, or no tasks found. Please verify the task names."
- )
-
- if gen_kwargs is not None:
- gen_kwargs = simple_parse_args_string(gen_kwargs)
- eval_logger.warning(
- "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
- "Ensure 'do_sample=True' for non-greedy decoding!"
+ **kwargs):
+
+ try:
+ from auto_round import AutoRoundConfig
+ except:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+
+ if model_args is None:
+ model_args = ""
+
+ if isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
- if gen_kwargs == "":
- gen_kwargs = None
- if isinstance(model, str):
- if model_args is None:
- model_args = ""
-
- if isinstance(model_args, dict):
- lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
-
- else:
- lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
else:
- if not isinstance(model, lm_eval.api.model.LM):
- raise TypeError
- lm = model
-
- if use_cache is not None:
- eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
- lm = lm_eval.api.model.CachingLM(
- lm,
- use_cache
- # each rank receives a different cache db.
- # necessary to avoid multiple writes to cache at once
- + "_rank"
- + str(lm.rank)
- + ".db",
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
if user_model is not None:
lm._model = user_model
+ return lm_simple_evaluate(
+ model=lm,
+ model_args=model_args,
+ batch_size=batch_size,
+ max_batch_size=max_batch_size,
+ device=device,
+ **kwargs)
- if task_manager is None:
- task_manager = TaskManager(verbosity)
-
- task_dict = get_task_dict(tasks, task_manager)
- for task_name in task_dict.keys():
- task_obj = task_dict[task_name]
- if isinstance(task_obj, tuple):
- _, task_obj = task_obj
- if task_obj is None:
- continue
-
- if task_obj.get_config("output_type") == "generate_until":
- if gen_kwargs is not None:
- task_obj.set_config(
- key="generation_kwargs", value=gen_kwargs, update=True
- )
-
- if predict_only:
- log_samples = True
- eval_logger.info(
- f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
- )
- # we have to change the class properties post-hoc. This is pretty hacky.
- task_obj.override_metric(metric_name="bypass")
- # override tasks' fewshot values to the provided num_fewshot arg value
- # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
- if num_fewshot is not None:
- if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
- eval_logger.info(
- f"num_fewshot has been set to 0 for {task_name} in its config."
- " Manual configuration will be ignored."
- )
- else:
- eval_logger.warning(
- f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
- )
- task_obj.set_config(key="num_fewshot", value=num_fewshot)
- else:
- # if num_fewshot not provided, and the task does not define a default one, default to 0
- if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
- task_obj.set_config(key="num_fewshot", value=0)
-
- if check_integrity:
- run_task_tests(task_list=tasks)
-
- results = evaluate(
- lm=lm,
- task_dict=task_dict,
- limit=limit,
- cache_requests=cache_requests,
- rewrite_requests_cache=rewrite_requests_cache,
- bootstrap_iters=bootstrap_iters,
- write_out=write_out,
- log_samples=log_samples,
- verbosity=verbosity,
- )
-
- if lm.rank == 0:
- if isinstance(model, str):
- model_name = model
- elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
- model_name = model.config._name_or_path
- else:
- model_name = type(model).__name__
-
- # add info about the model and few shot config
- results["config"] = {
- "model": model_name,
- "model_args": model_args,
- "batch_size": batch_size,
- "batch_sizes": (
- list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
- ),
- "device": device,
- "use_cache": use_cache,
- "limit": limit,
- "bootstrap_iters": bootstrap_iters,
- "gen_kwargs": gen_kwargs,
- }
- results["git_hash"] = get_git_commit_hash()
- results["date"] = start_date
- add_env_info(results) # additional environment info to results
- return results
- else:
- return None
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index da9184289..9ca72f561 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -63,9 +63,10 @@ reduce the seqlen to 512 (potential accuracy drop)
or combine them
- **Reduced CPU Memory Usage:**
-set "--low_cpu_mem_mode 1" to use block-wise mode, load the weights from disk of each block when tuning and release the memory of the block after tuning. (more tuning cost)
-set "--low_cpu_mem_mode 2" to use layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed.
+ - set "--low_cpu_mem_mode 1" to use block-wise mode, load the weights from disk of each block when tuning and release the memory of the block after tuning. (more tuning cost)
+
+ - set "--low_cpu_mem_mode 2" to use layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed.
- **Speedup the tuning:**
@@ -132,7 +133,7 @@ CUDA_VISIBLE_DEVICES=1,2 python3 eval/evaluation.py --model_name /save_model_pat
- User could also perform evaluation on Intel Gaudi-2 using the following script.
```bash
-python3 eval_042/evaluation.py --model_name /saved_model_path/ --tasks mmlu --device hpu
+python3 eval/evaluation.py --model_name /saved_model_path/ --tasks mmlu --device hpu
```
You can also utilize the official lm_eval [link](https://github.com/EleutherAI/lm-evaluation-harness/tree/main?tab=readme-ov-file#basic-usage).
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 3d542aa59..ab1aa24a6 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -1,280 +1,131 @@
-import itertools
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import logging
import random
import time
-from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
-from packaging.version import Version
-import pkg_resources
-LM_EVAL_VERSION = Version(pkg_resources.get_distribution('lm_eval').version)
-
-import numpy as np
-import torch
-
-import lm_eval.api.metrics
-import lm_eval.api.registry
-import lm_eval.models
-from lm_eval.evaluator import evaluate
-from lm_eval.caching.cache import delete_cache
-from lm_eval.evaluator_utils import run_task_tests
-if LM_EVAL_VERSION == Version('0.4.2'):
- from lm_eval.logging_utils import add_env_info, get_git_commit_hash
-else:
- from lm_eval.loggers.utils import add_env_info, get_git_commit_hash
-from lm_eval.tasks import TaskManager, get_task_dict
-from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string
+import lm_eval
+from lm_eval import simple_evaluate as lm_simple_evaluate
-if TYPE_CHECKING:
- from lm_eval.api.model import LM
- from lm_eval.tasks import Task
-
-@positional_deprecated
def simple_evaluate(
model,
model_args: Optional[Union[str, dict]] = None,
- tasks: Optional[List[Union[str, dict, object]]] = None,
- num_fewshot: Optional[int] = None,
+ user_model = None,
batch_size: Optional[int] = None,
max_batch_size: Optional[int] = None,
device: Optional[str] = None,
- use_cache: Optional[str] = None,
- cache_requests: bool = False,
- rewrite_requests_cache: bool = False,
- delete_requests_cache: bool = False,
- limit: Optional[Union[int, float]] = None,
- bootstrap_iters: int = 100000,
- check_integrity: bool = False,
- write_out: bool = False,
- log_samples: bool = True,
- gen_kwargs: Optional[str] = None,
- task_manager: Optional[TaskManager] = None,
- verbosity: str = "INFO",
- predict_only: bool = False,
- random_seed: int = 0,
- numpy_random_seed: int = 1234,
- torch_random_seed: int = 1234,
- user_model = None, ##user model does not support tensor parallelism
-):
- """Instantiate and evaluate a model on a list of tasks.
-
- :param model: Union[str, LM]
- Name of model or LM object, see lm_eval.models.get_model
- :param model_args: Optional[str, dict]
- String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
- Ignored if `model` argument is a LM object.
- :param tasks: list[Union[str, dict, Task]]
- List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME
- if defined and type(task).__name__ otherwise.
- :param num_fewshot: int
- Number of examples in few-shot context
- :param batch_size: int or str, optional
- Batch size for model
- :param max_batch_size: int, optional
- Maximal batch size to try with automatic batch size detection
- :param device: str, optional
- PyTorch device (e.g. "cpu" or "cuda:0") for running models
- :param use_cache: str, optional
- A path to a sqlite db file for caching model responses. `None` if not caching.
- :param cache_requests: bool, optional
- Speed up evaluation by caching the building of dataset requests. `None` if not caching.
- :param rewrite_requests_cache: bool, optional
- Rewrites all of the request cache if set to `True`. `None` if not desired.
- :param delete_requests_cache: bool, optional
- Deletes all of the request cache if set to `True`. `None` if not desired.
- :param limit: int or float, optional
- Limit the number of examples per task (only use this for testing), If <1,
- limit is a percentage of the total number of examples.
- :param bootstrap_iters:
- Number of iterations for bootstrap statistics
- :param check_integrity: bool
- Whether to run the relevant part of the test suite for the tasks
- :param write_out: bool
- If True, write out an example document and model input for checking task integrity
- :param log_samples: bool
- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
- :param gen_kwargs: str
- String arguments for model generation
- Ignored for all tasks with loglikelihood output_type
- :param predict_only: bool
- If true only model outputs will be generated and returned. Metrics will not be evaluated
- :param random_seed: int
- Random seed for python's random module. If set to None, the seed will not be set.
- :param numpy_random_seed: int
- Random seed for numpy. If set to None, the seed will not be set.
- :param torch_random_seed: int
- Random seed for torch. If set to None, the seed will not be set.
-
- :return
- Dictionary of results
- """
- from auto_round.auto_quantizer import AutoHfQuantizer
- eval_logger.setLevel(getattr(logging, f"{verbosity}"))
- start_date = time.time()
-
- if delete_requests_cache:
- eval_logger.info("Deleting requests cache...")
- delete_cache()
-
- seed_message = []
- if random_seed is not None:
- # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
- seed_message.append(f"Setting random seed to {random_seed}")
- random.seed(random_seed)
-
- if numpy_random_seed is not None:
- seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
- np.random.seed(numpy_random_seed)
-
- if torch_random_seed is not None:
- seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
- torch.manual_seed(torch_random_seed)
-
- if seed_message:
- eval_logger.info(" | ".join(seed_message))
-
- if tasks is None:
- tasks = []
- if len(tasks) == 0:
- raise ValueError(
- "No tasks specified, or no tasks found. Please verify the task names."
+ **kwargs):
+
+ try:
+ from auto_round import AutoRoundConfig
+ except:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+
+ if model_args is None:
+ model_args = ""
+
+ if isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
- if gen_kwargs is not None:
- gen_kwargs = simple_parse_args_string(gen_kwargs)
- eval_logger.warning(
- "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
- "Ensure 'do_sample=True' for non-greedy decoding!"
- )
- if gen_kwargs == "":
- gen_kwargs = None
-
- if isinstance(model, str):
- if model_args is None:
- model_args = ""
-
- if isinstance(model_args, dict):
- lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
-
- else:
- lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
else:
- if not isinstance(model, lm_eval.api.model.LM):
- raise TypeError
- lm = model
-
- if use_cache is not None:
- eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
- lm = lm_eval.api.model.CachingLM(
- lm,
- use_cache
- # each rank receives a different cache db.
- # necessary to avoid multiple writes to cache at once
- + "_rank"
- + str(lm.rank)
- + ".db",
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
if user_model is not None:
lm._model = user_model
+ return lm_simple_evaluate(
+ model=lm,
+ model_args=model_args,
+ batch_size=batch_size,
+ max_batch_size=max_batch_size,
+ device=device,
+ **kwargs)
- if task_manager is None:
- task_manager = TaskManager(verbosity)
-
- task_dict = get_task_dict(tasks, task_manager)
- for task_name in task_dict.keys():
- task_obj = task_dict[task_name]
- if isinstance(task_obj, tuple):
- _, task_obj = task_obj
- if task_obj is None:
- continue
- if task_obj.get_config("output_type") == "generate_until":
- if gen_kwargs is not None:
- task_obj.set_config(
- key="generation_kwargs", value=gen_kwargs, update=True
- )
- if predict_only:
- log_samples = True
- eval_logger.info(
- f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
- )
- # we have to change the class properties post-hoc. This is pretty hacky.
- task_obj.override_metric(metric_name="bypass")
+if __name__ == "__main__":
- # override tasks' fewshot values to the provided num_fewshot arg value
- # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
- if num_fewshot is not None:
- if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
- eval_logger.info(
- f"num_fewshot has been set to 0 for {task_name} in its config."
- " Manual configuration will be ignored."
- )
- else:
- eval_logger.warning(
- f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
- )
- task_obj.set_config(key="num_fewshot", value=num_fewshot)
- else:
- # if num_fewshot not provided, and the task does not define a default one, default to 0
- if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
- task_obj.set_config(key="num_fewshot", value=0)
+ import sys
- if check_integrity:
- run_task_tests(task_list=tasks)
+ sys.path.insert(0, '../../')
+ import time
+ import argparse
- results = evaluate(
- lm=lm,
- task_dict=task_dict,
- limit=limit,
- cache_requests=cache_requests,
- rewrite_requests_cache=rewrite_requests_cache,
- bootstrap_iters=bootstrap_iters,
- write_out=write_out,
- log_samples=log_samples,
- verbosity=verbosity,
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name", default="facebook/opt-125m/"
)
-
- if lm.rank == 0:
- if isinstance(model, str):
- model_name = model
- elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
- model_name = model.config._name_or_path
- else:
- model_name = type(model).__name__
-
- # add info about the model and few shot config
- results["config"] = {
- "model": model_name,
- "model_args": model_args,
- "batch_size": batch_size,
- "batch_sizes": (
- list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
- ),
- "device": device,
- "use_cache": use_cache,
- "limit": limit,
- "bootstrap_iters": bootstrap_iters,
- "gen_kwargs": gen_kwargs,
- }
- results["git_hash"] = get_git_commit_hash()
- results["date"] = start_date
- add_env_info(results) # additional environment info to results
- return results
- else:
- return None
\ No newline at end of file
+ parser.add_argument(
+ "--eval_bs", default=1,
+ )
+ parser.add_argument(
+ "--trust_remote_code", action='store_true',
+ help="Whether to enable trust_remote_code"
+ )
+ parser.add_argument(
+ "--device", default="cuda:0",
+ help="PyTorch device (e.g. cpu/cuda:0/hpu) for evaluation."
+ )
+ parser.add_argument("--tasks",
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
+ "openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks for lm_eval version 0.4.2")
+
+ args = parser.parse_args()
+ s = time.time()
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
+
+ if hasattr(config, "quantization_config"):
+ quantization_config = config.quantization_config
+ if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":
+ if args.device == "hpu":
+ from auto_round.auto_quantizer import AutoHfQuantizer
+
+ test_tasks = args.tasks
+ if isinstance(test_tasks, str):
+ test_tasks = test_tasks.split(',')
+ model_name = args.model_name.rstrip('/')
+ from lm_eval.utils import make_table
+
+ model_args = f"pretrained={args.model_name}"
+ model_args += ",dtype=float16"
+ if args.trust_remote_code:
+ model_args += f",trust_remote_code=True"
+ result = simple_evaluate(model="hf",
+ model_args=model_args,
+ tasks=test_tasks,
+ device=args.device,
+ batch_size=args.eval_bs)
+ print(make_table(result))
+
+ print("cost time: ", time.time() - s)
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 3d514e75e..ce7f03c37 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -94,7 +94,7 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
- parser.add_argument("--deployment_device", default='auto_round', type=str,
+ parser.add_argument("--deployment_device", default=None, type=str,
help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."
"default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")
@@ -183,6 +183,8 @@ if __name__ == '__main__':
args.format = "auto_round"
if args.deployment_device:
+ warnings.warn("The deployment_device is deprecated and will be removed in future version."
+ "Please use format instead", DeprecationWarning)
if "gpu" in args.deployment_device and args.sym is False:
print(
"warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --format='auto_round'")
@@ -246,10 +248,8 @@ if __name__ == '__main__':
# align with GPTQ to eval ppl
if "opt" in model_name:
seqlen = model.config.max_position_embeddings
- model.seqlen = model.config.max_position_embeddings
else:
seqlen = 2048
- model.seqlen = seqlen
seqlen = args.seqlen
if args.model_dtype != None: