Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add runable script for autoround (#225)

n1ck-guoe15cf35aae50
diff --git a/README.md b/README.md
index 8a3f3a93e..51b7c09ec 100644
--- a/README.md
+++ b/README.md
@@ -48,9 +48,22 @@ pip install -vvv --no-build-isolation -e .
pip install auto-round
```
-## Model quantization
-### Gaudi2/ CPU/ GPU
+
+## Model Quantization
+### Basic Usage
+AutoRound support Gaudi2, CPU and GPU. A user guide detailing the full list of supported arguments is provided by calling ```auto_round -h``` on the terminal. Alternatively, you can use ```auto-round``` instead of ```auto_round```. (**auto-round version > 0.3.0**)
+
+
+```bash
+auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --format auto_round \
+ --output_dir ./tmp_autoround
+```
+
+### API Usage
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -127,6 +140,34 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)
- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.
</details>
+<br>
+We provide two recipes for best accuracy and fast running speed with low memory. Details as below.
+<details>
+ <summary>Other Recipes</summary>
+
+ ```bash
+## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower
+ auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --nsamples 512 \
+ --iters 1000 \
+ --low_gpu_mem_usage
+ ```
+
+ ```bash
+## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128
+ auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --nsamples 128 \
+ --iters 200 \
+ --seqlen 512 \
+ --batch_size 4
+ ```
+
+</details>
+
#### Formats
@@ -144,7 +185,8 @@ Additionally, symmetric quantization tends to perform poorly at 2-bit precision.
within the community. Asymmetric quantization typically improves accuracy but may reduce inference speed. It features
specialized layer fusion tailored for Llama models. However, it supports only 4-bit asymmetric quantization. Currently, please manually install autoawq via `pip install autoawq` before exporting.
-## Model inference
+## Model Inference
+
Please run the quantization code first
@@ -160,6 +202,8 @@ text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
```
+
+
### AutoRound format
**CPU**: no extra operations
@@ -204,6 +248,19 @@ text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
```
+<br>
+<details>
+ <summary>Evaluation</summary>
+
+```bash
+## version > 0.3.0
+auto_round --model saved_quantized_model \
+ --eval \
+ --task lambada_openai \
+ --eval_bs 1
+```
+</details>
+
## Support List
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
new file mode 100644
index 000000000..d1ea08db9
--- /dev/null
+++ b/auto_round/__main__.py
@@ -0,0 +1,397 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import argparse
+
+import torch
+import subprocess
+import transformers
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+torch.use_deterministic_algorithms(True, warn_only=True)
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+from lm_eval.utils import make_table # pylint: disable=E0401
+
+from auto_round.utils import logger
+from auto_round import AutoRoundConfig
+from auto_round.eval.evaluation import simple_evaluate
+
+def setup_parser():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--model", default="facebook/opt-125m"
+ )
+
+ parser.add_argument('--eval', action='store_true',
+ help="whether to use eval mode.")
+
+ parser.add_argument("--bits", default=4, type=int,
+ help="number of bits")
+
+ parser.add_argument("--group_size", default=128, type=int,
+ help="group size")
+
+ parser.add_argument("--batch_size", default=8, type=int,
+ help="train batch size")
+
+ parser.add_argument("--eval_bs", default=1, type=int,
+ help="eval batch size")
+
+ parser.add_argument("--device", default="auto", type=str,
+ help="The device to be used for tuning. The default is set to auto/None,"
+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
+
+ parser.add_argument("--sym", action='store_true',
+ help=" sym quantization")
+
+ parser.add_argument("--iters", default=200, type=int,
+ help=" iters")
+
+ parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,
+ help="The dataset for quantization training. It can be a custom one.")
+
+ parser.add_argument("--lr", default=None, type=float,
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
+
+ parser.add_argument("--minmax_lr", default=None, type=float,
+ help="minmax learning rate, if None,it will beset to be the same with lr")
+
+ parser.add_argument("--seed", default=42, type=int,
+ help="seed")
+
+ parser.add_argument("--adam", action='store_true',
+ help="adam")
+
+ parser.add_argument("--seqlen", default=2048, type=int,
+ help="sequence length")
+
+ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+
+ parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
+
+ parser.add_argument("--nsamples", default=128, type=int,
+ help="number of samples")
+
+ parser.add_argument("--low_gpu_mem_usage", action='store_true',
+ help="lower gpu memory but 50%-100% slower")
+
+ parser.add_argument("--format", default=None, type=str,
+ help="The format in which to save the model. "
+ "The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."
+ "default to 'auto_round."
+ )
+
+ parser.add_argument("--data_type", default='int',
+ help="data type for tuning, 'int', 'mx_fp' and etc.")
+
+ parser.add_argument("--scale_dtype", default='fp16',
+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
+
+ parser.add_argument("--tasks",
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks for lm_eval version 0.4")
+
+ parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
+ help="Where to store the final model.")
+
+ parser.add_argument("--disable_eval", action='store_true',
+ help="Whether to do lmeval evaluation.")
+
+ parser.add_argument("--disable_amp", action='store_true',
+ help="disable amp")
+
+ parser.add_argument("--disable_minmax_tuning", action='store_true',
+ help="whether disable enable weight minmax tuning")
+
+ parser.add_argument("--enable_norm_bias_tuning", action='store_true',
+ help="whether enable norm bias tuning")
+
+ parser.add_argument("--disable_trust_remote_code", action='store_true',
+ help="Whether to disable trust_remote_code")
+
+ parser.add_argument("--disable_quanted_input", action='store_true',
+ help="whether to disuse the output of quantized block to tune the next block")
+
+ parser.add_argument("--quant_lm_head", action='store_true',
+ help="quant_lm_head")
+
+ parser.add_argument("--low_cpu_mem_mode", default=0, type=int,
+ help="Choose which low cpu memory mode to use. "
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+
+ parser.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,
+ help="temp work space to store the temporary files "
+ "when using low cpu memory mode. Will remove after tuning.")
+
+ parser.add_argument("--model_dtype", default=None, type=str,
+ help="force to convert the dtype, some backends supports fp16 dtype better")
+
+ parser.add_argument("--act_bits", default=32, type=int,
+ help="activation bits")
+
+ parser.add_argument("--fp_layers_list", default="", type=str,
+ help="List of Layers to maintain original data type")
+
+ args = parser.parse_args()
+ return args
+
+def tune(args):
+ tasks = args.tasks
+ use_eval_legacy = False
+ if args.format is None:
+ args.format = "auto_round"
+
+ model_name = args.model
+ if model_name[-1] == "/":
+ model_name = model_name[:-1]
+ print(model_name, flush=True)
+
+ from auto_round.utils import detect_device
+
+ device_str = detect_device(args.device)
+ torch_dtype = "auto"
+ if "hpu" in device_str:
+ torch_dtype = torch.bfloat16
+ torch_device = torch.device(device_str)
+
+ is_glm = bool(re.search("chatglm", model_name.lower()))
+ low_cpu_mem_usage = False
+ model_cls = AutoModel if is_glm else AutoModelForCausalLM
+ if args.low_cpu_mem_tmp_dir is None:
+ args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")
+ if args.low_cpu_mem_mode == 2:
+ from auto_round.low_cpu_mem.utils import load_model_with_hooks
+
+ model = load_model_with_hooks(
+ model_name,
+ model_cls,
+ device=device_str,
+ clean_weight=True,
+ saved_path=args.low_cpu_mem_tmp_dir,
+ torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code
+ )
+ elif args.low_cpu_mem_mode == 1:
+ from auto_round.low_cpu_mem.utils import load_empty_model
+
+ low_cpu_mem_usage = True
+ model = load_empty_model(
+ model_name,
+ model_cls,
+ device=device_str,
+ saved_path=args.low_cpu_mem_tmp_dir,
+ torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code
+ )
+ else:
+ model = model_cls.from_pretrained(
+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code
+ )
+
+ from auto_round import (AutoRound,
+ AutoAdamRound)
+
+ model = model.eval()
+ # align with GPTQ to eval ppl
+ if "opt" in model_name:
+ seqlen = model.config.max_position_embeddings
+ model.seqlen = model.config.max_position_embeddings
+ else:
+ seqlen = 2048
+ model.seqlen = seqlen
+ seqlen = args.seqlen
+
+ if args.model_dtype != None:
+ if args.model_dtype == "float16" or args.model_dtype == "fp16":
+ model = model.to(torch.float16)
+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":
+ model = model.to(torch.bfloat16)
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+
+ if hasattr(tokenizer, "model_max_length"):
+ if tokenizer.model_max_length < seqlen:
+ print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
+ flush=True)
+ seqlen = min(seqlen, tokenizer.model_max_length)
+ args.seqlen = seqlen
+
+ excel_name = f"{model_name}_{args.bits}_{args.group_size}"
+ if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
+ dtype = 'bfloat16'
+ else:
+ if "cpu" not in device_str:
+ dtype = 'float16'
+ else:
+ dtype = 'float32'
+
+ excel_name = f"{model_name}_{args.bits}_{args.group_size}"
+ if "bloom" in model_name:
+ args.low_gpu_mem_usage = False
+
+ round = AutoRound
+ if args.adam:
+ round = AutoAdamRound
+
+ layer_config = {}
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
+ layer_config[n] = {"bits": 32}
+ print(
+ f"{n} will not be quantized due to its shape not being divisible by 32,"
+ " resulting in an exporting issue to autogptq")
+ fp_layers_list = args.fp_layers_list.split(",")
+ if bool(fp_layers_list):
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ name = n.split('.')[-1]
+ if n in fp_layers_list or name in fp_layers_list:
+ layer_config[n] = {"bits": 32}
+ print(
+ f"{n} will not be quantized.")
+ lm_head_layer_name = "lm_head"
+ for n, _ in model.named_modules():
+ lm_head_layer_name = n
+ if args.quant_lm_head:
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):
+ tied_keys = model._tied_weights_keys
+ for item in tied_keys:
+ if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model
+ args.quant_lm_head = False
+ print(
+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "
+ f"supported currently")
+ break
+ if args.quant_lm_head:
+ layer_config[lm_head_layer_name] = {"bits": args.bits}
+ transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
+ if transformers_version[0] == 4 and transformers_version[1] < 38:
+ error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
+ raise EnvironmentError(error_message)
+
+ if args.quant_lm_head and args.low_gpu_mem_usage:
+ print(
+ f"warning, low_gpu_mem_usage=False is strongly recommended"
+ " if the whole model could be loaded to gpu")
+
+ autoround = round(
+ model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.batch_size,
+ dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input,
+ device=device_str, amp=not args.disable_amp, nsamples=args.nsamples, seed=args.seed,
+ low_gpu_mem_usage=args.low_gpu_mem_usage, scale_dtype=args.scale_dtype,
+ gradient_accumulate_steps=args.gradient_accumulate_steps, layer_config=layer_config,
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
+ low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,
+ enable_norm_bias_tuning=args.enable_norm_bias_tuning)
+ model, _ = autoround.quantize()
+ model_name = args.model_name.rstrip("/")
+ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
+ import shutil
+
+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
+
+ model.eval()
+ if "cpu" not in device_str:
+ torch.cuda.empty_cache()
+
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
+
+
+ format_list = args.format.replace(' ', '').split(',')
+ inplace = False if len(format_list) > 1 else True
+ for format_ in format_list:
+ eval_folder = f'{export_dir}-{format_}'
+ if 'auto_awq' in format_:
+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace, model_path=model_name)
+ else:
+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace)
+
+
+ def get_library_version(library_name):
+ try:
+ version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
+ return version
+ except subprocess.CalledProcessError:
+ return "Library not found"
+
+
+ lm_eval_version = get_library_version("lm-eval")
+
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+
+ if not args.disable_eval:
+ print(f"Using the latest {lm_eval_version}")
+ model_args = f"pretrained={eval_folder}"
+ model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"
+ user_model = None
+ if args.act_bits <= 8:
+ user_model = model.to(device_str)
+
+ res = simple_evaluate(
+ model="hf",
+ model_args=model_args,
+ tasks=tasks,
+ batch_size=args.eval_bs,
+ user_model=user_model)
+
+ print(make_table(res))
+
+
+def eval(args):
+ quantization_config = AutoRoundConfig(backend=args.device)
+ user_model = AutoModelForCausalLM.from_pretrained(
+ args.model,
+ device_map=args.device, quantization_config=quantization_config)
+ model_args = f"pretrained={args.model},trust_remote_code={not args.disable_trust_remote_code}"
+ if isinstance(args.tasks, str):
+ tasks = args.tasks.split(',')
+ res = simple_evaluate(
+ model="hf",
+ model_args=model_args,
+ user_model=user_model,
+ tasks=tasks,
+ device=args.device,
+ batch_size=args.eval_bs)
+
+ from lm_eval.utils import make_table # pylint: disable=E0401
+
+ print(make_table(res))
+
+
+def run():
+ args = setup_parser()
+ if args.eval:
+ eval(args)
+ else:
+ tune(args)
+
+
+if __name__ == '__main__':
+ run()
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 318d708a9..b930c1c48 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -250,7 +250,7 @@ class AutoRound(object):
if "mx_fp" in self.data_type:
logger.warning(
- "please save the quantized model to fake format"
+ "please save the quantized model to fake format "
"as real deployment is not supported for mx_fp datatype currently")
if "mx_fp" in self.data_type and self.group_size != 32:
diff --git a/auto_round/eval/__init__.py b/auto_round/eval/__init__.py
new file mode 100644
index 000000000..e3fdc07b1
--- /dev/null
+++ b/auto_round/eval/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/examples/language-modeling/eval_042/evaluation.py b/auto_round/eval/evaluation.py
similarity index 86%
rename from examples/language-modeling/eval_042/evaluation.py
rename to auto_round/eval/evaluation.py
index 40cd8e099..da29c34ee 100644
--- a/examples/language-modeling/eval_042/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import itertools
import logging
import random
@@ -8,21 +22,21 @@ from typing import TYPE_CHECKING, List, Optional, Union
import numpy as np
import torch
-import lm_eval.api.metrics
-import lm_eval.api.registry
-import lm_eval.models
-from lm_eval.caching.cache import delete_cache
-from lm_eval.evaluator_utils import (
+import lm_eval.api.metrics # pylint: disable=E0401
+import lm_eval.api.registry # pylint: disable=E0401
+import lm_eval.models # pylint: disable=E0401
+from lm_eval.caching.cache import delete_cache # pylint: disable=E0401
+from lm_eval.evaluator_utils import ( # pylint: disable=E0401
consolidate_results,
get_sample_size,
get_task_list,
prepare_print_tasks,
print_writeout,
run_task_tests,
-)
-from lm_eval.logging_utils import add_env_info, get_git_commit_hash
-from lm_eval.tasks import TaskManager, get_task_dict
-from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string
+) # pylint: disable=E0401
+from lm_eval.loggers.utils import add_env_info, get_git_commit_hash # pylint: disable=E0401
+from lm_eval.tasks import TaskManager, get_task_dict # pylint: disable=E0401
+from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string # pylint: disable=E0401
if TYPE_CHECKING:
from lm_eval.api.model import LM
@@ -64,7 +78,8 @@ def simple_evaluate(
String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
Ignored if `model` argument is a LM object.
:param tasks: list[Union[str, dict, Task]]
- List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME
+ if defined and type(task).__name__ otherwise.
:param num_fewshot: int
Number of examples in few-shot context
:param batch_size: int or str, optional
@@ -82,7 +97,8 @@ def simple_evaluate(
:param delete_requests_cache: bool, optional
Deletes all of the request cache if set to `True`. `None` if not desired.
:param limit: int or float, optional
- Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ Limit the number of examples per task (only use this for testing), If <1,
+ limit is a percentage of the total number of examples.
:param bootstrap_iters:
Number of iterations for bootstrap statistics
:param check_integrity: bool
@@ -219,7 +235,8 @@ def simple_evaluate(
if num_fewshot is not None:
if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
eval_logger.info(
- f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
+ f"num_fewshot has been set to 0 for {task_name} in its config."
+ " Manual configuration will be ignored."
)
else:
eval_logger.warning(
@@ -323,7 +340,7 @@ def evaluate(
):
raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
for task_output in eval_tasks:
- task: Task = task_output.task
+ task: Task = task_output.task # pylint: disable=E0601
limit = get_sample_size(task, limit)
task.build_all_requests(
limit=limit,
@@ -463,7 +480,7 @@ def evaluate(
# aggregate results ; run bootstrap CIs
for task_output in eval_tasks:
task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
- results, samples, configs, versions, num_fewshot = consolidate_results(
+ results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(
eval_tasks
)
@@ -515,9 +532,6 @@ def evaluate(
results[group][
stderr
] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
- # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
- # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
- # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)
results[group]["samples"] = sum(sizes)
@@ -559,73 +573,3 @@ def evaluate(
else:
return None
-
-
-if __name__ == "__main__":
-
- import sys
-
- sys.path.insert(0, '../../')
- import time
- import argparse
-
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--model_name", default="/models/opt-125m/"
- )
- parser.add_argument(
- "--eval_bs", default=1,
- )
- parser.add_argument(
- "--trust_remote_code", action='store_true',
- help="Whether to enable trust_remote_code"
- )
- parser.add_argument(
- "--device", default="cuda:0",
- help="PyTorch device (e.g. cpu/cuda:0/hpu) for evaluation."
- )
- parser.add_argument("--tasks",
- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
- "openbookqa,boolq,rte,arc_easy,arc_challenge",
- help="lm-eval tasks for lm_eval version 0.4.2")
-
- parser.add_argument("--parallelize", action='store_true',
- help="Whether to enable parallelize."
- )
-
- args = parser.parse_args()
- s = time.time()
- from transformers import AutoConfig
-
- config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
-
- if hasattr(config, "quantization_config"):
- quantization_config = config.quantization_config
- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
- from auto_round.auto_quantizer import AutoHfQuantizer
- elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":
- if args.device == "hpu":
- from auto_round.auto_quantizer import AutoHfQuantizer
-
- test_tasks = args.tasks
- if isinstance(test_tasks, str):
- test_tasks = test_tasks.split(',')
- model_name = args.model_name.rstrip('/')
- from lm_eval.utils import make_table
-
- model_args = f"pretrained={args.model_name}"
- model_args += ",dtype=float16"
- if args.trust_remote_code:
- model_args += f",trust_remote_code=True"
- if args.parallelize:
- model_args += f",parallelize=True"
- result = simple_evaluate(model="hf",
- model_args=model_args,
- tasks=test_tasks,
- device=args.device,
- batch_size=args.eval_bs)
- print(make_table(result))
-
- print("cost time: ", time.time() - s)
-
-
diff --git a/examples/language-modeling/eval/__init__.py b/examples/language-modeling/eval/__init__.py
index 07f1ae76d..e69de29bb 100644
--- a/examples/language-modeling/eval/__init__.py
+++ b/examples/language-modeling/eval/__init__.py
@@ -1,5 +0,0 @@
-if __name__ == "__main__":
- import sys
- sys.path.insert(0, './')
-from .evaluation import eval_model
-
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 2e498a260..c8a228d0e 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -1,100 +1,72 @@
-
-import os
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-import os.path
-import torch
+import itertools
import logging
-import pprint
-import re
-import shutil
-import transformers
-from typing import TYPE_CHECKING, Optional, Union
+import random
import time
-
-if __name__ == "__main__":
- import sys
- sys.path.insert(0, './')
-
-EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
-fewshots_dict = {}
-fewshots_dict['paper'] = {
- "lambada_openai": [0],
- "hellaswag": [0],
- "winogrande": [0],
- "piqa": [0],
- "mmlu": [0],
- "wikitext": [0],
- "truthfulqa_mc1": [0],
- "truthfulqa_mc2": [0],
- "openbookqa": [0],
- "boolq": [0],
- "rte": [0],
- "arc_easy": [0],
- "arc_challenge": [0],
- "gsm8k": [0],
- "ceval-valid": [0],
- "cmmlu": [0],
-}
-fewshots_dict['leadboard'] = {
- "hellaswag": [10],
- "winogrande": [5],
- "arc_easy": [25],
- "arc_challenge": [25],
- "mmlu": [5],
- "drop": [3],
- "gsm8k": [5],
-}
-fewshots_dict['all'] = {
- "lambada_openai": [0],
- "hellaswag": [0, 10],
- "winogrande": [0, 5],
- "piqa": [0],
- "coqa": [], ## coqa is not enabled in llamav1 models
- "truthfulqa_mc1": [0],
- "truthfulqa_mc2": [0],
- "openbookqa": [0],
- "boolq": [0],
- "rte": [0],
- "arc_easy": [0, 25],
- "arc_challenge": [0, 25],
- "mmlu": [0, 5],
- "wikitext": [0],
- "drop": [3],
- "gsm8k": [5]
-}
+from collections import defaultdict
+from typing import TYPE_CHECKING, List, Optional, Union
+
+from packaging.version import Version
+import pkg_resources
+LM_EVAL_VERSION = Version(pkg_resources.get_distribution('lm_eval').version)
+
+import numpy as np
+import torch
+
+import lm_eval.api.metrics
+import lm_eval.api.registry
+import lm_eval.models
+from lm_eval.caching.cache import delete_cache
+from lm_eval.evaluator_utils import (
+ consolidate_results,
+ get_sample_size,
+ get_task_list,
+ prepare_print_tasks,
+ print_writeout,
+ run_task_tests,
+)
+if LM_EVAL_VERSION == Version('0.4.2'):
+ from lm_eval.logging_utils import add_env_info, get_git_commit_hash
+else:
+ from lm_eval.loggers.utils import add_env_info, get_git_commit_hash
+from lm_eval.tasks import TaskManager, get_task_dict
+from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string
+
+if TYPE_CHECKING:
+ from lm_eval.api.model import LM
+ from lm_eval.tasks import Task
+@positional_deprecated
def simple_evaluate(
- model,
- model_args: Optional[Union[str, dict, None]] = None,
- tasks=None,
- num_fewshot: Optional[int] = None,
- batch_size: Optional[int] = None,
- max_batch_size: Optional[int] = None,
- device: Optional[str] = None,
- use_cache: Optional[str] = None,
- cache_requests: bool = False,
- rewrite_requests_cache: bool = False,
- delete_requests_cache: bool = False,
- limit: Optional[Union[int, float]] = None,
- bootstrap_iters: int = 100000,
- check_integrity: bool = False,
- decontamination_ngrams_path=None,
- write_out: bool = False,
- log_samples: bool = True,
- gen_kwargs: str = None,
- task_manager=None,
- verbosity: str = "INFO",
- predict_only: bool = False,
- random_seed: int = 1234,
- numpy_random_seed: int = 1234,
- torch_random_seed: int = 1234,
- lm=None
+ model,
+ model_args: Optional[Union[str, dict]] = None,
+ tasks: Optional[List[Union[str, dict, object]]] = None,
+ num_fewshot: Optional[int] = None,
+ batch_size: Optional[int] = None,
+ max_batch_size: Optional[int] = None,
+ device: Optional[str] = None,
+ use_cache: Optional[str] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ delete_requests_cache: bool = False,
+ limit: Optional[Union[int, float]] = None,
+ bootstrap_iters: int = 100000,
+ check_integrity: bool = False,
+ write_out: bool = False,
+ log_samples: bool = True,
+ gen_kwargs: Optional[str] = None,
+ task_manager: Optional[TaskManager] = None,
+ verbosity: str = "INFO",
+ predict_only: bool = False,
+ random_seed: int = 0,
+ numpy_random_seed: int = 1234,
+ torch_random_seed: int = 1234,
+ user_model = None, ##user model does not support tensor parallelism
):
"""Instantiate and evaluate a model on a list of tasks.
:param model: Union[str, LM]
- Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model
+ Name of model or LM object, see lm_eval.models.get_model
:param model_args: Optional[str, dict]
String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
Ignored if `model` argument is a LM object.
@@ -141,112 +113,92 @@ def simple_evaluate(
:return
Dictionary of results
"""
- from lm_eval.tasks import TaskManager
- import random
- import numpy as np
- import transformers
- import lm_eval.api
- import lm_eval.tasks
- import lm_eval.models
- import lm_eval.api.metrics
- import lm_eval.api.registry
- from lm_eval.evaluator import evaluate
- from lm_eval.logging_utils import add_env_info, get_git_commit_hash
- from lm_eval.tasks import get_task_dict
-
- from lm_eval.utils import (
- eval_logger,
- positional_deprecated,
- run_task_tests,
- simple_parse_args_string,
- )
-
+ from auto_round.auto_quantizer import AutoHfQuantizer
eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+ start_date = time.time()
if delete_requests_cache:
eval_logger.info("Deleting requests cache...")
delete_cache()
-
+
+ seed_message = []
if random_seed is not None:
# See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
- eval_logger.info(f"Setting random seed to {random_seed}")
+ seed_message.append(f"Setting random seed to {random_seed}")
random.seed(random_seed)
if numpy_random_seed is not None:
- eval_logger.info(f"Setting numpy seed to {numpy_random_seed}")
+ seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
np.random.seed(numpy_random_seed)
if torch_random_seed is not None:
- eval_logger.info(f"Setting torch manual seed to {torch_random_seed}")
+ seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
torch.manual_seed(torch_random_seed)
-
+
+ if seed_message:
+ eval_logger.info(" | ".join(seed_message))
+
if tasks is None:
tasks = []
- assert (
- tasks != []
- ), "No tasks specified, or no tasks found. Please verify the task names."
-
- if lm == None:
- if gen_kwargs is not None:
- gen_kwargs = simple_parse_args_string(gen_kwargs)
- eval_logger.warning(
- "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!"
+ if len(tasks) == 0:
+ raise ValueError(
+ "No tasks specified, or no tasks found. Please verify the task names."
+ )
+
+ if gen_kwargs is not None:
+ gen_kwargs = simple_parse_args_string(gen_kwargs)
+ eval_logger.warning(
+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
+ "Ensure 'do_sample=True' for non-greedy decoding!"
+ )
+ if gen_kwargs == "":
+ gen_kwargs = None
+
+ if isinstance(model, str):
+ if model_args is None:
+ model_args = ""
+
+ if isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
- if gen_kwargs == "":
- gen_kwargs = None
- if isinstance(model, str):
- if model_args is None:
- model_args = ""
-
- elif isinstance(model_args, dict):
- lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
-
- else:
- lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
- elif isinstance(model, transformers.PreTrainedModel):
- lm = lm_eval.api.registry.get_model("hf")(
- pretrained=model,
- batch_size=batch_size,
- max_batch_size=max_batch_size,
- )
- use_cache = None
else:
- assert isinstance(model, lm_eval.api.model.LM)
- lm = model
-
- if use_cache is not None:
- print(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
- lm = lm_eval.api.model.CachingLM(
- lm,
- "lm_cache/"
- + (model if isinstance(model, str) else model.model.config._name_or_path)
- + "_"
- + model_args.replace("=", "-").replace(",", "_").replace("/", "-")
- + ".db",
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
)
-
+ else:
+ if not isinstance(model, lm_eval.api.model.LM):
+ raise TypeError
+ lm = model
+
+ if use_cache is not None:
+ eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
+ lm = lm_eval.api.model.CachingLM(
+ lm,
+ use_cache
+ # each rank receives a different cache db.
+ # necessary to avoid multiple writes to cache at once
+ + "_rank"
+ + str(lm.rank)
+ + ".db",
+ )
+ if user_model is not None:
+ lm._model = user_model
+
if task_manager is None:
task_manager = TaskManager(verbosity)
- eval_logger.info(
- "get_task_dict has been updated to accept an optional argument, `task_manager`"
- "Read more here:https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage"
- )
task_dict = get_task_dict(tasks, task_manager)
for task_name in task_dict.keys():
task_obj = task_dict[task_name]
@@ -261,14 +213,16 @@ def simple_evaluate(
key="generation_kwargs", value=gen_kwargs, update=True
)
- if predict_only:
- log_samples = True
- eval_logger.info(
- f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
- )
- # we have to change the class properties post-hoc. This is pretty hacky.
- task_obj.override_metric(metric_name="bypass")
+ if predict_only:
+ log_samples = True
+ eval_logger.info(
+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+ )
+ # we have to change the class properties post-hoc. This is pretty hacky.
+ task_obj.override_metric(metric_name="bypass")
+ # override tasks' fewshot values to the provided num_fewshot arg value
+ # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
if num_fewshot is not None:
if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
eval_logger.info(
@@ -279,6 +233,10 @@ def simple_evaluate(
f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
)
task_obj.set_config(key="num_fewshot", value=num_fewshot)
+ else:
+ # if num_fewshot not provided, and the task does not define a default one, default to 0
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
+ task_obj.set_config(key="num_fewshot", value=0)
if check_integrity:
run_task_tests(task_list=tasks)
@@ -290,7 +248,6 @@ def simple_evaluate(
cache_requests=cache_requests,
rewrite_requests_cache=rewrite_requests_cache,
bootstrap_iters=bootstrap_iters,
- decontamination_ngrams_path=decontamination_ngrams_path,
write_out=write_out,
log_samples=log_samples,
verbosity=verbosity,
@@ -319,207 +276,368 @@ def simple_evaluate(
"gen_kwargs": gen_kwargs,
}
results["git_hash"] = get_git_commit_hash()
+ results["date"] = start_date
add_env_info(results) # additional environment info to results
- return results, lm
+ return results
else:
return None
-def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
- eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,
- device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
- print("evaluation with official lm-eval", flush=True)
- try:
- import lm_eval.api
- import lm_eval.tasks
- import lm_eval.models
- import lm_eval.api.metrics
- import lm_eval.api.registry
- from lm_eval.evaluator import evaluate
-
- except:
- raise ImportError("""follow requirements to install dependencies.""")
-
- org_s = time.time()
- if dtype == None:
- from eval.utils import convert_dtype_torch2str_hf
- from transformers import AutoConfig
- config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
- if hasattr(config, "torch_dtype"):
- dtype = convert_dtype_torch2str_hf(config.torch_dtype)
+@positional_deprecated
+def evaluate(
+ lm: "LM",
+ task_dict,
+ limit: Optional[int] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ bootstrap_iters: Optional[int] = 100000,
+ write_out: bool = False,
+ log_samples: bool = True,
+ verbosity: str = "INFO",
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param lm: obj
+ Language Model
+ :param task_dict: dict[str, Task]
+ Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
+ :param limit: int, optional
+ Limit the number of examples per task (only use this for testing)
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics
+ :param write_out: bool
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :return
+ Dictionary of results
+ """
+
+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+ # tracks all Instances/requests a model must generate output on.
+ requests = defaultdict(list)
+ # stores the amount to pad out reqs per req. type so that
+ # number of fwd passes per distributed rank is equal
+ padding_requests = defaultdict(int)
+
+ # get lists of group hierarchy and each type of request
+ task_hierarchy, eval_tasks = get_task_list(task_dict)
+ if not log_samples:
+ if not all(
+ "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
+ for task_output in eval_tasks
+ ):
+ raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
+ for task_output in eval_tasks:
+ task: Task = task_output.task
+ limit = get_sample_size(task, limit)
+ task.build_all_requests(
+ limit=limit,
+ rank=lm.rank,
+ world_size=lm.world_size,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
+ )
+ eval_logger.debug(
+ f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
+ )
+
+ if write_out:
+ print_writeout(task)
+ # aggregate Instances by LM method requested to get output.
+ for instance in task.instances:
+ reqtype = instance.request_type
+ requests[reqtype].append(instance)
+
+ if lm.world_size > 1:
+ instances_rnk = torch.tensor(len(task._instances), device=lm.device)
+ gathered_item = (
+ lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
+ )
+ # "multiple_choice" task types dispatch (several) "loglikelihood" request types
+ reqtype = (
+ "loglikelihood"
+ if task.OUTPUT_TYPE == "multiple_choice"
+ else task.OUTPUT_TYPE
+ )
+ # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
+ numpad = max(gathered_item) - gathered_item[lm.rank]
+ # todo: may not account for padding in cases like SquadV2 which has multiple req types
+ padding_requests[reqtype] += numpad
+
+ ### Run LM on inputs, get all outputs ###
+ # execute each type of request
+ for reqtype, reqs in requests.items():
+ eval_logger.info(f"Running {reqtype} requests")
+ # create `K` copies of each request `req` based off `K = req.repeats`
+ cloned_reqs = []
+ for req in reqs:
+ cloned_reqs.extend([req] * req.repeats)
+
+ if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
+ for _ in range(padding_requests[reqtype]):
+ cloned_reqs.extend([req] * req.repeats)
+
+ # run requests through model
+ resps = getattr(lm, reqtype)(cloned_reqs)
+
+ # put responses from model into a list of length K for each request.
+ for x, req in zip(resps, cloned_reqs):
+ req.resps.append(x)
+
+ if lm.world_size > 1:
+ lm.accelerator.wait_for_everyone()
+
+ RANK = lm.rank
+ WORLD_SIZE = lm.world_size
+ ### Postprocess outputs ###
+ # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
+ for task_output in eval_tasks:
+ task = task_output.task
+ task.apply_filters()
+
+ ### Collect values of metrics on all datapoints ###
+ # # unpack results and sort back in order and return control to Task
+ # TODO: make it possible to use a different metric per filter
+ # Pre-process task.instances to group by doc_id
+ instances_by_doc_id = defaultdict(list)
+ for instance in task.instances:
+ instances_by_doc_id[instance.doc_id].append(instance)
+ # Sort instances within each group
+ for instances in instances_by_doc_id.values():
+ instances.sort(key=lambda x: x.idx)
+ # iterate over different filters used
+ for filter_key in task.instances[0].filtered_resps.keys():
+ doc_iterator = task.doc_iterator(
+ rank=RANK, limit=limit, world_size=WORLD_SIZE
+ )
+ for doc_id, doc in doc_iterator:
+ requests = instances_by_doc_id[doc_id]
+ metrics = task.process_results(
+ doc, [req.filtered_resps[filter_key] for req in requests]
+ )
+ if log_samples:
+ target = task.doc_to_target(doc)
+ example = {
+ "doc_id": doc_id,
+ "doc": doc,
+ "target": target,
+ "arguments": [req.args for req in requests],
+ "resps": [req.resps for req in requests],
+ "filtered_resps": [
+ req.filtered_resps[filter_key] for req in requests
+ ],
+ }
+ example.update(metrics)
+ task_output.logged_samples.append(example)
+ for metric, value in metrics.items():
+ task_output.sample_metrics[(metric, filter_key)].append(value)
+
+ if WORLD_SIZE > 1:
+ # if multigpu, then gather data across all ranks to rank 0
+ # first gather logged samples across all ranks
+ for task_output in eval_tasks:
+ if log_samples:
+ # for task_name, task_samples in list(samples.items()):
+ full_samples = [None] * WORLD_SIZE if RANK == 0 else None
+ torch.distributed.gather_object(
+ obj=task_output.logged_samples,
+ object_gather_list=full_samples,
+ dst=0,
+ )
+
+ if RANK == 0:
+ task_output.logged_samples = list(
+ itertools.chain.from_iterable(full_samples)
+ )
+
+ # then collect metrics across all ranks
+ for metrics in task_output.sample_metrics:
+ metric_list = [None] * WORLD_SIZE if RANK == 0 else None
+ torch.distributed.gather_object(
+ obj=task_output.sample_metrics[metrics],
+ object_gather_list=metric_list,
+ dst=0,
+ )
+ if RANK == 0:
+ task_output.sample_metrics[metrics] = list(
+ itertools.chain.from_iterable(metric_list)
+ )
+
+ if RANK == 0:
+ ### Aggregate results over all datapoints ###
+ # aggregate results ; run bootstrap CIs
+ for task_output in eval_tasks:
+ task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
+ if LM_EVAL_VERSION == Version('0.4.2'):
+ results, samples, configs, versions, num_fewshot = consolidate_results(
+ eval_tasks
+ )
else:
- dtype = "float16"
- print(f"Using {dtype} as evaluation data type.")
-
- external_tasks = []
- if isinstance(tasks, str):
- tasks = tasks.split(',')
- for each in EXT_TASKS:
- if each in tasks:
- external_tasks.append(each)
- tasks.remove(each)
-
- results = {}
- model = None
- lm = None
-
- for tmp_tasks in tasks:
- try:
- num_fewshot = fewshots_dict[mark][tmp_tasks]
- print(f'********* {tmp_tasks} evaluate ************')
- task_s = time.time()
- for shot in num_fewshot:
- model_type = "hf"
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
- if 'gpu' in model_path:
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code={trust_remote_code}'
- if use_accelerate: # bool(re.search("chatglm", model_path.lower()))
- model_args += f',parallelize=True'
-
- if "wikitext" in tmp_tasks:
- tmp_eval_bs = 1
- else:
- tmp_eval_bs = eval_bs
- tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,
- num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
- max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
- if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
- sub_name = f'hendrycksTest-* {shot}-shot'
- else:
- sub_name = f'{tmp_tasks} {shot}-shot'
- print(f'{sub_name}: ')
- pprint.pprint(tmp_results["results"])
- print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
- results[sub_name] = {}
- if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
- for cata in ['humanities', 'other', 'stem', 'sociology']:
- results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']
- results[sub_name]['avg'] = tmp_results['results'][f'mmlu']
- else:
- results[sub_name] = tmp_results['results']
- except Exception as e:
- print(f'********* {tmp_tasks} ERROR ************')
- print(str(e))
- continue
-
- tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)
- model = lm.model
- # for external tasks
- # maybe adjust for specific model
- # if hasattr(lm.model.config, "max_position_embeddings"):
- # lm.model.seqlen = lm.model.config.max_position_embeddings
- # else:
- # ## for llama-1, opt
- # lm.model.seqlen = 2048
-
- # if "opt" in model_name:
- # seqlen = model.config.max_position_embeddings
- # model.seqlen = model.config.max_position_embeddings
- # else:
- # seqlen = 2048
- # model.seqlen = seqlen
-
- model.seqlen = 2048
- from eval.utils import get_loaders, eval_ppl_same_with_gptq
- for dataset in external_tasks:
- try:
- dataloader, testloader = get_loaders(
- dataset, nsamples=nsamples, seed=seed,
- tokenizer=tokenizer, seqlen=model.seqlen
+ results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(
+ eval_tasks
)
- ppl = eval_ppl_same_with_gptq(model, testloader, device)
- print(dataset, ppl)
-
- results.update({dataset: ppl})
- except Exception as e:
- print(str(e))
- continue
-
- print(results, flush=True)
- print("cost time: ", time.time() - org_s)
- import pickle
- from collections import OrderedDict
- new_dict = OrderedDict()
- new_dict["model"] = "tmp"
- new_dict["paper-avg"] = 0
- new_dict["leaderboard-avg"] = 0
- new_dict["wikitext2"] = 0
- new_dict["ptb-new"] = 0
- new_dict["c4-new"] = 0
- new_dict["wikitext 0-shot_word_perplexity"] = 0
- new_dict["wikitext 0-shot_byte_perplexity"] = 0
- new_dict["wikitext 0-shot_bits_per_byte"] = 0
-
- # Special handling of mmlu for compatibility with the old excel results sequence
- search_str = 'hendrycksTest'
- mmlu_matching_keys = [key for key in results.keys() if search_str.lower() in key.lower()]
- for key in mmlu_matching_keys:
- data = results.pop(key)
- for sub_key in data.keys():
- for sub_sub_key in data[sub_key].keys():
- new_key = key + "-" + sub_key + "-" + sub_sub_key
- if "std" in new_key or "alias" in new_key:
- continue
- new_key = new_key.split(",")[0]
- new_dict[new_key] = data[sub_key][sub_sub_key]
- new_key = new_key + "_norm"
- new_dict[new_key] = data[sub_key][sub_sub_key]
-
- for key in results.keys():
- if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
- continue
- data = results[key]
- if not isinstance(data, dict):
- new_dict[key] = results[key]
- continue
- for sub_key in data.keys():
- for sub_sub_key in data[sub_key].keys():
- new_key = key + "_" + sub_sub_key
- if "std" in new_key or "alias" in new_key:
+
+ ### Calculate group metrics ###
+ if bool(results):
+ for group, task_list in reversed(task_hierarchy.items()):
+ if len(task_list) == 0:
+ # task_hierarchy entries are either
+ # `group_name: [subtask1, subtask2, ...]`
+ # or `task_name: []`.
+ # we only want to operate on groups here.
continue
- new_key = new_key.split(",")[0]
- new_dict[new_key] = data[sub_key][sub_sub_key]
-
+ metric_list = list(
+ {
+ key
+ for task in task_list
+ for key in results[task].keys()
+ if "_stderr" not in key and key not in ["alias", "samples"]
+ }
+ )
+ for metric in metric_list:
+ stderr = "_stderr,".join(metric.split(","))
+
+ # gather metrics, sizes, and stderrs from subtasks
+ metrics = [
+ results[task][metric]
+ for task in task_list
+ if metric in results[task]
+ ] # TODO: copy?
+ stderrs = [
+ results[task][stderr]
+ for task in task_list
+ if stderr in results[task]
+ ]
+ sizes = [
+ results[task]["samples"]
+ for task in task_list
+ if metric in results[task]
+ ]
+
+ # compute group's pooled metric and stderr
+ results[group][
+ metric
+ ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
+ # TODO: calculate grouped metric using aggregation fn
+ if "N/A" in stderrs:
+ results[group][stderr] = "N/A"
+ else:
+ results[group][
+ stderr
+ ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
+ # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
+ # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
+ # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)
+
+ results[group]["samples"] = sum(sizes)
+
+ results_agg = defaultdict(dict)
+ groups_agg = defaultdict(dict)
+ all_tasks_list = list(task_hierarchy.keys())
+ while True:
+ add_tasks_list = list(k for k in results_agg.keys())
+ left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
+ if len(left_tasks_list) == 0:
+ break
+
+ _task_hierarchy = {
+ k: v for k, v in task_hierarchy.items() if k in left_tasks_list
+ }
+ _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
+
+ results_agg = {**results_agg, **_results_agg}
+ groups_agg = {**groups_agg, **_groups_agg}
+
+ for group_name, task_list in task_hierarchy.items():
+ if task_list:
+ num_fewshot[group_name] = num_fewshot[
+ task_list[0]
+ ] # TODO: validate this
+
+ results_dict = {
+ "results": dict(results_agg.items()),
+ **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
+ "group_subtasks": dict(reversed(task_hierarchy.items())),
+ "configs": dict(sorted(configs.items())),
+ "versions": dict(sorted(versions.items())),
+ "n-shot": dict(sorted(num_fewshot.items())),
+ }
+ if log_samples:
+ results_dict["samples"] = dict(samples)
+
+ return results_dict
- import pandas as pd
- df = pd.DataFrame(data=new_dict, index=[0])
- df.to_excel(excel_file)
+ else:
+ return None
if __name__ == "__main__":
+
+ import sys
+
+ sys.path.insert(0, '../../')
import time
import argparse
+
parser = argparse.ArgumentParser()
parser.add_argument(
- "--model_name", default="/models/opt-125m/"
+ "--model_name", default="facebook/opt-125m"
)
parser.add_argument(
"--eval_bs", default=1,
)
parser.add_argument(
- "--tasks", default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
- "arc_easy", "arc_challenge"]
+ "--trust_remote_code", action='store_true',
+ help="Whether to enable trust_remote_code"
)
parser.add_argument(
- "--excel_path", default=None,
- help="The path to save eval results with excel format."
+ "--device", default="cuda:0",
+ help="PyTorch device (e.g. cpu/cuda:0/hpu) for evaluation."
+ )
+ parser.add_argument("--tasks",
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
+ "openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks for lm_eval version 0.4.2")
+
+ parser.add_argument("--parallelize", action='store_true',
+ help="Whether to enable parallelize."
)
args = parser.parse_args()
s = time.time()
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
+
+ if hasattr(config, "quantization_config"):
+ quantization_config = config.quantization_config
+ if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":
+ if args.device == "hpu":
+ from auto_round.auto_quantizer import AutoHfQuantizer
test_tasks = args.tasks
if isinstance(test_tasks, str):
test_tasks = test_tasks.split(',')
model_name = args.model_name.rstrip('/')
- if args.excel_path is None:
- excel_name = model_name.split('/')[-1] + ".xlsx"
- else:
- excel_name = args.excel_path
- eval_model(model_path=args.model_name,
- tasks=test_tasks,
- eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
+ from lm_eval.utils import make_table
+
+ model_args = f"pretrained={args.model_name}"
+ model_args += ",dtype=float16"
+ if args.trust_remote_code:
+ model_args += f",trust_remote_code=True"
+ if args.parallelize:
+ model_args += f",parallelize=True"
+ result = simple_evaluate(model="hf",
+ model_args=model_args,
+ tasks=test_tasks,
+ device=args.device,
+ batch_size=args.eval_bs)
+ print(make_table(result))
print("cost time: ", time.time() - s)
+
diff --git a/examples/language-modeling/eval_042/__init__.py b/examples/language-modeling/eval_042/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/examples/language-modeling/eval_legacy/__init__.py b/examples/language-modeling/eval_legacy/__init__.py
index 07f1ae76d..35dde853f 100644
--- a/examples/language-modeling/eval_legacy/__init__.py
+++ b/examples/language-modeling/eval_legacy/__init__.py
@@ -2,4 +2,4 @@ if __name__ == "__main__":
import sys
sys.path.insert(0, './')
from .evaluation import eval_model
-
+from .evaluation_legacy import eval_model_legacy
diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.py
index c9f5b88e5..7a0666b70 100644
--- a/examples/language-modeling/eval_legacy/evaluation.py
+++ b/examples/language-modeling/eval_legacy/evaluation.py
@@ -3,16 +3,18 @@ import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import os.path
import torch
-
+import logging
import pprint
import re
import shutil
import transformers
+from typing import TYPE_CHECKING, Optional, Union
+import time
+
if __name__ == "__main__":
import sys
sys.path.insert(0, './')
-from eval_legacy.parse_results import result_parser
-import time
+
EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
fewshots_dict = {}
fewshots_dict['paper'] = {
@@ -20,21 +22,25 @@ fewshots_dict['paper'] = {
"hellaswag": [0],
"winogrande": [0],
"piqa": [0],
- "hendrycksTest-*": [0],
+ "mmlu": [0],
"wikitext": [0],
- "truthfulqa_mc": [0],
+ "truthfulqa_mc1": [0],
+ "truthfulqa_mc2": [0],
"openbookqa": [0],
"boolq": [0],
"rte": [0],
"arc_easy": [0],
"arc_challenge": [0],
+ "gsm8k": [0],
+ "ceval-valid": [0],
+ "cmmlu": [0],
}
fewshots_dict['leadboard'] = {
"hellaswag": [10],
"winogrande": [5],
"arc_easy": [25],
"arc_challenge": [25],
- "hendrycksTest-*": [5],
+ "mmlu": [5],
"drop": [3],
"gsm8k": [5],
}
@@ -44,13 +50,14 @@ fewshots_dict['all'] = {
"winogrande": [0, 5],
"piqa": [0],
"coqa": [], ## coqa is not enabled in llamav1 models
- "truthfulqa_mc": [0],
+ "truthfulqa_mc1": [0],
+ "truthfulqa_mc2": [0],
"openbookqa": [0],
"boolq": [0],
"rte": [0],
"arc_easy": [0, 25],
"arc_challenge": [0, 25],
- "hendrycksTest-*": [0, 5],
+ "mmlu": [0, 5],
"wikitext": [0],
"drop": [3],
"gsm8k": [5]
@@ -58,31 +65,40 @@ fewshots_dict['all'] = {
def simple_evaluate(
- model,
- model_args=None,
- tasks=[],
- num_fewshot=0,
- batch_size=None,
- max_batch_size=None,
- device=None,
- no_cache=True,
- limit=None,
- bootstrap_iters=100000,
- description_dict=None,
- check_integrity=False,
- decontamination_ngrams_path=None,
- write_out=False,
- output_base_path=None,
- lm=None
+ model,
+ model_args: Optional[Union[str, dict, None]] = None,
+ tasks=None,
+ num_fewshot: Optional[int] = None,
+ batch_size: Optional[int] = None,
+ max_batch_size: Optional[int] = None,
+ device: Optional[str] = None,
+ use_cache: Optional[str] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ delete_requests_cache: bool = False,
+ limit: Optional[Union[int, float]] = None,
+ bootstrap_iters: int = 100000,
+ check_integrity: bool = False,
+ decontamination_ngrams_path=None,
+ write_out: bool = False,
+ log_samples: bool = True,
+ gen_kwargs: str = None,
+ task_manager=None,
+ verbosity: str = "INFO",
+ predict_only: bool = False,
+ random_seed: int = 1234,
+ numpy_random_seed: int = 1234,
+ torch_random_seed: int = 1234,
+ lm=None
):
"""Instantiate and evaluate a model on a list of tasks.
:param model: Union[str, LM]
- Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model
- :param model_args: Optional[str]
- String arguments for each model class, see LM.create_from_arg_string.
+ Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model
+ :param model_args: Optional[str, dict]
+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
Ignored if `model` argument is a LM object.
- :param tasks: list[Union[str, Task]]
+ :param tasks: list[Union[str, dict, Task]]
List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
:param num_fewshot: int
Number of examples in few-shot context
@@ -92,66 +108,130 @@ def simple_evaluate(
Maximal batch size to try with automatic batch size detection
:param device: str, optional
PyTorch device (e.g. "cpu" or "cuda:0") for running models
- :param no_cache: bool
- Whether or not to cache
+ :param use_cache: str, optional
+ A path to a sqlite db file for caching model responses. `None` if not caching.
+ :param cache_requests: bool, optional
+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.
+ :param rewrite_requests_cache: bool, optional
+ Rewrites all of the request cache if set to `True`. `None` if not desired.
+ :param delete_requests_cache: bool, optional
+ Deletes all of the request cache if set to `True`. `None` if not desired.
:param limit: int or float, optional
Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
:param bootstrap_iters:
Number of iterations for bootstrap statistics
- :param description_dict: dict[str, str]
- Dictionary of custom task descriptions of the form: `task_name: description`
:param check_integrity: bool
Whether to run the relevant part of the test suite for the tasks
:param write_out: bool
- If True, write details about prompts and logits to json for all tasks
- :param output_base_path: str, optional
- Directory to which detailed eval info will be written. Defaults to present working dir.
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :param gen_kwargs: str
+ String arguments for model generation
+ Ignored for all tasks with loglikelihood output_type
+ :param predict_only: bool
+ If true only model outputs will be generated and returned. Metrics will not be evaluated
+ :param random_seed: int
+ Random seed for python's random module. If set to None, the seed will not be set.
+ :param numpy_random_seed: int
+ Random seed for numpy. If set to None, the seed will not be set.
+ :param torch_random_seed: int
+ Random seed for torch. If set to None, the seed will not be set.
+
:return
Dictionary of results
"""
- import collections
- import itertools
+ from lm_eval.tasks import TaskManager
import random
-
- import lm_eval.metrics
- import lm_eval.models
- import lm_eval.tasks
- import lm_eval.base
- from lm_eval.utils import positional_deprecated, run_task_tests
- from lm_eval.models.gpt2 import HFLM
-
import numpy as np
import transformers
+ import lm_eval.api
+ import lm_eval.tasks
+ import lm_eval.models
+ import lm_eval.api.metrics
+ import lm_eval.api.registry
from lm_eval.evaluator import evaluate
- random.seed(1234)
- np.random.seed(1234)
+ from lm_eval.logging_utils import add_env_info, get_git_commit_hash
+ from lm_eval.tasks import get_task_dict
+
+ from lm_eval.utils import (
+ eval_logger,
+ positional_deprecated,
+ run_task_tests,
+ simple_parse_args_string,
+ )
+
+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+ if delete_requests_cache:
+ eval_logger.info("Deleting requests cache...")
+ delete_cache()
+
+ if random_seed is not None:
+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
+ eval_logger.info(f"Setting random seed to {random_seed}")
+ random.seed(random_seed)
+
+ if numpy_random_seed is not None:
+ eval_logger.info(f"Setting numpy seed to {numpy_random_seed}")
+ np.random.seed(numpy_random_seed)
+
+ if torch_random_seed is not None:
+ eval_logger.info(f"Setting torch manual seed to {torch_random_seed}")
+ torch.manual_seed(torch_random_seed)
+
+ if tasks is None:
+ tasks = []
+ assert (
+ tasks != []
+ ), "No tasks specified, or no tasks found. Please verify the task names."
- assert tasks != [], "No tasks specified"
if lm == None:
+ if gen_kwargs is not None:
+ gen_kwargs = simple_parse_args_string(gen_kwargs)
+ eval_logger.warning(
+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!"
+ )
+ if gen_kwargs == "":
+ gen_kwargs = None
+
if isinstance(model, str):
if model_args is None:
model_args = ""
- lm = lm_eval.models.get_model(model).create_from_arg_string(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
+
+ elif isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+
+ else:
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
elif isinstance(model, transformers.PreTrainedModel):
- lm = lm_eval.models.get_model("hf-causal")(
+ lm = lm_eval.api.registry.get_model("hf")(
pretrained=model,
batch_size=batch_size,
max_batch_size=max_batch_size,
)
- no_cache = True
+ use_cache = None
else:
- assert isinstance(model, lm_eval.base.LM)
+ assert isinstance(model, lm_eval.api.model.LM)
lm = model
- if not no_cache:
- lm = lm_eval.base.CachingLM(
+ if use_cache is not None:
+ print(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
+ lm = lm_eval.api.model.CachingLM(
lm,
"lm_cache/"
+ (model if isinstance(model, str) else model.model.config._name_or_path)
@@ -159,8 +239,46 @@ def simple_evaluate(
+ model_args.replace("=", "-").replace(",", "_").replace("/", "-")
+ ".db",
)
+
+ if task_manager is None:
+ task_manager = TaskManager(verbosity)
- task_dict = lm_eval.tasks.get_task_dict(tasks)
+ eval_logger.info(
+ "get_task_dict has been updated to accept an optional argument, `task_manager`"
+ "Read more here:https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage"
+ )
+ task_dict = get_task_dict(tasks, task_manager)
+ for task_name in task_dict.keys():
+ task_obj = task_dict[task_name]
+ if isinstance(task_obj, tuple):
+ _, task_obj = task_obj
+ if task_obj is None:
+ continue
+
+ if task_obj.get_config("output_type") == "generate_until":
+ if gen_kwargs is not None:
+ task_obj.set_config(
+ key="generation_kwargs", value=gen_kwargs, update=True
+ )
+
+ if predict_only:
+ log_samples = True
+ eval_logger.info(
+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+ )
+ # we have to change the class properties post-hoc. This is pretty hacky.
+ task_obj.override_metric(metric_name="bypass")
+
+ if num_fewshot is not None:
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
+ eval_logger.info(
+ f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
+ )
+ else:
+ eval_logger.warning(
+ f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
+ )
+ task_obj.set_config(key="num_fewshot", value=num_fewshot)
if check_integrity:
run_task_tests(task_list=tasks)
@@ -168,48 +286,57 @@ def simple_evaluate(
results = evaluate(
lm=lm,
task_dict=task_dict,
- num_fewshot=num_fewshot,
limit=limit,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
bootstrap_iters=bootstrap_iters,
- description_dict=description_dict,
decontamination_ngrams_path=decontamination_ngrams_path,
write_out=write_out,
- output_base_path=output_base_path,
+ log_samples=log_samples,
+ verbosity=verbosity,
)
- # add info about the model and few shot config
- model_name = None
- if isinstance(model, str):
- model_name = model
- elif isinstance(model, transformers.PreTrainedModel):
- model_name = "pretrained=" + model.config._name_or_path
- results["config"] = {
- "model": model_name,
- "model_args": model_args,
- "num_fewshot": num_fewshot,
- "batch_size": batch_size,
- "batch_sizes": list(lm.batch_sizes.values())
- if hasattr(lm, "batch_sizes")
- else [],
- "device": device,
- "no_cache": no_cache,
- "limit": limit,
- "bootstrap_iters": bootstrap_iters,
- "description_dict": description_dict,
- }
-
- return results, lm
-
-
-def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+ if lm.rank == 0:
+ if isinstance(model, str):
+ model_name = model
+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
+ model_name = model.config._name_or_path
+ else:
+ model_name = type(model).__name__
+
+ # add info about the model and few shot config
+ results["config"] = {
+ "model": model_name,
+ "model_args": model_args,
+ "batch_size": batch_size,
+ "batch_sizes": (
+ list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
+ ),
+ "device": device,
+ "use_cache": use_cache,
+ "limit": limit,
+ "bootstrap_iters": bootstrap_iters,
+ "gen_kwargs": gen_kwargs,
+ }
+ results["git_hash"] = get_git_commit_hash()
+ add_env_info(results) # additional environment info to results
+ return results, lm
+ else:
+ return None
+
+
+def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,
device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
- import lm_eval
- from lm_eval import evaluator
- from lm_eval.tasks import ALL_TASKS, get_task_dict
- import pandas as pd
+ import lm_eval.api
+ import lm_eval.tasks
+ import lm_eval.models
+ import lm_eval.api.metrics
+ import lm_eval.api.registry
+ from lm_eval.evaluator import evaluate
+
except:
raise ImportError("""follow requirements to install dependencies.""")
@@ -224,7 +351,6 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
dtype = "float16"
print(f"Using {dtype} as evaluation data type.")
-
external_tasks = []
if isinstance(tasks, str):
tasks = tasks.split(',')
@@ -236,32 +362,41 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
results = {}
model = None
lm = None
+
for tmp_tasks in tasks:
try:
num_fewshot = fewshots_dict[mark][tmp_tasks]
- task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)
print(f'********* {tmp_tasks} evaluate ************')
task_s = time.time()
for shot in num_fewshot:
- if bool(re.search("chatglm", model_path.lower())):
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
- model_type = "hf-causal"
- else:
- model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code={trust_remote_code}'
- model_type = "hf-causal-experimental"
-
- if "wikitext" in task_names:
+ model_type = "hf"
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
+ if 'gpu' in model_path:
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},autogptq=True,gptq_use_triton=True,trust_remote_code={trust_remote_code}'
+ if use_accelerate: # bool(re.search("chatglm", model_path.lower()))
+ model_args += f',parallelize=True'
+
+ if "wikitext" in tmp_tasks:
tmp_eval_bs = 1
else:
tmp_eval_bs = eval_bs
- tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,
+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=tmp_tasks,
num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
- sub_name = f'{tmp_tasks} {shot}-shot'
+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
+ sub_name = f'hendrycksTest-* {shot}-shot'
+ else:
+ sub_name = f'{tmp_tasks} {shot}-shot'
print(f'{sub_name}: ')
pprint.pprint(tmp_results["results"])
print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
- results[sub_name] = tmp_results
+ results[sub_name] = {}
+ if 'mmlu' in tmp_tasks and 'cmmlu' not in tmp_tasks:
+ for cata in ['humanities', 'other', 'stem', 'sociology']:
+ results[sub_name][cata] = tmp_results['results'][f'mmlu_{cata}']
+ results[sub_name]['avg'] = tmp_results['results'][f'mmlu']
+ else:
+ results[sub_name] = tmp_results['results']
except Exception as e:
print(f'********* {tmp_tasks} ERROR ************')
print(str(e))
@@ -301,7 +436,6 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
continue
print(results, flush=True)
- new_results = result_parser(results)
print("cost time: ", time.time() - org_s)
import pickle
from collections import OrderedDict
@@ -315,25 +449,39 @@ def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrand
new_dict["wikitext 0-shot_word_perplexity"] = 0
new_dict["wikitext 0-shot_byte_perplexity"] = 0
new_dict["wikitext 0-shot_bits_per_byte"] = 0
- for key in new_results.keys():
+
+ # Special handling of mmlu for compatibility with the old excel results sequence
+ search_str = 'hendrycksTest'
+ mmlu_matching_keys = [key for key in results.keys() if search_str.lower() in key.lower()]
+ for key in mmlu_matching_keys:
+ data = results.pop(key)
+ for sub_key in data.keys():
+ for sub_sub_key in data[sub_key].keys():
+ new_key = key + "-" + sub_key + "-" + sub_sub_key
+ if "std" in new_key or "alias" in new_key:
+ continue
+ new_key = new_key.split(",")[0]
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+ new_key = new_key + "_norm"
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+
+ for key in results.keys():
if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
continue
- data = new_results[key]
+ data = results[key]
if not isinstance(data, dict):
- new_dict[key] = new_results[key]
+ new_dict[key] = results[key]
continue
for sub_key in data.keys():
for sub_sub_key in data[sub_key].keys():
- if "hendry" in key:
- new_key = key + "-" + sub_key + "-" + sub_sub_key
- else:
- new_key = key + "_" + sub_sub_key
- if "std" in new_key:
+ new_key = key + "_" + sub_sub_key
+ if "std" in new_key or "alias" in new_key:
continue
- # if "norm" in new_key:
- # continue
+ new_key = new_key.split(",")[0]
new_dict[new_key] = data[sub_key][sub_sub_key]
+
+ import pandas as pd
df = pd.DataFrame(data=new_dict, index=[0])
df.to_excel(excel_file)
@@ -343,31 +491,35 @@ if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
- "--model_name", default="/models/opt-125m/"
+ "--model_name", default="facebook/opt-125m"
)
parser.add_argument(
"--eval_bs", default=1,
)
+ parser.add_argument(
+ "--tasks", default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
+ "arc_easy", "arc_challenge"]
+ )
+ parser.add_argument(
+ "--excel_path", default=None,
+ help="The path to save eval results with excel format."
+ )
args = parser.parse_args()
s = time.time()
- test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",
- "arc_challenge"]
- seen = set()
- tmp_tasks = test_tasks
- test_tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
+ test_tasks = args.tasks
+ if isinstance(test_tasks, str):
+ test_tasks = test_tasks.split(',')
model_name = args.model_name.rstrip('/')
- excel_name = model_name.split('/')[-1] + ".xlsx"
+ if args.excel_path is None:
+ excel_name = model_name.split('/')[-1] + ".xlsx"
+ else:
+ excel_name = args.excel_path
eval_model(model_path=args.model_name,
tasks=test_tasks,
eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
print("cost time: ", time.time() - s)
-
-
-
-
-
diff --git a/examples/language-modeling/eval_legacy/evaluation_legacy.py b/examples/language-modeling/eval_legacy/evaluation_legacy.py
new file mode 100644
index 000000000..68656a7d9
--- /dev/null
+++ b/examples/language-modeling/eval_legacy/evaluation_legacy.py
@@ -0,0 +1,373 @@
+
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+import os.path
+import torch
+
+import pprint
+import re
+import shutil
+import transformers
+if __name__ == "__main__":
+ import sys
+ sys.path.insert(0, './')
+from eval_legacy.parse_results import result_parser
+import time
+EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
+fewshots_dict = {}
+fewshots_dict['paper'] = {
+ "lambada_openai": [0],
+ "hellaswag": [0],
+ "winogrande": [0],
+ "piqa": [0],
+ "hendrycksTest-*": [0],
+ "wikitext": [0],
+ "truthfulqa_mc": [0],
+ "openbookqa": [0],
+ "boolq": [0],
+ "rte": [0],
+ "arc_easy": [0],
+ "arc_challenge": [0],
+}
+fewshots_dict['leadboard'] = {
+ "hellaswag": [10],
+ "winogrande": [5],
+ "arc_easy": [25],
+ "arc_challenge": [25],
+ "hendrycksTest-*": [5],
+ "drop": [3],
+ "gsm8k": [5],
+}
+fewshots_dict['all'] = {
+ "lambada_openai": [0],
+ "hellaswag": [0, 10],
+ "winogrande": [0, 5],
+ "piqa": [0],
+ "coqa": [], ## coqa is not enabled in llamav1 models
+ "truthfulqa_mc": [0],
+ "openbookqa": [0],
+ "boolq": [0],
+ "rte": [0],
+ "arc_easy": [0, 25],
+ "arc_challenge": [0, 25],
+ "hendrycksTest-*": [0, 5],
+ "wikitext": [0],
+ "drop": [3],
+ "gsm8k": [5]
+}
+
+
+def simple_evaluate(
+ model,
+ model_args=None,
+ tasks=[],
+ num_fewshot=0,
+ batch_size=None,
+ max_batch_size=None,
+ device=None,
+ no_cache=True,
+ limit=None,
+ bootstrap_iters=100000,
+ description_dict=None,
+ check_integrity=False,
+ decontamination_ngrams_path=None,
+ write_out=False,
+ output_base_path=None,
+ lm=None
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param model: Union[str, LM]
+ Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model
+ :param model_args: Optional[str]
+ String arguments for each model class, see LM.create_from_arg_string.
+ Ignored if `model` argument is a LM object.
+ :param tasks: list[Union[str, Task]]
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ :param num_fewshot: int
+ Number of examples in few-shot context
+ :param batch_size: int or str, optional
+ Batch size for model
+ :param max_batch_size: int, optional
+ Maximal batch size to try with automatic batch size detection
+ :param device: str, optional
+ PyTorch device (e.g. "cpu" or "cuda:0") for running models
+ :param no_cache: bool
+ Whether or not to cache
+ :param limit: int or float, optional
+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics
+ :param description_dict: dict[str, str]
+ Dictionary of custom task descriptions of the form: `task_name: description`
+ :param check_integrity: bool
+ Whether to run the relevant part of the test suite for the tasks
+ :param write_out: bool
+ If True, write details about prompts and logits to json for all tasks
+ :param output_base_path: str, optional
+ Directory to which detailed eval info will be written. Defaults to present working dir.
+ :return
+ Dictionary of results
+ """
+ import collections
+ import itertools
+ import random
+
+ import lm_eval.metrics
+ import lm_eval.models
+ import lm_eval.tasks
+ import lm_eval.base
+ from lm_eval.utils import positional_deprecated, run_task_tests
+ from lm_eval.models.gpt2 import HFLM
+
+ import numpy as np
+ import transformers
+ from lm_eval.evaluator import evaluate
+ random.seed(1234)
+ np.random.seed(1234)
+
+ assert tasks != [], "No tasks specified"
+ if lm == None:
+ if isinstance(model, str):
+ if model_args is None:
+ model_args = ""
+ lm = lm_eval.models.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+ elif isinstance(model, transformers.PreTrainedModel):
+ lm = lm_eval.models.get_model("hf-causal")(
+ pretrained=model,
+ batch_size=batch_size,
+ max_batch_size=max_batch_size,
+ )
+ no_cache = True
+ else:
+ assert isinstance(model, lm_eval.base.LM)
+ lm = model
+
+ if not no_cache:
+ lm = lm_eval.base.CachingLM(
+ lm,
+ "lm_cache/"
+ + (model if isinstance(model, str) else model.model.config._name_or_path)
+ + "_"
+ + model_args.replace("=", "-").replace(",", "_").replace("/", "-")
+ + ".db",
+ )
+
+ task_dict = lm_eval.tasks.get_task_dict(tasks)
+
+ if check_integrity:
+ run_task_tests(task_list=tasks)
+
+ results = evaluate(
+ lm=lm,
+ task_dict=task_dict,
+ num_fewshot=num_fewshot,
+ limit=limit,
+ bootstrap_iters=bootstrap_iters,
+ description_dict=description_dict,
+ decontamination_ngrams_path=decontamination_ngrams_path,
+ write_out=write_out,
+ output_base_path=output_base_path,
+ )
+
+ # add info about the model and few shot config
+ model_name = None
+ if isinstance(model, str):
+ model_name = model
+ elif isinstance(model, transformers.PreTrainedModel):
+ model_name = "pretrained=" + model.config._name_or_path
+ results["config"] = {
+ "model": model_name,
+ "model_args": model_args,
+ "num_fewshot": num_fewshot,
+ "batch_size": batch_size,
+ "batch_sizes": list(lm.batch_sizes.values())
+ if hasattr(lm, "batch_sizes")
+ else [],
+ "device": device,
+ "no_cache": no_cache,
+ "limit": limit,
+ "bootstrap_iters": bootstrap_iters,
+ "description_dict": description_dict,
+ }
+
+ return results, lm
+
+
+def eval_model_legacy(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+ eval_bs=32, use_accelerate=True, dtype=None, limit=None, trust_remote_code=True,
+ device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
+ print("evaluation with official lm-eval", flush=True)
+ try:
+ import lm_eval
+ from lm_eval import evaluator
+ from lm_eval.tasks import ALL_TASKS, get_task_dict
+ import pandas as pd
+ except:
+ raise ImportError("""follow requirements to install dependencies.""")
+
+ org_s = time.time()
+ if dtype == None:
+ from eval.utils import convert_dtype_torch2str_hf
+ from transformers import AutoConfig
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
+ if hasattr(config, "torch_dtype"):
+ dtype = convert_dtype_torch2str_hf(config.torch_dtype)
+ else:
+ dtype = "float16"
+ print(f"Using {dtype} as evaluation data type.")
+
+
+ external_tasks = []
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+ for each in EXT_TASKS:
+ if each in tasks:
+ external_tasks.append(each)
+ tasks.remove(each)
+
+ results = {}
+ model = None
+ lm = None
+ for tmp_tasks in tasks:
+ try:
+ num_fewshot = fewshots_dict[mark][tmp_tasks]
+ task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)
+ print(f'********* {tmp_tasks} evaluate ************')
+ task_s = time.time()
+ for shot in num_fewshot:
+ if bool(re.search("chatglm", model_path.lower())):
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code={trust_remote_code}'
+ model_type = "hf-causal"
+ else:
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code={trust_remote_code}'
+ model_type = "hf-causal-experimental"
+
+ if "wikitext" in task_names:
+ tmp_eval_bs = 1
+ else:
+ tmp_eval_bs = eval_bs
+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,
+ num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
+ max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
+ sub_name = f'{tmp_tasks} {shot}-shot'
+ print(f'{sub_name}: ')
+ pprint.pprint(tmp_results["results"])
+ print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
+ results[sub_name] = tmp_results
+ except Exception as e:
+ print(f'********* {tmp_tasks} ERROR ************')
+ print(str(e))
+ continue
+
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)
+ model = lm.model
+ # for external tasks
+ # maybe adjust for specific model
+ # if hasattr(lm.model.config, "max_position_embeddings"):
+ # lm.model.seqlen = lm.model.config.max_position_embeddings
+ # else:
+ # ## for llama-1, opt
+ # lm.model.seqlen = 2048
+
+ # if "opt" in model_name:
+ # seqlen = model.config.max_position_embeddings
+ # model.seqlen = model.config.max_position_embeddings
+ # else:
+ # seqlen = 2048
+ # model.seqlen = seqlen
+
+ model.seqlen = 2048
+ from eval.utils import get_loaders, eval_ppl_same_with_gptq
+ for dataset in external_tasks:
+ try:
+ dataloader, testloader = get_loaders(
+ dataset, nsamples=nsamples, seed=seed,
+ tokenizer=tokenizer, seqlen=model.seqlen
+ )
+ ppl = eval_ppl_same_with_gptq(model, testloader, device)
+ print(dataset, ppl)
+
+ results.update({dataset: ppl})
+ except Exception as e:
+ print(str(e))
+ continue
+
+ print(results, flush=True)
+ new_results = result_parser(results)
+ print("cost time: ", time.time() - org_s)
+ import pickle
+ from collections import OrderedDict
+ new_dict = OrderedDict()
+ new_dict["model"] = "tmp"
+ new_dict["paper-avg"] = 0
+ new_dict["leaderboard-avg"] = 0
+ new_dict["wikitext2"] = 0
+ new_dict["ptb-new"] = 0
+ new_dict["c4-new"] = 0
+ new_dict["wikitext 0-shot_word_perplexity"] = 0
+ new_dict["wikitext 0-shot_byte_perplexity"] = 0
+ new_dict["wikitext 0-shot_bits_per_byte"] = 0
+ for key in new_results.keys():
+ if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
+ continue
+ data = new_results[key]
+ if not isinstance(data, dict):
+ new_dict[key] = new_results[key]
+ continue
+ for sub_key in data.keys():
+ for sub_sub_key in data[sub_key].keys():
+ if "hendry" in key:
+ new_key = key + "-" + sub_key + "-" + sub_sub_key
+ else:
+ new_key = key + "_" + sub_sub_key
+ if "std" in new_key:
+ continue
+ # if "norm" in new_key:
+ # continue
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+
+ df = pd.DataFrame(data=new_dict, index=[0])
+ df.to_excel(excel_file)
+
+
+if __name__ == "__main__":
+ import time
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name", default="facebook/opt-125m"
+ )
+ parser.add_argument(
+ "--eval_bs", default=1,
+ )
+
+ args = parser.parse_args()
+ s = time.time()
+
+ test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",
+ "arc_challenge"]
+ seen = set()
+ tmp_tasks = test_tasks
+ test_tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
+ model_name = args.model_name.rstrip('/')
+ excel_name = model_name.split('/')[-1] + ".xlsx"
+ eval_model_legacy(model_path=args.model_name,
+ tasks=test_tasks,
+ eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
+
+ print("cost time: ", time.time() - s)
+
+
+
+
+
+
diff --git a/examples/language-modeling/eval/utils.py b/examples/language-modeling/eval_legacy/utils.py
similarity index 100%
rename from examples/language-modeling/eval/utils.py
rename to examples/language-modeling/eval_legacy/utils.py
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 9dbf2ea67..98062ce87 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -1,18 +1,19 @@
-import argparse
+import os
+import re
import sys
+import argparse
+import subprocess
+from packaging import version
sys.path.insert(0, '../..')
parser = argparse.ArgumentParser()
import torch
-import os
import transformers
-import subprocess
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=True)
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
-import re
os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -81,9 +82,15 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
- parser.add_argument("--deployment_device", default='auto_round', type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."
- "default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")
+ parser.add_argument("--deployment_device", default=None, type=str,
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'."
+ "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
+
+ parser.add_argument("--format", default=None, type=str,
+ help="The format in which to save the model. "
+ "The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."
+ "default to 'auto_round."
+ )
parser.add_argument("--data_type", default='int',
help="data type for tuning, 'int', 'mx_fp' and etc.")
@@ -157,12 +164,19 @@ if __name__ == '__main__':
tasks = args.tasks
use_eval_legacy = False
- if "gpu" in args.deployment_device and args.sym is False:
- print(
- "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")
+ if args.format and args.deployment_device:
+ assert False, "please only specify one of format and deployment_device"
+
+ if args.deployment_device is None and args.format is None:
+ args.format == "auto_round"
+
+ if args.deployment_device:
+ if "gpu" in args.deployment_device and args.sym is False:
+ print(
+ "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --format='auto_round'")
- if "marlin" in args.deployment_device and args.sym is False:
- assert False, "marlin backend only supports sym quantization, please set --sym"
+ if "marlin" in args.deployment_device and args.sym is False:
+ assert False, "marlin backend only supports sym quantization, please set --sym"
model_name = args.model_name
@@ -310,7 +324,7 @@ if __name__ == '__main__':
scale_dtype=args.scale_dtype, layer_config=layer_config,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type, enable_norm_bias_tuning=args.enable_norm_bias_tuning)
- model, _ = autoround.quantize()
+ # model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
@@ -324,62 +338,77 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
- deployment_device = args.deployment_device.split(',')
- gpu_formats = []
- for item in deployment_device:
- if "gpu" in item or "auto_gptq" in item or "auto_round" in item or "auto_awq" in item:
- gpu_formats.append(item)
-
- if 'gpu' in deployment_device:
- if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":
- gpu_formats.append("auto_round")
- else:
- gpu_formats.append("auto_gptq")
- gpu_formats = list(set(gpu_formats))
-
- inplace = True if len(deployment_device) < 2 else False
eval_folder = None
- for gpu_format in gpu_formats:
- if "round" in gpu_format:
- eval_folder = f'{export_dir}-round'
- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
- elif "gptq" in gpu_format:
- eval_folder = f'{export_dir}-gpu'
- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
- elif "auto_awq" in gpu_format: # pragma: no cover
- eval_folder = f'{export_dir}-awq'
- autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)
-
- if 'xpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)
- if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
- if "fake" in deployment_device:
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
- if eval_folder is None:
- eval_folder = output_dir
-
+ if args.format:
+ format_list = args.format.replace(' ', '').split(',')
+ inplace = False if len(format_list) > 1 else True
+ for format_ in format_list:
+ eval_folder = f'{export_dir}-{format_}'
+ if 'auto_awq' in format_:
+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace, model_path=model_name)
+ else:
+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace)
+ else:
+ deployment_device = args.deployment_device.split(',')
+ gpu_formats = []
+ for item in deployment_device:
+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item or "auto_awq" in item:
+ gpu_formats.append(item)
+
+ if 'gpu' in deployment_device:
+ if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":
+ gpu_formats.append("auto_round")
+ else:
+ gpu_formats.append("auto_gptq")
+ gpu_formats = list(set(gpu_formats))
+
+ inplace = True if len(deployment_device) < 2 else False
+ for gpu_format in gpu_formats:
+ if "round" in gpu_format:
+ eval_folder = f'{export_dir}-round'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ elif "gptq" in gpu_format:
+ eval_folder = f'{export_dir}-gpu'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ elif "auto_awq" in gpu_format: # pragma: no cover
+ eval_folder = f'{export_dir}-awq'
+ autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)
+
+ if 'xpu' in deployment_device:
+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)
+ if "cpu" in deployment_device:
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
+ if "fake" in deployment_device:
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ tokenizer.save_pretrained(output_dir)
+ if eval_folder is None:
+ eval_folder = output_dir
+ if not ('gpu' in deployment_device or len(gpu_formats) > 0) or 'fake' not in deployment_device:
+ print('does not support cpu, xpu model evaluation.')
+ exit() ## does not support cpu,xpu model eval
def get_library_version(library_name):
+ import pkg_resources
try:
- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
+ version = pkg_resources.get_distribution(library_name).version
return version
- except subprocess.CalledProcessError:
- return "Library not found"
-
-
- lm_eval_version = get_library_version("lm-eval")
- if lm_eval_version == "0.3.0":
+ except pkg_resources.DistributionNotFound:
+ return f"{library_name} is not installed"
+
+
+ from packaging.version import Version
+ lm_eval_version = get_library_version("lm_eval")
+ lm_eval_version = Version(lm_eval_version)
+ if lm_eval_version == Version("0.3.0"):
use_eval_legacy = True
if isinstance(tasks, str):
tasks = tasks.split(',')
if not use_eval_legacy:
- from eval import eval_model
- else:
from eval_legacy import eval_model
+ else:
+ from eval_legacy import eval_model_legacy as eval_model
if isinstance(tasks, list):
if "mmlu" in tasks:
@@ -392,13 +421,18 @@ if __name__ == '__main__':
tmp_tasks = tasks
tasks = [x for x in tmp_tasks if not (x in seen or seen.add(x))]
- if 'fake' in args.deployment_device and not args.disable_eval:
+ use_qdq = False
+ if args.deployment_device and 'fake' in args.deployment_device:
+ use_qdq = True
+ if args.format and ('fake' in args.format or 'qdq' in args.format):
+ use_qdq = True
+ if use_qdq and not args.disable_eval:
if use_eval_legacy:
print("Using the legacy lm_eval(0.3.0)")
else:
print(f"Using the latest {lm_eval_version}")
- if not args.disable_eval and "fake" in deployment_device and lm_eval_version != "0.4.2":
+ if not args.disable_eval and use_qdq and lm_eval_version < Version("0.4.2"):
excel_name = f"{output_dir}_result.xlsx"
output_dir += "/"
print(excel_name, flush=True)
@@ -407,15 +441,10 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name,
trust_remote_code=not args.disable_trust_remote_code)
- if not args.disable_eval and lm_eval_version == "0.4.2":
- from eval_042.evaluation import simple_evaluate
+ if not args.disable_eval and lm_eval_version >= Version("0.4.2"):
+ from eval.evaluation import simple_evaluate
- if 'gpu' in deployment_device or len(gpu_formats) > 0:
- model_args = f"pretrained={eval_folder}"
- elif "fake" in deployment_device:
- model_args = f"pretrained={eval_folder}"
- else:
- exit() ## does not support cpu,xpu model eval
+ model_args = f"pretrained={eval_folder}"
model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"
user_model = None
if args.act_bits <= 8:
diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txt
index dab669f97..6db0fc595 100644
--- a/examples/language-modeling/requirements.txt
+++ b/examples/language-modeling/requirements.txt
@@ -1,6 +1,6 @@
transformers
torch
-lm-eval==0.4.2
+lm-eval==0.4.3
##for lm-eval 0.4.1 we usually used
##git+https://github.com/EleutherAI/lm-evaluation-harness.git@96d185fa6232a5ab685ba7c43e45d1dbb3bb906d
# For the paper results use the old lm_eval (0.3.0)
diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
index ac9f7636e..dd440f106 100644
--- a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
@@ -571,7 +571,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
- "--model_name", default="/models/opt-125m/"
+ "--model_name", default="facebook/opt-125m"
)
parser.add_argument(
"--eval_bs", default=1,
diff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
index 08f7170a4..4e1446705 100644
--- a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
+++ b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
@@ -10,7 +10,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
- "--model_name", default="/models/opt-125m/"
+ "--model_name", default="facebook/opt-125m"
)
parser.add_argument(
"--eval_bs", default=4, type=int,
diff --git a/requirements.txt b/requirements.txt
index 1edf3fd76..4400fcfe0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,5 +7,6 @@ transformers
triton
numpy < 2.0
threadpoolctl
+lm-eval==0.4.3
intel-extension-for-transformers
tqdm
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index b00c17fa1..9de8ea928 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,3 +2,8 @@
license_files =
LICENSE
third-party-programs.txt
+
+[options.entry_points]
+console_scripts =
+ auto_round = auto_round.__main__:run
+ auto-round = auto_round.__main__:run