Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix example bug (#245)

n1ck-guo0d60ad211fa5
diff --git a/README.md b/README.md
index fe41b2ebb..fada849a0 100644
--- a/README.md
+++ b/README.md
@@ -77,44 +77,6 @@ output_dir = "./tmp_autoround"
autoround.save_quantized(output_dir, format='auto_round', inplace=True)
```
-### Basic Usage
-AutoRound support Gaudi2, CPU and GPU. A user guide detailing the full list of supported arguments is provided by calling ```auto_round -h``` on the terminal. Alternatively, you can use ```auto-round``` instead of ```auto_round```. (**auto-round version > 0.3.0**)
-
-
-```bash
-auto_round --model facebook/opt-125m \
- --bits 4 \
- --group_size 128 \
- --format auto_round \
- --output_dir ./tmp_autoround
-```
-We provide two recipes for best accuracy and fast running speed with low memory. Details as below.
-<details>
- <summary>Other Recipes</summary>
-
- ```bash
-## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower
- auto_round --model facebook/opt-125m \
- --bits 4 \
- --group_size 128 \
- --nsamples 512 \
- --iters 1000 \
- --low_gpu_mem_usage
- ```
-
- ```bash
-## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128
- auto_round --model facebook/opt-125m \
- --bits 4 \
- --group_size 128 \
- --nsamples 128 \
- --iters 200 \
- --seqlen 512 \
- --batch_size 4
- ```
-</details>
-<br>
-
<details>
<summary>Detailed Hyperparameters</summary>
@@ -166,9 +128,44 @@ We provide two recipes for best accuracy and fast running speed with low memory.
- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.
</details>
-<br>
+### Basic Usage (auto-round version > 0.3.0)
+AutoRound support Gaudi2, CPU and GPU. A user guide detailing the full list of supported arguments is provided by calling ```auto_round -h``` on the terminal. Alternatively, you can use ```auto-round``` instead of ```auto_round```.
+
+
+```bash
+auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --format auto_round \
+ --disable_eval \
+ --output_dir ./tmp_autoround
+```
+We provide two recipes for best accuracy and fast running speed with low memory. Details as below.
+<details>
+ <summary>Other Recipes</summary>
+ ```bash
+## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower
+ auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --nsamples 512 \
+ --iters 1000 \
+ --low_gpu_mem_usage
+ ```
+
+ ```bash
+## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128
+ auto_round --model facebook/opt-125m \
+ --bits 4 \
+ --group_size 128 \
+ --nsamples 128 \
+ --iters 200 \
+ --seqlen 512 \
+ --batch_size 4
+ ```
+</details>
#### Formats
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index d1ea08db9..6a235dfd8 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -24,9 +24,9 @@ torch.use_deterministic_algorithms(True, warn_only=True)
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
from lm_eval.utils import make_table # pylint: disable=E0401
-from auto_round.utils import logger
from auto_round import AutoRoundConfig
from auto_round.eval.evaluation import simple_evaluate
+from auto_round.utils import detect_device
def setup_parser():
parser = argparse.ArgumentParser()
@@ -156,7 +156,6 @@ def setup_parser():
def tune(args):
tasks = args.tasks
- use_eval_legacy = False
if args.format is None:
args.format = "auto_round"
@@ -165,7 +164,6 @@ def tune(args):
model_name = model_name[:-1]
print(model_name, flush=True)
- from auto_round.utils import detect_device
device_str = detect_device(args.device)
torch_dtype = "auto"
@@ -236,16 +234,6 @@ def tune(args):
seqlen = min(seqlen, tokenizer.model_max_length)
args.seqlen = seqlen
- excel_name = f"{model_name}_{args.bits}_{args.group_size}"
- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
- dtype = 'bfloat16'
- else:
- if "cpu" not in device_str:
- dtype = 'float16'
- else:
- dtype = 'float32'
-
- excel_name = f"{model_name}_{args.bits}_{args.group_size}"
if "bloom" in model_name:
args.low_gpu_mem_usage = False
@@ -309,7 +297,7 @@ def tune(args):
low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,
enable_norm_bias_tuning=args.enable_norm_bias_tuning)
model, _ = autoround.quantize()
- model_name = args.model_name.rstrip("/")
+ model_name = args.model.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
@@ -320,7 +308,6 @@ def tune(args):
torch.cuda.empty_cache()
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
format_list = args.format.replace(' ', '').split(',')
@@ -366,9 +353,10 @@ def tune(args):
def eval(args):
quantization_config = AutoRoundConfig(backend=args.device)
+ device_str = detect_device(args.device)
user_model = AutoModelForCausalLM.from_pretrained(
args.model,
- device_map=args.device, quantization_config=quantization_config)
+ device_map=device_str, quantization_config=quantization_config)
model_args = f"pretrained={args.model},trust_remote_code={not args.disable_trust_remote_code}"
if isinstance(args.tasks, str):
tasks = args.tasks.split(',')
@@ -377,7 +365,6 @@ def eval(args):
model_args=model_args,
user_model=user_model,
tasks=tasks,
- device=args.device,
batch_size=args.eval_bs)
from lm_eval.utils import make_table # pylint: disable=E0401
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index da29c34ee..1b4db5aea 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -12,11 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import itertools
import logging
import random
import time
-from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
import numpy as np
@@ -26,14 +24,8 @@ import lm_eval.api.metrics # pylint: disable=E0401
import lm_eval.api.registry # pylint: disable=E0401
import lm_eval.models # pylint: disable=E0401
from lm_eval.caching.cache import delete_cache # pylint: disable=E0401
-from lm_eval.evaluator_utils import ( # pylint: disable=E0401
- consolidate_results,
- get_sample_size,
- get_task_list,
- prepare_print_tasks,
- print_writeout,
- run_task_tests,
-) # pylint: disable=E0401
+from lm_eval.evaluator import evaluate # pylint: disable=E0401
+from lm_eval.evaluator_utils import run_task_tests # pylint: disable=E0401
from lm_eval.loggers.utils import add_env_info, get_git_commit_hash # pylint: disable=E0401
from lm_eval.tasks import TaskManager, get_task_dict # pylint: disable=E0401
from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string # pylint: disable=E0401
@@ -291,285 +283,3 @@ def simple_evaluate(
return results
else:
return None
-
-
-@positional_deprecated
-def evaluate(
- lm: "LM",
- task_dict,
- limit: Optional[int] = None,
- cache_requests: bool = False,
- rewrite_requests_cache: bool = False,
- bootstrap_iters: Optional[int] = 100000,
- write_out: bool = False,
- log_samples: bool = True,
- verbosity: str = "INFO",
-):
- """Instantiate and evaluate a model on a list of tasks.
-
- :param lm: obj
- Language Model
- :param task_dict: dict[str, Task]
- Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
- :param limit: int, optional
- Limit the number of examples per task (only use this for testing)
- :param bootstrap_iters:
- Number of iterations for bootstrap statistics
- :param write_out: bool
- If True, write out an example document and model input for checking task integrity
- :param log_samples: bool
- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
- :return
- Dictionary of results
- """
-
- eval_logger.setLevel(getattr(logging, f"{verbosity}"))
-
- # tracks all Instances/requests a model must generate output on.
- requests = defaultdict(list)
- # stores the amount to pad out reqs per req. type so that
- # number of fwd passes per distributed rank is equal
- padding_requests = defaultdict(int)
-
- # get lists of group hierarchy and each type of request
- task_hierarchy, eval_tasks = get_task_list(task_dict)
- if not log_samples:
- if not all(
- "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
- for task_output in eval_tasks
- ):
- raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
- for task_output in eval_tasks:
- task: Task = task_output.task # pylint: disable=E0601
- limit = get_sample_size(task, limit)
- task.build_all_requests(
- limit=limit,
- rank=lm.rank,
- world_size=lm.world_size,
- cache_requests=cache_requests,
- rewrite_requests_cache=rewrite_requests_cache,
- )
- eval_logger.debug(
- f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
- )
-
- if write_out:
- print_writeout(task)
- # aggregate Instances by LM method requested to get output.
- for instance in task.instances:
- reqtype = instance.request_type
- requests[reqtype].append(instance)
-
- if lm.world_size > 1:
- instances_rnk = torch.tensor(len(task._instances), device=lm.device)
- gathered_item = (
- lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
- )
- # "multiple_choice" task types dispatch (several) "loglikelihood" request types
- reqtype = (
- "loglikelihood"
- if task.OUTPUT_TYPE == "multiple_choice"
- else task.OUTPUT_TYPE
- )
- # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
- numpad = max(gathered_item) - gathered_item[lm.rank]
- # todo: may not account for padding in cases like SquadV2 which has multiple req types
- padding_requests[reqtype] += numpad
-
- ### Run LM on inputs, get all outputs ###
- # execute each type of request
- for reqtype, reqs in requests.items():
- eval_logger.info(f"Running {reqtype} requests")
- # create `K` copies of each request `req` based off `K = req.repeats`
- cloned_reqs = []
- for req in reqs:
- cloned_reqs.extend([req] * req.repeats)
-
- if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
- for _ in range(padding_requests[reqtype]):
- cloned_reqs.extend([req] * req.repeats)
-
- # run requests through model
- resps = getattr(lm, reqtype)(cloned_reqs)
-
- # put responses from model into a list of length K for each request.
- for x, req in zip(resps, cloned_reqs):
- req.resps.append(x)
-
- if lm.world_size > 1:
- lm.accelerator.wait_for_everyone()
-
- RANK = lm.rank
- WORLD_SIZE = lm.world_size
- ### Postprocess outputs ###
- # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
- for task_output in eval_tasks:
- task = task_output.task
- task.apply_filters()
-
- ### Collect values of metrics on all datapoints ###
- # # unpack results and sort back in order and return control to Task
- # TODO: make it possible to use a different metric per filter
- # Pre-process task.instances to group by doc_id
- instances_by_doc_id = defaultdict(list)
- for instance in task.instances:
- instances_by_doc_id[instance.doc_id].append(instance)
- # Sort instances within each group
- for instances in instances_by_doc_id.values():
- instances.sort(key=lambda x: x.idx)
- # iterate over different filters used
- for filter_key in task.instances[0].filtered_resps.keys():
- doc_iterator = task.doc_iterator(
- rank=RANK, limit=limit, world_size=WORLD_SIZE
- )
- for doc_id, doc in doc_iterator:
- requests = instances_by_doc_id[doc_id]
- metrics = task.process_results(
- doc, [req.filtered_resps[filter_key] for req in requests]
- )
- if log_samples:
- target = task.doc_to_target(doc)
- example = {
- "doc_id": doc_id,
- "doc": doc,
- "target": target,
- "arguments": [req.args for req in requests],
- "resps": [req.resps for req in requests],
- "filtered_resps": [
- req.filtered_resps[filter_key] for req in requests
- ],
- }
- example.update(metrics)
- task_output.logged_samples.append(example)
- for metric, value in metrics.items():
- task_output.sample_metrics[(metric, filter_key)].append(value)
-
- if WORLD_SIZE > 1:
- # if multigpu, then gather data across all ranks to rank 0
- # first gather logged samples across all ranks
- for task_output in eval_tasks:
- if log_samples:
- # for task_name, task_samples in list(samples.items()):
- full_samples = [None] * WORLD_SIZE if RANK == 0 else None
- torch.distributed.gather_object(
- obj=task_output.logged_samples,
- object_gather_list=full_samples,
- dst=0,
- )
-
- if RANK == 0:
- task_output.logged_samples = list(
- itertools.chain.from_iterable(full_samples)
- )
-
- # then collect metrics across all ranks
- for metrics in task_output.sample_metrics:
- metric_list = [None] * WORLD_SIZE if RANK == 0 else None
- torch.distributed.gather_object(
- obj=task_output.sample_metrics[metrics],
- object_gather_list=metric_list,
- dst=0,
- )
- if RANK == 0:
- task_output.sample_metrics[metrics] = list(
- itertools.chain.from_iterable(metric_list)
- )
-
- if RANK == 0:
- ### Aggregate results over all datapoints ###
- # aggregate results ; run bootstrap CIs
- for task_output in eval_tasks:
- task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
- results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(
- eval_tasks
- )
-
- ### Calculate group metrics ###
- if bool(results):
- for group, task_list in reversed(task_hierarchy.items()):
- if len(task_list) == 0:
- # task_hierarchy entries are either
- # `group_name: [subtask1, subtask2, ...]`
- # or `task_name: []`.
- # we only want to operate on groups here.
- continue
- metric_list = list(
- {
- key
- for task in task_list
- for key in results[task].keys()
- if "_stderr" not in key and key not in ["alias", "samples"]
- }
- )
- for metric in metric_list:
- stderr = "_stderr,".join(metric.split(","))
-
- # gather metrics, sizes, and stderrs from subtasks
- metrics = [
- results[task][metric]
- for task in task_list
- if metric in results[task]
- ] # TODO: copy?
- stderrs = [
- results[task][stderr]
- for task in task_list
- if stderr in results[task]
- ]
- sizes = [
- results[task]["samples"]
- for task in task_list
- if metric in results[task]
- ]
-
- # compute group's pooled metric and stderr
- results[group][
- metric
- ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
- # TODO: calculate grouped metric using aggregation fn
- if "N/A" in stderrs:
- results[group][stderr] = "N/A"
- else:
- results[group][
- stderr
- ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
-
- results[group]["samples"] = sum(sizes)
-
- results_agg = defaultdict(dict)
- groups_agg = defaultdict(dict)
- all_tasks_list = list(task_hierarchy.keys())
- while True:
- add_tasks_list = list(k for k in results_agg.keys())
- left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
- if len(left_tasks_list) == 0:
- break
-
- _task_hierarchy = {
- k: v for k, v in task_hierarchy.items() if k in left_tasks_list
- }
- _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
-
- results_agg = {**results_agg, **_results_agg}
- groups_agg = {**groups_agg, **_groups_agg}
-
- for group_name, task_list in task_hierarchy.items():
- if task_list:
- num_fewshot[group_name] = num_fewshot[
- task_list[0]
- ] # TODO: validate this
-
- results_dict = {
- "results": dict(results_agg.items()),
- **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
- "group_subtasks": dict(reversed(task_hierarchy.items())),
- "configs": dict(sorted(configs.items())),
- "versions": dict(sorted(versions.items())),
- "n-shot": dict(sorted(num_fewshot.items())),
- }
- if log_samples:
- results_dict["samples"] = dict(samples)
-
- return results_dict
-
- else:
- return None
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index c8a228d0e..3d542aa59 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -15,15 +15,9 @@ import torch
import lm_eval.api.metrics
import lm_eval.api.registry
import lm_eval.models
+from lm_eval.evaluator import evaluate
from lm_eval.caching.cache import delete_cache
-from lm_eval.evaluator_utils import (
- consolidate_results,
- get_sample_size,
- get_task_list,
- prepare_print_tasks,
- print_writeout,
- run_task_tests,
-)
+from lm_eval.evaluator_utils import run_task_tests
if LM_EVAL_VERSION == Version('0.4.2'):
from lm_eval.logging_utils import add_env_info, get_git_commit_hash
else:
@@ -71,7 +65,8 @@ def simple_evaluate(
String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
Ignored if `model` argument is a LM object.
:param tasks: list[Union[str, dict, Task]]
- List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME
+ if defined and type(task).__name__ otherwise.
:param num_fewshot: int
Number of examples in few-shot context
:param batch_size: int or str, optional
@@ -89,7 +84,8 @@ def simple_evaluate(
:param delete_requests_cache: bool, optional
Deletes all of the request cache if set to `True`. `None` if not desired.
:param limit: int or float, optional
- Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ Limit the number of examples per task (only use this for testing), If <1,
+ limit is a percentage of the total number of examples.
:param bootstrap_iters:
Number of iterations for bootstrap statistics
:param check_integrity: bool
@@ -226,7 +222,8 @@ def simple_evaluate(
if num_fewshot is not None:
if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
eval_logger.info(
- f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
+ f"num_fewshot has been set to 0 for {task_name} in its config."
+ " Manual configuration will be ignored."
)
else:
eval_logger.warning(
@@ -280,364 +277,4 @@ def simple_evaluate(
add_env_info(results) # additional environment info to results
return results
else:
- return None
-
-
-@positional_deprecated
-def evaluate(
- lm: "LM",
- task_dict,
- limit: Optional[int] = None,
- cache_requests: bool = False,
- rewrite_requests_cache: bool = False,
- bootstrap_iters: Optional[int] = 100000,
- write_out: bool = False,
- log_samples: bool = True,
- verbosity: str = "INFO",
-):
- """Instantiate and evaluate a model on a list of tasks.
-
- :param lm: obj
- Language Model
- :param task_dict: dict[str, Task]
- Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
- :param limit: int, optional
- Limit the number of examples per task (only use this for testing)
- :param bootstrap_iters:
- Number of iterations for bootstrap statistics
- :param write_out: bool
- If True, write out an example document and model input for checking task integrity
- :param log_samples: bool
- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
- :return
- Dictionary of results
- """
-
- eval_logger.setLevel(getattr(logging, f"{verbosity}"))
-
- # tracks all Instances/requests a model must generate output on.
- requests = defaultdict(list)
- # stores the amount to pad out reqs per req. type so that
- # number of fwd passes per distributed rank is equal
- padding_requests = defaultdict(int)
-
- # get lists of group hierarchy and each type of request
- task_hierarchy, eval_tasks = get_task_list(task_dict)
- if not log_samples:
- if not all(
- "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
- for task_output in eval_tasks
- ):
- raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
- for task_output in eval_tasks:
- task: Task = task_output.task
- limit = get_sample_size(task, limit)
- task.build_all_requests(
- limit=limit,
- rank=lm.rank,
- world_size=lm.world_size,
- cache_requests=cache_requests,
- rewrite_requests_cache=rewrite_requests_cache,
- )
- eval_logger.debug(
- f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
- )
-
- if write_out:
- print_writeout(task)
- # aggregate Instances by LM method requested to get output.
- for instance in task.instances:
- reqtype = instance.request_type
- requests[reqtype].append(instance)
-
- if lm.world_size > 1:
- instances_rnk = torch.tensor(len(task._instances), device=lm.device)
- gathered_item = (
- lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
- )
- # "multiple_choice" task types dispatch (several) "loglikelihood" request types
- reqtype = (
- "loglikelihood"
- if task.OUTPUT_TYPE == "multiple_choice"
- else task.OUTPUT_TYPE
- )
- # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
- numpad = max(gathered_item) - gathered_item[lm.rank]
- # todo: may not account for padding in cases like SquadV2 which has multiple req types
- padding_requests[reqtype] += numpad
-
- ### Run LM on inputs, get all outputs ###
- # execute each type of request
- for reqtype, reqs in requests.items():
- eval_logger.info(f"Running {reqtype} requests")
- # create `K` copies of each request `req` based off `K = req.repeats`
- cloned_reqs = []
- for req in reqs:
- cloned_reqs.extend([req] * req.repeats)
-
- if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
- for _ in range(padding_requests[reqtype]):
- cloned_reqs.extend([req] * req.repeats)
-
- # run requests through model
- resps = getattr(lm, reqtype)(cloned_reqs)
-
- # put responses from model into a list of length K for each request.
- for x, req in zip(resps, cloned_reqs):
- req.resps.append(x)
-
- if lm.world_size > 1:
- lm.accelerator.wait_for_everyone()
-
- RANK = lm.rank
- WORLD_SIZE = lm.world_size
- ### Postprocess outputs ###
- # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
- for task_output in eval_tasks:
- task = task_output.task
- task.apply_filters()
-
- ### Collect values of metrics on all datapoints ###
- # # unpack results and sort back in order and return control to Task
- # TODO: make it possible to use a different metric per filter
- # Pre-process task.instances to group by doc_id
- instances_by_doc_id = defaultdict(list)
- for instance in task.instances:
- instances_by_doc_id[instance.doc_id].append(instance)
- # Sort instances within each group
- for instances in instances_by_doc_id.values():
- instances.sort(key=lambda x: x.idx)
- # iterate over different filters used
- for filter_key in task.instances[0].filtered_resps.keys():
- doc_iterator = task.doc_iterator(
- rank=RANK, limit=limit, world_size=WORLD_SIZE
- )
- for doc_id, doc in doc_iterator:
- requests = instances_by_doc_id[doc_id]
- metrics = task.process_results(
- doc, [req.filtered_resps[filter_key] for req in requests]
- )
- if log_samples:
- target = task.doc_to_target(doc)
- example = {
- "doc_id": doc_id,
- "doc": doc,
- "target": target,
- "arguments": [req.args for req in requests],
- "resps": [req.resps for req in requests],
- "filtered_resps": [
- req.filtered_resps[filter_key] for req in requests
- ],
- }
- example.update(metrics)
- task_output.logged_samples.append(example)
- for metric, value in metrics.items():
- task_output.sample_metrics[(metric, filter_key)].append(value)
-
- if WORLD_SIZE > 1:
- # if multigpu, then gather data across all ranks to rank 0
- # first gather logged samples across all ranks
- for task_output in eval_tasks:
- if log_samples:
- # for task_name, task_samples in list(samples.items()):
- full_samples = [None] * WORLD_SIZE if RANK == 0 else None
- torch.distributed.gather_object(
- obj=task_output.logged_samples,
- object_gather_list=full_samples,
- dst=0,
- )
-
- if RANK == 0:
- task_output.logged_samples = list(
- itertools.chain.from_iterable(full_samples)
- )
-
- # then collect metrics across all ranks
- for metrics in task_output.sample_metrics:
- metric_list = [None] * WORLD_SIZE if RANK == 0 else None
- torch.distributed.gather_object(
- obj=task_output.sample_metrics[metrics],
- object_gather_list=metric_list,
- dst=0,
- )
- if RANK == 0:
- task_output.sample_metrics[metrics] = list(
- itertools.chain.from_iterable(metric_list)
- )
-
- if RANK == 0:
- ### Aggregate results over all datapoints ###
- # aggregate results ; run bootstrap CIs
- for task_output in eval_tasks:
- task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
- if LM_EVAL_VERSION == Version('0.4.2'):
- results, samples, configs, versions, num_fewshot = consolidate_results(
- eval_tasks
- )
- else:
- results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(
- eval_tasks
- )
-
- ### Calculate group metrics ###
- if bool(results):
- for group, task_list in reversed(task_hierarchy.items()):
- if len(task_list) == 0:
- # task_hierarchy entries are either
- # `group_name: [subtask1, subtask2, ...]`
- # or `task_name: []`.
- # we only want to operate on groups here.
- continue
- metric_list = list(
- {
- key
- for task in task_list
- for key in results[task].keys()
- if "_stderr" not in key and key not in ["alias", "samples"]
- }
- )
- for metric in metric_list:
- stderr = "_stderr,".join(metric.split(","))
-
- # gather metrics, sizes, and stderrs from subtasks
- metrics = [
- results[task][metric]
- for task in task_list
- if metric in results[task]
- ] # TODO: copy?
- stderrs = [
- results[task][stderr]
- for task in task_list
- if stderr in results[task]
- ]
- sizes = [
- results[task]["samples"]
- for task in task_list
- if metric in results[task]
- ]
-
- # compute group's pooled metric and stderr
- results[group][
- metric
- ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
- # TODO: calculate grouped metric using aggregation fn
- if "N/A" in stderrs:
- results[group][stderr] = "N/A"
- else:
- results[group][
- stderr
- ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
- # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
- # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
- # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)
-
- results[group]["samples"] = sum(sizes)
-
- results_agg = defaultdict(dict)
- groups_agg = defaultdict(dict)
- all_tasks_list = list(task_hierarchy.keys())
- while True:
- add_tasks_list = list(k for k in results_agg.keys())
- left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
- if len(left_tasks_list) == 0:
- break
-
- _task_hierarchy = {
- k: v for k, v in task_hierarchy.items() if k in left_tasks_list
- }
- _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
-
- results_agg = {**results_agg, **_results_agg}
- groups_agg = {**groups_agg, **_groups_agg}
-
- for group_name, task_list in task_hierarchy.items():
- if task_list:
- num_fewshot[group_name] = num_fewshot[
- task_list[0]
- ] # TODO: validate this
-
- results_dict = {
- "results": dict(results_agg.items()),
- **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
- "group_subtasks": dict(reversed(task_hierarchy.items())),
- "configs": dict(sorted(configs.items())),
- "versions": dict(sorted(versions.items())),
- "n-shot": dict(sorted(num_fewshot.items())),
- }
- if log_samples:
- results_dict["samples"] = dict(samples)
-
- return results_dict
-
- else:
- return None
-
-
-if __name__ == "__main__":
-
- import sys
-
- sys.path.insert(0, '../../')
- import time
- import argparse
-
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--model_name", default="facebook/opt-125m"
- )
- parser.add_argument(
- "--eval_bs", default=1,
- )
- parser.add_argument(
- "--trust_remote_code", action='store_true',
- help="Whether to enable trust_remote_code"
- )
- parser.add_argument(
- "--device", default="cuda:0",
- help="PyTorch device (e.g. cpu/cuda:0/hpu) for evaluation."
- )
- parser.add_argument("--tasks",
- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
- "openbookqa,boolq,rte,arc_easy,arc_challenge",
- help="lm-eval tasks for lm_eval version 0.4.2")
-
- parser.add_argument("--parallelize", action='store_true',
- help="Whether to enable parallelize."
- )
-
- args = parser.parse_args()
- s = time.time()
- from transformers import AutoConfig
-
- config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
-
- if hasattr(config, "quantization_config"):
- quantization_config = config.quantization_config
- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
- from auto_round.auto_quantizer import AutoHfQuantizer
- elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":
- if args.device == "hpu":
- from auto_round.auto_quantizer import AutoHfQuantizer
-
- test_tasks = args.tasks
- if isinstance(test_tasks, str):
- test_tasks = test_tasks.split(',')
- model_name = args.model_name.rstrip('/')
- from lm_eval.utils import make_table
-
- model_args = f"pretrained={args.model_name}"
- model_args += ",dtype=float16"
- if args.trust_remote_code:
- model_args += f",trust_remote_code=True"
- if args.parallelize:
- model_args += f",parallelize=True"
- result = simple_evaluate(model="hf",
- model_args=model_args,
- tasks=test_tasks,
- device=args.device,
- batch_size=args.eval_bs)
- print(make_table(result))
-
- print("cost time: ", time.time() - s)
-
-
+ return None
\ No newline at end of file
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 98062ce87..bbda9171f 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -168,7 +168,7 @@ if __name__ == '__main__':
assert False, "please only specify one of format and deployment_device"
if args.deployment_device is None and args.format is None:
- args.format == "auto_round"
+ args.format = "auto_round"
if args.deployment_device:
if "gpu" in args.deployment_device and args.sym is False:
@@ -324,7 +324,7 @@ if __name__ == '__main__':
scale_dtype=args.scale_dtype, layer_config=layer_config,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type, enable_norm_bias_tuning=args.enable_norm_bias_tuning)
- # model, _ = autoround.quantize()
+ model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txt
index 6db0fc595..d7e848413 100644
--- a/examples/language-modeling/requirements.txt
+++ b/examples/language-modeling/requirements.txt
@@ -1,6 +1,6 @@
transformers
torch
-lm-eval==0.4.3
+lm-eval==0.4.4
##for lm-eval 0.4.1 we usually used
##git+https://github.com/EleutherAI/lm-evaluation-harness.git@96d185fa6232a5ab685ba7c43e45d1dbb3bb906d
# For the paper results use the old lm_eval (0.3.0)
diff --git a/requirements.txt b/requirements.txt
index 4400fcfe0..31b70312a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,6 @@ transformers
triton
numpy < 2.0
threadpoolctl
-lm-eval==0.4.3
+lm-eval==0.4.4
intel-extension-for-transformers
tqdm
\ No newline at end of file