PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix example bug (#245)
n1ck-guo0d60ad211fa5
diff --git a/README.md b/README.mdindex fe41b2ebb..fada849a0 100644--- a/README.md+++ b/README.md@@ -77,44 +77,6 @@ output_dir = "./tmp_autoround"autoround.save_quantized(output_dir, format='auto_round', inplace=True)```-### Basic Usage-AutoRound support Gaudi2, CPU and GPU. A user guide detailing the full list of supported arguments is provided by calling ```auto_round -h``` on the terminal. Alternatively, you can use ```auto-round``` instead of ```auto_round```. (**auto-round version > 0.3.0**)---```bash-auto_round --model facebook/opt-125m \- --bits 4 \- --group_size 128 \- --format auto_round \- --output_dir ./tmp_autoround-```-We provide two recipes for best accuracy and fast running speed with low memory. Details as below.-<details>- <summary>Other Recipes</summary>-- ```bash-## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower- auto_round --model facebook/opt-125m \- --bits 4 \- --group_size 128 \- --nsamples 512 \- --iters 1000 \- --low_gpu_mem_usage- ```-- ```bash-## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128- auto_round --model facebook/opt-125m \- --bits 4 \- --group_size 128 \- --nsamples 128 \- --iters 200 \- --seqlen 512 \- --batch_size 4- ```-</details>-<br>-<details><summary>Detailed Hyperparameters</summary>@@ -166,9 +128,44 @@ We provide two recipes for best accuracy and fast running speed with low memory.- `device`: The device to be used for tuning. The default is set to 'auto', allowing for automatic detection.</details>-<br>+### Basic Usage (auto-round version > 0.3.0)+AutoRound support Gaudi2, CPU and GPU. A user guide detailing the full list of supported arguments is provided by calling ```auto_round -h``` on the terminal. Alternatively, you can use ```auto-round``` instead of ```auto_round```.+++```bash+auto_round --model facebook/opt-125m \+ --bits 4 \+ --group_size 128 \+ --format auto_round \+ --disable_eval \+ --output_dir ./tmp_autoround+```+We provide two recipes for best accuracy and fast running speed with low memory. Details as below.+<details>+ <summary>Other Recipes</summary>+ ```bash+## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower+ auto_round --model facebook/opt-125m \+ --bits 4 \+ --group_size 128 \+ --nsamples 512 \+ --iters 1000 \+ --low_gpu_mem_usage+ ```++ ```bash+## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128+ auto_round --model facebook/opt-125m \+ --bits 4 \+ --group_size 128 \+ --nsamples 128 \+ --iters 200 \+ --seqlen 512 \+ --batch_size 4+ ```+</details>#### Formatsdiff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex d1ea08db9..6a235dfd8 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -24,9 +24,9 @@ torch.use_deterministic_algorithms(True, warn_only=True)from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModelfrom lm_eval.utils import make_table # pylint: disable=E0401-from auto_round.utils import loggerfrom auto_round import AutoRoundConfigfrom auto_round.eval.evaluation import simple_evaluate+from auto_round.utils import detect_devicedef setup_parser():parser = argparse.ArgumentParser()@@ -156,7 +156,6 @@ def setup_parser():def tune(args):tasks = args.tasks- use_eval_legacy = Falseif args.format is None:args.format = "auto_round"@@ -165,7 +164,6 @@ def tune(args):model_name = model_name[:-1]print(model_name, flush=True)- from auto_round.utils import detect_devicedevice_str = detect_device(args.device)torch_dtype = "auto"@@ -236,16 +234,6 @@ def tune(args):seqlen = min(seqlen, tokenizer.model_max_length)args.seqlen = seqlen- excel_name = f"{model_name}_{args.bits}_{args.group_size}"- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):- dtype = 'bfloat16'- else:- if "cpu" not in device_str:- dtype = 'float16'- else:- dtype = 'float32'-- excel_name = f"{model_name}_{args.bits}_{args.group_size}"if "bloom" in model_name:args.low_gpu_mem_usage = False@@ -309,7 +297,7 @@ def tune(args):low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,enable_norm_bias_tuning=args.enable_norm_bias_tuning)model, _ = autoround.quantize()- model_name = args.model_name.rstrip("/")+ model_name = args.model.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutil@@ -320,7 +308,6 @@ def tune(args):torch.cuda.empty_cache()export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"format_list = args.format.replace(' ', '').split(',')@@ -366,9 +353,10 @@ def tune(args):def eval(args):quantization_config = AutoRoundConfig(backend=args.device)+ device_str = detect_device(args.device)user_model = AutoModelForCausalLM.from_pretrained(args.model,- device_map=args.device, quantization_config=quantization_config)+ device_map=device_str, quantization_config=quantization_config)model_args = f"pretrained={args.model},trust_remote_code={not args.disable_trust_remote_code}"if isinstance(args.tasks, str):tasks = args.tasks.split(',')@@ -377,7 +365,6 @@ def eval(args):model_args=model_args,user_model=user_model,tasks=tasks,- device=args.device,batch_size=args.eval_bs)from lm_eval.utils import make_table # pylint: disable=E0401diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex da29c34ee..1b4db5aea 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -12,11 +12,9 @@# See the License for the specific language governing permissions and# limitations under the License.-import itertoolsimport loggingimport randomimport time-from collections import defaultdictfrom typing import TYPE_CHECKING, List, Optional, Unionimport numpy as np@@ -26,14 +24,8 @@ import lm_eval.api.metrics # pylint: disable=E0401import lm_eval.api.registry # pylint: disable=E0401import lm_eval.models # pylint: disable=E0401from lm_eval.caching.cache import delete_cache # pylint: disable=E0401-from lm_eval.evaluator_utils import ( # pylint: disable=E0401- consolidate_results,- get_sample_size,- get_task_list,- prepare_print_tasks,- print_writeout,- run_task_tests,-) # pylint: disable=E0401+from lm_eval.evaluator import evaluate # pylint: disable=E0401+from lm_eval.evaluator_utils import run_task_tests # pylint: disable=E0401from lm_eval.loggers.utils import add_env_info, get_git_commit_hash # pylint: disable=E0401from lm_eval.tasks import TaskManager, get_task_dict # pylint: disable=E0401from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string # pylint: disable=E0401@@ -291,285 +283,3 @@ def simple_evaluate(return resultselse:return None---@positional_deprecated-def evaluate(- lm: "LM",- task_dict,- limit: Optional[int] = None,- cache_requests: bool = False,- rewrite_requests_cache: bool = False,- bootstrap_iters: Optional[int] = 100000,- write_out: bool = False,- log_samples: bool = True,- verbosity: str = "INFO",-):- """Instantiate and evaluate a model on a list of tasks.-- :param lm: obj- Language Model- :param task_dict: dict[str, Task]- Dictionary of tasks. Tasks will be taken to have name type(task).config.task .- :param limit: int, optional- Limit the number of examples per task (only use this for testing)- :param bootstrap_iters:- Number of iterations for bootstrap statistics- :param write_out: bool- If True, write out an example document and model input for checking task integrity- :param log_samples: bool- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis- :return- Dictionary of results- """-- eval_logger.setLevel(getattr(logging, f"{verbosity}"))-- # tracks all Instances/requests a model must generate output on.- requests = defaultdict(list)- # stores the amount to pad out reqs per req. type so that- # number of fwd passes per distributed rank is equal- padding_requests = defaultdict(int)-- # get lists of group hierarchy and each type of request- task_hierarchy, eval_tasks = get_task_list(task_dict)- if not log_samples:- if not all(- "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()- for task_output in eval_tasks- ):- raise ValueError("log_samples must be True for 'bypass' metric-only tasks")- for task_output in eval_tasks:- task: Task = task_output.task # pylint: disable=E0601- limit = get_sample_size(task, limit)- task.build_all_requests(- limit=limit,- rank=lm.rank,- world_size=lm.world_size,- cache_requests=cache_requests,- rewrite_requests_cache=rewrite_requests_cache,- )- eval_logger.debug(- f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"- )-- if write_out:- print_writeout(task)- # aggregate Instances by LM method requested to get output.- for instance in task.instances:- reqtype = instance.request_type- requests[reqtype].append(instance)-- if lm.world_size > 1:- instances_rnk = torch.tensor(len(task._instances), device=lm.device)- gathered_item = (- lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()- )- # "multiple_choice" task types dispatch (several) "loglikelihood" request types- reqtype = (- "loglikelihood"- if task.OUTPUT_TYPE == "multiple_choice"- else task.OUTPUT_TYPE- )- # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)- numpad = max(gathered_item) - gathered_item[lm.rank]- # todo: may not account for padding in cases like SquadV2 which has multiple req types- padding_requests[reqtype] += numpad-- ### Run LM on inputs, get all outputs ###- # execute each type of request- for reqtype, reqs in requests.items():- eval_logger.info(f"Running {reqtype} requests")- # create `K` copies of each request `req` based off `K = req.repeats`- cloned_reqs = []- for req in reqs:- cloned_reqs.extend([req] * req.repeats)-- if (lm.world_size > 1) and (padding_requests[reqtype] > 0):- for _ in range(padding_requests[reqtype]):- cloned_reqs.extend([req] * req.repeats)-- # run requests through model- resps = getattr(lm, reqtype)(cloned_reqs)-- # put responses from model into a list of length K for each request.- for x, req in zip(resps, cloned_reqs):- req.resps.append(x)-- if lm.world_size > 1:- lm.accelerator.wait_for_everyone()-- RANK = lm.rank- WORLD_SIZE = lm.world_size- ### Postprocess outputs ###- # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)- for task_output in eval_tasks:- task = task_output.task- task.apply_filters()-- ### Collect values of metrics on all datapoints ###- # # unpack results and sort back in order and return control to Task- # TODO: make it possible to use a different metric per filter- # Pre-process task.instances to group by doc_id- instances_by_doc_id = defaultdict(list)- for instance in task.instances:- instances_by_doc_id[instance.doc_id].append(instance)- # Sort instances within each group- for instances in instances_by_doc_id.values():- instances.sort(key=lambda x: x.idx)- # iterate over different filters used- for filter_key in task.instances[0].filtered_resps.keys():- doc_iterator = task.doc_iterator(- rank=RANK, limit=limit, world_size=WORLD_SIZE- )- for doc_id, doc in doc_iterator:- requests = instances_by_doc_id[doc_id]- metrics = task.process_results(- doc, [req.filtered_resps[filter_key] for req in requests]- )- if log_samples:- target = task.doc_to_target(doc)- example = {- "doc_id": doc_id,- "doc": doc,- "target": target,- "arguments": [req.args for req in requests],- "resps": [req.resps for req in requests],- "filtered_resps": [- req.filtered_resps[filter_key] for req in requests- ],- }- example.update(metrics)- task_output.logged_samples.append(example)- for metric, value in metrics.items():- task_output.sample_metrics[(metric, filter_key)].append(value)-- if WORLD_SIZE > 1:- # if multigpu, then gather data across all ranks to rank 0- # first gather logged samples across all ranks- for task_output in eval_tasks:- if log_samples:- # for task_name, task_samples in list(samples.items()):- full_samples = [None] * WORLD_SIZE if RANK == 0 else None- torch.distributed.gather_object(- obj=task_output.logged_samples,- object_gather_list=full_samples,- dst=0,- )-- if RANK == 0:- task_output.logged_samples = list(- itertools.chain.from_iterable(full_samples)- )-- # then collect metrics across all ranks- for metrics in task_output.sample_metrics:- metric_list = [None] * WORLD_SIZE if RANK == 0 else None- torch.distributed.gather_object(- obj=task_output.sample_metrics[metrics],- object_gather_list=metric_list,- dst=0,- )- if RANK == 0:- task_output.sample_metrics[metrics] = list(- itertools.chain.from_iterable(metric_list)- )-- if RANK == 0:- ### Aggregate results over all datapoints ###- # aggregate results ; run bootstrap CIs- for task_output in eval_tasks:- task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)- results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(- eval_tasks- )-- ### Calculate group metrics ###- if bool(results):- for group, task_list in reversed(task_hierarchy.items()):- if len(task_list) == 0:- # task_hierarchy entries are either- # `group_name: [subtask1, subtask2, ...]`- # or `task_name: []`.- # we only want to operate on groups here.- continue- metric_list = list(- {- key- for task in task_list- for key in results[task].keys()- if "_stderr" not in key and key not in ["alias", "samples"]- }- )- for metric in metric_list:- stderr = "_stderr,".join(metric.split(","))-- # gather metrics, sizes, and stderrs from subtasks- metrics = [- results[task][metric]- for task in task_list- if metric in results[task]- ] # TODO: copy?- stderrs = [- results[task][stderr]- for task in task_list- if stderr in results[task]- ]- sizes = [- results[task]["samples"]- for task in task_list- if metric in results[task]- ]-- # compute group's pooled metric and stderr- results[group][- metric- ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)- # TODO: calculate grouped metric using aggregation fn- if "N/A" in stderrs:- results[group][stderr] = "N/A"- else:- results[group][- stderr- ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)-- results[group]["samples"] = sum(sizes)-- results_agg = defaultdict(dict)- groups_agg = defaultdict(dict)- all_tasks_list = list(task_hierarchy.keys())- while True:- add_tasks_list = list(k for k in results_agg.keys())- left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))- if len(left_tasks_list) == 0:- break-- _task_hierarchy = {- k: v for k, v in task_hierarchy.items() if k in left_tasks_list- }- _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)-- results_agg = {**results_agg, **_results_agg}- groups_agg = {**groups_agg, **_groups_agg}-- for group_name, task_list in task_hierarchy.items():- if task_list:- num_fewshot[group_name] = num_fewshot[- task_list[0]- ] # TODO: validate this-- results_dict = {- "results": dict(results_agg.items()),- **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),- "group_subtasks": dict(reversed(task_hierarchy.items())),- "configs": dict(sorted(configs.items())),- "versions": dict(sorted(versions.items())),- "n-shot": dict(sorted(num_fewshot.items())),- }- if log_samples:- results_dict["samples"] = dict(samples)-- return results_dict-- else:- return Nonediff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.pyindex c8a228d0e..3d542aa59 100644--- a/examples/language-modeling/eval/evaluation.py+++ b/examples/language-modeling/eval/evaluation.py@@ -15,15 +15,9 @@ import torchimport lm_eval.api.metricsimport lm_eval.api.registryimport lm_eval.models+from lm_eval.evaluator import evaluatefrom lm_eval.caching.cache import delete_cache-from lm_eval.evaluator_utils import (- consolidate_results,- get_sample_size,- get_task_list,- prepare_print_tasks,- print_writeout,- run_task_tests,-)+from lm_eval.evaluator_utils import run_task_testsif LM_EVAL_VERSION == Version('0.4.2'):from lm_eval.logging_utils import add_env_info, get_git_commit_hashelse:@@ -71,7 +65,8 @@ def simple_evaluate(String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.Ignored if `model` argument is a LM object.:param tasks: list[Union[str, dict, Task]]- List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME+ if defined and type(task).__name__ otherwise.:param num_fewshot: intNumber of examples in few-shot context:param batch_size: int or str, optional@@ -89,7 +84,8 @@ def simple_evaluate(:param delete_requests_cache: bool, optionalDeletes all of the request cache if set to `True`. `None` if not desired.:param limit: int or float, optional- Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.+ Limit the number of examples per task (only use this for testing), If <1,+ limit is a percentage of the total number of examples.:param bootstrap_iters:Number of iterations for bootstrap statistics:param check_integrity: bool@@ -226,7 +222,8 @@ def simple_evaluate(if num_fewshot is not None:if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:eval_logger.info(- f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."+ f"num_fewshot has been set to 0 for {task_name} in its config."+ " Manual configuration will be ignored.")else:eval_logger.warning(@@ -280,364 +277,4 @@ def simple_evaluate(add_env_info(results) # additional environment info to resultsreturn resultselse:- return None---@positional_deprecated-def evaluate(- lm: "LM",- task_dict,- limit: Optional[int] = None,- cache_requests: bool = False,- rewrite_requests_cache: bool = False,- bootstrap_iters: Optional[int] = 100000,- write_out: bool = False,- log_samples: bool = True,- verbosity: str = "INFO",-):- """Instantiate and evaluate a model on a list of tasks.-- :param lm: obj- Language Model- :param task_dict: dict[str, Task]- Dictionary of tasks. Tasks will be taken to have name type(task).config.task .- :param limit: int, optional- Limit the number of examples per task (only use this for testing)- :param bootstrap_iters:- Number of iterations for bootstrap statistics- :param write_out: bool- If True, write out an example document and model input for checking task integrity- :param log_samples: bool- If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis- :return- Dictionary of results- """-- eval_logger.setLevel(getattr(logging, f"{verbosity}"))-- # tracks all Instances/requests a model must generate output on.- requests = defaultdict(list)- # stores the amount to pad out reqs per req. type so that- # number of fwd passes per distributed rank is equal- padding_requests = defaultdict(int)-- # get lists of group hierarchy and each type of request- task_hierarchy, eval_tasks = get_task_list(task_dict)- if not log_samples:- if not all(- "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()- for task_output in eval_tasks- ):- raise ValueError("log_samples must be True for 'bypass' metric-only tasks")- for task_output in eval_tasks:- task: Task = task_output.task- limit = get_sample_size(task, limit)- task.build_all_requests(- limit=limit,- rank=lm.rank,- world_size=lm.world_size,- cache_requests=cache_requests,- rewrite_requests_cache=rewrite_requests_cache,- )- eval_logger.debug(- f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"- )-- if write_out:- print_writeout(task)- # aggregate Instances by LM method requested to get output.- for instance in task.instances:- reqtype = instance.request_type- requests[reqtype].append(instance)-- if lm.world_size > 1:- instances_rnk = torch.tensor(len(task._instances), device=lm.device)- gathered_item = (- lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()- )- # "multiple_choice" task types dispatch (several) "loglikelihood" request types- reqtype = (- "loglikelihood"- if task.OUTPUT_TYPE == "multiple_choice"- else task.OUTPUT_TYPE- )- # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)- numpad = max(gathered_item) - gathered_item[lm.rank]- # todo: may not account for padding in cases like SquadV2 which has multiple req types- padding_requests[reqtype] += numpad-- ### Run LM on inputs, get all outputs ###- # execute each type of request- for reqtype, reqs in requests.items():- eval_logger.info(f"Running {reqtype} requests")- # create `K` copies of each request `req` based off `K = req.repeats`- cloned_reqs = []- for req in reqs:- cloned_reqs.extend([req] * req.repeats)-- if (lm.world_size > 1) and (padding_requests[reqtype] > 0):- for _ in range(padding_requests[reqtype]):- cloned_reqs.extend([req] * req.repeats)-- # run requests through model- resps = getattr(lm, reqtype)(cloned_reqs)-- # put responses from model into a list of length K for each request.- for x, req in zip(resps, cloned_reqs):- req.resps.append(x)-- if lm.world_size > 1:- lm.accelerator.wait_for_everyone()-- RANK = lm.rank- WORLD_SIZE = lm.world_size- ### Postprocess outputs ###- # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)- for task_output in eval_tasks:- task = task_output.task- task.apply_filters()-- ### Collect values of metrics on all datapoints ###- # # unpack results and sort back in order and return control to Task- # TODO: make it possible to use a different metric per filter- # Pre-process task.instances to group by doc_id- instances_by_doc_id = defaultdict(list)- for instance in task.instances:- instances_by_doc_id[instance.doc_id].append(instance)- # Sort instances within each group- for instances in instances_by_doc_id.values():- instances.sort(key=lambda x: x.idx)- # iterate over different filters used- for filter_key in task.instances[0].filtered_resps.keys():- doc_iterator = task.doc_iterator(- rank=RANK, limit=limit, world_size=WORLD_SIZE- )- for doc_id, doc in doc_iterator:- requests = instances_by_doc_id[doc_id]- metrics = task.process_results(- doc, [req.filtered_resps[filter_key] for req in requests]- )- if log_samples:- target = task.doc_to_target(doc)- example = {- "doc_id": doc_id,- "doc": doc,- "target": target,- "arguments": [req.args for req in requests],- "resps": [req.resps for req in requests],- "filtered_resps": [- req.filtered_resps[filter_key] for req in requests- ],- }- example.update(metrics)- task_output.logged_samples.append(example)- for metric, value in metrics.items():- task_output.sample_metrics[(metric, filter_key)].append(value)-- if WORLD_SIZE > 1:- # if multigpu, then gather data across all ranks to rank 0- # first gather logged samples across all ranks- for task_output in eval_tasks:- if log_samples:- # for task_name, task_samples in list(samples.items()):- full_samples = [None] * WORLD_SIZE if RANK == 0 else None- torch.distributed.gather_object(- obj=task_output.logged_samples,- object_gather_list=full_samples,- dst=0,- )-- if RANK == 0:- task_output.logged_samples = list(- itertools.chain.from_iterable(full_samples)- )-- # then collect metrics across all ranks- for metrics in task_output.sample_metrics:- metric_list = [None] * WORLD_SIZE if RANK == 0 else None- torch.distributed.gather_object(- obj=task_output.sample_metrics[metrics],- object_gather_list=metric_list,- dst=0,- )- if RANK == 0:- task_output.sample_metrics[metrics] = list(- itertools.chain.from_iterable(metric_list)- )-- if RANK == 0:- ### Aggregate results over all datapoints ###- # aggregate results ; run bootstrap CIs- for task_output in eval_tasks:- task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)- if LM_EVAL_VERSION == Version('0.4.2'):- results, samples, configs, versions, num_fewshot = consolidate_results(- eval_tasks- )- else:- results, samples, configs, versions, num_fewshot, higher_is_better = consolidate_results(- eval_tasks- )-- ### Calculate group metrics ###- if bool(results):- for group, task_list in reversed(task_hierarchy.items()):- if len(task_list) == 0:- # task_hierarchy entries are either- # `group_name: [subtask1, subtask2, ...]`- # or `task_name: []`.- # we only want to operate on groups here.- continue- metric_list = list(- {- key- for task in task_list- for key in results[task].keys()- if "_stderr" not in key and key not in ["alias", "samples"]- }- )- for metric in metric_list:- stderr = "_stderr,".join(metric.split(","))-- # gather metrics, sizes, and stderrs from subtasks- metrics = [- results[task][metric]- for task in task_list- if metric in results[task]- ] # TODO: copy?- stderrs = [- results[task][stderr]- for task in task_list- if stderr in results[task]- ]- sizes = [- results[task]["samples"]- for task in task_list- if metric in results[task]- ]-- # compute group's pooled metric and stderr- results[group][- metric- ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)- # TODO: calculate grouped metric using aggregation fn- if "N/A" in stderrs:- results[group][stderr] = "N/A"- else:- results[group][- stderr- ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)- # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility- # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:- # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)-- results[group]["samples"] = sum(sizes)-- results_agg = defaultdict(dict)- groups_agg = defaultdict(dict)- all_tasks_list = list(task_hierarchy.keys())- while True:- add_tasks_list = list(k for k in results_agg.keys())- left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))- if len(left_tasks_list) == 0:- break-- _task_hierarchy = {- k: v for k, v in task_hierarchy.items() if k in left_tasks_list- }- _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)-- results_agg = {**results_agg, **_results_agg}- groups_agg = {**groups_agg, **_groups_agg}-- for group_name, task_list in task_hierarchy.items():- if task_list:- num_fewshot[group_name] = num_fewshot[- task_list[0]- ] # TODO: validate this-- results_dict = {- "results": dict(results_agg.items()),- **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),- "group_subtasks": dict(reversed(task_hierarchy.items())),- "configs": dict(sorted(configs.items())),- "versions": dict(sorted(versions.items())),- "n-shot": dict(sorted(num_fewshot.items())),- }- if log_samples:- results_dict["samples"] = dict(samples)-- return results_dict-- else:- return None---if __name__ == "__main__":-- import sys-- sys.path.insert(0, '../../')- import time- import argparse-- parser = argparse.ArgumentParser()- parser.add_argument(- "--model_name", default="facebook/opt-125m"- )- parser.add_argument(- "--eval_bs", default=1,- )- parser.add_argument(- "--trust_remote_code", action='store_true',- help="Whether to enable trust_remote_code"- )- parser.add_argument(- "--device", default="cuda:0",- help="PyTorch device (e.g. cpu/cuda:0/hpu) for evaluation."- )- parser.add_argument("--tasks",- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \- "openbookqa,boolq,rte,arc_easy,arc_challenge",- help="lm-eval tasks for lm_eval version 0.4.2")-- parser.add_argument("--parallelize", action='store_true',- help="Whether to enable parallelize."- )-- args = parser.parse_args()- s = time.time()- from transformers import AutoConfig-- config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)-- if hasattr(config, "quantization_config"):- quantization_config = config.quantization_config- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:- from auto_round.auto_quantizer import AutoHfQuantizer- elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":- if args.device == "hpu":- from auto_round.auto_quantizer import AutoHfQuantizer-- test_tasks = args.tasks- if isinstance(test_tasks, str):- test_tasks = test_tasks.split(',')- model_name = args.model_name.rstrip('/')- from lm_eval.utils import make_table-- model_args = f"pretrained={args.model_name}"- model_args += ",dtype=float16"- if args.trust_remote_code:- model_args += f",trust_remote_code=True"- if args.parallelize:- model_args += f",parallelize=True"- result = simple_evaluate(model="hf",- model_args=model_args,- tasks=test_tasks,- device=args.device,- batch_size=args.eval_bs)- print(make_table(result))-- print("cost time: ", time.time() - s)--+ return None\ No newline at end of filediff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 98062ce87..bbda9171f 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -168,7 +168,7 @@ if __name__ == '__main__':assert False, "please only specify one of format and deployment_device"if args.deployment_device is None and args.format is None:- args.format == "auto_round"+ args.format = "auto_round"if args.deployment_device:if "gpu" in args.deployment_device and args.sym is False:@@ -324,7 +324,7 @@ if __name__ == '__main__':scale_dtype=args.scale_dtype, layer_config=layer_config,enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type, enable_norm_bias_tuning=args.enable_norm_bias_tuning)- # model, _ = autoround.quantize()+ model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutildiff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txtindex 6db0fc595..d7e848413 100644--- a/examples/language-modeling/requirements.txt+++ b/examples/language-modeling/requirements.txt@@ -1,6 +1,6 @@transformerstorch-lm-eval==0.4.3+lm-eval==0.4.4##for lm-eval 0.4.1 we usually used##git+https://github.com/EleutherAI/lm-evaluation-harness.git@96d185fa6232a5ab685ba7c43e45d1dbb3bb906d# For the paper results use the old lm_eval (0.3.0)diff --git a/requirements.txt b/requirements.txtindex 4400fcfe0..31b70312a 100644--- a/requirements.txt+++ b/requirements.txt@@ -7,6 +7,6 @@ transformerstritonnumpy < 2.0threadpoolctl-lm-eval==0.4.3+lm-eval==0.4.4intel-extension-for-transformerstqdm\ No newline at end of file