🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
Refactor Evaluator
didi17f3cd4955cd
diff --git a/examples/aflow/scripts/evaluator.py b/examples/aflow/scripts/evaluator.pyindex d19be02a8..285627b75 100644--- a/examples/aflow/scripts/evaluator.py+++ b/examples/aflow/scripts/evaluator.py@@ -3,7 +3,8 @@# @Author : all# @Desc : Evaluation for different datasets-from typing import Literal+from typing import Literal, Tuple, Optional+import asynciofrom examples.aflow.benchmark.gsm8k import optimize_gsm8k_evaluationfrom examples.aflow.benchmark.math import optimize_math_evaluation@@ -22,155 +23,51 @@ class Evaluator:def __init__(self, eval_path: str):self.eval_path = eval_path+ self.dataset_configs = {+ "GSM8K": {"name": "GSM8K", "eval_func": optimize_gsm8k_evaluation},+ "MATH": {"name": "MATH", "eval_func": optimize_math_evaluation},+ "HumanEval": {"name": "HumanEval", "eval_func": optimize_humaneval_evaluation},+ "HotpotQA": {"name": "HotpotQA", "eval_func": optimize_hotpotqa_evaluation},+ "MBPP": {"name": "MBPP", "eval_func": optimize_mbpp_evaluation},+ "DROP": {"name": "DROP", "eval_func": optimize_drop_evaluation},+ }def graph_evaluate(self, dataset: DatasetType, graph, params: dict, path, is_test=False):"""Evaluates on validation dataset."""- if dataset == "GSM8K":- return self._gsm8k_eval(graph, params, path, is_test)- elif dataset == "MATH":- return self._math_eval(graph, params, path, is_test)- elif dataset == "HumanEval":- return self._humaneval_eval(graph, params, path, is_test)- elif dataset == "HotpotQA":- return self._hotpotqa_eval(graph, params, path, is_test)- elif dataset == "MBPP":- return self._mbpp_eval(graph, params, path, is_test)- elif dataset == "DROP":- return self._drop_eval(graph, params, path, is_test)+ if dataset in self.dataset_configs:+ return self._generic_eval(dataset, graph, params, path, is_test)else:return None- async def _gsm8k_eval(self, graph_class, params, path, test=False):+ async def _generic_eval(self, dataset: DatasetType, graph_class, params: dict, path: str, test: bool = False) -> Tuple[float, float, float]:"""- Evaluate GSM8K dataset.+ Generic evaluation function for all datasets."""async def load_graph():- dataset = params["dataset"]+ dataset_config = params["dataset"]llm_config = params["llm_config"]- return graph_class(name="GSM8K", llm_config=llm_config, dataset=dataset)-- if test:- data_path = "examples/aflow/data/gsm8k_test.jsonl" # Replace with your JSONL file path- va_list = None- else:- data_path = "examples/aflow/data/gsm8k_validate.jsonl" # Replace with your JSONL file path- va_list = [1,2,3] # Replace with the filtered index list+ return graph_class(name=self.dataset_configs[dataset]["name"], llm_config=llm_config, dataset=dataset_config)+ data_path, va_list = self._get_data_path_and_va_list(dataset, test)graph = await load_graph()- avg_score, avg_cost, total_cost = await optimize_gsm8k_evaluation(graph, data_path, path, va_list)+ eval_func = self.dataset_configs[dataset]["eval_func"]+ avg_score, avg_cost, total_cost = await eval_func(graph, data_path, path, va_list)return avg_score, avg_cost, total_cost- async def _math_eval(self, graph_class, params, path, test=False):+ def _get_data_path_and_va_list(self, dataset: DatasetType, test: bool) -> Tuple[str, Optional[list]]:"""- Evaluate MATH dataset.+ Get data path and validation list based on dataset and test flag."""- async def load_graph():- dataset = params["dataset"]- llm_config = params["llm_config"]- return graph_class(name="MATH", llm_config=llm_config, dataset=dataset)-+ base_path = f"examples/aflow/data/{dataset.lower()}"if test:- data_path = "examples/aflow/data/math_test.jsonl"- va_list = None+ return f"{base_path}_test.jsonl", Noneelse:- data_path = "examples/aflow/data/math_validate.jsonl"- va_list = [1,2,3] # Replace with the filtered index list-- graph = await load_graph()-- avg_score, avg_cost, total_cost = await optimize_math_evaluation(graph, data_path, path, va_list)-- return avg_score, avg_cost, total_cost-- async def _humaneval_eval(self, graph_class, params, path, test=False):- """- Evaluate HumanEval dataset.- """- async def load_graph():- dataset = params["dataset"]- llm_config = params["llm_config"]- return graph_class(name="HumanEval", llm_config=llm_config, dataset=dataset)+ return f"{base_path}_validate.jsonl", [1, 2, 3] # Replace with the actual filtered index list- if test:- data_path = "examples/aflow/data/human-eval_test.jsonl" # Replace with your JSONL file path- va_list = None- else:- data_path = "examples/aflow/data/human-eval_validate.jsonl" # Replace with your JSONL file path- va_list = [1,2,3] # Replace with the filtered index list-- graph = await load_graph()-- avg_score, avg_cost, total_cost = await optimize_humaneval_evaluation(graph, data_path, path, va_list)-- return avg_score, avg_cost, total_cost-- async def _mbpp_eval(self, graph_class, params, path, test=False):- """- Evaluate MBPP dataset.- """- async def load_graph():- dataset = params["dataset"]- llm_config = params["llm_config"]- return graph_class(name="MBPP", llm_config=llm_config, dataset=dataset)-- if test:- data_path = "examples/aflow/data/mbpp_test.jsonl"- va_list = None- else:- data_path = "examples/aflow/data/mbpp_validate.jsonl"- va_list = [1,2,3] # Replace with the filtered index list-- graph = await load_graph()-- avg_score, avg_cost, total_cost = await optimize_mbpp_evaluation(graph, data_path, path, va_list)-- return avg_score, avg_cost, total_cost-- async def _hotpotqa_eval(self, graph_class, params, path, test=False):- """- Evaluate HotpotQA dataset.- """- async def load_graph():- dataset = params["dataset"]- llm_config = params["llm_config"]- return graph_class(name="HotpotQA", llm_config=llm_config, dataset=dataset)-- if test:- data_path = "examples/aflow/data/hotpotqa_test.jsonl"- va_list = None- else:- data_path = "examples/aflow/data/hotpotqa_validate.jsonl"- va_list = [1,2,3] # Replace with the filtered index list-- graph = await load_graph()-- avg_score, avg_cost, total_cost = await optimize_hotpotqa_evaluation(graph, data_path, path, va_list)-- return avg_score, avg_cost, total_cost--- async def _drop_eval(self, graph_class, params, path, test=False):- """- Evaluate DROP dataset.- """- async def load_graph():- dataset = params["dataset"]- llm_config = params["llm_config"]- return graph_class(name="DROP", llm_config=llm_config, dataset=dataset)-- if test:- data_path = "examples/aflow/data/drop_test.jsonl"- va_list = None- else:- data_path = "examples/aflow/data/drop_validate.jsonl"- va_list = [1,2,3] # Replace with the filtered index list-- graph = await load_graph()-- avg_score, avg_cost, total_cost = await optimize_drop_evaluation(graph, data_path, path, va_list)-- return avg_score, avg_cost, total_cost\ No newline at end of file+# Alias methods for backward compatibility+for dataset in ["gsm8k", "math", "humaneval", "mbpp", "hotpotqa", "drop"]:+ setattr(Evaluator, f"_{dataset}_eval", lambda self, *args, dataset=dataset.upper(), **kwargs: self._generic_eval(dataset, *args, **kwargs))\ No newline at end of file