Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
Make the metric call function async in the evaluation module (#758)
ZiTao-Li811fb28a3dfd
diff --git a/docs/tutorial/en/src/task_eval.py b/docs/tutorial/en/src/task_eval.pyindex d7f3902de..43b50a8d4 100644--- a/docs/tutorial/en/src/task_eval.py+++ b/docs/tutorial/en/src/task_eval.py@@ -101,7 +101,7 @@ class CheckEqual(MetricBase):)self.ground_truth = ground_truth- def __call__(+ async def __call__(self,solution: SolutionOutput,) -> MetricResult:@@ -239,7 +239,7 @@ async def toy_solution_generation(async def main() -> None:evaluator = GeneralEvaluator(- name="ACEbench evaluation",+ name="Toy benchmark evaluation",benchmark=ToyBenchmark(),# Repeat how many timesn_repeat=1,diff --git a/docs/tutorial/zh_CN/src/task_eval.py b/docs/tutorial/zh_CN/src/task_eval.pyindex 2bf881464..f58ecc4e4 100644--- a/docs/tutorial/zh_CN/src/task_eval.py+++ b/docs/tutorial/zh_CN/src/task_eval.py@@ -100,7 +100,7 @@ class CheckEqual(MetricBase):)self.ground_truth = ground_truth- def __call__(+ async def __call__(self,solution: SolutionOutput,) -> MetricResult:@@ -237,7 +237,7 @@ async def toy_solution_generation(async def main() -> None:evaluator = GeneralEvaluator(- name="ACEbench evaluation",+ name="Toy benchmark evaluation",benchmark=ToyBenchmark(),# 重复多少次n_repeat=1,diff --git a/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py b/src/agentscope/evaluate/_ace_benchmark/_ace_metric.pyindex d25b15263..492577867 100644--- a/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py+++ b/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py@@ -20,7 +20,7 @@ class ACEProcessAccuracy(MetricBase):)self.mile_stone = mile_stone- def __call__(+ async def __call__(self,solution: SolutionOutput,) -> MetricResult:@@ -82,7 +82,7 @@ class ACEAccuracy(MetricBase):)self.state = state- def __call__(+ async def __call__(self,solution: SolutionOutput,) -> MetricResult:diff --git a/src/agentscope/evaluate/_evaluator/_general_evaluator.py b/src/agentscope/evaluate/_evaluator/_general_evaluator.pyindex ce121c151..40edc62de 100644--- a/src/agentscope/evaluate/_evaluator/_general_evaluator.py+++ b/src/agentscope/evaluate/_evaluator/_general_evaluator.py@@ -39,14 +39,14 @@ class GeneralEvaluator(EvaluatorBase):self.n_repeat = n_repeatself.n_workers = n_workers- def run_evaluation(+ async def run_evaluation(self,task: Task,repeat_id: str,solution_output: SolutionOutput,) -> None:"""Run the evaluation for a task and solution result."""- evaluation_results = task.evaluate(solution_output)+ evaluation_results = await task.evaluate(solution_output)# store the evaluation resultfor result in evaluation_results:self.storage.save_evaluation_result(@@ -91,7 +91,7 @@ class GeneralEvaluator(EvaluatorBase):repeat_id,metric.name,):- self.run_evaluation(+ await self.run_evaluation(task,repeat_id,solution_result,diff --git a/src/agentscope/evaluate/_evaluator/_ray_evaluator.py b/src/agentscope/evaluate/_evaluator/_ray_evaluator.pyindex a8f4a0147..1b7b86530 100644--- a/src/agentscope/evaluate/_evaluator/_ray_evaluator.py+++ b/src/agentscope/evaluate/_evaluator/_ray_evaluator.py@@ -15,57 +15,29 @@ from .._task import Taskfrom .._evaluator_storage import EvaluatorStorageBase-def _lazy_ray_remote(func: Callable) -> Callable:- """Decorator to lazily initialize ray.remote."""- try:- import ray as lazy_ray-- return lazy_ray.remote(func)-- except ImportError:- return func---class RayEvaluator(EvaluatorBase):- """The ray-based evaluator that supports distributed and parallel- evaluation."""-- def __init__(- self,- name: str,- benchmark: BenchmarkBase,- n_repeat: int,- storage: EvaluatorStorageBase,- n_workers: int,- ) -> None:- """Initialize the evaluator."""- super().__init__(- name=name,- benchmark=benchmark,- n_repeat=n_repeat,- storage=storage,- )-- assert isinstance(benchmark, BenchmarkBase)-- assert n_repeat >= 1, "n_repeat must be at least 1"-- assert n_workers >= 1, "n_workers must be at least 1"-- self.benchmark = benchmark- self.n_repeat = n_repeat- self.n_workers = n_workers+class RayEvaluationActor:+ """+ Actor class for running evaluation with ray remote.+ """@staticmethod- @_lazy_ray_remote- def run_evaluation(+ async def run(storage: EvaluatorStorageBase,task: Task,repeat_id: str,solution_output: SolutionOutput,) -> None:- """Run the evaluation for a task and solution result."""- evaluation_results = task.evaluate(solution_output)+ """+ Run the evaluation for a task and solution result.++ Args:+ storage (EvaluatorStorageBase): Evaluator storage.+ task (Task): Task to be evaluated.+ repeat_id (str): Repeat ID+ solution_output (SolutionOutput): output data after execute agents.+ """+ evaluation_results = await task.evaluate(solution_output)# store the evaluation resultfor result in evaluation_results:storage.save_evaluation_result(@@ -74,9 +46,20 @@ class RayEvaluator(EvaluatorBase):evaluation=result,)- @staticmethod- @_lazy_ray_remote- def run_solution(++class RaySolutionActor:+ """+ Actor class for running agent solutions with ray remote.+ """++ def __init__(self, n_workers: int = 1):+ self.eval_actor = RayEvaluationActor.options(+ max_concurrency=n_workers,+ ).remote()++ async def run(+ self,storage: EvaluatorStorageBase,repeat_id: str,task: Task,@@ -85,7 +68,16 @@ class RayEvaluator(EvaluatorBase):Coroutine[Any, Any, SolutionOutput],],) -> None:- """Generate a solution to a task and evaluate."""+ """Generate a solution to a task and evaluate.++ Args:+ storage (EvaluatorStorageBase): Evaluator storage.+ repeat_id (str): Repeat ID.+ task (Task): Task to be evaluated.+ solution+ (Callable[[Task, Callable], Awaitable[SolutionOutput, Any]]):+ callable function to execute agents and generate results.+ """if storage.solution_result_exists(task.id, repeat_id):# Obtain from storagesolution_result = storage.get_solution_result(@@ -95,15 +87,14 @@ class RayEvaluator(EvaluatorBase):else:# Run the solution- solution_result = asyncio.run(- solution(- task,- storage.get_agent_pre_print_hook(- task.id,- repeat_id,- ),+ solution_result = await solution(+ task,+ storage.get_agent_pre_print_hook(+ task.id,+ repeat_id,),)+storage.save_solution_result(task.id,repeat_id,@@ -119,14 +110,46 @@ class RayEvaluator(EvaluatorBase):metric.name,):futures.append(- RayEvaluator.run_evaluation.remote(+ self.eval_actor.run.remote(storage,task,repeat_id,solution_result,),)- ray.get(futures)+ if futures:+ await asyncio.gather(*futures)+++class RayEvaluator(EvaluatorBase):+ """The ray-based evaluator that supports distributed and parallel+ evaluation."""++ def __init__(+ self,+ name: str,+ benchmark: BenchmarkBase,+ n_repeat: int,+ storage: EvaluatorStorageBase,+ n_workers: int,+ ) -> None:+ """Initialize the evaluator."""+ super().__init__(+ name=name,+ benchmark=benchmark,+ n_repeat=n_repeat,+ storage=storage,+ )++ assert isinstance(benchmark, BenchmarkBase)++ assert n_repeat >= 1, "n_repeat must be at least 1"++ assert n_workers >= 1, "n_workers must be at least 1"++ self.benchmark = benchmark+ self.n_repeat = n_repeat+ self.n_workers = n_workersasync def run(self,@@ -147,16 +170,20 @@ class RayEvaluator(EvaluatorBase):await self._save_evaluation_meta()futures = []+ solution_actor = RaySolutionActor.options(+ max_concurrency=self.n_workers,+ ).remote(n_workers=self.n_workers)for repeat_id in range(self.n_repeat):for task in self.benchmark:futures.append(- RayEvaluator.run_solution.remote(+ solution_actor.run.remote(self.storage,str(repeat_id),task,solution,),)- ray.get(futures)+ if futures:+ await asyncio.gather(*futures)await self.aggregate()diff --git a/src/agentscope/evaluate/_metric_base.py b/src/agentscope/evaluate/_metric_base.pyindex 649b48d22..edab99105 100644--- a/src/agentscope/evaluate/_metric_base.py+++ b/src/agentscope/evaluate/_metric_base.py@@ -79,7 +79,7 @@ class MetricBase(ABC):self.categories = categories@abstractmethod- def __call__(+ async def __call__(self,*args: Any,**kwargs: Any,diff --git a/src/agentscope/evaluate/_task.py b/src/agentscope/evaluate/_task.pyindex ed33ba304..8ad1f9a7d 100644--- a/src/agentscope/evaluate/_task.py+++ b/src/agentscope/evaluate/_task.py@@ -35,7 +35,7 @@ class Task:)"""Additional metadata for the task."""- def evaluate(self, solution: SolutionOutput) -> list[MetricResult]:+ async def evaluate(self, solution: SolutionOutput) -> list[MetricResult]:"""Evaluate the task with the given solution.Args:@@ -48,6 +48,6 @@ class Task:"""evaluations = []for metric in self.metrics:- result = metric(solution)+ result = await metric(solution)evaluations.append(result)return evaluationsdiff --git a/tests/evaluation_test.py b/tests/evaluation_test.pynew file mode 100644index 000000000..29529def9--- /dev/null+++ b/tests/evaluation_test.py@@ -0,0 +1,287 @@+# -*- coding: utf-8 -*-+"""Evaluation module tests in agentscope."""+import os+import sys+import shutil+from typing import Generator, Callable, Any, cast+from unittest.async_case import IsolatedAsyncioTestCase+import ray++from agentscope.agent import AgentBase+from agentscope.message import Msg+from agentscope.evaluate import (+ SolutionOutput,+ MetricBase,+ MetricResult,+ MetricType,+ Task,+ BenchmarkBase,+ GeneralEvaluator,+ RayEvaluator,+ FileEvaluatorStorage,+)+++TASK_ID_1 = "math_problem_1"+TASK_ID_2 = "math_problem_2"++TOY_BENCHMARK = [+ {+ "id": TASK_ID_1,+ "question": "What is 2 + 2?",+ "ground_truth": 4.0,+ "tags": {+ "difficulty": "easy",+ "category": "math",+ },+ },+ {+ "id": TASK_ID_2,+ "question": "What is 12345 + 54321 + 6789 + 9876?",+ "ground_truth": 83331,+ "tags": {+ "difficulty": "medium",+ "category": "math",+ },+ },+]+++METRIC_NAME = "math_check_number_equal"+++class CheckEqual(MetricBase):+ """Metric to check whether the provide answer is equal to ground truth."""++ def __init__(+ self,+ ground_truth: float,+ ):+ super().__init__(+ name=METRIC_NAME,+ metric_type=MetricType.NUMERICAL,+ description="Toy metric checking if two numbers are equal",+ categories=[],+ )+ self.ground_truth = ground_truth++ async def __call__(+ self,+ solution: SolutionOutput,+ ) -> MetricResult:+ if solution.output == self.ground_truth:+ return MetricResult(+ name=self.name,+ result=1.0,+ message="Correct",+ )+ else:+ return MetricResult(+ name=self.name,+ result=0.0,+ message="Incorrect",+ )+++class ToyBenchmark(BenchmarkBase):+ """A toy benchmark for testing"""++ def __init__(self) -> None:+ super().__init__(+ name="Toy bench",+ description="A toy benchmark for testing "+ "the evaluation module.",+ )+ self.dataset = self._load_data()++ @staticmethod+ def _load_data() -> list[Task]:+ dataset = []+ for item in TOY_BENCHMARK:+ dataset.append(+ Task(+ id=item["id"],+ input=item["question"],+ ground_truth=item["ground_truth"],+ tags=item.get("tags", {}),+ metrics=[+ CheckEqual(cast(float, item["ground_truth"])),+ ],+ metadata={},+ ),+ )+ return dataset++ def __iter__(self) -> Generator[Task, None, None]:+ """Iterate over the benchmark."""+ for task in self.dataset:+ yield task++ def __getitem__(self, index: int) -> Task:+ """Get a task by index."""+ return self.dataset[index]++ def __len__(self) -> int:+ """Get the length of the benchmark."""+ return len(self.dataset)+++class EvalTestAgent(AgentBase):+ """Test agent class for testing hooks."""++ def __init__(self) -> None:+ """Initialize the test agent."""+ super().__init__()+ self.records: list[str] = []+ self.memory: list[Msg] = []++ async def reply(self, msg: Msg) -> Msg:+ """Reply to the message."""+ return Msg(+ name="test_eval_agent",+ content=msg.content,+ role="assistant",+ metadata={"answer_as_number": 4.0},+ )++ async def handle_interrupt(+ self,+ *args: Any,+ **kwargs: Any,+ ) -> Msg:+ """Dummy handle interrupt."""+ print(args, kwargs)++ async def observe(self, msg: Msg | list[Msg] | None) -> None:+ """Dummy observe function."""+ print(msg.content)+++async def dummy_solution_generation(+ task: Task,+ pre_hook: Callable, # pylint: disable=W0613+) -> SolutionOutput:+ """Solution generation function for test"""+ agent = EvalTestAgent()++ msg_input = Msg("user", task.input, role="user")+ res = await agent(msg_input)+ return SolutionOutput(+ success=True,+ output=res.metadata.get("answer_as_number", None),+ trajectory=[],+ )+++class EvaluatorTest(IsolatedAsyncioTestCase):+ """Test for evaluators in AS"""++ async def asyncSetUp(self) -> None:+ """Set up the test environment."""+ current_dir = os.path.dirname(os.path.abspath(__file__))+ self.file_storage_general = FileEvaluatorStorage(+ save_dir=os.path.join(+ current_dir,+ "general_results",+ ),+ )+ self.file_storage_ray = FileEvaluatorStorage(+ save_dir=os.path.join(+ current_dir,+ "ray_results",+ ),+ )+ # Initialize Ray with proper serialization settings+ if not ray.is_initialized():+ # Add the current directory to Python path for Ray workers+ current_dir = os.path.dirname(os.path.abspath(__file__))+ if current_dir not in sys.path:+ sys.path.insert(0, current_dir)++ ray.init(+ _temp_dir=None, # Use default temp directory+ ignore_reinit_error=True, # Allow re-initialization+ runtime_env={+ "working_dir": current_dir,+ "py_modules": [__file__], # Include this test file+ },+ )++ async def test_general_evaluator(self) -> None:+ """Test general evaluator."""+ evaluator = GeneralEvaluator(+ name="Test evaluation",+ benchmark=ToyBenchmark(),+ # Repeat how many times+ n_repeat=1,+ storage=self.file_storage_general,+ # How many workers to use+ n_workers=1,+ )++ # Run the evaluation+ await evaluator.run(dummy_solution_generation)+ metric_result_1 = self.file_storage_general.get_evaluation_result(+ task_id=TASK_ID_1,+ repeat_id="0",+ metric_name=METRIC_NAME,+ )+ self.assertEqual(+ metric_result_1.result,+ 1.0,+ )+ metric_result_2 = self.file_storage_general.get_evaluation_result(+ task_id=TASK_ID_2,+ repeat_id="0",+ metric_name=METRIC_NAME,+ )+ self.assertEqual(+ metric_result_2.result,+ 0.0,+ )++ async def test_ray_evaluator(self) -> None:+ """Test ray evaluator."""+ evaluator = RayEvaluator(+ name="Test evaluation",+ benchmark=ToyBenchmark(),+ # Repeat how many times+ n_repeat=1,+ storage=self.file_storage_ray,+ # How many workers to use+ n_workers=1,+ )++ # Run the evaluation+ await evaluator.run(dummy_solution_generation)+ metric_result_1 = self.file_storage_ray.get_evaluation_result(+ task_id=TASK_ID_1,+ repeat_id="0",+ metric_name=METRIC_NAME,+ )+ self.assertEqual(+ metric_result_1.result,+ 1.0,+ )+ metric_result_2 = self.file_storage_ray.get_evaluation_result(+ task_id=TASK_ID_2,+ repeat_id="0",+ metric_name=METRIC_NAME,+ )+ self.assertEqual(+ metric_result_2.result,+ 0.0,+ )++ async def asyncTearDown(self) -> None:+ """Clean up the test environment."""+ # Shutdown Ray if it was initialized+ if ray.is_initialized():+ ray.shutdown()++ # clean written files+ if os.path.exists(self.file_storage_general.save_dir):+ shutil.rmtree(self.file_storage_general.save_dir)++ if os.path.exists(self.file_storage_ray.save_dir):+ shutil.rmtree(self.file_storage_ray.save_dir)