Skip to content

Build and run agents you can see, understand and trust.

Make the metric call function async in the evaluation module (#758)

ZiTao-Li811fb28a3dfd
diff --git a/docs/tutorial/en/src/task_eval.py b/docs/tutorial/en/src/task_eval.py
index d7f3902de..43b50a8d4 100644
--- a/docs/tutorial/en/src/task_eval.py
+++ b/docs/tutorial/en/src/task_eval.py
@@ -101,7 +101,7 @@ class CheckEqual(MetricBase):
)
self.ground_truth = ground_truth
- def __call__(
+ async def __call__(
self,
solution: SolutionOutput,
) -> MetricResult:
@@ -239,7 +239,7 @@ async def toy_solution_generation(
async def main() -> None:
evaluator = GeneralEvaluator(
- name="ACEbench evaluation",
+ name="Toy benchmark evaluation",
benchmark=ToyBenchmark(),
# Repeat how many times
n_repeat=1,
diff --git a/docs/tutorial/zh_CN/src/task_eval.py b/docs/tutorial/zh_CN/src/task_eval.py
index 2bf881464..f58ecc4e4 100644
--- a/docs/tutorial/zh_CN/src/task_eval.py
+++ b/docs/tutorial/zh_CN/src/task_eval.py
@@ -100,7 +100,7 @@ class CheckEqual(MetricBase):
)
self.ground_truth = ground_truth
- def __call__(
+ async def __call__(
self,
solution: SolutionOutput,
) -> MetricResult:
@@ -237,7 +237,7 @@ async def toy_solution_generation(
async def main() -> None:
evaluator = GeneralEvaluator(
- name="ACEbench evaluation",
+ name="Toy benchmark evaluation",
benchmark=ToyBenchmark(),
# 重复多少次
n_repeat=1,
diff --git a/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py b/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py
index d25b15263..492577867 100644
--- a/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py
+++ b/src/agentscope/evaluate/_ace_benchmark/_ace_metric.py
@@ -20,7 +20,7 @@ class ACEProcessAccuracy(MetricBase):
)
self.mile_stone = mile_stone
- def __call__(
+ async def __call__(
self,
solution: SolutionOutput,
) -> MetricResult:
@@ -82,7 +82,7 @@ class ACEAccuracy(MetricBase):
)
self.state = state
- def __call__(
+ async def __call__(
self,
solution: SolutionOutput,
) -> MetricResult:
diff --git a/src/agentscope/evaluate/_evaluator/_general_evaluator.py b/src/agentscope/evaluate/_evaluator/_general_evaluator.py
index ce121c151..40edc62de 100644
--- a/src/agentscope/evaluate/_evaluator/_general_evaluator.py
+++ b/src/agentscope/evaluate/_evaluator/_general_evaluator.py
@@ -39,14 +39,14 @@ class GeneralEvaluator(EvaluatorBase):
self.n_repeat = n_repeat
self.n_workers = n_workers
- def run_evaluation(
+ async def run_evaluation(
self,
task: Task,
repeat_id: str,
solution_output: SolutionOutput,
) -> None:
"""Run the evaluation for a task and solution result."""
- evaluation_results = task.evaluate(solution_output)
+ evaluation_results = await task.evaluate(solution_output)
# store the evaluation result
for result in evaluation_results:
self.storage.save_evaluation_result(
@@ -91,7 +91,7 @@ class GeneralEvaluator(EvaluatorBase):
repeat_id,
metric.name,
):
- self.run_evaluation(
+ await self.run_evaluation(
task,
repeat_id,
solution_result,
diff --git a/src/agentscope/evaluate/_evaluator/_ray_evaluator.py b/src/agentscope/evaluate/_evaluator/_ray_evaluator.py
index a8f4a0147..1b7b86530 100644
--- a/src/agentscope/evaluate/_evaluator/_ray_evaluator.py
+++ b/src/agentscope/evaluate/_evaluator/_ray_evaluator.py
@@ -15,57 +15,29 @@ from .._task import Task
from .._evaluator_storage import EvaluatorStorageBase
-def _lazy_ray_remote(func: Callable) -> Callable:
- """Decorator to lazily initialize ray.remote."""
- try:
- import ray as lazy_ray
-
- return lazy_ray.remote(func)
-
- except ImportError:
- return func
-
-
-class RayEvaluator(EvaluatorBase):
- """The ray-based evaluator that supports distributed and parallel
- evaluation."""
-
- def __init__(
- self,
- name: str,
- benchmark: BenchmarkBase,
- n_repeat: int,
- storage: EvaluatorStorageBase,
- n_workers: int,
- ) -> None:
- """Initialize the evaluator."""
- super().__init__(
- name=name,
- benchmark=benchmark,
- n_repeat=n_repeat,
- storage=storage,
- )
-
- assert isinstance(benchmark, BenchmarkBase)
-
- assert n_repeat >= 1, "n_repeat must be at least 1"
-
- assert n_workers >= 1, "n_workers must be at least 1"
-
- self.benchmark = benchmark
- self.n_repeat = n_repeat
- self.n_workers = n_workers
+class RayEvaluationActor:
+ """
+ Actor class for running evaluation with ray remote.
+ """
@staticmethod
- @_lazy_ray_remote
- def run_evaluation(
+ async def run(
storage: EvaluatorStorageBase,
task: Task,
repeat_id: str,
solution_output: SolutionOutput,
) -> None:
- """Run the evaluation for a task and solution result."""
- evaluation_results = task.evaluate(solution_output)
+ """
+ Run the evaluation for a task and solution result.
+
+ Args:
+ storage (EvaluatorStorageBase): Evaluator storage.
+ task (Task): Task to be evaluated.
+ repeat_id (str): Repeat ID
+ solution_output (SolutionOutput): output data after execute agents.
+ """
+ evaluation_results = await task.evaluate(solution_output)
# store the evaluation result
for result in evaluation_results:
storage.save_evaluation_result(
@@ -74,9 +46,20 @@ class RayEvaluator(EvaluatorBase):
evaluation=result,
)
- @staticmethod
- @_lazy_ray_remote
- def run_solution(
+
+class RaySolutionActor:
+ """
+ Actor class for running agent solutions with ray remote.
+ """
+
+ def __init__(self, n_workers: int = 1):
+ self.eval_actor = RayEvaluationActor.options(
+ max_concurrency=n_workers,
+ ).remote()
+
+ async def run(
+ self,
storage: EvaluatorStorageBase,
repeat_id: str,
task: Task,
@@ -85,7 +68,16 @@ class RayEvaluator(EvaluatorBase):
Coroutine[Any, Any, SolutionOutput],
],
) -> None:
- """Generate a solution to a task and evaluate."""
+ """Generate a solution to a task and evaluate.
+
+ Args:
+ storage (EvaluatorStorageBase): Evaluator storage.
+ repeat_id (str): Repeat ID.
+ task (Task): Task to be evaluated.
+ solution
+ (Callable[[Task, Callable], Awaitable[SolutionOutput, Any]]):
+ callable function to execute agents and generate results.
+ """
if storage.solution_result_exists(task.id, repeat_id):
# Obtain from storage
solution_result = storage.get_solution_result(
@@ -95,15 +87,14 @@ class RayEvaluator(EvaluatorBase):
else:
# Run the solution
- solution_result = asyncio.run(
- solution(
- task,
- storage.get_agent_pre_print_hook(
- task.id,
- repeat_id,
- ),
+ solution_result = await solution(
+ task,
+ storage.get_agent_pre_print_hook(
+ task.id,
+ repeat_id,
),
)
+
storage.save_solution_result(
task.id,
repeat_id,
@@ -119,14 +110,46 @@ class RayEvaluator(EvaluatorBase):
metric.name,
):
futures.append(
- RayEvaluator.run_evaluation.remote(
+ self.eval_actor.run.remote(
storage,
task,
repeat_id,
solution_result,
),
)
- ray.get(futures)
+ if futures:
+ await asyncio.gather(*futures)
+
+
+class RayEvaluator(EvaluatorBase):
+ """The ray-based evaluator that supports distributed and parallel
+ evaluation."""
+
+ def __init__(
+ self,
+ name: str,
+ benchmark: BenchmarkBase,
+ n_repeat: int,
+ storage: EvaluatorStorageBase,
+ n_workers: int,
+ ) -> None:
+ """Initialize the evaluator."""
+ super().__init__(
+ name=name,
+ benchmark=benchmark,
+ n_repeat=n_repeat,
+ storage=storage,
+ )
+
+ assert isinstance(benchmark, BenchmarkBase)
+
+ assert n_repeat >= 1, "n_repeat must be at least 1"
+
+ assert n_workers >= 1, "n_workers must be at least 1"
+
+ self.benchmark = benchmark
+ self.n_repeat = n_repeat
+ self.n_workers = n_workers
async def run(
self,
@@ -147,16 +170,20 @@ class RayEvaluator(EvaluatorBase):
await self._save_evaluation_meta()
futures = []
+ solution_actor = RaySolutionActor.options(
+ max_concurrency=self.n_workers,
+ ).remote(n_workers=self.n_workers)
for repeat_id in range(self.n_repeat):
for task in self.benchmark:
futures.append(
- RayEvaluator.run_solution.remote(
+ solution_actor.run.remote(
self.storage,
str(repeat_id),
task,
solution,
),
)
- ray.get(futures)
+ if futures:
+ await asyncio.gather(*futures)
await self.aggregate()
diff --git a/src/agentscope/evaluate/_metric_base.py b/src/agentscope/evaluate/_metric_base.py
index 649b48d22..edab99105 100644
--- a/src/agentscope/evaluate/_metric_base.py
+++ b/src/agentscope/evaluate/_metric_base.py
@@ -79,7 +79,7 @@ class MetricBase(ABC):
self.categories = categories
@abstractmethod
- def __call__(
+ async def __call__(
self,
*args: Any,
**kwargs: Any,
diff --git a/src/agentscope/evaluate/_task.py b/src/agentscope/evaluate/_task.py
index ed33ba304..8ad1f9a7d 100644
--- a/src/agentscope/evaluate/_task.py
+++ b/src/agentscope/evaluate/_task.py
@@ -35,7 +35,7 @@ class Task:
)
"""Additional metadata for the task."""
- def evaluate(self, solution: SolutionOutput) -> list[MetricResult]:
+ async def evaluate(self, solution: SolutionOutput) -> list[MetricResult]:
"""Evaluate the task with the given solution.
Args:
@@ -48,6 +48,6 @@ class Task:
"""
evaluations = []
for metric in self.metrics:
- result = metric(solution)
+ result = await metric(solution)
evaluations.append(result)
return evaluations
diff --git a/tests/evaluation_test.py b/tests/evaluation_test.py
new file mode 100644
index 000000000..29529def9
--- /dev/null
+++ b/tests/evaluation_test.py
@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+"""Evaluation module tests in agentscope."""
+import os
+import sys
+import shutil
+from typing import Generator, Callable, Any, cast
+from unittest.async_case import IsolatedAsyncioTestCase
+import ray
+
+from agentscope.agent import AgentBase
+from agentscope.message import Msg
+from agentscope.evaluate import (
+ SolutionOutput,
+ MetricBase,
+ MetricResult,
+ MetricType,
+ Task,
+ BenchmarkBase,
+ GeneralEvaluator,
+ RayEvaluator,
+ FileEvaluatorStorage,
+)
+
+
+TASK_ID_1 = "math_problem_1"
+TASK_ID_2 = "math_problem_2"
+
+TOY_BENCHMARK = [
+ {
+ "id": TASK_ID_1,
+ "question": "What is 2 + 2?",
+ "ground_truth": 4.0,
+ "tags": {
+ "difficulty": "easy",
+ "category": "math",
+ },
+ },
+ {
+ "id": TASK_ID_2,
+ "question": "What is 12345 + 54321 + 6789 + 9876?",
+ "ground_truth": 83331,
+ "tags": {
+ "difficulty": "medium",
+ "category": "math",
+ },
+ },
+]
+
+
+METRIC_NAME = "math_check_number_equal"
+
+
+class CheckEqual(MetricBase):
+ """Metric to check whether the provide answer is equal to ground truth."""
+
+ def __init__(
+ self,
+ ground_truth: float,
+ ):
+ super().__init__(
+ name=METRIC_NAME,
+ metric_type=MetricType.NUMERICAL,
+ description="Toy metric checking if two numbers are equal",
+ categories=[],
+ )
+ self.ground_truth = ground_truth
+
+ async def __call__(
+ self,
+ solution: SolutionOutput,
+ ) -> MetricResult:
+ if solution.output == self.ground_truth:
+ return MetricResult(
+ name=self.name,
+ result=1.0,
+ message="Correct",
+ )
+ else:
+ return MetricResult(
+ name=self.name,
+ result=0.0,
+ message="Incorrect",
+ )
+
+
+class ToyBenchmark(BenchmarkBase):
+ """A toy benchmark for testing"""
+
+ def __init__(self) -> None:
+ super().__init__(
+ name="Toy bench",
+ description="A toy benchmark for testing "
+ "the evaluation module.",
+ )
+ self.dataset = self._load_data()
+
+ @staticmethod
+ def _load_data() -> list[Task]:
+ dataset = []
+ for item in TOY_BENCHMARK:
+ dataset.append(
+ Task(
+ id=item["id"],
+ input=item["question"],
+ ground_truth=item["ground_truth"],
+ tags=item.get("tags", {}),
+ metrics=[
+ CheckEqual(cast(float, item["ground_truth"])),
+ ],
+ metadata={},
+ ),
+ )
+ return dataset
+
+ def __iter__(self) -> Generator[Task, None, None]:
+ """Iterate over the benchmark."""
+ for task in self.dataset:
+ yield task
+
+ def __getitem__(self, index: int) -> Task:
+ """Get a task by index."""
+ return self.dataset[index]
+
+ def __len__(self) -> int:
+ """Get the length of the benchmark."""
+ return len(self.dataset)
+
+
+class EvalTestAgent(AgentBase):
+ """Test agent class for testing hooks."""
+
+ def __init__(self) -> None:
+ """Initialize the test agent."""
+ super().__init__()
+ self.records: list[str] = []
+ self.memory: list[Msg] = []
+
+ async def reply(self, msg: Msg) -> Msg:
+ """Reply to the message."""
+ return Msg(
+ name="test_eval_agent",
+ content=msg.content,
+ role="assistant",
+ metadata={"answer_as_number": 4.0},
+ )
+
+ async def handle_interrupt(
+ self,
+ *args: Any,
+ **kwargs: Any,
+ ) -> Msg:
+ """Dummy handle interrupt."""
+ print(args, kwargs)
+
+ async def observe(self, msg: Msg | list[Msg] | None) -> None:
+ """Dummy observe function."""
+ print(msg.content)
+
+
+async def dummy_solution_generation(
+ task: Task,
+ pre_hook: Callable, # pylint: disable=W0613
+) -> SolutionOutput:
+ """Solution generation function for test"""
+ agent = EvalTestAgent()
+
+ msg_input = Msg("user", task.input, role="user")
+ res = await agent(msg_input)
+ return SolutionOutput(
+ success=True,
+ output=res.metadata.get("answer_as_number", None),
+ trajectory=[],
+ )
+
+
+class EvaluatorTest(IsolatedAsyncioTestCase):
+ """Test for evaluators in AS"""
+
+ async def asyncSetUp(self) -> None:
+ """Set up the test environment."""
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ self.file_storage_general = FileEvaluatorStorage(
+ save_dir=os.path.join(
+ current_dir,
+ "general_results",
+ ),
+ )
+ self.file_storage_ray = FileEvaluatorStorage(
+ save_dir=os.path.join(
+ current_dir,
+ "ray_results",
+ ),
+ )
+ # Initialize Ray with proper serialization settings
+ if not ray.is_initialized():
+ # Add the current directory to Python path for Ray workers
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ if current_dir not in sys.path:
+ sys.path.insert(0, current_dir)
+
+ ray.init(
+ _temp_dir=None, # Use default temp directory
+ ignore_reinit_error=True, # Allow re-initialization
+ runtime_env={
+ "working_dir": current_dir,
+ "py_modules": [__file__], # Include this test file
+ },
+ )
+
+ async def test_general_evaluator(self) -> None:
+ """Test general evaluator."""
+ evaluator = GeneralEvaluator(
+ name="Test evaluation",
+ benchmark=ToyBenchmark(),
+ # Repeat how many times
+ n_repeat=1,
+ storage=self.file_storage_general,
+ # How many workers to use
+ n_workers=1,
+ )
+
+ # Run the evaluation
+ await evaluator.run(dummy_solution_generation)
+ metric_result_1 = self.file_storage_general.get_evaluation_result(
+ task_id=TASK_ID_1,
+ repeat_id="0",
+ metric_name=METRIC_NAME,
+ )
+ self.assertEqual(
+ metric_result_1.result,
+ 1.0,
+ )
+ metric_result_2 = self.file_storage_general.get_evaluation_result(
+ task_id=TASK_ID_2,
+ repeat_id="0",
+ metric_name=METRIC_NAME,
+ )
+ self.assertEqual(
+ metric_result_2.result,
+ 0.0,
+ )
+
+ async def test_ray_evaluator(self) -> None:
+ """Test ray evaluator."""
+ evaluator = RayEvaluator(
+ name="Test evaluation",
+ benchmark=ToyBenchmark(),
+ # Repeat how many times
+ n_repeat=1,
+ storage=self.file_storage_ray,
+ # How many workers to use
+ n_workers=1,
+ )
+
+ # Run the evaluation
+ await evaluator.run(dummy_solution_generation)
+ metric_result_1 = self.file_storage_ray.get_evaluation_result(
+ task_id=TASK_ID_1,
+ repeat_id="0",
+ metric_name=METRIC_NAME,
+ )
+ self.assertEqual(
+ metric_result_1.result,
+ 1.0,
+ )
+ metric_result_2 = self.file_storage_ray.get_evaluation_result(
+ task_id=TASK_ID_2,
+ repeat_id="0",
+ metric_name=METRIC_NAME,
+ )
+ self.assertEqual(
+ metric_result_2.result,
+ 0.0,
+ )
+
+ async def asyncTearDown(self) -> None:
+ """Clean up the test environment."""
+ # Shutdown Ray if it was initialized
+ if ray.is_initialized():
+ ray.shutdown()
+
+ # clean written files
+ if os.path.exists(self.file_storage_general.save_dir):
+ shutil.rmtree(self.file_storage_general.save_dir)
+
+ if os.path.exists(self.file_storage_ray.save_dir):
+ shutil.rmtree(self.file_storage_ray.save_dir)