A programming framework for agentic AI
Initial work porting WebArena to async (#325)
afourney8b13d59b59a9
diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.pyindex c94f1d59e..aeb19b8de 100644--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.py+++ b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.py@@ -7,12 +7,13 @@ import importlibimport jsonimport timeimport urllib+import inspectfrom pathlib import Pathfrom typing import Any, Tuple, Union, TypedDict, Dictfrom beartype import beartypefrom nltk.tokenize import word_tokenize # type: ignore-from playwright.sync_api import CDPSession, Page+from playwright.async_api import CDPSession, Pageimport numpy as npimport numpy.typing as npt@@ -58,7 +59,7 @@ class Evaluator(object):self.eval_tag = eval_tag@beartype- def __call__(+ async def __call__(self,trajectory: Trajectory,config_file: Path | str,@@ -136,7 +137,7 @@ class StringEvaluator(Evaluator):def ua_match(ref: str, pred: str, intent: str, azure_config: dict[str, Any] | None) -> float:return llm_ua_match(pred, ref, intent, azure_config)- def __call__(+ async def __call__(self,trajectory: Trajectory,config_file: Path | str,@@ -192,7 +193,7 @@ class URLEvaluator(Evaluator):"""Check URL matching"""@beartype- def __call__(+ async def __call__(self,trajectory: Trajectory,config_file: Path | str,@@ -256,7 +257,7 @@ class HTMLContentEvaluator(Evaluator):"""Check whether the contents appear in the page"""@beartype- def __call__(+ async def __call__(self,trajectory: Trajectory,config_file: Path | str,@@ -276,12 +277,14 @@ class HTMLContentEvaluator(Evaluator):func = target_url.split("func:")[1]func = func.replace("__last_url__", page.url)target_url = eval(func)+ if inspect.isawaitable(target_url):+ target_url = await target_urllocator: str = target["locator"] # js element locator# navigate to that urlif target_url != "last":- page.goto(target_url)+ await page.goto(target_url)time.sleep(3) # TODO [shuyanzh]: fix this hard-coded sleep# empty, use the full page@@ -292,11 +295,11 @@ class HTMLContentEvaluator(Evaluator):if "prep_actions" in target:try:for prep_action in target["prep_actions"]:- page.evaluate(f"() => {prep_action}")+ await page.evaluate(f"() => {prep_action}")except Exception:passtry:- selected_element = str(page.evaluate(f"() => {locator}"))+ selected_element = str(await page.evaluate(f"() => {locator}"))if not selected_element:selected_element = ""except Exception:@@ -307,6 +310,8 @@ class HTMLContentEvaluator(Evaluator):func = locator.split("func:")[1]func = func.replace("__page__", "page")selected_element = eval(func)+ if inspect.isawaitable(selected_element):+ selected_element = await selected_elementelse:raise ValueError(f"Unknown locator: {locator}")@@ -344,7 +349,7 @@ class EvaluatorComb:self.evaluators = evaluators@beartype- def __call__(+ async def __call__(self,trajectory: Trajectory,config_file: Path | str,@@ -354,7 +359,7 @@ class EvaluatorComb:) -> float:score = 1.0for evaluator in self.evaluators:- cur_score = evaluator(trajectory, config_file, page, client, azure_config)+ cur_score = await evaluator(trajectory, config_file, page, client, azure_config)score *= cur_scorereturn scorediff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.pyindex acb874a2a..a2e20d622 100644--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.py+++ b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.py@@ -5,7 +5,7 @@ from typing import Anyfrom urllib.parse import urlparseimport requests-from playwright.sync_api import CDPSession, Page+from playwright.async_api import Pagefrom .env_config import (ACCOUNTS,@@ -110,10 +110,10 @@ def reddit_get_post_url(url: str) -> str:return post_url-def gitlab_get_project_memeber_role(page: Page, account_name: str) -> str:+async def gitlab_get_project_memeber_role(page: Page, account_name: str) -> str:# get the account indextry:- account_idx = page.evaluate(+ account_idx = await page.evaluate(f"""(() => {{const elements = document.querySelectorAll("td[data-label='Account'] span.gl-avatar-labeled-sublabel");let index = -1; // Default value if not found@@ -130,7 +130,7 @@ def gitlab_get_project_memeber_role(page: Page, account_name: str) -> str:)# get the role- role: str = page.evaluate(+ role: str = await page.evaluate(f"""(() => {{return document.querySelectorAll("td.col-max-role span")[{account_idx}].outerText;}})()"""diff --git a/python/benchmarks/WebArena/Templates/TeamOne/scenario.py b/python/benchmarks/WebArena/Templates/TeamOne/scenario.pyindex cc37f8b07..bf38ab25f 100644--- a/python/benchmarks/WebArena/Templates/TeamOne/scenario.py+++ b/python/benchmarks/WebArena/Templates/TeamOne/scenario.py@@ -120,7 +120,6 @@ async def main() -> None:runtime = SingleThreadedAgentRuntime()# Create the AzureOpenAI client, with AAD auth- # token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")client = AzureOpenAIChatCompletionClient(api_version="2024-02-15-preview",azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",@@ -236,18 +235,17 @@ Once the user has taken the final necessary action to complete the task, and you########## EVALUATION ##########context = actual_surfer._contextpage = actual_surfer._page- cdp_session = context.new_cdp_session(page)+ cdp_session = await context.new_cdp_session(page)config_file = "full_task.json"evaluator = evaluation_harness.evaluator_router(config_file)- score = "N/A"- #score = evaluator(- # trajectory=evaluation_harness.make_answer_trajecotry(final_answer),- # config_file=config_file,- # page=page,- # client=cdp_session,+ score = await evaluator(+ trajectory=evaluation_harness.make_answer_trajecotry(final_answer),+ config_file=config_file,+ page=page,+ client=cdp_session,# azure_config=llm_config,- #)+ )print("FINAL SCORE: " + str(score))diff --git a/python/teams/team-one/src/team_one/agents/base_orchestrator.py b/python/teams/team-one/src/team_one/agents/base_orchestrator.pyindex 8a86c1a33..b8a03565e 100644--- a/python/teams/team-one/src/team_one/agents/base_orchestrator.py+++ b/python/teams/team-one/src/team_one/agents/base_orchestrator.py@@ -5,7 +5,7 @@ from agnext.application.logging import EVENT_LOGGER_NAMEfrom agnext.components.models import AssistantMessage, LLMMessage, UserMessagefrom agnext.core import AgentProxy, CancellationToken-from ..messages import BroadcastMessage, OrchestrationEvent, RequestReplyMessage+from ..messages import BroadcastMessage, OrchestrationEvent, RequestReplyMessage, ResetMessagefrom ..utils import message_content_to_strfrom .base_agent import TeamOneBaseAgent@@ -82,3 +82,10 @@ class BaseOrchestrator(TeamOneBaseAgent):def get_max_rounds(self) -> int:return self._max_rounds++ async def _handle_reset(self, message: ResetMessage, cancellation_token: CancellationToken) -> None:+ """Handle a reset message."""+ await self._reset(cancellation_token)++ async def _reset(self, cancellation_token: CancellationToken) -> None:+ pass