Skip to content

A programming framework for agentic AI

add webarena in samples (#2114)

olgavrouaf9b300be39b
diff --git a/samples/tools/webarena/README.md b/samples/tools/webarena/README.md
new file mode 100644
index 000000000..9c1841629
--- /dev/null
+++ b/samples/tools/webarena/README.md
@@ -0,0 +1,32 @@
+# WebArena Benchmark
+
+This directory helps run AutoGen agents on the [WebArena](https://arxiv.org/pdf/2307.13854.pdf) benchmark.
+
+## Installing WebArena
+
+WebArena can be installed by following the instructions from [WebArena's GitHub repository]([email protected]:web-arena-x/webarena.git)
+
+If using WebArena with AutoGen there is a clash on the versions of OpenAI and some code changes are needed in WebArena to be compatible with AutoGen's OpenAI version:
+
+- webarena's openai version is `openai==0.27.0`
+- autogen's openai version is: `openai>=1.3`
+
+Prior to installation, in the WebArena codebase, any file containing `openai.error` needs to be replaced with `openai`.
+
+## Running with AutoGen agents
+
+You can use the `run.py` file in the `webarena` directory to run WebArena with AutoGen. The OpenAI (or AzureOpenAI or other model) configuration can be setup via `OAI_CONFIG_LIST`. The config list will be filtered by whatever model is passed in the `--model` argument.
+
+e.g. of running `run.py`:
+
+```
+mkdir myresultdir
+python run.py --instruction_path agent/prompts/jsons/p_cot_id_actree_2s.json --test_start_idx 27 --test_end_idx 28 --model gpt-4 --result_dir myresultdir
+```
+
+The original `run.py` file has been modified to use AutoGen agents which are defined in the `webarena_agents.py` file.
+
+## References
+**WebArena: A Realistic Web Environment for Building Autonomous Agents**<br/>
+Zhou, Shuyan and Xu, Frank F and Zhu, Hao and Zhou, Xuhui and Lo, Robert and Sridhar, Abishek and Cheng, Xianyi and Bisk, Yonatan and Fried, Daniel and Alon, Uri and others<br/>
+[https://arxiv.org/pdf/2307.13854.pdf](https://arxiv.org/pdf/2307.13854.pdf)
diff --git a/samples/tools/webarena/webarena/run.py b/samples/tools/webarena/webarena/run.py
new file mode 100644
index 000000000..a9411ea12
--- /dev/null
+++ b/samples/tools/webarena/webarena/run.py
@@ -0,0 +1,354 @@
+"""Script to run end-to-end evaluation on the benchmark"""
+
+import argparse
+import glob
+import json
+import logging
+import os
+import random
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+import openai
+import autogen
+from webarena_agents import ActionTakingCapability, EnvironmentAgent
+
+from agent import (
+ Agent,
+ PromptAgent,
+ TeacherForcingAgent,
+ construct_agent,
+)
+
+from agent.prompts import *
+from browser_env import (
+ ScriptBrowserEnv,
+)
+from browser_env.auto_login import get_site_comb_from_filepath
+from browser_env.helper_functions import (
+ RenderHelper,
+)
+from evaluation_harness import evaluator_router
+
+LOG_FOLDER = "log_files"
+Path(LOG_FOLDER).mkdir(parents=True, exist_ok=True)
+LOG_FILE_NAME = f"{LOG_FOLDER}/log_{time.strftime('%Y%m%d%H%M%S', time.localtime())}_{random.randint(0, 10000)}.log"
+
+logger = logging.getLogger("logger")
+logger.setLevel(logging.INFO)
+
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.DEBUG)
+logger.addHandler(console_handler)
+
+file_handler = logging.FileHandler(LOG_FILE_NAME)
+file_handler.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+# Set the log format
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+console_handler.setFormatter(formatter)
+file_handler.setFormatter(formatter)
+
+
+def config() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Run end-to-end evaluation on the benchmark")
+ parser.add_argument("--render", action="store_true", help="Render the browser")
+ parser.add_argument(
+ "--slow_mo",
+ type=int,
+ default=0,
+ help="Slow down the browser by the specified amount",
+ )
+ parser.add_argument("--action_set_tag", default="id_accessibility_tree", help="Action type")
+ parser.add_argument(
+ "--observation_type",
+ choices=["accessibility_tree", "html", "image"],
+ default="accessibility_tree",
+ help="Observation type",
+ )
+ parser.add_argument(
+ "--current_viewport_only",
+ action="store_true",
+ help="Only use the current viewport for the observation",
+ )
+ parser.add_argument("--viewport_width", type=int, default=1280)
+ parser.add_argument("--viewport_height", type=int, default=720)
+ parser.add_argument("--save_trace_enabled", action="store_true")
+ parser.add_argument("--sleep_after_execution", type=float, default=0.0)
+
+ parser.add_argument("--max_steps", type=int, default=30)
+
+ # agent config
+ parser.add_argument("--agent_type", type=str, default="prompt")
+ parser.add_argument(
+ "--instruction_path",
+ type=str,
+ default="agents/prompts/state_action_agent.json",
+ )
+ parser.add_argument(
+ "--parsing_failure_th",
+ help="When concesecutive parsing failure exceeds this threshold, the agent will stop",
+ type=int,
+ default=3,
+ )
+ parser.add_argument(
+ "--repeating_action_failure_th",
+ help="When concesecutive repeating action exceeds this threshold, the agent will stop",
+ type=int,
+ default=3,
+ )
+
+ # lm config
+ parser.add_argument("--provider", type=str, default="openai")
+ parser.add_argument("--model", type=str, default="gpt-3.5-turbo-0613")
+ parser.add_argument("--mode", type=str, default="chat")
+ parser.add_argument("--temperature", type=float, default=1.0)
+ parser.add_argument("--top_p", type=float, default=0.9)
+ parser.add_argument("--context_length", type=int, default=0)
+ parser.add_argument("--max_tokens", type=int, default=384)
+ parser.add_argument("--stop_token", type=str, default=None)
+ parser.add_argument(
+ "--max_retry",
+ type=int,
+ help="max retry times to perform generations when parsing fails",
+ default=1,
+ )
+ parser.add_argument(
+ "--max_obs_length",
+ type=int,
+ help="when not zero, will truncate the observation to this length before feeding to the model",
+ default=1920,
+ )
+ parser.add_argument(
+ "--model_endpoint",
+ help="huggingface model endpoint",
+ type=str,
+ default="",
+ )
+
+ # example config
+ parser.add_argument("--test_start_idx", type=int, default=0)
+ parser.add_argument("--test_end_idx", type=int, default=1000)
+
+ # logging related
+ parser.add_argument("--result_dir", type=str, default="")
+ args = parser.parse_args()
+
+ # check the whether the action space is compatible with the observation space
+ if args.action_set_tag == "id_accessibility_tree" and args.observation_type != "accessibility_tree":
+ raise ValueError(
+ f"Action type {args.action_set_tag} is incompatible with the observation type {args.observation_type}"
+ )
+
+ return args
+
+
+def test(
+ args: argparse.Namespace,
+ agent: Agent | PromptAgent | TeacherForcingAgent,
+ config_file_list: list[str],
+) -> None:
+ scores = []
+ max_steps = args.max_steps
+
+ early_stop_thresholds = {
+ "parsing_failure": args.parsing_failure_th,
+ "repeating_action": args.repeating_action_failure_th,
+ }
+
+ env = ScriptBrowserEnv(
+ headless=not args.render,
+ slow_mo=args.slow_mo,
+ observation_type=args.observation_type,
+ current_viewport_only=args.current_viewport_only,
+ viewport_size={
+ "width": args.viewport_width,
+ "height": args.viewport_height,
+ },
+ save_trace_enabled=args.save_trace_enabled,
+ sleep_after_execution=args.sleep_after_execution,
+ )
+
+ for config_file in config_file_list:
+ try:
+ render_helper = RenderHelper(config_file, args.result_dir, args.action_set_tag)
+
+ # get intent
+ with open(config_file) as f:
+ _c = json.load(f)
+ intent = _c["intent"]
+ task_id = _c["task_id"]
+ # automatically login
+ if _c["storage_state"]:
+ cookie_file_name = os.path.basename(_c["storage_state"])
+ comb = get_site_comb_from_filepath(cookie_file_name)
+ temp_dir = tempfile.mkdtemp()
+ # subprocess to renew the cookie
+ subprocess.run(
+ [
+ "python",
+ "browser_env/auto_login.py",
+ "--auth_folder",
+ temp_dir,
+ "--site_list",
+ *comb,
+ ]
+ )
+ _c["storage_state"] = f"{temp_dir}/{cookie_file_name}"
+ assert os.path.exists(_c["storage_state"])
+ # update the config file
+ config_file = f"{temp_dir}/{os.path.basename(config_file)}"
+ with open(config_file, "w") as f:
+ json.dump(_c, f)
+
+ logger.info(f"[Config file]: {config_file}")
+ logger.info(f"[Intent]: {intent}")
+
+ agent.reset(config_file)
+
+ env_agent = EnvironmentAgent(
+ name="env_agent",
+ env=env,
+ config_file=config_file,
+ result_dir=args.result_dir,
+ action_set_tag=args.action_set_tag,
+ llm_config=False,
+ code_execution_config=False,
+ )
+
+ config_list = autogen.config_list_from_json(
+ "OAI_CONFIG_LIST",
+ filter_dict={"model": [args.model]},
+ )
+
+ action_agent = autogen.AssistantAgent(
+ "action_agent",
+ llm_config={"config_list": config_list, "cache_seed": 32},
+ system_message="",
+ )
+
+ action_taking_capability = ActionTakingCapability(
+ prompt_constructor=agent.prompt_constructor,
+ action_set_tag=args.action_set_tag,
+ max_steps=max_steps,
+ early_stop_thresholds=early_stop_thresholds,
+ )
+
+ action_taking_capability.add_to_agent(action_agent)
+
+ env_agent.initiate_chat(
+ action_agent,
+ message={
+ "content": {"intent": intent},
+ },
+ )
+
+ evaluator = evaluator_router(config_file)
+ score = evaluator(
+ trajectory=env_agent.trajectory,
+ config_file=config_file,
+ page=env.page,
+ client=env.get_page_client(env.page),
+ )
+
+ scores.append(score)
+
+ if score == 1:
+ logger.info(f"[Result] (PASS) {config_file}")
+ else:
+ logger.info(f"[Result] (FAIL) {config_file}")
+
+ if args.save_trace_enabled:
+ env.save_trace(Path(args.result_dir) / "traces" / f"{task_id}.zip")
+
+ except openai.OpenAIError as e:
+ logger.info(f"[OpenAI Error] {repr(e)}")
+ except Exception as e:
+ logger.info(f"[Unhandled Error] {repr(e)}]")
+ import traceback
+
+ # write to error file
+ with open(Path(args.result_dir) / "error.txt", "a") as f:
+ f.write(f"[Config file]: {config_file}\n")
+ f.write(f"[Unhandled Error] {repr(e)}\n")
+ f.write(traceback.format_exc()) # write stack trace to file
+
+ render_helper.close()
+
+ env.close()
+ if len(scores) > 0:
+ logger.info(f"Average score: {sum(scores) / len(scores)}")
+
+
+def prepare(args: argparse.Namespace) -> None:
+ # convert prompt python files to json
+ from agent.prompts import to_json
+
+ to_json.run()
+
+ # prepare result dir
+ result_dir = args.result_dir
+ if not result_dir:
+ result_dir = f"cache/results_{time.strftime('%Y%m%d%H%M%S', time.localtime())}"
+ if not Path(result_dir).exists():
+ Path(result_dir).mkdir(parents=True, exist_ok=True)
+ args.result_dir = result_dir
+ logger.info(f"Create result dir: {result_dir}")
+
+ if not (Path(result_dir) / "traces").exists():
+ (Path(result_dir) / "traces").mkdir(parents=True)
+
+ # log the log file
+ with open(os.path.join(result_dir, "log_files.txt"), "a+") as f:
+ f.write(f"{LOG_FILE_NAME}\n")
+
+
+def get_unfinished(config_files: list[str], result_dir: str) -> list[str]:
+ result_files = glob.glob(f"{result_dir}/*.html")
+ task_ids = [os.path.basename(f).split(".")[0].split("_")[1] for f in result_files]
+ unfinished_configs = []
+ for config_file in config_files:
+ task_id = os.path.basename(config_file).split(".")[0]
+ if task_id not in task_ids:
+ unfinished_configs.append(config_file)
+ return unfinished_configs
+
+
+def dump_config(args: argparse.Namespace) -> None:
+ config_file = Path(args.result_dir) / "config.json"
+ if not config_file.exists():
+ with open(config_file, "w") as f:
+ json.dump(vars(args), f, indent=4)
+ logger.info(f"Dump config to {config_file}")
+
+
+# def run():
+if __name__ == "__main__":
+ args = config()
+ args.sleep_after_execution = 2.0
+ prepare(args)
+
+ test_file_list = []
+ st_idx = args.test_start_idx
+ ed_idx = args.test_end_idx
+ for i in range(st_idx, ed_idx):
+ test_file_list.append(f"config_files/{i}.json")
+ if "debug" not in args.result_dir:
+ test_file_list = get_unfinished(test_file_list, args.result_dir)
+
+ if len(test_file_list) == 0:
+ logger.info("No task left to run")
+ else:
+ print(f"Total {len(test_file_list)} tasks left")
+ args.render = False
+ args.render_screenshot = True
+ args.save_trace_enabled = True
+
+ args.current_viewport_only = True
+ dump_config(args)
+
+ agent = construct_agent(args)
+ test(args, agent, test_file_list)
diff --git a/samples/tools/webarena/webarena/webarena_agents.py b/samples/tools/webarena/webarena/webarena_agents.py
new file mode 100644
index 000000000..103057259
--- /dev/null
+++ b/samples/tools/webarena/webarena/webarena_agents.py
@@ -0,0 +1,205 @@
+import autogen
+
+from browser_env.actions import (
+ ActionParsingError,
+ Action,
+ create_none_action,
+ create_id_based_action,
+ create_playwright_action,
+)
+
+from browser_env import (
+ ActionTypes,
+ StateInfo,
+ Trajectory,
+ create_stop_action,
+)
+
+from browser_env.helper_functions import (
+ RenderHelper,
+ get_action_description,
+)
+
+from browser_env.actions import is_equivalent
+
+
+def early_stop(trajectory: Trajectory, max_steps: int, thresholds: dict[str, int]) -> tuple[bool, str]:
+ """Check whether need to early stop"""
+
+ # reach the max step
+ num_steps = (len(trajectory) - 1) / 2
+ if num_steps >= max_steps:
+ return True, f"Reach max steps {max_steps}"
+
+ last_k_actions: list[Action]
+ action_seq: list[Action]
+
+ # Case: parsing failure for k times
+ k = thresholds["parsing_failure"]
+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]
+ if len(last_k_actions) >= k:
+ if all([action["action_type"] == ActionTypes.NONE for action in last_k_actions]):
+ return True, f"Failed to parse actions for {k} times"
+
+ # Case: same action for k times
+ k = thresholds["repeating_action"]
+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]
+ action_seq = trajectory[1::2] # type: ignore[assignment]
+
+ if len(action_seq) == 0:
+ return False, ""
+
+ last_action: Action = action_seq[-1]
+
+ if last_action["action_type"] != ActionTypes.TYPE:
+ if len(last_k_actions) >= k:
+ if all([is_equivalent(action, last_action) for action in last_k_actions]):
+ return True, f"Same action for {k} times"
+
+ else:
+ # check the action sequence
+ if sum([is_equivalent(action, last_action) for action in action_seq]) >= k:
+ return True, f"Same typing action for {k} times"
+
+ return False, ""
+
+
+class EnvironmentAgent(autogen.ConversableAgent):
+ def __init__(
+ self,
+ env,
+ config_file,
+ result_dir,
+ action_set_tag,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.env = env
+ self.trajectory: Trajectory = []
+ obs, info = env.reset(options={"config_file": config_file})
+ self.state_info: StateInfo = {"observation": obs, "info": info}
+ self.trajectory.append(self.state_info)
+ self.meta_data = {"action_history": ["None"]}
+ self.action_set_tag = action_set_tag
+ self.render_helper = RenderHelper(config_file, result_dir, action_set_tag)
+ self.register_reply([autogen.Agent, None], EnvironmentAgent.generate_env_reply, position=1)
+ self.register_hook(
+ hookable_method="process_message_before_send",
+ hook=self.process_message_before_send,
+ )
+
+ def process_message_before_send(self, sender, message, recipient, silent):
+ if "context" in message:
+ raise ValueError("Message should not contain context freom generate_reply")
+
+ message["context"] = {
+ "trajectory": self.trajectory,
+ "meta_data": self.meta_data,
+ "state_info": self.state_info,
+ }
+
+ if "intent" in message["content"]:
+ intent = message["content"]["intent"]
+ obs = self.state_info["observation"]["text"]
+ message["content"] = f"{obs} \n\n {intent}"
+ message["context"]["intent"] = intent
+ return message
+
+ def generate_env_reply(
+ self,
+ messages=None,
+ sender=None,
+ config=None,
+ ):
+ if messages is None:
+ messages = self.chat_messages[sender]
+
+ action = messages[-1]["context"]
+ action_str = messages[-1]["content"]
+ self.trajectory.append(action)
+
+ self.render_helper.render(action, self.state_info, self.meta_data, True)
+ self.meta_data["action_history"].append(action_str)
+
+ if action["action_type"] == ActionTypes.STOP:
+ return True, {"content": "TERMINATE"}
+
+ obs, _, terminated, _, info = self.env.step(action)
+ self.state_info = {"observation": obs, "info": info}
+ self.trajectory.append(self.state_info)
+
+ if terminated:
+ self.trajectory.append(create_stop_action(""))
+ return True, "TERMINATE"
+
+ return True, {"content": obs["text"]}
+
+
+class ActionTakingCapability:
+ def __init__(self, prompt_constructor, action_set_tag, max_steps, early_stop_thresholds):
+ self.action_set_tag = action_set_tag
+ self.prompt_constructor = prompt_constructor
+ self.max_steps = max_steps
+ self.early_stop_thresholds = early_stop_thresholds
+
+ def add_to_agent(self, agent):
+ agent.register_hook(
+ hookable_method="process_all_messages_before_reply",
+ hook=self.process_all_messages_before_reply,
+ )
+ agent.register_hook(
+ hookable_method="process_message_before_send",
+ hook=self.process_message_before_send,
+ )
+
+ def process_all_messages_before_reply(self, messages):
+ if "context" in messages[-1]:
+ m = messages[-1]["context"]
+ trajectory = m["trajectory"]
+ meta_data = m["meta_data"]
+ intent = messages[0]["context"]["intent"]
+ else:
+ if "TERMINATE" in messages[-1]["content"]:
+ return messages
+
+ # take action
+ prompt = self.prompt_constructor.construct(trajectory, intent, meta_data)
+ return prompt
+
+ def process_message_before_send(self, sender, message, recipient, silent):
+ force_prefix = self.prompt_constructor.instruction["meta_data"].get("force_prefix", "")
+ response = f"{force_prefix}{message}"
+ try:
+ parsed_response = self.prompt_constructor.extract_action(response)
+ if self.action_set_tag == "id_accessibility_tree":
+ action = create_id_based_action(parsed_response)
+ elif self.action_set_tag == "playwright":
+ action = create_playwright_action(parsed_response)
+ else:
+ action = create_stop_action(f"ERROR: action tag not recognised: {self.action_set_tag}")
+ except ActionParsingError:
+ action = create_none_action()
+ action["raw_prediction"] = response
+ except Exception as e:
+ action = create_stop_action(f"ERROR: {str(e)}")
+ action["raw_prediction"] = response
+
+ messages = sender.chat_messages[recipient]
+ if "context" in messages[-1]:
+ m = messages[-1]["context"]
+ state_info = m["state_info"]
+ trajectory = m["trajectory"]
+ early_stop_flag, stop_info = early_stop(trajectory, self.max_steps, self.early_stop_thresholds)
+
+ if early_stop_flag:
+ action = create_stop_action(f"Early stop: {stop_info}")
+
+ action_str = get_action_description(
+ action,
+ state_info["info"]["observation_metadata"],
+ action_set_tag=self.action_set_tag,
+ prompt_constructor=self.prompt_constructor,
+ )
+ return {"content": action_str, "context": action}
+ return {"content": "", "context": action}