A programming framework for agentic AI
add webarena in samples (#2114)
olgavrouaf9b300be39b
diff --git a/samples/tools/webarena/README.md b/samples/tools/webarena/README.mdnew file mode 100644index 000000000..9c1841629--- /dev/null+++ b/samples/tools/webarena/README.md@@ -0,0 +1,32 @@+# WebArena Benchmark++This directory helps run AutoGen agents on the [WebArena](https://arxiv.org/pdf/2307.13854.pdf) benchmark.++## Installing WebArena++WebArena can be installed by following the instructions from [WebArena's GitHub repository]([email protected]:web-arena-x/webarena.git)++If using WebArena with AutoGen there is a clash on the versions of OpenAI and some code changes are needed in WebArena to be compatible with AutoGen's OpenAI version:++- webarena's openai version is `openai==0.27.0`+- autogen's openai version is: `openai>=1.3`++Prior to installation, in the WebArena codebase, any file containing `openai.error` needs to be replaced with `openai`.++## Running with AutoGen agents++You can use the `run.py` file in the `webarena` directory to run WebArena with AutoGen. The OpenAI (or AzureOpenAI or other model) configuration can be setup via `OAI_CONFIG_LIST`. The config list will be filtered by whatever model is passed in the `--model` argument.++e.g. of running `run.py`:++```+mkdir myresultdir+python run.py --instruction_path agent/prompts/jsons/p_cot_id_actree_2s.json --test_start_idx 27 --test_end_idx 28 --model gpt-4 --result_dir myresultdir+```++The original `run.py` file has been modified to use AutoGen agents which are defined in the `webarena_agents.py` file.++## References+**WebArena: A Realistic Web Environment for Building Autonomous Agents**<br/>+Zhou, Shuyan and Xu, Frank F and Zhu, Hao and Zhou, Xuhui and Lo, Robert and Sridhar, Abishek and Cheng, Xianyi and Bisk, Yonatan and Fried, Daniel and Alon, Uri and others<br/>+[https://arxiv.org/pdf/2307.13854.pdf](https://arxiv.org/pdf/2307.13854.pdf)diff --git a/samples/tools/webarena/webarena/run.py b/samples/tools/webarena/webarena/run.pynew file mode 100644index 000000000..a9411ea12--- /dev/null+++ b/samples/tools/webarena/webarena/run.py@@ -0,0 +1,354 @@+"""Script to run end-to-end evaluation on the benchmark"""++import argparse+import glob+import json+import logging+import os+import random+import subprocess+import tempfile+import time+from pathlib import Path++import openai+import autogen+from webarena_agents import ActionTakingCapability, EnvironmentAgent++from agent import (+ Agent,+ PromptAgent,+ TeacherForcingAgent,+ construct_agent,+)++from agent.prompts import *+from browser_env import (+ ScriptBrowserEnv,+)+from browser_env.auto_login import get_site_comb_from_filepath+from browser_env.helper_functions import (+ RenderHelper,+)+from evaluation_harness import evaluator_router++LOG_FOLDER = "log_files"+Path(LOG_FOLDER).mkdir(parents=True, exist_ok=True)+LOG_FILE_NAME = f"{LOG_FOLDER}/log_{time.strftime('%Y%m%d%H%M%S', time.localtime())}_{random.randint(0, 10000)}.log"++logger = logging.getLogger("logger")+logger.setLevel(logging.INFO)++console_handler = logging.StreamHandler()+console_handler.setLevel(logging.DEBUG)+logger.addHandler(console_handler)++file_handler = logging.FileHandler(LOG_FILE_NAME)+file_handler.setLevel(logging.DEBUG)+logger.addHandler(file_handler)++# Set the log format+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")+console_handler.setFormatter(formatter)+file_handler.setFormatter(formatter)+++def config() -> argparse.Namespace:+ parser = argparse.ArgumentParser(description="Run end-to-end evaluation on the benchmark")+ parser.add_argument("--render", action="store_true", help="Render the browser")+ parser.add_argument(+ "--slow_mo",+ type=int,+ default=0,+ help="Slow down the browser by the specified amount",+ )+ parser.add_argument("--action_set_tag", default="id_accessibility_tree", help="Action type")+ parser.add_argument(+ "--observation_type",+ choices=["accessibility_tree", "html", "image"],+ default="accessibility_tree",+ help="Observation type",+ )+ parser.add_argument(+ "--current_viewport_only",+ action="store_true",+ help="Only use the current viewport for the observation",+ )+ parser.add_argument("--viewport_width", type=int, default=1280)+ parser.add_argument("--viewport_height", type=int, default=720)+ parser.add_argument("--save_trace_enabled", action="store_true")+ parser.add_argument("--sleep_after_execution", type=float, default=0.0)++ parser.add_argument("--max_steps", type=int, default=30)++ # agent config+ parser.add_argument("--agent_type", type=str, default="prompt")+ parser.add_argument(+ "--instruction_path",+ type=str,+ default="agents/prompts/state_action_agent.json",+ )+ parser.add_argument(+ "--parsing_failure_th",+ help="When concesecutive parsing failure exceeds this threshold, the agent will stop",+ type=int,+ default=3,+ )+ parser.add_argument(+ "--repeating_action_failure_th",+ help="When concesecutive repeating action exceeds this threshold, the agent will stop",+ type=int,+ default=3,+ )++ # lm config+ parser.add_argument("--provider", type=str, default="openai")+ parser.add_argument("--model", type=str, default="gpt-3.5-turbo-0613")+ parser.add_argument("--mode", type=str, default="chat")+ parser.add_argument("--temperature", type=float, default=1.0)+ parser.add_argument("--top_p", type=float, default=0.9)+ parser.add_argument("--context_length", type=int, default=0)+ parser.add_argument("--max_tokens", type=int, default=384)+ parser.add_argument("--stop_token", type=str, default=None)+ parser.add_argument(+ "--max_retry",+ type=int,+ help="max retry times to perform generations when parsing fails",+ default=1,+ )+ parser.add_argument(+ "--max_obs_length",+ type=int,+ help="when not zero, will truncate the observation to this length before feeding to the model",+ default=1920,+ )+ parser.add_argument(+ "--model_endpoint",+ help="huggingface model endpoint",+ type=str,+ default="",+ )++ # example config+ parser.add_argument("--test_start_idx", type=int, default=0)+ parser.add_argument("--test_end_idx", type=int, default=1000)++ # logging related+ parser.add_argument("--result_dir", type=str, default="")+ args = parser.parse_args()++ # check the whether the action space is compatible with the observation space+ if args.action_set_tag == "id_accessibility_tree" and args.observation_type != "accessibility_tree":+ raise ValueError(+ f"Action type {args.action_set_tag} is incompatible with the observation type {args.observation_type}"+ )++ return args+++def test(+ args: argparse.Namespace,+ agent: Agent | PromptAgent | TeacherForcingAgent,+ config_file_list: list[str],+) -> None:+ scores = []+ max_steps = args.max_steps++ early_stop_thresholds = {+ "parsing_failure": args.parsing_failure_th,+ "repeating_action": args.repeating_action_failure_th,+ }++ env = ScriptBrowserEnv(+ headless=not args.render,+ slow_mo=args.slow_mo,+ observation_type=args.observation_type,+ current_viewport_only=args.current_viewport_only,+ viewport_size={+ "width": args.viewport_width,+ "height": args.viewport_height,+ },+ save_trace_enabled=args.save_trace_enabled,+ sleep_after_execution=args.sleep_after_execution,+ )++ for config_file in config_file_list:+ try:+ render_helper = RenderHelper(config_file, args.result_dir, args.action_set_tag)++ # get intent+ with open(config_file) as f:+ _c = json.load(f)+ intent = _c["intent"]+ task_id = _c["task_id"]+ # automatically login+ if _c["storage_state"]:+ cookie_file_name = os.path.basename(_c["storage_state"])+ comb = get_site_comb_from_filepath(cookie_file_name)+ temp_dir = tempfile.mkdtemp()+ # subprocess to renew the cookie+ subprocess.run(+ [+ "python",+ "browser_env/auto_login.py",+ "--auth_folder",+ temp_dir,+ "--site_list",+ *comb,+ ]+ )+ _c["storage_state"] = f"{temp_dir}/{cookie_file_name}"+ assert os.path.exists(_c["storage_state"])+ # update the config file+ config_file = f"{temp_dir}/{os.path.basename(config_file)}"+ with open(config_file, "w") as f:+ json.dump(_c, f)++ logger.info(f"[Config file]: {config_file}")+ logger.info(f"[Intent]: {intent}")++ agent.reset(config_file)++ env_agent = EnvironmentAgent(+ name="env_agent",+ env=env,+ config_file=config_file,+ result_dir=args.result_dir,+ action_set_tag=args.action_set_tag,+ llm_config=False,+ code_execution_config=False,+ )++ config_list = autogen.config_list_from_json(+ "OAI_CONFIG_LIST",+ filter_dict={"model": [args.model]},+ )++ action_agent = autogen.AssistantAgent(+ "action_agent",+ llm_config={"config_list": config_list, "cache_seed": 32},+ system_message="",+ )++ action_taking_capability = ActionTakingCapability(+ prompt_constructor=agent.prompt_constructor,+ action_set_tag=args.action_set_tag,+ max_steps=max_steps,+ early_stop_thresholds=early_stop_thresholds,+ )++ action_taking_capability.add_to_agent(action_agent)++ env_agent.initiate_chat(+ action_agent,+ message={+ "content": {"intent": intent},+ },+ )++ evaluator = evaluator_router(config_file)+ score = evaluator(+ trajectory=env_agent.trajectory,+ config_file=config_file,+ page=env.page,+ client=env.get_page_client(env.page),+ )++ scores.append(score)++ if score == 1:+ logger.info(f"[Result] (PASS) {config_file}")+ else:+ logger.info(f"[Result] (FAIL) {config_file}")++ if args.save_trace_enabled:+ env.save_trace(Path(args.result_dir) / "traces" / f"{task_id}.zip")++ except openai.OpenAIError as e:+ logger.info(f"[OpenAI Error] {repr(e)}")+ except Exception as e:+ logger.info(f"[Unhandled Error] {repr(e)}]")+ import traceback++ # write to error file+ with open(Path(args.result_dir) / "error.txt", "a") as f:+ f.write(f"[Config file]: {config_file}\n")+ f.write(f"[Unhandled Error] {repr(e)}\n")+ f.write(traceback.format_exc()) # write stack trace to file++ render_helper.close()++ env.close()+ if len(scores) > 0:+ logger.info(f"Average score: {sum(scores) / len(scores)}")+++def prepare(args: argparse.Namespace) -> None:+ # convert prompt python files to json+ from agent.prompts import to_json++ to_json.run()++ # prepare result dir+ result_dir = args.result_dir+ if not result_dir:+ result_dir = f"cache/results_{time.strftime('%Y%m%d%H%M%S', time.localtime())}"+ if not Path(result_dir).exists():+ Path(result_dir).mkdir(parents=True, exist_ok=True)+ args.result_dir = result_dir+ logger.info(f"Create result dir: {result_dir}")++ if not (Path(result_dir) / "traces").exists():+ (Path(result_dir) / "traces").mkdir(parents=True)++ # log the log file+ with open(os.path.join(result_dir, "log_files.txt"), "a+") as f:+ f.write(f"{LOG_FILE_NAME}\n")+++def get_unfinished(config_files: list[str], result_dir: str) -> list[str]:+ result_files = glob.glob(f"{result_dir}/*.html")+ task_ids = [os.path.basename(f).split(".")[0].split("_")[1] for f in result_files]+ unfinished_configs = []+ for config_file in config_files:+ task_id = os.path.basename(config_file).split(".")[0]+ if task_id not in task_ids:+ unfinished_configs.append(config_file)+ return unfinished_configs+++def dump_config(args: argparse.Namespace) -> None:+ config_file = Path(args.result_dir) / "config.json"+ if not config_file.exists():+ with open(config_file, "w") as f:+ json.dump(vars(args), f, indent=4)+ logger.info(f"Dump config to {config_file}")+++# def run():+if __name__ == "__main__":+ args = config()+ args.sleep_after_execution = 2.0+ prepare(args)++ test_file_list = []+ st_idx = args.test_start_idx+ ed_idx = args.test_end_idx+ for i in range(st_idx, ed_idx):+ test_file_list.append(f"config_files/{i}.json")+ if "debug" not in args.result_dir:+ test_file_list = get_unfinished(test_file_list, args.result_dir)++ if len(test_file_list) == 0:+ logger.info("No task left to run")+ else:+ print(f"Total {len(test_file_list)} tasks left")+ args.render = False+ args.render_screenshot = True+ args.save_trace_enabled = True++ args.current_viewport_only = True+ dump_config(args)++ agent = construct_agent(args)+ test(args, agent, test_file_list)diff --git a/samples/tools/webarena/webarena/webarena_agents.py b/samples/tools/webarena/webarena/webarena_agents.pynew file mode 100644index 000000000..103057259--- /dev/null+++ b/samples/tools/webarena/webarena/webarena_agents.py@@ -0,0 +1,205 @@+import autogen++from browser_env.actions import (+ ActionParsingError,+ Action,+ create_none_action,+ create_id_based_action,+ create_playwright_action,+)++from browser_env import (+ ActionTypes,+ StateInfo,+ Trajectory,+ create_stop_action,+)++from browser_env.helper_functions import (+ RenderHelper,+ get_action_description,+)++from browser_env.actions import is_equivalent+++def early_stop(trajectory: Trajectory, max_steps: int, thresholds: dict[str, int]) -> tuple[bool, str]:+ """Check whether need to early stop"""++ # reach the max step+ num_steps = (len(trajectory) - 1) / 2+ if num_steps >= max_steps:+ return True, f"Reach max steps {max_steps}"++ last_k_actions: list[Action]+ action_seq: list[Action]++ # Case: parsing failure for k times+ k = thresholds["parsing_failure"]+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]+ if len(last_k_actions) >= k:+ if all([action["action_type"] == ActionTypes.NONE for action in last_k_actions]):+ return True, f"Failed to parse actions for {k} times"++ # Case: same action for k times+ k = thresholds["repeating_action"]+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]+ action_seq = trajectory[1::2] # type: ignore[assignment]++ if len(action_seq) == 0:+ return False, ""++ last_action: Action = action_seq[-1]++ if last_action["action_type"] != ActionTypes.TYPE:+ if len(last_k_actions) >= k:+ if all([is_equivalent(action, last_action) for action in last_k_actions]):+ return True, f"Same action for {k} times"++ else:+ # check the action sequence+ if sum([is_equivalent(action, last_action) for action in action_seq]) >= k:+ return True, f"Same typing action for {k} times"++ return False, ""+++class EnvironmentAgent(autogen.ConversableAgent):+ def __init__(+ self,+ env,+ config_file,+ result_dir,+ action_set_tag,+ *args,+ **kwargs,+ ):+ super().__init__(*args, **kwargs)+ self.env = env+ self.trajectory: Trajectory = []+ obs, info = env.reset(options={"config_file": config_file})+ self.state_info: StateInfo = {"observation": obs, "info": info}+ self.trajectory.append(self.state_info)+ self.meta_data = {"action_history": ["None"]}+ self.action_set_tag = action_set_tag+ self.render_helper = RenderHelper(config_file, result_dir, action_set_tag)+ self.register_reply([autogen.Agent, None], EnvironmentAgent.generate_env_reply, position=1)+ self.register_hook(+ hookable_method="process_message_before_send",+ hook=self.process_message_before_send,+ )++ def process_message_before_send(self, sender, message, recipient, silent):+ if "context" in message:+ raise ValueError("Message should not contain context freom generate_reply")++ message["context"] = {+ "trajectory": self.trajectory,+ "meta_data": self.meta_data,+ "state_info": self.state_info,+ }++ if "intent" in message["content"]:+ intent = message["content"]["intent"]+ obs = self.state_info["observation"]["text"]+ message["content"] = f"{obs} \n\n {intent}"+ message["context"]["intent"] = intent+ return message++ def generate_env_reply(+ self,+ messages=None,+ sender=None,+ config=None,+ ):+ if messages is None:+ messages = self.chat_messages[sender]++ action = messages[-1]["context"]+ action_str = messages[-1]["content"]+ self.trajectory.append(action)++ self.render_helper.render(action, self.state_info, self.meta_data, True)+ self.meta_data["action_history"].append(action_str)++ if action["action_type"] == ActionTypes.STOP:+ return True, {"content": "TERMINATE"}++ obs, _, terminated, _, info = self.env.step(action)+ self.state_info = {"observation": obs, "info": info}+ self.trajectory.append(self.state_info)++ if terminated:+ self.trajectory.append(create_stop_action(""))+ return True, "TERMINATE"++ return True, {"content": obs["text"]}+++class ActionTakingCapability:+ def __init__(self, prompt_constructor, action_set_tag, max_steps, early_stop_thresholds):+ self.action_set_tag = action_set_tag+ self.prompt_constructor = prompt_constructor+ self.max_steps = max_steps+ self.early_stop_thresholds = early_stop_thresholds++ def add_to_agent(self, agent):+ agent.register_hook(+ hookable_method="process_all_messages_before_reply",+ hook=self.process_all_messages_before_reply,+ )+ agent.register_hook(+ hookable_method="process_message_before_send",+ hook=self.process_message_before_send,+ )++ def process_all_messages_before_reply(self, messages):+ if "context" in messages[-1]:+ m = messages[-1]["context"]+ trajectory = m["trajectory"]+ meta_data = m["meta_data"]+ intent = messages[0]["context"]["intent"]+ else:+ if "TERMINATE" in messages[-1]["content"]:+ return messages++ # take action+ prompt = self.prompt_constructor.construct(trajectory, intent, meta_data)+ return prompt++ def process_message_before_send(self, sender, message, recipient, silent):+ force_prefix = self.prompt_constructor.instruction["meta_data"].get("force_prefix", "")+ response = f"{force_prefix}{message}"+ try:+ parsed_response = self.prompt_constructor.extract_action(response)+ if self.action_set_tag == "id_accessibility_tree":+ action = create_id_based_action(parsed_response)+ elif self.action_set_tag == "playwright":+ action = create_playwright_action(parsed_response)+ else:+ action = create_stop_action(f"ERROR: action tag not recognised: {self.action_set_tag}")+ except ActionParsingError:+ action = create_none_action()+ action["raw_prediction"] = response+ except Exception as e:+ action = create_stop_action(f"ERROR: {str(e)}")+ action["raw_prediction"] = response++ messages = sender.chat_messages[recipient]+ if "context" in messages[-1]:+ m = messages[-1]["context"]+ state_info = m["state_info"]+ trajectory = m["trajectory"]+ early_stop_flag, stop_info = early_stop(trajectory, self.max_steps, self.early_stop_thresholds)++ if early_stop_flag:+ action = create_stop_action(f"Early stop: {stop_info}")++ action_str = get_action_description(+ action,+ state_info["info"]["observation_metadata"],+ action_set_tag=self.action_set_tag,+ prompt_constructor=self.prompt_constructor,+ )+ return {"content": action_str, "context": action}+ return {"content": "", "context": action}