A programming framework for agentic AI
Moved the benchmarks to another repo. (#349)
afourney7b99b84dabef
diff --git a/python/benchmarks/GAIA/ENV.json.sample b/python/benchmarks/GAIA/ENV.json.sampledeleted file mode 100644index 468df6813..000000000--- a/python/benchmarks/GAIA/ENV.json.sample+++ /dev/null@@ -1,7 +0,0 @@-{- "CHAT_COMPLETION_PROVIDER": "azure",- "CHAT_COMPLETION_KWARGS_JSON": "{\"api_version\": \"2024-05-01-preview\", \"azure_endpoint\": \"YOUR_ENDPOINT_HERE\", \"model_capabilities\": {\"function_calling\": true, \"json_output\": true, \"vision\": true}, \"model\": \"gpt-4o-2024-05-13\"}",- "BING_API_KEY": "YOUR_KEY_KEY",- "HOMEPAGE": "https://www.bing.com/",- "WEB_SURFER_DEBUG_DIR": "/autogen/debug"-}diff --git a/python/benchmarks/GAIA/Scripts/custom_tabulate.py b/python/benchmarks/GAIA/Scripts/custom_tabulate.pydeleted file mode 100644index ec51863e9..000000000--- a/python/benchmarks/GAIA/Scripts/custom_tabulate.py+++ /dev/null@@ -1,197 +0,0 @@-import os-import sys-import re-from agbench.tabulate_cmd import default_tabulate-import json-import pandas as pd-import sqlite3-import glob-import numpy as np--EXCLUDE_DIR_NAMES = ["__pycache__"]---def normalize_answer(a):- # Lower case- # Trim (left and right)- # standardize comma separated values- # Replace multiple spaces with one space- # Remove trailing punctuation- norm_answer = ", ".join(a.strip().lower().split(","))- norm_answer = re.sub(r"[\.\!\?]+$", "", re.sub(r"\s+", " ", norm_answer))- return norm_answer---def scorer(instance_dir):- # Read the expected answer- expected_answer_file = os.path.join(instance_dir, "expected_answer.txt")- if not os.path.isfile(expected_answer_file):- return None-- expected_answer = None- with open(expected_answer_file, "rt") as fh:- expected_answer = fh.read().strip()-- # Read the console- console_log_file = os.path.join(instance_dir, "console_log.txt")- if not os.path.isfile(console_log_file):- return None-- console_log = ""- with open(console_log_file, "rt") as fh:- console_log = fh.read()-- final_answer = None- m = re.search(r"FINAL ANSWER:(.*?)\n", console_log, re.DOTALL)- if m:- final_answer = m.group(1).strip()-- # Missing the final answer line- if final_answer is None:- return None-- # Return true if they are equal after normalization- n_ex = normalize_answer(expected_answer)- n_final = normalize_answer(final_answer)- return (- (n_ex != "" and n_ex == n_final),- n_ex,- n_final- )---def get_number_of_chat_messages(chat_messages_dir):- result = 0- for file in glob.glob(f"{chat_messages_dir}/*_messages.json"):- with open(file, "r") as f:- content = json.load(f)- for agent, messages in content.items():- result += len(messages)- return result---def main(args):- parsed_args, all_results = default_tabulate(args, scorer=scorer)- excel_path = parsed_args.excel-- if excel_path:- excel_dir = os.path.dirname(excel_path) or "."- if not os.path.exists(excel_dir):- os.makedirs(excel_dir, exist_ok=True)-- if not excel_path.endswith((".xlsx", ".xls")):- excel_path += ".xlsx"-- runlogs = parsed_args.runlogs if parsed_args.runlogs.endswith("/") else parsed_args.runlogs + "/"-- if os.path.isdir(runlogs):- task_ids = sorted(- [task_id for task_id in os.listdir(runlogs) if task_id not in EXCLUDE_DIR_NAMES],- key=lambda s: os.path.getmtime(os.path.join(parsed_args.runlogs, s)),- )- else:- raise ValueError("please input a valid directory to tabulate result")-- trials = sorted(os.listdir(f"{runlogs}{task_ids[0]}"), key=lambda x: int(x)) if len(task_ids) > 0 else []- dbnames = [[f"{runlogs}{task_id}/{trial}/telemetry.db" for task_id in task_ids] for trial in trials]-- query = """- SELECT cost, session_id, response, start_time, end_time- FROM (- SELECT invocation_id, cost, session_id, response, start_time, end_time,- ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY start_time) as rn- FROM chat_completions- )- WHERE rn = 1;- """-- with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:- for trial_index, each_trial in enumerate(dbnames):- result_df = pd.DataFrame(- columns=[- "id",- "status",- "expected_answer",- "final_answer",- "cost",- "latency",- "num_of_llm_requests",- "num_of_chat_messages",- "prompt_tokens",- "completion_tokens",- "total_tokens",- "model",- ]- )-- result_df_type_mapping = {- "id": str,- "status": bool,- "expected_answer": str,- "final_answer": str,- "cost": float,- "latency": float,- "num_of_llm_requests": int,- "num_of_chat_messages": int,- "prompt_tokens": int,- "completion_tokens": int,- "total_tokens": int,- }-- for dbname, scorer_results in zip(each_trial, all_results):- task_id = scorer_results[0]- scorer_result = scorer_results[trial_index + 1]-- status, expected_answer, final_answer = scorer_result if scorer_result else (False,"","")-- con = sqlite3.connect(dbname)-- # TODO: if large amount of data, add chunksize- telemetry_df = pd.read_sql_query(query, con)-- earliest_starttime = pd.to_datetime(telemetry_df["start_time"], format="%Y-%m-%d %H:%M:%S.%f").min()- latest_endtime = pd.to_datetime(telemetry_df["end_time"], format="%Y-%m-%d %H:%M:%S.%f").max()-- num_of_chat_messages = get_number_of_chat_messages(chat_messages_dir=os.path.dirname(dbname))- result = {- "id": task_id,- "status": status,- "expected_answer": expected_answer,- "final_answer": final_answer,- "cost": telemetry_df["cost"].sum(),- "latency": (latest_endtime - earliest_starttime).total_seconds(),- "num_of_llm_requests": len(telemetry_df),- "num_of_chat_messages": num_of_chat_messages,- "prompt_tokens": telemetry_df["response"]- .apply(- lambda x: json.loads(x)["usage"]["prompt_tokens"]- if "usage" in json.loads(x) and "prompt_tokens" in json.loads(x)["usage"]- else 0- )- .sum(),- "completion_tokens": telemetry_df["response"]- .apply(- lambda x: json.loads(x)["usage"]["completion_tokens"]- if "usage" in json.loads(x) and "completion_tokens" in json.loads(x)["usage"]- else 0- )- .sum(),- "total_tokens": telemetry_df["response"]- .apply(- lambda x: json.loads(x)["usage"]["total_tokens"]- if "usage" in json.loads(x) and "total_tokens" in json.loads(x)["usage"]- else 0- )- .sum(),- "model": telemetry_df["response"]- .apply(lambda x: json.loads(x)["model"] if "model" in json.loads(x) else "")- .unique(),- }-- result_df = result_df.astype(result_df_type_mapping)- result_df = pd.concat([result_df, pd.DataFrame([result])], ignore_index=True)- result_df.to_excel(writer, sheet_name=f"trial_{trial_index}", index=False)---if __name__ == "__main__" and __package__ is None:- main(sys.argv)diff --git a/python/benchmarks/GAIA/Scripts/init_tasks.py b/python/benchmarks/GAIA/Scripts/init_tasks.pydeleted file mode 100644index 3db67a37f..000000000--- a/python/benchmarks/GAIA/Scripts/init_tasks.py+++ /dev/null@@ -1,157 +0,0 @@-#-# Run this file to download the human_eval dataset, and create a corresponding testbed scenario:-# (default: ../scenarios/human_eval_two_agents_gpt4.jsonl and ./scenarios/human_eval_two_agents_gpt35.jsonl)-#--import json-import os-import re-import sys--from huggingface_hub import snapshot_download--SCRIPT_PATH = os.path.realpath(__file__)-SCRIPT_NAME = os.path.basename(SCRIPT_PATH)-SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)--SCENARIO_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.path.pardir))-TEMPLATES_DIR = os.path.join(SCENARIO_DIR, "Templates")-TASKS_DIR = os.path.join(SCENARIO_DIR, "Tasks")-DOWNLOADS_DIR = os.path.join(SCENARIO_DIR, "Downloads")-REPO_DIR = os.path.join(DOWNLOADS_DIR, "GAIA")---def download_gaia():- """Download the GAIA benchmark from Hugging Face."""-- if not os.path.isdir(DOWNLOADS_DIR):- os.mkdir(DOWNLOADS_DIR)-- """Download the GAIA dataset from Hugging Face Hub"""- snapshot_download(- repo_id="gaia-benchmark/GAIA",- repo_type="dataset",- local_dir=REPO_DIR,- local_dir_use_symlinks=True,- )---def create_jsonl(name, tasks, files_dir, template):- """Creates a JSONL scenario file with a given name, and template path."""-- if not os.path.isdir(TASKS_DIR):- os.mkdir(TASKS_DIR)-- with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:- for task in tasks:- print(f"Converting: [{name}] {task['task_id']}")-- # Figure out what files we need to copy- template_cp_list = [template]- if len(task["file_name"].strip()) > 0:- template_cp_list.append(- [- os.path.join(files_dir, task["file_name"].strip()),- task["file_name"].strip(),- #os.path.join("coding", task["file_name"].strip()),- ]- )-- record = {- "id": task["task_id"],- "template": template_cp_list,- "substitutions": {- "scenario.py": {- "__FILE_NAME__": task["file_name"],- },- "expected_answer.txt": {"__EXPECTED_ANSWER__": task["Final answer"]},- "prompt.txt": {"__PROMPT__": task["Question"]},- },- }-- fh.write(json.dumps(record).strip() + "\n")---###############################################################################-def main():- download_gaia()-- gaia_validation_files = os.path.join(REPO_DIR, "2023", "validation")- gaia_test_files = os.path.join(REPO_DIR, "2023", "test")-- if not os.path.isdir(gaia_validation_files) or not os.path.isdir(gaia_test_files):- sys.exit(f"Error: '{REPO_DIR}' does not appear to be a copy of the GAIA repository.")-- # Load the GAIA data- gaia_validation_tasks = [[], [], []]- with open(os.path.join(gaia_validation_files, "metadata.jsonl")) as fh:- for line in fh:- data = json.loads(line)- gaia_validation_tasks[data["Level"] - 1].append(data)-- gaia_test_tasks = [[], [], []]- with open(os.path.join(gaia_test_files, "metadata.jsonl")) as fh:- for line in fh:- data = json.loads(line)-- # A welcome message -- not a real task- if data["task_id"] == "0-0-0-0-0":- continue-- gaia_test_tasks[data["Level"] - 1].append(data)-- # list all directories in the Templates directory- # and populate a dictionary with the name and path- templates = {}- for entry in os.scandir(TEMPLATES_DIR):- if entry.is_dir():- templates[re.sub(r"\s", "", entry.name)] = entry.path-- # Add coding directories if needed (these are usually empty and left out of the repo)- #for template in templates.values():- # code_dir_path = os.path.join(template, "coding")- # if not os.path.isdir(code_dir_path):- # os.mkdir(code_dir_path)-- # Create the various combinations of [models] x [templates]- for t in templates.items():- create_jsonl(- f"gaia_validation_level_1__{t[0]}",- gaia_validation_tasks[0],- gaia_validation_files,- t[1],- )- create_jsonl(- f"gaia_validation_level_2__{t[0]}",- gaia_validation_tasks[1],- gaia_validation_files,- t[1],- )- create_jsonl(- f"gaia_validation_level_3__{t[0]}",- gaia_validation_tasks[2],- gaia_validation_files,- t[1],- )- create_jsonl(- f"gaia_test_level_1__{t[0]}",- gaia_test_tasks[0],- gaia_test_files,- t[1],- )- create_jsonl(- f"gaia_test_level_2__{t[0]}",- gaia_test_tasks[1],- gaia_test_files,- t[1],- )- create_jsonl(- f"gaia_test_level_3__{t[0]}",- gaia_test_tasks[2],- gaia_test_files,- t[1],- )---if __name__ == "__main__" and __package__ is None:- main()diff --git a/python/benchmarks/GAIA/Templates/TeamOne/expected_answer.txt b/python/benchmarks/GAIA/Templates/TeamOne/expected_answer.txtdeleted file mode 100644index 8153c2bf8..000000000--- a/python/benchmarks/GAIA/Templates/TeamOne/expected_answer.txt+++ /dev/null@@ -1 +0,0 @@-__EXPECTED_ANSWER__diff --git a/python/benchmarks/GAIA/Templates/TeamOne/prompt.txt b/python/benchmarks/GAIA/Templates/TeamOne/prompt.txtdeleted file mode 100644index 482f50dca..000000000--- a/python/benchmarks/GAIA/Templates/TeamOne/prompt.txt+++ /dev/null@@ -1 +0,0 @@-__PROMPT__diff --git a/python/benchmarks/GAIA/Templates/TeamOne/requirements.txt b/python/benchmarks/GAIA/Templates/TeamOne/requirements.txtdeleted file mode 100644index ffb79dd43..000000000--- a/python/benchmarks/GAIA/Templates/TeamOne/requirements.txt+++ /dev/null@@ -1 +0,0 @@-/agnext/teams/team-onediff --git a/python/benchmarks/GAIA/Templates/TeamOne/scenario.py b/python/benchmarks/GAIA/Templates/TeamOne/scenario.pydeleted file mode 100644index 1a84bee76..000000000--- a/python/benchmarks/GAIA/Templates/TeamOne/scenario.py+++ /dev/null@@ -1,217 +0,0 @@-import asyncio-import logging-import os--from openai import AzureOpenAI--from typing import List-from agnext.application import SingleThreadedAgentRuntime-from agnext.application.logging import EVENT_LOGGER_NAME-from agnext.components.models import (- AzureOpenAIChatCompletionClient,- ChatCompletionClient,- ModelCapabilities,- UserMessage,- LLMMessage,-)-from agnext.components.code_executor import LocalCommandLineCodeExecutor-from agnext.application.logging import EVENT_LOGGER_NAME-from team_one.markdown_browser import MarkdownConverter, UnsupportedFormatException-from team_one.agents.coder import Coder, Executor-from team_one.agents.orchestrator import LedgerOrchestrator-from team_one.messages import BroadcastMessage-from team_one.agents.multimodal_web_surfer import MultimodalWebSurfer-from team_one.agents.file_surfer import FileSurfer-from team_one.utils import LogHandler, message_content_to_str--import re-import tiktoken--from agnext.components.models import AssistantMessage--encoding = None-def count_token(value: str) -> int:- # TODO:: Migrate to model_client.count_tokens- global encoding- if encoding is None:- encoding = tiktoken.encoding_for_model("gpt-4o-2024-05-13")- return len(encoding.encode(value))--async def response_preparer(task: str, source: str, client: ChatCompletionClient, transcript: List[LLMMessage]) -> str:- messages: List[LLMMessage] = []-- # copy them to this context- for message in transcript:- messages.append(- UserMessage(- content = message_content_to_str(message.content),- # TODO fix this -> remove type ignore- source=message.source, # type: ignore- )- )-- # Remove messages until we are within 2k of the context window limit- while len(messages) and client.remaining_tokens( messages ) < 2000:- messages.pop(0)-- # Add the preamble- messages.insert(0,- UserMessage(- content=f"Earlier you were asked the following:\n\n{task}\n\nYour team then worked diligently to address that request. Here is a transcript of that conversation:",- source=source,- )- )-- # ask for the final answer- messages.append(- UserMessage(- content= f"""-Read the above conversation and output a FINAL ANSWER to the question. The question is repeated here for convenience:--{task}--To output the final answer, use the following template: FINAL ANSWER: [YOUR FINAL ANSWER]-Your FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.-ADDITIONALLY, your FINAL ANSWER MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)-If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.-If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.-If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.-If you are unable to determine the final answer, output 'FINAL ANSWER: Unable to determine'-""",- source=source,- )- )--- response = await client.create(messages)- assert isinstance(response.content, str)-- # No answer- if "unable to determine" in response.content.lower():- messages.append( AssistantMessage(content=response.content, source="self" ) )- messages.append(- UserMessage(- content= f"""-I understand that a definitive answer could not be determined. Please make a well-informed EDUCATED GUESS based on the conversation.--To output the educated guess, use the following template: EDUCATED GUESS: [YOUR EDUCATED GUESS]-Your EDUCATED GUESS should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. DO NOT OUTPUT 'I don't know', 'Unable to determine', etc.-ADDITIONALLY, your EDUCATED GUESS MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)-If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.-If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.-If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.-""".strip(),- source=source,- )- )-- response = await client.create(messages)- assert isinstance(response.content, str)- return re.sub(r"EDUCATED GUESS:", "FINAL ANSWER:", response.content)-- else:- return response.content---async def main() -> None:- # Read the prompt- prompt = ""- with open("prompt.txt", "rt") as fh:- prompt = fh.read().strip()- filename = "__FILE_NAME__".strip()-- # Create the runtime.- runtime = SingleThreadedAgentRuntime()-- # Create the AzureOpenAI client, with AAD auth- client = AzureOpenAIChatCompletionClient(- api_version="2024-02-15-preview",- azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",- model="gpt-4o-2024-05-13",- temperature=0.1,- model_capabilities=ModelCapabilities(- function_calling=True, json_output=True, vision=True- ),- )-- mlm_client = AzureOpenAI(- api_version="2024-02-15-preview",- azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",- )-- # Register agents.- coder = await runtime.register_and_get_proxy(- "Assistant",- lambda: Coder(model_client=client),- )-- executor = await runtime.register_and_get_proxy(- "ComputerTerminal",- lambda: Executor(executor=LocalCommandLineCodeExecutor()),- )-- file_surfer = await runtime.register_and_get_proxy(- "FileSurfer",- lambda: FileSurfer(model_client=client),- )-- web_surfer = await runtime.register_and_get_proxy(- "WebSurfer",- lambda: MultimodalWebSurfer(), # Configuration is set later by init()- )-- orchestrator = await runtime.register_and_get_proxy("Orchestrator", lambda: LedgerOrchestrator(- agents=[coder, executor, file_surfer, web_surfer],- model_client=client,- max_rounds=30,- max_time=25*60,- ))-- run_context = runtime.start()-- actual_surfer = await runtime.try_get_underlying_agent_instance(web_surfer.id, type=MultimodalWebSurfer)- await actual_surfer.init(model_client=client, downloads_folder=os.getcwd(), browser_channel="chromium")-- filename_prompt = ""- if len(filename) > 0:- #relpath = os.path.join("coding", filename)- #file_uri = pathlib.Path(os.path.abspath(os.path.expanduser(relpath))).as_uri()-- filename_prompt = f"The question is about a file, document or image, which can be accessed by the filename '{filename}' in the current working directory."-- mlm_prompt = f"""Write a detailed caption for this image. Pay special attention to any details that might be useful for someone answering the following:--{prompt}-""".strip()-- try:- mdconverter = MarkdownConverter(mlm_client=mlm_client, mlm_model="gpt-4o-2024-05-13")- res = mdconverter.convert(filename, mlm_prompt=mlm_prompt)- if res.text_content:- if count_token(res.text_content) < 8000: # Don't put overly-large documents into the prompt- filename_prompt += "\n\nHere are the file's contents:\n\n" + res.text_content- except UnsupportedFormatException:- pass-- task = f"{prompt}\n\n{filename_prompt}"-- await runtime.publish_message(- BroadcastMessage(content=UserMessage(content=task.strip(), source="human")),- namespace="default",- )-- await run_context.stop_when_idle()-- # Output the final answer- actual_orchestrator = await runtime.try_get_underlying_agent_instance(orchestrator.id, type=LedgerOrchestrator)- transcript: List[LLMMessage] = actual_orchestrator._chat_history # type: ignore- print(await response_preparer(task=task, source=(await orchestrator.metadata)["type"], client=client, transcript=transcript))----if __name__ == "__main__":- logger = logging.getLogger(EVENT_LOGGER_NAME)- logger.setLevel(logging.INFO)- log_handler = LogHandler()- logger.handlers = [log_handler]- asyncio.run(main())diff --git a/python/benchmarks/HumanEval/README.md b/python/benchmarks/HumanEval/README.mddeleted file mode 100644index 61a33fb05..000000000--- a/python/benchmarks/HumanEval/README.md+++ /dev/null@@ -1,21 +0,0 @@-# HumanEval Benchmark--This scenario implements a modified version of the [HumanEval](https://arxiv.org/abs/2107.03374) benchmark.-Compared to the original benchmark, there are **two key differences** here:--- A chat model rather than a completion model is used.-- The agents get pass/fail feedback about their implementations, and can keep trying until they succeed or run out of tokens or turns.--## Running the tasks--```-autogenbench run Tasks/human_eval_two_agents.jsonl-autogenbench tabulate Results/human_eval_two_agents-```--For faster development and iteration, a reduced HumanEval set is available via `Tasks/r_human_eval_two_agents.jsonl`, and contains only 26 problems of varying difficulty.--## References-**Evaluating Large Language Models Trained on Code**<br/>-Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba<br/>-[https://arxiv.org/abs/2107.03374](https://arxiv.org/abs/2107.03374)diff --git a/python/benchmarks/HumanEval/Scripts/custom_tabulate.py b/python/benchmarks/HumanEval/Scripts/custom_tabulate.pydeleted file mode 100644index 8d689a9f1..000000000--- a/python/benchmarks/HumanEval/Scripts/custom_tabulate.py+++ /dev/null@@ -1,12 +0,0 @@-import os-import sys--from agbench.tabulate_cmd import default_tabulate---def main(args):- default_tabulate(args)---if __name__ == "__main__" and __package__ is None:- main(sys.argv)diff --git a/python/benchmarks/HumanEval/Scripts/init_tasks.py b/python/benchmarks/HumanEval/Scripts/init_tasks.pydeleted file mode 100644index df4e6b194..000000000--- a/python/benchmarks/HumanEval/Scripts/init_tasks.py+++ /dev/null@@ -1,124 +0,0 @@-#-# Run this file to download the human_eval dataset, and create a corresponding testbed scenario:-# (default: ../scenarios/human_eval_two_agents_gpt4.jsonl and ./scenarios/human_eval_two_agents_gpt35.jsonl)-#--import base64-import gzip-import io-import json-import os-import re--import requests--URL = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz"--SCRIPT_PATH = os.path.realpath(__file__)-SCRIPT_NAME = os.path.basename(SCRIPT_PATH)-SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)--SCENARIO_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.path.pardir))-TEMPLATES_DIR = os.path.join(SCENARIO_DIR, "Templates")-TASKS_DIR = os.path.join(SCENARIO_DIR, "Tasks")--# A selected subset of HumanEval problems to work with during development--# Deprecated 2/5/2024 -- Use subsample instead-REDUCED_SET = [- "HumanEval/2",- "HumanEval/26",- "HumanEval/32",- "HumanEval/33",- "HumanEval/36",- "HumanEval/38",- "HumanEval/41",- "HumanEval/50",- "HumanEval/56",- "HumanEval/65",- "HumanEval/67",- "HumanEval/84",- "HumanEval/85",- "HumanEval/86",- "HumanEval/89",- "HumanEval/99",- "HumanEval/104",- "HumanEval/113",- "HumanEval/115",- "HumanEval/120",- "HumanEval/124",- "HumanEval/126",- "HumanEval/132",- "HumanEval/135",- "HumanEval/140",- "HumanEval/146",-]---def download_human_eval():- """Download the HumanEval dataset, un-gzips it, and returns a list of its parsed JSON objects."""-- # Send a HTTP request to the URL of the file- response = requests.get(URL)-- # Ensure we raise an error if the download failed- response.raise_for_status()-- # Create a BytesIO object from the response content- buffer = io.BytesIO(response.content)-- # Read the file, line by line, populating a list of parsed JSON objects- results = []- with gzip.GzipFile(fileobj=buffer) as f_in:- for line in f_in:- # Parse each line as JSON- results.append(json.loads(line))-- return results---def create_jsonl(name, tasks, template):- """Creates a JSONL scenario file with a given name, list of HumanEval tasks, and template path."""-- # Create a task directory if it doesn't exist- if not os.path.isdir(TASKS_DIR):- os.mkdir(TASKS_DIR)-- # Create the jsonl file- with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:- for task in tasks:- print(f"Converting: [{name}] {task['task_id']}")-- record = {- "id": task["task_id"].replace("/", "_"),- "template": template,- "substitutions": {- "scenario.py": {"__ENTRY_POINT__": task["entry_point"]},- "prompt.txt": {"__PROMPT__": task["prompt"]},- "unit_tests.py": {"__TEST__": task["test"]},- },- }-- fh.write(json.dumps(record).strip() + "\n")---###############################################################################-def main():- human_eval = download_human_eval()- # Deprecated: reduced_human_eval = [t for t in human_eval if t["task_id"] in REDUCED_SET]-- # list all directories in the Templates directory- # and populate a dictionary with the name and path- templates = {}- for entry in os.scandir(TEMPLATES_DIR):- if entry.is_dir():- templates[re.sub(r"\s", "", entry.name)] = entry.path-- # Create the various combinations of [models] x [templates]- for t in templates.items():- create_jsonl(f"human_eval_{t[0]}", human_eval, t[1])- # Deprecated: create_jsonl(f"r_human_eval_{t[0]}", reduced_human_eval, t[1])---if __name__ == "__main__" and __package__ is None:- main()diff --git a/python/benchmarks/HumanEval/Templates/TeamOne/prompt.txt b/python/benchmarks/HumanEval/Templates/TeamOne/prompt.txtdeleted file mode 100644index 482f50dca..000000000--- a/python/benchmarks/HumanEval/Templates/TeamOne/prompt.txt+++ /dev/null@@ -1 +0,0 @@-__PROMPT__diff --git a/python/benchmarks/HumanEval/Templates/TeamOne/requirements.txt b/python/benchmarks/HumanEval/Templates/TeamOne/requirements.txtdeleted file mode 100644index ffb79dd43..000000000--- a/python/benchmarks/HumanEval/Templates/TeamOne/requirements.txt+++ /dev/null@@ -1 +0,0 @@-/agnext/teams/team-onediff --git a/python/benchmarks/HumanEval/Templates/TeamOne/scenario.py b/python/benchmarks/HumanEval/Templates/TeamOne/scenario.pydeleted file mode 100644index 8f9924c53..000000000--- a/python/benchmarks/HumanEval/Templates/TeamOne/scenario.py+++ /dev/null@@ -1,106 +0,0 @@-import asyncio-import logging--# from typing import Any, Dict, List, Tuple, Union--from agnext.application import SingleThreadedAgentRuntime-from agnext.components.models import (- AzureOpenAIChatCompletionClient,- ModelCapabilities,- UserMessage,-)-from agnext.components.code_executor import LocalCommandLineCodeExecutor-from agnext.application.logging import EVENT_LOGGER_NAME--from team_one.agents.coder import Coder, Executor-from team_one.agents.orchestrator import RoundRobinOrchestrator-from team_one.messages import BroadcastMessage, OrchestrationEvent---async def main() -> None:- # Create the runtime.- runtime = SingleThreadedAgentRuntime()-- # Create the AzureOpenAI client, with AAD auth- # token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")- client = AzureOpenAIChatCompletionClient(- api_version="2024-02-15-preview",- azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",- model="gpt-4o-2024-05-13",- model_capabilities=ModelCapabilities(- function_calling=True, json_output=True, vision=True- ),- # azure_ad_token_provider=token_provider- )-- # Register agents.- coder = await runtime.register_and_get_proxy(- "Coder",- lambda: Coder(model_client=client),- )- executor = await runtime.register_and_get_proxy(- "Executor",- lambda: Executor(- "A agent for executing code", executor=LocalCommandLineCodeExecutor()- ),- )-- await runtime.register("orchestrator", lambda: RoundRobinOrchestrator([coder, executor]))-- prompt = ""- with open("prompt.txt", "rt") as fh:- prompt = fh.read()-- entry_point = "__ENTRY_POINT__"-- task = f"""-The following python code imports the `run_tests` function from unit_tests.py, and runs-it on the function `{entry_point}`. This will run a set of automated unit tests to verify the-correct implementation of `{entry_point}`. However, `{entry_point}` is only partially-implemented in the code below. Complete the implementation of `{entry_point}` and then execute-a new stand-alone code block that contains everything needed to run the tests, including: importing-`unit_tests`, calling `run_tests({entry_point})`, as well as {entry_point}'s complete definition,-such that this code block can be run directly in Python.--```python-from unit_tests import run_tests--{prompt}--# Run the unit tests-run_tests({entry_point})-```-""".strip()-- run_context = runtime.start()-- await runtime.publish_message(- BroadcastMessage(content=UserMessage(content=task, source="human")),- namespace="default",- )-- await run_context.stop_when_idle()---class MyHandler(logging.Handler):- def __init__(self) -> None:- super().__init__()-- def emit(self, record: logging.LogRecord) -> None:- try:- if isinstance(record.msg, OrchestrationEvent):- print(f"""----------------------------------------------------------------------------\033[91m{record.msg.source}:\033[0m--{record.msg.message}""", flush=True)- except Exception:- self.handleError(record)---if __name__ == "__main__":-- logger = logging.getLogger(EVENT_LOGGER_NAME)- logger.setLevel(logging.INFO)- my_handler = MyHandler()- logger.handlers = [my_handler]- asyncio.run(main())diff --git a/python/benchmarks/HumanEval/Templates/TeamOne/unit_tests.py b/python/benchmarks/HumanEval/Templates/TeamOne/unit_tests.pydeleted file mode 100644index 55be54d41..000000000--- a/python/benchmarks/HumanEval/Templates/TeamOne/unit_tests.py+++ /dev/null@@ -1,15 +0,0 @@-# Disable ruff linter for template files-# ruff: noqa: F821 E722-import sys---__TEST__---def run_tests(candidate):- try:- check(candidate)- # We can search for this string in the output- print("ALL TESTS PASSED !#!#")- except AssertionError:- sys.exit("SOME TESTS FAILED - TRY AGAIN !#!#")diff --git a/python/benchmarks/HumanEval/Templates/TwoAgents/prompt.txt b/python/benchmarks/HumanEval/Templates/TwoAgents/prompt.txtdeleted file mode 100644index 482f50dca..000000000--- a/python/benchmarks/HumanEval/Templates/TwoAgents/prompt.txt+++ /dev/null@@ -1 +0,0 @@-__PROMPT__diff --git a/python/benchmarks/HumanEval/Templates/TwoAgents/requirements.txt b/python/benchmarks/HumanEval/Templates/TwoAgents/requirements.txtdeleted file mode 100644index a26b4b119..000000000--- a/python/benchmarks/HumanEval/Templates/TwoAgents/requirements.txt+++ /dev/null@@ -1 +0,0 @@-/agnextdiff --git a/python/benchmarks/HumanEval/Templates/TwoAgents/scenario.py b/python/benchmarks/HumanEval/Templates/TwoAgents/scenario.pydeleted file mode 100644index fedc66484..000000000--- a/python/benchmarks/HumanEval/Templates/TwoAgents/scenario.py+++ /dev/null@@ -1,266 +0,0 @@-import asyncio-import re-import uuid-from dataclasses import dataclass-from typing import Dict, List, Union--from agnext.application import SingleThreadedAgentRuntime-from agnext.components import TypeRoutedAgent, message_handler-from agnext.components.code_executor import (- CodeBlock,- CodeExecutor,- LocalCommandLineCodeExecutor,-)-from agnext.components.models import (- AssistantMessage,- AzureOpenAIChatCompletionClient,- ChatCompletionClient,- LLMMessage,- ModelCapabilities,- SystemMessage,- UserMessage,-)-from agnext.core import CancellationToken--# from azure.identity import DefaultAzureCredential, get_bearer_token_provider---@dataclass-class TaskMessage:- content: str---@dataclass-class CodeExecutionRequestMessage:- session_id: str- execution_request: str---@dataclass-class CodeExecutionResultMessage:- session_id: str- output: str- exit_code: int---class Coder(TypeRoutedAgent):- """An agent that uses tools to write, execute, and debug Python code."""-- DEFAULT_DESCRIPTION = "A Python coder assistant."-- DEFAULT_SYSTEM_MESSAGES = [- SystemMessage(- """You are a helpful AI assistant. Solve tasks using your Python coding skills. The code you output must be formatted in Markdown code blocks demarcated by triple backticks (```). As an example:--```python--def main():- print("Hello world.")--if __name__ == "__main__":- main()-```--The user cannot provide any feedback or perform any other action beyond executing the code you suggest. In particular, the user can't modify your code, and can't copy and paste anything, and can't fill in missing values. Thus, do not suggest incomplete code which requires users to perform any of these actions.--Check the execution result returned by the user. If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes -- code blocks must stand alone and be ready to execute without modification. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, and think of a different approach to try.--If the code has executed successfully, and the problem is stolved, reply "TERMINATE".-"""- )- ]-- def __init__(- self,- model_client: ChatCompletionClient,- description: str = DEFAULT_DESCRIPTION,- system_messages: List[SystemMessage] = DEFAULT_SYSTEM_MESSAGES,- max_turns: int | None = None,- ) -> None:- super().__init__(description)- self._model_client = model_client- self._system_messages = system_messages- self._session_memory: Dict[str, List[LLMMessage]] = {}- self._max_turns = max_turns-- @message_handler- async def handle_user_message(- self, message: TaskMessage, cancellation_token: CancellationToken- ) -> None:- """Handle a user message, execute the model and tools, and returns the response."""- # Create a new session.- session_id = str(uuid.uuid4())- self._session_memory.setdefault(session_id, []).append(- UserMessage(content=message.content, source="user")- )-- # Make an inference to the model.- response = await self._model_client.create(- self._system_messages + self._session_memory[session_id]- )- assert isinstance(response.content, str)- self._session_memory[session_id].append(- AssistantMessage(content=response.content, source=self.metadata["type"])- )-- await self.publish_message(- CodeExecutionRequestMessage(- execution_request=response.content, session_id=session_id- ),- cancellation_token=cancellation_token,- )-- @message_handler- async def handle_code_execution_result(- self, message: CodeExecutionResultMessage, cancellation_token: CancellationToken- ) -> None:-- execution_result = f"The script ran, then exited with Unix exit code: {message.exit_code}\nIts output was:\n{message.output}"-- # Store the code execution output.- self._session_memory[message.session_id].append(- UserMessage(content=execution_result, source="user")- )-- # Count the number of rounds so far- if self._max_turns is not None:- n_turns = sum(- 1- for message in self._session_memory[message.session_id]- if isinstance(message, AssistantMessage)- )- if n_turns >= self._max_turns:- return-- # Make an inference to the model.- response = await self._model_client.create(- self._system_messages + self._session_memory[message.session_id]- )- assert isinstance(response.content, str)- self._session_memory[message.session_id].append(- AssistantMessage(content=response.content, source=self.metadata["type"])- )-- if "TERMINATE" in response.content:- return- else:- await self.publish_message(- CodeExecutionRequestMessage(- execution_request=response.content, session_id=message.session_id- ),- cancellation_token=cancellation_token,- )---class Executor(TypeRoutedAgent):-- def __init__(self, description: str, executor: CodeExecutor) -> None:- super().__init__(description)- self._executor = executor-- @message_handler- async def handle_code_execution(- self,- message: CodeExecutionRequestMessage,- cancellation_token: CancellationToken,- ) -> None:-- # Extract code block from the message.- code = self._extract_execution_request(message.execution_request)- if code is not None:- execution_requests = [CodeBlock(code=code, language="python")]- result = await self._executor.execute_code_blocks(execution_requests, cancellation_token)- await self.publish_message(- CodeExecutionResultMessage(- output=result.output,- exit_code=result.exit_code,- session_id=message.session_id,- )- )- else:- await self.publish_message(- CodeExecutionResultMessage(- output="No code block detected. Please provide a markdown-encoded code block to execute.",- exit_code=1,- session_id=message.session_id,- )- )-- def _extract_execution_request(self, markdown_text: str) -> Union[str, None]:- pattern = r"```(\w+)\n(.*?)\n```"- # Search for the pattern in the markdown text- match = re.search(pattern, markdown_text, re.DOTALL)- # Extract the language and code block if a match is found- if match:- return match.group(2)- return None---async def main() -> None:- # Create the runtime.- runtime = SingleThreadedAgentRuntime()-- # Create the AzureOpenAI client, with AAD auth- # token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")- client = AzureOpenAIChatCompletionClient(- api_version="2024-02-15-preview",- azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",- model="gpt-4o-2024-05-13",- model_capabilities=ModelCapabilities(- function_calling=True, json_output=True, vision=True- ),- # azure_ad_token_provider=token_provider- )-- # Register agents.- coder = await runtime.register_and_get(- "Coder",- lambda: Coder(model_client=client),- )- await runtime.register(- "Executor",- lambda: Executor(- "A agent for executing code", executor=LocalCommandLineCodeExecutor()- ),- )-- prompt = ""- with open("prompt.txt", "rt") as fh:- prompt = fh.read()-- entry_point = "__ENTRY_POINT__"-- task = TaskMessage(- f"""-The following python code imports the `run_tests` function from unit_tests.py, and runs-it on the function `{entry_point}`. This will run a set of automated unit tests to verify the-correct implementation of `{entry_point}`. However, `{entry_point}` is only partially-implemented in the code below. Complete the implementation of `{entry_point}` and then execute-a new stand-alone code block that contains everything needed to run the tests, including: importing-`unit_tests`, calling `run_tests({entry_point})`, as well as {entry_point}'s complete definition,-such that this code block can be run directly in Python.--```python-from unit_tests import run_tests--{prompt}--# Run the unit tests-run_tests({entry_point})-```-""".strip()- )-- # Run the runtime until the task is completed.- run_context = runtime.start()- # Send a task to the tool user.- await runtime.send_message(task, coder)- await run_context.stop_when_idle()---if __name__ == "__main__":- import logging-- logging.basicConfig(level=logging.WARNING)- logging.getLogger("agnext").setLevel(logging.DEBUG)- asyncio.run(main())diff --git a/python/benchmarks/HumanEval/Templates/TwoAgents/unit_tests.py b/python/benchmarks/HumanEval/Templates/TwoAgents/unit_tests.pydeleted file mode 100644index 55be54d41..000000000--- a/python/benchmarks/HumanEval/Templates/TwoAgents/unit_tests.py+++ /dev/null@@ -1,15 +0,0 @@-# Disable ruff linter for template files-# ruff: noqa: F821 E722-import sys---__TEST__---def run_tests(candidate):- try:- check(candidate)- # We can search for this string in the output- print("ALL TESTS PASSED !#!#")- except AssertionError:- sys.exit("SOME TESTS FAILED - TRY AGAIN !#!#")diff --git a/python/benchmarks/README.md b/python/benchmarks/README.mdnew file mode 100644index 000000000..b4dd6c51f--- /dev/null+++ b/python/benchmarks/README.md@@ -0,0 +1 @@+Moved to another repo.diff --git a/python/benchmarks/WebArena/.gitignore b/python/benchmarks/WebArena/.gitignoredeleted file mode 100644index 8e419f867..000000000--- a/python/benchmarks/WebArena/.gitignore+++ /dev/null@@ -1,2 +0,0 @@-Results-ENV.jsondiff --git a/python/benchmarks/WebArena/ENV.sample.json b/python/benchmarks/WebArena/ENV.sample.jsondeleted file mode 100644index 0b8311b13..000000000--- a/python/benchmarks/WebArena/ENV.sample.json+++ /dev/null@@ -1,28 +0,0 @@-{- "CHAT_COMPLETION_PROVIDER": "azure",- "CHAT_COMPLETION_KWARGS_JSON": "{\"api_version\": \"2024-05-01-preview\", \"azure_endpoint\": \"YOUR_ENDPOINT_HERE\", \"model_capabilities\": {\"function_calling\": true, \"json_output\": true, \"vision\": true}, \"model\": \"gpt-4o-2024-05-13\"}",-- "SHOPPING": "",- "SHOPPING_ADMIN": "",- "REDDIT": "",- "GITLAB": "",- "MAP": "",- "WIKIPEDIA": "",- "HOMEPAGE": "",-- "REDDIT_USERNAME": "",- "REDDIT_PASSWORD": "",-- "GITLAB_USERNAME": "",- "GITLAB_PASSWORD": "",-- "SHOPPING_USERNAME": "",- "SHOPPING_PASSWORD": "",-- "SHOPPING_ADMIN_USERNAME": "",- "SHOPPING_ADMIN_PASSWORD": "",-- "WEB_SURFER_DEBUG_DIR": "/autogen/debug",-- "OPENAI_API_KEY": ""-}diff --git a/python/benchmarks/WebArena/Scripts/custom_tabulate.py b/python/benchmarks/WebArena/Scripts/custom_tabulate.pydeleted file mode 100644index daedc2786..000000000--- a/python/benchmarks/WebArena/Scripts/custom_tabulate.py+++ /dev/null@@ -1,15 +0,0 @@-import os-import sys-from autogenbench.tabulate_cmd import default_tabulate, default_scorer---def scorer(instance_dir, success_strings=["FINAL SCORE: 1"]):- return default_scorer(instance_dir, success_strings=success_strings)---def main(args):- default_tabulate(args, scorer=scorer)---if __name__ == "__main__" and __package__ is None:- main(sys.argv)diff --git a/python/benchmarks/WebArena/Scripts/init_tasks.py b/python/benchmarks/WebArena/Scripts/init_tasks.pydeleted file mode 100644index 5ba3fd4d0..000000000--- a/python/benchmarks/WebArena/Scripts/init_tasks.py+++ /dev/null@@ -1,122 +0,0 @@-#-# Run this file to download the human_eval dataset, and create a corresponding testbed scenario:-# (default: ../scenarios/human_eval_two_agents_gpt4.jsonl and ./scenarios/human_eval_two_agents_gpt35.jsonl)-#--import requests-import tarfile-import hashlib-import io-import json-import os-import re-import sys--URL = "https://raw.githubusercontent.com/web-arena-x/webarena/main/config_files/test.raw.json"--SCRIPT_PATH = os.path.realpath(__file__)-SCRIPT_NAME = os.path.basename(SCRIPT_PATH)-SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)--SCENARIO_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.path.pardir))-TEMPLATES_DIR = os.path.join(SCENARIO_DIR, "Templates")-TASKS_DIR = os.path.join(SCENARIO_DIR, "Tasks")-DOWNLOADS_DIR = os.path.join(SCENARIO_DIR, "Downloads")---def download():- """Download the WebArena dataset (if not already downloaded).- Return a JSON list of problem instances."""-- if not os.path.isdir(DOWNLOADS_DIR):- os.mkdir(DOWNLOADS_DIR)-- json_file = os.path.join(DOWNLOADS_DIR, "test.raw.json")-- if not os.path.isfile(json_file):- # Send a HTTP request to the URL- response = requests.get(URL, stream=True)- response.raise_for_status()-- # If the HTTP request returns a status code 200, proceed- with open(json_file, "wb") as fh:- for chunk in response.iter_content(chunk_size=512):- fh.write(chunk)-- # Load the problems- problems = None- with open(json_file, "rb") as fh:- problems = json.load(fh)- return problems---def create_jsonl(name, tasks, template):- """Creates a JSONL scenario file with a given name, dictionary of MATH problems, and template path."""-- # Create a task directory if it doesn't exist- if not os.path.isdir(TASKS_DIR):- os.mkdir(TASKS_DIR)-- # Create the jsonl file- prompt_fields = ["task_id", "intent_template_id", "sites", "require_login", "start_url", "geolocation", "intent"]- with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:- for task in tasks:- print(f"Converting: {name}, {task['task_id']}")-- task_prompt = {}- for field in prompt_fields:- task_prompt[field] = task[field]-- record = {- "id": str(task["task_id"]),- "template": [os.path.join(TEMPLATES_DIR, "Common"), template],- "substitutions": {- "task_prompt.json.txt": {"__TASK_PROMPT__": json.dumps(task_prompt, indent=4)},- "full_task.json.txt": {"__FULL_TASK__": json.dumps(task, indent=4)},- },- }-- fh.write(json.dumps(record).strip() + "\n")---###############################################################################-def main():- tasks = download()-- # list all directories in the Templates directory- # and populate a dictionary with the name and path- templates = {}- for entry in os.scandir(TEMPLATES_DIR):- if entry.is_dir():- if entry.name == "Common": # Skip the common template, which will be included in all- continue- templates[re.sub(r"\s", "", entry.name)] = entry.path-- # Divide the tasks by their websites and if they are validation or test- page_groups = dict()- for task in tasks:-- # We don't know how the intent ids are distributed, so hash them to get a uniform distribution- template_hash = hashlib.md5(str(task["intent_template_id"]).encode("utf-8")).hexdigest()-- # The full hash will consist of 32 hexadecimal digits. We can get a 50/50 split by checking if the first digit is in the range (0-7) vs (8-F)- task_set = "validation" if template_hash[0] in "01234567" else "test"-- key = task["sites"][0]- if len(task["sites"]) > 1:- key = "several_sites"- key = task_set + "_" + key-- # key = "__".join(sorted([s for s in task["sites"]]))- if key not in page_groups:- page_groups[key] = list()- page_groups[key].append(task)-- # Create the json files- for t in templates.items():- for pg in page_groups:- create_jsonl(f"webarena__{pg}_{t[0]}", page_groups[pg], t[1])---if __name__ == "__main__" and __package__ is None:- main()diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/ATTRIBUTION b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/ATTRIBUTIONdeleted file mode 100644index 0713904fb..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/ATTRIBUTION+++ /dev/null@@ -1,28 +0,0 @@-The contents of this `evaluation_harness` folder are adapted from:-- https://github.com/web-arena-x/webarena--under the following license:--=========================================================================================================--Copyright (c) 2024 Jing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, and Po-Yu Huang--Permission is hereby granted, free of charge, to any person obtaining-a copy of this software and associated documentation files (the-"Software"), to deal in the Software without restriction, including-without limitation the rights to use, copy, modify, merge, publish,-distribute, sublicense, and/or sell copies of the Software, and to-permit persons to whom the Software is furnished to do so, subject to-the following conditions:--The above copyright notice and this permission notice shall be-included in all copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/__init__.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/__init__.pydeleted file mode 100644index e942c1066..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/__init__.py+++ /dev/null@@ -1,6 +0,0 @@-from .evaluators import *-from .helper_functions import (- shopping_get_latest_order_url,- shopping_get_sku_latest_review_author,- shopping_get_sku_latest_review_rating,-)diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/env_config.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/env_config.pydeleted file mode 100644index ed84ae473..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/env_config.py+++ /dev/null@@ -1,90 +0,0 @@-# websites domain-import os--REDDIT = os.environ.get("REDDIT", "")-SHOPPING = os.environ.get("SHOPPING", "")-SHOPPING_ADMIN = os.environ.get("SHOPPING_ADMIN", "")-GITLAB = os.environ.get("GITLAB", "")-WIKIPEDIA = os.environ.get("WIKIPEDIA", "")-MAP = os.environ.get("MAP", "")-HOMEPAGE = os.environ.get("HOMEPAGE", "")--REDDIT_USERNAME = os.environ.get("REDDIT_USERNAME", "")-REDDIT_PASSWORD = os.environ.get("REDDIT_PASSWORD", "")--GITLAB_USERNAME = os.environ.get("GITLAB_USERNAME", "")-GITLAB_PASSWORD = os.environ.get("GITLAB_PASSWORD", "")--SHOPPING_USERNAME = os.environ.get("SHOPPING_USERNAME", "")-SHOPPING_PASSWORD = os.environ.get("SHOPPING_PASSWORD", "")--SHOPPING_ADMIN_USERNAME = os.environ.get("SHOPPING_ADMIN_USERNAME", "")-SHOPPING_ADMIN_PASSWORD = os.environ.get("SHOPPING_ADMIN_PASSWORD", "")--assert REDDIT and SHOPPING and SHOPPING_ADMIN and GITLAB and WIKIPEDIA and MAP and HOMEPAGE, (- "Please setup the URLs to each site. Current: \n"- + f"Reddit: {REDDIT}\n"- + f"Shopping: {SHOPPING}\n"- + f"Shopping Admin: {SHOPPING_ADMIN}\n"- + f"Gitlab: {GITLAB}\n"- + f"Wikipedia: {WIKIPEDIA}\n"- + f"Map: {MAP}\n"- + f"Homepage: {HOMEPAGE}\n"-)--ACCOUNTS = {- "reddit": {"username": REDDIT_USERNAME, "password": REDDIT_PASSWORD},- "gitlab": {"username": GITLAB_USERNAME, "password": GITLAB_PASSWORD},- "shopping": {"username": SHOPPING_USERNAME, "password": SHOPPING_PASSWORD},- "shopping_admin": {"username": SHOPPING_ADMIN_USERNAME, "password": SHOPPING_ADMIN_PASSWORD},- "shopping_site_admin": {"username": SHOPPING_ADMIN_USERNAME, "password": SHOPPING_ADMIN_PASSWORD},-}--URL_MAPPINGS = {- REDDIT: "http://reddit.com",- SHOPPING: "http://onestopmarket.com",- SHOPPING_ADMIN: "http://luma.com/admin",- GITLAB: "http://gitlab.com",- WIKIPEDIA: "http://wikipedia.org",- MAP: "http://openstreetmap.org",- HOMEPAGE: "http://homepage.com",-}--# ADDED BY MSR Frontiers-#########################-SITE_URLS = {- "reddit": REDDIT,- "gitlab": GITLAB,- "shopping": SHOPPING,- "shopping_admin": SHOPPING_ADMIN,- "shopping_site_admin": SHOPPING_ADMIN,- "map": MAP,- "wikipedia": WIKIPEDIA,-}--LOGIN_PROMPTS = {- "reddit": f"Type '{REDDIT}' into the address bar to navigate to the site. Click 'Log in', type the username '{ACCOUNTS['reddit']['username']}', and password is '{ACCOUNTS['reddit']['password']}'. Finally click the login button.",- "gitlab": f"Type '{GITLAB}' into the address bar to navigate to the site. At the log in prompt, type the username '{ACCOUNTS['gitlab']['username']}', and the password '{ACCOUNTS['gitlab']['password']}'. Finally click the 'Sign in' button.",- "shopping": f"Type '{SHOPPING}' into the address bar to navigate to the site. Click 'Sign In' at the top of the page. Enter the Email '{ACCOUNTS['shopping']['username']}', and password '{ACCOUNTS['shopping']['password']}'. Finally click the 'Sign In' button.",- "shopping_admin": f"Type '{SHOPPING_ADMIN}' into the address bar to navigate to the site. At the log in prompt, enter the username '{ACCOUNTS['shopping_admin']['username']}', and the password '{ACCOUNTS['shopping_admin']['password']}'. Finally click the 'Sign In' button.",-}--SITE_DESCRIPTIONS = {- "reddit": "a Postmill forum populated with a large sample of data crawled from Reddit. Postmill is similar to Reddit, but the UI is distinct, and 'subreddits' begin with /f/ rather than /r/",- "gitlab": "a Gitlab site populated with various programming projects. Gitlab is similar to GitHub, though the UIs are slightly different",- "shopping": "an online store built with the Magento open source eCommerce platform",- "shopping_admin": "the content management admin portal for an online store running the Magento open source eCommerce software",-}---def url_to_sitename(url):- if url.startswith(REDDIT):- return "reddit"- elif url.startswith(GITLAB):- return "gitlab"- elif url.startswith(SHOPPING):- return "shopping"- elif url.startswith(SHOPPING_ADMIN):- return "shopping_admin"- else:- return Nonediff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.pydeleted file mode 100644index aeb19b8de..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/evaluators.py+++ /dev/null@@ -1,386 +0,0 @@-"""base class for evaluation"""--# answer string match-import collections-import html-import importlib-import json-import time-import urllib-import inspect-from pathlib import Path-from typing import Any, Tuple, Union, TypedDict, Dict--from beartype import beartype-from nltk.tokenize import word_tokenize # type: ignore-from playwright.async_api import CDPSession, Page--import numpy as np-import numpy.typing as npt--from .helper_functions import (- PseudoPage,- gitlab_get_project_memeber_role,- llm_fuzzy_match,- llm_ua_match,- reddit_get_post_url,- shopping_get_latest_order_url,- shopping_get_sku_latest_review_author,- shopping_get_sku_latest_review_rating,-)---# Subset used for evaluation (added by: adamfo)-#####################################################################-class Action(TypedDict):- answer: str---Observation = str | npt.NDArray[np.uint8]---class StateInfo(TypedDict):- observation: dict[str, Observation]- info: Dict[str, Any]---Trajectory = list[Union[Action, StateInfo]]---def make_answer_trajecotry(answer: str) -> Trajectory:- ans = Action()- ans["answer"] = answer- return [ans]---#####################################################################-class Evaluator(object):- def __init__(self, eval_tag: str = "") -> None:- self.eval_tag = eval_tag-- @beartype- async def __call__(- self,- trajectory: Trajectory,- config_file: Path | str,- page: Page | PseudoPage,- client: CDPSession,- azure_config: dict[str, Any] | None = None,- ) -> float:- raise NotImplementedError-- @staticmethod- def get_last_action(trajectory: Trajectory) -> Action:- try:- # is_bearable(trajectory[-1], Action)- last_action = trajectory[-1]- except Exception:- raise ValueError("The last element of trajectory should be an action, add a fake stop action if needed")-- return last_action # type: ignore[return-value]-- @staticmethod- def get_last_state(trajectory: Trajectory) -> StateInfo:- try:- # is_bearable(trajectory[-2], StateInfo)- last_state = trajectory[-2]- except Exception:- raise ValueError(- "The second last element of trajectory should be a state, add a fake stop action if needed"- )-- return last_state # type: ignore[return-value]---class StringEvaluator(Evaluator):- """Check whether the answer is correct with:- exact match: the answer is exactly the same as the reference answer- must include: each phrase in the reference answer must be included in the answer- fuzzy match: the answer is similar to the reference answer, using LLM judge- """-- @staticmethod- @beartype- def clean_answer(answer: str) -> str:- answer = answer.strip()- if answer.startswith("'") and answer.endswith("'"):- answer = answer[1:-1]- elif answer.startswith('"') and answer.endswith('"'):- answer = answer[1:-1]- return answer.lower()-- @staticmethod- @beartype- def exact_match(ref: str, pred: str) -> float:- return float(StringEvaluator.clean_answer(pred) == StringEvaluator.clean_answer(ref))-- @staticmethod- @beartype- def must_include(ref: str, pred: str, tokenize: bool = False) -> float:- clean_ref = StringEvaluator.clean_answer(ref)- clean_pred = StringEvaluator.clean_answer(pred)- # tokenize the answer if the ref is a single word- # prevent false positive (e.g, 0)- if tokenize and len(clean_ref) == 1 and len(word_tokenize(clean_ref)) == 1:- tok_pred = word_tokenize(clean_pred)- return float(clean_ref in tok_pred)- else:- return float(clean_ref in clean_pred)-- @staticmethod- @beartype- def fuzzy_match(ref: str, pred: str, intent: str, azure_config: dict[str, Any] | None) -> float:- return llm_fuzzy_match(pred, ref, intent, azure_config)-- @staticmethod- @beartype- def ua_match(ref: str, pred: str, intent: str, azure_config: dict[str, Any] | None) -> float:- return llm_ua_match(pred, ref, intent, azure_config)-- async def __call__(- self,- trajectory: Trajectory,- config_file: Path | str,- page: Page | PseudoPage | None = None,- client: CDPSession | None = None,- azure_config: dict[str, Any] | None = None,- ) -> float:- with open(config_file, "r") as f:- configs = json.load(f)-- last_action = self.get_last_action(trajectory)- pred = self.clean_answer(last_action["answer"])-- score = 1.0- for approach, value in configs["eval"]["reference_answers"].items():- match approach:- case "exact_match":- score *= self.exact_match(ref=value, pred=pred)-- case "must_include":- assert isinstance(value, list)- for must_value in value:- score *= self.must_include(- ref=must_value,- pred=pred,- tokenize=(len(value) == 1),- )- case "fuzzy_match":- intent = configs["intent"]- if value == "N/A":- # if the instruction only asks the model to generate N/A when encountering an unachievable task- # without more concrete reasons- score *= self.exact_match(ref=value, pred=pred)- # if the instruction also asks the model to generate the reason why the task is unachievable- # this should be the default as it will prevent false positive N/A`- if score != 1:- score = 1.0 * self.ua_match(- intent=configs["intent"],- ref=configs["eval"]["string_note"],- pred=pred,- azure_config=azure_config,- )- else:- assert isinstance(value, list)- for reference in value:- score *= self.fuzzy_match(- ref=reference, pred=pred, intent=intent, azure_config=azure_config- )- return score---class URLEvaluator(Evaluator):- """Check URL matching"""-- @beartype- async def __call__(- self,- trajectory: Trajectory,- config_file: Path | str,- page: Page | PseudoPage,- client: CDPSession | None = None,- azure_config: dict[str, Any] | None = None,- ) -> float:- with open(config_file, "r") as f:- configs = json.load(f)-- def clean_url(url: str) -> str:- url = str(url)- url = url.rstrip("/")- return url-- def parse_url(url: str) -> tuple[str, dict[str, list[str]]]:- """Parse a URL into its base, path, and query components."""- parsed_url = urllib.parse.urlparse(url)- base_path = parsed_url.netloc + parsed_url.path- query = urllib.parse.parse_qs(parsed_url.query)- return base_path, query-- def parse_urls(- urls: list[str],- ) -> tuple[list[str], dict[str, set[str]]]:- """Parse a list of URLs."""- base_paths = []- queries = collections.defaultdict(set)- for url in urls:- base_path, query = parse_url(url)- base_paths.append(base_path)- for k, v in query.items():- queries[k].update(v)- return base_paths, queries-- pred = clean_url(page.url)- ref_urls = configs["eval"]["reference_url"].split(" |OR| ")- ref_urls = [clean_url(url) for url in ref_urls]- matching_rule = configs["eval"].get("url_note", "GOLD in PRED")- if matching_rule == "GOLD in PRED":- print(f"Pred: {pred}")- print(f"Ref: {ref_urls}")- ref_base_paths, ref_queries = parse_urls(ref_urls)- pred_base_paths, pred_query = parse_url(pred)-- base_score = float(any([ref_base_path in pred_base_paths for ref_base_path in ref_base_paths]))- query_score = 1.0- for k, possible_values in ref_queries.items():- query_score *= float(- any(possible_ref_value in pred_query.get(k, []) for possible_ref_value in possible_values)- )- score = base_score * query_score-- else:- raise ValueError(f"Unknown matching rule: {matching_rule}")-- return score---class HTMLContentEvaluator(Evaluator):- """Check whether the contents appear in the page"""-- @beartype- async def __call__(- self,- trajectory: Trajectory,- config_file: Path | str,- page: Page | PseudoPage,- client: CDPSession | None = None,- azure_config: dict[str, Any] | None = None,- ) -> float:- with open(config_file, "r") as f:- configs = json.load(f)-- targets = configs["eval"]["program_html"]-- score = 1.0- for target in targets:- target_url: str = target["url"] # which url to check- if target_url.startswith("func"):- func = target_url.split("func:")[1]- func = func.replace("__last_url__", page.url)- target_url = eval(func)- if inspect.isawaitable(target_url):- target_url = await target_url-- locator: str = target["locator"] # js element locator-- # navigate to that url- if target_url != "last":- await page.goto(target_url)- time.sleep(3) # TODO [shuyanzh]: fix this hard-coded sleep-- # empty, use the full page- if not locator.strip():- selected_element = page.content()- # use JS to select the element- elif locator.startswith("document.") or locator.startswith("[...document."):- if "prep_actions" in target:- try:- for prep_action in target["prep_actions"]:- await page.evaluate(f"() => {prep_action}")- except Exception:- pass- try:- selected_element = str(await page.evaluate(f"() => {locator}"))- if not selected_element:- selected_element = ""- except Exception:- # the page is wrong, return empty- selected_element = ""- # run program to call API- elif locator.startswith("func:"): # a helper function- func = locator.split("func:")[1]- func = func.replace("__page__", "page")- selected_element = eval(func)- if inspect.isawaitable(selected_element):- selected_element = await selected_element- else:- raise ValueError(f"Unknown locator: {locator}")-- selected_element = html.unescape(selected_element)-- if "exact_match" in target["required_contents"]:- required_contents = target["required_contents"]["exact_match"]- cur_score = StringEvaluator.exact_match(ref=required_contents, pred=selected_element)- score *= float(cur_score)- # print(f"[exact match] {cur_score}, selected element: {selected_element}, required contents: {required_contents}")- elif "must_include" in target["required_contents"]:- required_contents = target["required_contents"]["must_include"]- assert isinstance(required_contents, list)- for content in required_contents:- content_or = content.split(" |OR| ")- cur_score = any(- [- StringEvaluator.must_include(- ref=content,- pred=selected_element,- tokenize=False,- )- for content in content_or- ]- )- score *= float(cur_score)- # print(f"[must include] {cur_score}, selected element: {selected_element}, required contents: {content_or}")- else:- raise ValueError(f"Unknown required_contents: {target['required_contents'].keys()}")- return score---class EvaluatorComb:- def __init__(self, evaluators: list[Evaluator]) -> None:- self.evaluators = evaluators-- @beartype- async def __call__(- self,- trajectory: Trajectory,- config_file: Path | str,- page: Page | PseudoPage,- client: CDPSession,- azure_config: dict[str, Any] | None = None,- ) -> float:- score = 1.0- for evaluator in self.evaluators:- cur_score = await evaluator(trajectory, config_file, page, client, azure_config)- score *= cur_score- return score---@beartype-def evaluator_router(config_file: Path | str) -> EvaluatorComb:- """Router to get the evaluator class"""- with open(config_file, "r") as f:- configs = json.load(f)-- eval_types = configs["eval"]["eval_types"]- evaluators: list[Evaluator] = []- for eval_type in eval_types:- match eval_type:- case "string_match":- evaluators.append(StringEvaluator())- case "url_match":- evaluators.append(URLEvaluator())- case "program_html":- evaluators.append(HTMLContentEvaluator())- case _:- raise ValueError(f"eval_type {eval_type} is not supported")-- return EvaluatorComb(evaluators)diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.pydeleted file mode 100644index a2e20d622..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/helper_functions.py+++ /dev/null@@ -1,233 +0,0 @@-"""Implements helper functions to assist evaluation cases where other evaluators are not suitable."""--import json-from typing import Any-from urllib.parse import urlparse--import requests-from playwright.async_api import Page--from .env_config import (- ACCOUNTS,- GITLAB,- MAP,- REDDIT,- SHOPPING,- SHOPPING_ADMIN,- WIKIPEDIA,-)--from .openai_utils import (- generate_from_openai_chat_completion,-)--import autogen---def shopping_get_auth_token() -> str:- response = requests.post(- url=f"{SHOPPING}/rest/default/V1/integration/admin/token",- headers={"content-type": "application/json"},- data=json.dumps(- {- "username": ACCOUNTS["shopping_site_admin"]["username"],- "password": ACCOUNTS["shopping_site_admin"]["password"],- }- ),- )- token: str = response.json()- return token---def shopping_get_latest_order_url() -> str:- """Get the latest order url from the shopping website."""-- header = {- "Authorization": f"Bearer {shopping_get_auth_token()}",- "Content-Type": "application/json",- }-- params = {- "searchCriteria[sortOrders][0][field]": "created_at",- "searchCriteria[sortOrders][0][direction]": "DESC",- "searchCriteria[pageSize]": "1",- }-- response = requests.get(f"{SHOPPING}/rest/V1/orders", params=params, headers=header)- assert response.status_code == 200- response_obj = response.json()["items"][0]- order_id = int(response_obj["increment_id"])- order_url = f"{SHOPPING}/sales/order/view/order_id/{order_id}/"- return order_url---def shopping_get_sku_latest_review_author(sku: str) -> str:- """Get the latest review for shopping admin."""- header = {- "Authorization": f"Bearer {shopping_get_auth_token()}",- "Content-Type": "application/json",- }- response = requests.get(f"{SHOPPING}/rest/V1/products/{sku}/reviews", headers=header)- assert response.status_code == 200- response_obj = response.json()- if len(response_obj) == 0:- return ""- author: str = response_obj[-1]["nickname"]- return author---def shopping_get_sku_latest_review_rating(sku: str) -> str:- """Get the latest review for shopping admin."""- header = {- "Authorization": f"Bearer {shopping_get_auth_token()}",- "Content-Type": "application/json",- }- response = requests.get(f"{SHOPPING}/rest/V1/products/{sku}/reviews", headers=header)- assert response.status_code == 200- response_obj = response.json()- if len(response_obj) == 0:- return ""- assert response_obj[0]["ratings"][0]["rating_name"] == "Rating"- rating: str = str(response_obj[-1]["ratings"][0]["percent"])- return rating---def reddit_get_post_url(url: str) -> str:- """Get the post url"""- # Url is http://domain/f/subreddit/post_id/...- # get domain, subreddit, post_id- domain = urlparse(url).netloc- tok_url = urlparse(url).path.split("/")- # not a valid post/comment url, return the url as is- if len(tok_url) < 4:- return url- if tok_url[1] != "f":- return url- subreddit = urlparse(url).path.split("/")[2]- post_id = urlparse(url).path.split("/")[3]- scheme = urlparse(url).scheme- post_url = f"{scheme}://{domain}/f/{subreddit}/{post_id}/"- return post_url---async def gitlab_get_project_memeber_role(page: Page, account_name: str) -> str:- # get the account index- try:- account_idx = await page.evaluate(- f"""(() => {{- const elements = document.querySelectorAll("td[data-label='Account'] span.gl-avatar-labeled-sublabel");- let index = -1; // Default value if not found-- for(let i = 0; i < elements.length; i++) {{- if(elements[i].outerText === '@{account_name}') {{- index = i;- break;- }}- }}-- return index;- }})()"""- )-- # get the role- role: str = await page.evaluate(- f"""(() => {{- return document.querySelectorAll("td.col-max-role span")[{account_idx}].outerText;- }})()"""- )- except Exception:- role = ""-- return role---def llm_fuzzy_match(pred: str, reference: str, question: str, azure_config: dict[str, Any] | None) -> float:- """Check whether the prediction matches the reference with GPT4-turbo"""- messages: list[dict[str, Any]] = []- # construct the question to ask- message = "Help a teacher to grade the answer of a student given a question. Keep in mind that the student may use different phrasing or wording to answer the question. The goal is to evaluate whether the answer is semantically equivalent to the reference answer.\n"- message += f"question: {question}\n"- message += f"reference answer: {reference}\n"- message += "all the string 'N/A' that you see is a special sequence that means 'not achievable'\n"- message += f"student answer: {pred}\n"- message += "Conclude the judgement by correct/incorrect/partially correct."- messages = [- {"role": "system", "content": "You are a helpful assistant"},- {"role": "user", "content": message},- ]-- response = None- if azure_config is None:- response = generate_from_openai_chat_completion(- model="gpt-4-1106-preview",- messages=messages,- temperature=0,- max_tokens=768,- top_p=1.0,- context_length=0,- ).lower()- else:- client = autogen.OpenAIWrapper(**azure_config)- raw_response = client.create(context=None, messages=messages)- response = client.extract_text_or_completion_object(raw_response)[0].lower()-- if "partially correct" in response or "incorrect" in response:- return 0.0- else:- assert "correct" in response- return 1.0---def llm_ua_match(pred: str, reference: str, question: str, azure_config: dict[str, Any] | None) -> float:- """Check whether the prediction matches the reference with GPT-turbo"""- messages: list[dict[str, Any]] = []- # construct the question to ask- message = ""- message += f"task: {question}\n"- message += f"actual unachievable reason: {reference}\n"- message += f"reported unachievable reason: {pred}\n"- message += (- "The task described above is inherently unachievable due to the reason specified under 'actual unachievable reason'. "- "An individual previously attempted this task and was unable to complete it. They provided a reason for their failure, "- "which is listed under 'reported unachievable reason'. Your role is to review both the actual and reported reasons. "- "Determine if the reported reason aligns with the actual reason, even if implicitly. "- "If the stated reason is in line with the actual reason, respond with 'same'. Otherwise, respond with 'different'."- )- messages = [- {"role": "system", "content": "You are a helpful assistant"},- {"role": "user", "content": message},- ]-- response = None- if azure_config is None:- response = generate_from_openai_chat_completion(- model="gpt-4-1106-preview",- messages=messages,- temperature=0,- max_tokens=768,- top_p=1.0,- context_length=0,- ).lower()- else:- client = autogen.OpenAIWrapper(**azure_config)- raw_response = client.create(context=None, messages=messages)- response = client.extract_text_or_completion_object(raw_response)[0].lower()-- if "different" in response:- return 0.0- else:- assert "same" in response- return 1.0---class PseudoPage:- def __init__(self, original_page: Page, url: str):- self.url = url- self.original_page = original_page-- def __getattr__(self, attr: str) -> Any:- # Delegate attribute access to the original page object- if attr not in ["url"]:- return getattr(self.original_page, attr)- else:- return getattr(self, attr)diff --git a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/openai_utils.py b/python/benchmarks/WebArena/Templates/Common/evaluation_harness/openai_utils.pydeleted file mode 100644index 1381f392c..000000000--- a/python/benchmarks/WebArena/Templates/Common/evaluation_harness/openai_utils.py+++ /dev/null@@ -1,275 +0,0 @@-"""Tools to generate from OpenAI prompts.-Adopted from https://github.com/zeno-ml/zeno-build/"""--import asyncio-import logging-import os-import random-import time-from typing import Any--import aiolimiter-import openai-from openai import AsyncOpenAI, OpenAI--client = None-aclient = None-if "OPENAI_API_KEY" not in os.environ and "OAI_CONFIG_LIST" not in os.environ:- raise ValueError("Neither OPENAI_API_KEY nor OAI_CONFIG_LIST is defined in the environment.")--if "OPENAI_API_KEY" in os.environ:- client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])- aclient = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])-from tqdm.asyncio import tqdm_asyncio---def retry_with_exponential_backoff( # type: ignore- func,- initial_delay: float = 1,- exponential_base: float = 2,- jitter: bool = True,- max_retries: int = 3,- errors: tuple[Any] = (- openai.RateLimitError,- openai.BadRequestError,- openai.InternalServerError,- ),-):- """Retry a function with exponential backoff."""-- def wrapper(*args, **kwargs): # type: ignore- # Initialize variables- num_retries = 0- delay = initial_delay-- # Loop until a successful response or max_retries is hit or an exception is raised- while True:- try:-- return func(*args, **kwargs)-- # Retry on specified errors- except errors:- # Increment retries- num_retries += 1-- # Check if max retries has been reached- if num_retries > max_retries:- raise Exception(f"Maximum number of retries ({max_retries}) exceeded.")-- # Increment the delay- delay *= exponential_base * (1 + jitter * random.random())-- # Sleep for the delay- time.sleep(delay)-- # Raise exceptions for any errors not specified- except Exception as e:- raise e-- return wrapper---async def _throttled_openai_completion_acreate(- engine: str,- prompt: str,- temperature: float,- max_tokens: int,- top_p: float,- limiter: aiolimiter.AsyncLimiter,-) -> dict[str, Any]:- async with limiter:- for _ in range(3):- try:- return await aclient.completions.create(- engine=engine,- prompt=prompt,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- )- except openai.RateLimitError:- logging.warning("OpenAI API rate limit exceeded. Sleeping for 10 seconds.")- await asyncio.sleep(10)- except openai.APIError as e:- logging.warning(f"OpenAI API error: {e}")- break- return {"choices": [{"message": {"content": ""}}]}---async def agenerate_from_openai_completion(- prompts: list[str],- engine: str,- temperature: float,- max_tokens: int,- top_p: float,- context_length: int,- requests_per_minute: int = 300,-) -> list[str]:- """Generate from OpenAI Completion API.-- Args:- prompts: list of prompts- temperature: Temperature to use.- max_tokens: Maximum number of tokens to generate.- top_p: Top p to use.- context_length: Length of context to use.- requests_per_minute: Number of requests per minute to allow.-- Returns:- List of generated responses.- """- if "OPENAI_API_KEY" not in os.environ:- raise ValueError("OPENAI_API_KEY environment variable must be set when using OpenAI API.")-- limiter = aiolimiter.AsyncLimiter(requests_per_minute)- async_responses = [- _throttled_openai_completion_acreate(- engine=engine,- prompt=prompt,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- limiter=limiter,- )- for prompt in prompts- ]- responses = await tqdm_asyncio.gather(*async_responses)- return [x["choices"][0]["text"] for x in responses]---@retry_with_exponential_backoff-def generate_from_openai_completion(- prompt: str,- engine: str,- temperature: float,- max_tokens: int,- top_p: float,- context_length: int,- stop_token: str | None = None,-) -> str:- if "OPENAI_API_KEY" not in os.environ:- raise ValueError("OPENAI_API_KEY environment variable must be set when using OpenAI API.")-- response = client.completions.create(- prompt=prompt,- engine=engine,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- stop=[stop_token],- )- answer: str = response["choices"][0]["text"]- return answer---async def _throttled_openai_chat_completion_acreate(- model: str,- messages: list[dict[str, str]],- temperature: float,- max_tokens: int,- top_p: float,- limiter: aiolimiter.AsyncLimiter,-) -> dict[str, Any]:- async with limiter:- for _ in range(3):- try:- return await aclient.chat.completions.create(- model=model,- messages=messages,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- )- except openai.RateLimitError:- logging.warning("OpenAI API rate limit exceeded. Sleeping for 10 seconds.")- await asyncio.sleep(10)- except asyncio.exceptions.TimeoutError:- logging.warning("OpenAI API timeout. Sleeping for 10 seconds.")- await asyncio.sleep(10)- except openai.APIError as e:- logging.warning(f"OpenAI API error: {e}")- break- return {"choices": [{"message": {"content": ""}}]}---async def agenerate_from_openai_chat_completion(- messages_list: list[list[dict[str, str]]],- engine: str,- temperature: float,- max_tokens: int,- top_p: float,- context_length: int,- requests_per_minute: int = 300,-) -> list[str]:- """Generate from OpenAI Chat Completion API.-- Args:- messages_list: list of message list- temperature: Temperature to use.- max_tokens: Maximum number of tokens to generate.- top_p: Top p to use.- context_length: Length of context to use.- requests_per_minute: Number of requests per minute to allow.-- Returns:- List of generated responses.- """- if "OPENAI_API_KEY" not in os.environ:- raise ValueError("OPENAI_API_KEY environment variable must be set when using OpenAI API.")-- limiter = aiolimiter.AsyncLimiter(requests_per_minute)- async_responses = [- _throttled_openai_chat_completion_acreate(- model=engine,- messages=message,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- limiter=limiter,- )- for message in messages_list- ]- responses = await tqdm_asyncio.gather(*async_responses)- return [x["choices"][0]["message"]["content"] for x in responses]---@retry_with_exponential_backoff-def generate_from_openai_chat_completion(- messages: list[dict[str, str]],- model: str,- temperature: float,- max_tokens: int,- top_p: float,- context_length: int,- stop_token: str | None = None,-) -> str:- if "OPENAI_API_KEY" not in os.environ:- raise ValueError("OPENAI_API_KEY environment variable must be set when using OpenAI API.")- response = client.chat.completions.create(- model=model,- messages=messages,- temperature=temperature,- max_tokens=max_tokens,- top_p=top_p,- )- answer: str = response.choices[0].message.content- return answer---@retry_with_exponential_backoff-# debug only-def fake_generate_from_openai_chat_completion(- messages: list[dict[str, str]],- model: str,- temperature: float,- max_tokens: int,- top_p: float,- context_length: int,- stop_token: str | None = None,-) -> str:- if "OPENAI_API_KEY" not in os.environ:- raise ValueError("OPENAI_API_KEY environment variable must be set when using OpenAI API.")-- answer = "Let's think step-by-step. This page shows a list of links and buttons. There is a search box with the label 'Search query'. I will click on the search box to type the query. So the action I will perform is \"click [60]\"."- return answerdiff --git a/python/benchmarks/WebArena/Templates/TeamOne/full_task.json.txt b/python/benchmarks/WebArena/Templates/TeamOne/full_task.json.txtdeleted file mode 100644index 2b53498b8..000000000--- a/python/benchmarks/WebArena/Templates/TeamOne/full_task.json.txt+++ /dev/null@@ -1 +0,0 @@-__FULL_TASK__diff --git a/python/benchmarks/WebArena/Templates/TeamOne/requirements.txt b/python/benchmarks/WebArena/Templates/TeamOne/requirements.txtdeleted file mode 100644index 6270e241e..000000000--- a/python/benchmarks/WebArena/Templates/TeamOne/requirements.txt+++ /dev/null@@ -1,2 +0,0 @@-/agnext/teams/team-one-pyautogendiff --git a/python/benchmarks/WebArena/Templates/TeamOne/scenario.py b/python/benchmarks/WebArena/Templates/TeamOne/scenario.pydeleted file mode 100644index 77de88d44..000000000--- a/python/benchmarks/WebArena/Templates/TeamOne/scenario.py+++ /dev/null@@ -1,260 +0,0 @@-import asyncio-import logging-import json-import os-import re-import nltk--from typing import Any, Dict, List, Tuple, Union-from agnext.application import SingleThreadedAgentRuntime-from agnext.application.logging import EVENT_LOGGER_NAME-from agnext.components.models import (- AzureOpenAIChatCompletionClient,- ChatCompletionClient,- ModelCapabilities,- UserMessage,- SystemMessage,- LLMMessage,-)-from agnext.components.code_executor import LocalCommandLineCodeExecutor-from agnext.application.logging import EVENT_LOGGER_NAME-from team_one.markdown_browser import MarkdownConverter, UnsupportedFormatException-from team_one.agents.coder import Coder, Executor-from team_one.agents.orchestrator import RoundRobinOrchestrator, LedgerOrchestrator-from team_one.messages import BroadcastMessage, OrchestrationEvent, RequestReplyMessage, ResetMessage, DeactivateMessage-from team_one.agents.multimodal_web_surfer import MultimodalWebSurfer-from team_one.agents.file_surfer import FileSurfer-from team_one.utils import LogHandler, message_content_to_str, create_completion_client_from_env---import evaluation_harness-from evaluation_harness.env_config import (- ACCOUNTS,- GITLAB,- MAP,- REDDIT,- SHOPPING,- SHOPPING_ADMIN,- WIKIPEDIA,- HOMEPAGE,- SITE_URLS,- LOGIN_PROMPTS,- SITE_DESCRIPTIONS,- url_to_sitename,-)--REPLACEMENTS = {- "__REDDIT__": REDDIT,- "__SHOPPING__": SHOPPING,- "__SHOPPING_ADMIN__": SHOPPING_ADMIN,- "__GITLAB__": GITLAB,- "__WIKIPEDIA__": WIKIPEDIA,- "__MAP__": MAP,- "__HOMEPAGE__": HOMEPAGE,-}--nltk.download("punkt")---async def response_preparer(task: str, source: str, client: ChatCompletionClient, transcript: List[LLMMessage]) -> str:- messages: List[LLMMessage] = [- UserMessage(- content=f"Earlier you were asked the following:\n\n{task}\n\nYour team then worked diligently to address that request. Here is a transcript of that conversation:",- source=source,- )- ]-- # copy them to this context- for message in transcript:- messages.append(- UserMessage(- content = message_content_to_str(message.content),- # TODO fix this -> remove type ignore- source=message.source, # type: ignore- )- )-- # ask for the final answer- messages.append(- UserMessage(- content= f"""-Read the above conversation and output a FINAL ANSWER to the original request. The original request is repeated here for convenience:--{task}--To output the final answer, use the following template: FINAL ANSWER: [YOUR FINAL ANSWER]-Your FINAL ANSWER should be as few words as possible.-If the original request was not a question, or you did not find a definitive answer, simply summarize the final state of the page or task as your FINAL ANSWER.""",- source=source,- )- )-- response = await client.create(messages)- assert isinstance(response.content, str)- return response.content---async def main() -> None:- # Expand the prompt and the full task- task_prompt = ""- TASK = None- with open("task_prompt.json.txt", "rt") as fh:- task_prompt = fh.read()- with open("task_prompt.json", "wt") as fh:- for k in REPLACEMENTS:- task_prompt = task_prompt.replace(k, REPLACEMENTS[k])- fh.write(task_prompt)- TASK = json.loads(task_prompt)- if TASK["start_url"] == REDDIT:- TASK["start_url"] = TASK["start_url"] + "/forums/all"-- full_task = ""- with open("full_task.json.txt", "rt") as fh:- full_task = fh.read()- with open("full_task.json", "wt") as fh:- for k in REPLACEMENTS:- full_task = full_task.replace(k, REPLACEMENTS[k])- fh.write(full_task)-- # Create the runtime.- runtime = SingleThreadedAgentRuntime()-- # Create the AzureOpenAI client, with AAD auth- client = AzureOpenAIChatCompletionClient(- api_version="2024-02-15-preview",- azure_endpoint="https://aif-complex-tasks-west-us-3.openai.azure.com/",- model="gpt-4o-2024-05-13",- model_capabilities=ModelCapabilities(- function_calling=True, json_output=True, vision=True- ),- # azure_ad_token_provider=token_provider- )-- # Login assistant- login_assistant = await runtime.register_and_get_proxy(- "LoginAssistant",- lambda: Coder(- model_client=client,- system_messages=[- SystemMessage("""You are a general-purpose AI assistant and can handle many questions -- but you don't have access to a web browser. However, the user you are talking to does have a browser, and you can see the screen. Provide short direct instructions to them to take you where you need to go to answer the initial question posed to you.--Once the user has taken the final necessary action to complete the task, and you have fully addressed the initial request, reply with the word TERMINATE.""",- )- ],- ),- )-- # Web surfer- web_surfer = await runtime.register_and_get_proxy(- "WebSurfer",- lambda: MultimodalWebSurfer(), # Configuration is set later by init()- )-- actual_surfer = await runtime.try_get_underlying_agent_instance(web_surfer.id, type=MultimodalWebSurfer)- await actual_surfer.init(model_client=client, downloads_folder=os.getcwd(), browser_channel="chromium")-- # Round-robin orchestrator- round_robin_orc = await runtime.register_and_get_proxy("round_robin_orc", lambda: RoundRobinOrchestrator(- agents=[web_surfer, login_assistant],- )- )-- # Login to the necessary websites- for site in TASK["sites"]:- if site in ["reddit", "gitlab", "shopping", "shopping_admin"]:- actual_surfer.start_page = SITE_URLS[site]-- run_context = runtime.start()- await runtime.publish_message(ResetMessage(), namespace="default")- await runtime.publish_message(- BroadcastMessage(content=UserMessage(content=LOGIN_PROMPTS[site], source="human")),- namespace="default",- )- await run_context.stop_when_idle()-- # Deactivate the login-related agents- run_context = runtime.start()- await runtime.send_message(DeactivateMessage(), login_assistant.id)- await runtime.send_message(DeactivateMessage(), round_robin_orc.id)- await run_context.stop_when_idle()-- # By this point, we should be logged in. Prepare for the main event- coder = await runtime.register_and_get_proxy(- "Coder",- lambda: Coder(model_client=client),- )-- executor = await runtime.register_and_get_proxy(- "Executor",- lambda: Executor(- "A agent for executing code", executor=LocalCommandLineCodeExecutor()- ),- )-- file_surfer = await runtime.register_and_get_proxy(- "file_surfer",- lambda: FileSurfer(model_client=client),- )-- orchestrator = await runtime.register_and_get_proxy("orchestrator", lambda: LedgerOrchestrator(- agents=[coder, executor, file_surfer, web_surfer],- model_client=client,- max_rounds=30,- max_time=25*60,- ))-- # The main event- actual_surfer.start_page = TASK["start_url"]- run_context = runtime.start()- await runtime.send_message(ResetMessage(), web_surfer.id)-- # Provide some background about the pages- site_description_prompt = ""- sitename = url_to_sitename(TASK["start_url"])- if sitename:- site_description_prompt = ", " + SITE_DESCRIPTIONS[sitename]- task = f"Your web browser is currently open to the website {TASK['start_url']}{site_description_prompt}. On this website, please complete the following task:\n\n{TASK['intent']}"-- await runtime.publish_message(- BroadcastMessage(content=UserMessage(content=task.strip(), source="human")),- namespace="default",- )-- await run_context.stop_when_idle()-- # Output the final answer- actual_orchestrator = await runtime.try_get_underlying_agent_instance(orchestrator.id, type=LedgerOrchestrator)- transcript: List[LLMMessage] = actual_orchestrator._chat_history # type: ignore- final_answer = await response_preparer(task=TASK["intent"], source=(await orchestrator.metadata)["name"], client=client, transcript=transcript)-- m = re.search("FINAL ANSWER:(.*)$", final_answer, re.DOTALL)- if m:- final_answer = m.group(1).strip()-- print('page.stop("' + final_answer + '")')- print("MAIN TASK COMPLETE !#!#")-- ########## EVALUATION ##########- context = actual_surfer._context- page = actual_surfer._page- cdp_session = await context.new_cdp_session(page)- config_file = "full_task.json"-- evaluator = evaluation_harness.evaluator_router(config_file)- score = await evaluator(- trajectory=evaluation_harness.make_answer_trajecotry(final_answer),- config_file=config_file,- page=page,- client=cdp_session,- # azure_config=llm_config,- )-- print("FINAL SCORE: " + str(score))---if __name__ == "__main__":- logger = logging.getLogger(EVENT_LOGGER_NAME)- logger.setLevel(logging.INFO)- log_handler = LogHandler()- logger.handlers = [log_handler]- asyncio.run(main())diff --git a/python/benchmarks/WebArena/Templates/TeamOne/task_prompt.json.txt b/python/benchmarks/WebArena/Templates/TeamOne/task_prompt.json.txtdeleted file mode 100644index 2ec4f7ee3..000000000--- a/python/benchmarks/WebArena/Templates/TeamOne/task_prompt.json.txt+++ /dev/null@@ -1 +0,0 @@-__TASK_PROMPT__