A programming framework for agentic AI
Add collate file and more tests from autogpt into testbed (#915)
LeoLjl2ee944df378e
diff --git a/samples/tools/testbed/README.md b/samples/tools/testbed/README.mdindex 506c8b083..8c34f3e04 100644--- a/samples/tools/testbed/README.md+++ b/samples/tools/testbed/README.md@@ -216,3 +216,20 @@ python ./run_scenarios.py ./scenarios/GAIA/gaia_validation_level_1__two_agents_g# Compute Metricspython utils/collate_gaia_csv.py ./results/gaia_validation_level_1__two_agents_gpt4 | python utils/metrics_gaia.py```++## (Example) Running tasks from AutoGPT++The Testbed supports running tasks proposed in [AutoGPT benchmark](https://github.com/Significant-Gravitas/AutoGPT/tree/master/benchmark/agbenchmark/challenges). In this scenario, the agents are prompted to handle a diverse range of tasks, including coding, question answering according to given tasks, web scraping. Similar to scenarios in HumanEval, the agents can call the unit test script to check if the task is successfully done.++Accessing this scenario-type requires converting tasks, running the Testbed, collating the results, and finally computing the metrics. The following commands will run each test instance with GPT-4:++```+# Convert tasks+python utils/prepare_autogpt.py++# Run all the scenarios with GPT-4+python run_scenarios.py scenarios/AutoGPT/autogpt_twoagent_gpt4.jsonl++# Compute metrics, the metric script shares the same one with HumanEval+python utils/collate_autogpt.py ./results/autogpt_twoagent_gpt4 | python metrics_human_eval.py+```diff --git a/samples/tools/testbed/includes/requirements.txt b/samples/tools/testbed/includes/requirements.txtindex 8f88664e8..33070268d 100644--- a/samples/tools/testbed/includes/requirements.txt+++ b/samples/tools/testbed/includes/requirements.txt@@ -1 +1,5 @@git+https://github.com/microsoft/autogen.git+pandas+beautifulsoup4+requests+pytestdiff --git a/samples/tools/testbed/scenarios/AutoGPT/README.md b/samples/tools/testbed/scenarios/AutoGPT/README.mdindex 5e737a087..db08a0af4 100644--- a/samples/tools/testbed/scenarios/AutoGPT/README.md+++ b/samples/tools/testbed/scenarios/AutoGPT/README.md@@ -1,3 +1,3 @@The AutoGPT style tasks are contained in folder `challenges`.-Run `python utils/prepare_data.py` to convert the tasks to jsonl format compatible for evaluation.+Run `python ../../utils/prepare_autogpt.py` to convert the tasks to jsonl format compatible for evaluation.diff --git a/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/check.py b/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/check.pyindex cc015278c..57043d569 100644--- a/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/check.py+++ b/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/check.py@@ -3,6 +3,7 @@ import globimport osimport subprocessimport sys+import shutildef scoring(content: str, should_contain: list, should_not_contain: list):@@ -28,7 +29,6 @@ def scoring(content: str, should_contain: list, should_not_contain: list):def check():- workspace = "coding"files_contents = []scores = []@@ -54,9 +54,11 @@ def check():for file_path in matching_files:if eval_type == "python":+ # copy the test file to working directory+ shutil.copy(f"../custom_python/{file_path}", "./")result = subprocess.run([sys.executable, file_path],- cwd=os.path.abspath(workspace),+ cwd=os.path.abspath("./"),capture_output=True,text=True,)diff --git a/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/scenario.py b/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/scenario.pyindex 4add9d00c..e43b8f05d 100644--- a/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/scenario.py+++ b/samples/tools/testbed/scenarios/AutoGPT/Templates/TwoAgents/scenario.py@@ -24,15 +24,15 @@ user_proxy = UserProxyAgent("work_dir": work_dir,"use_docker": False,},- max_consecutive_auto_reply=10,+ max_consecutive_auto_reply=5,# default_auto_reply="TERMINATE",)if target_folder:# The tasks involves reading from a file then do sth to it.message = """- Your task is to: __TASK__ The file you needed is located in this directory: '__TARGET_FOLDER__'. You should save the output files in this directory: './'- Use the following command to check if all the unit tests have passed:+ Here is the task description: __TASK__ The file you needed is located in this directory: '__TARGET_FOLDER__'. You should save the output files in the current directory: './'+ Run the following command to check if all the unit tests have passed:```bashpython ../check.py```@@ -40,8 +40,8 @@ if target_folder:"""else:message = """- Your task is to: __TASK__- Use the following command to check if all the unit tests have passed:+ Here is the task description: __TASK__+ Run the following command to check if all the unit tests have passed:```bashpython ../check.py```diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/custom_python/test_pwd.py b/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/custom_python/test_pwd.pynew file mode 100644index 000000000..86ce911ab--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/custom_python/test_pwd.py@@ -0,0 +1,25 @@+import unittest++import password_generator+++class TestPasswordGenerator(unittest.TestCase):+ def test_password_length(self):+ for i in range(8, 17):+ password = password_generator.generate_password(i)+ self.assertEqual(len(password), i)++ def test_value_error(self):+ with self.assertRaises(ValueError):+ password_generator.generate_password(7)+ with self.assertRaises(ValueError):+ password_generator.generate_password(17)++ def test_password_content(self):+ password = password_generator.generate_password()+ self.assertTrue(any(c.isdigit() for c in password))+ self.assertTrue(any(c in password_generator.string.punctuation for c in password))+++if __name__ == "__main__":+ unittest.main()diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/data.jsonnew file mode 100644index 000000000..40af0e3c1--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/10_password_generator/data.json@@ -0,0 +1,28 @@+{+ "category": [+ "coding"+ ],+ "cutoff": 90,+ "dependencies": [+ "TestThreeSum"+ ],+ "eval_id": "ac75c471-e0ce-400c-ba9a-fb72aaab444f",+ "ground": {+ "answer": "password_generator.py is created and satisfies the requirements.",+ "eval": {+ "type": "python"+ },+ "files": [+ "test_pwd.py"+ ],+ "should_contain": [],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can create a random password generator.",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "PasswordGenerator",+ "task": "Create a random password generator. The password should have between 8 and 16 characters and should contain at least one letter, number and symbol. The password should be printed to the console. If no length is specified, the password should be 8 characters long. The password_generator should be imported as a module and called as password = password_generator.generate_password(length=x). Any invalid input should raise a ValueError."+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/custom_python/test_file_organize.py b/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/custom_python/test_file_organize.pynew file mode 100644index 000000000..fa39119ed--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/custom_python/test_file_organize.py@@ -0,0 +1,41 @@+import os+import subprocess+import tempfile+import unittest+++class TestOrganizeFiles(unittest.TestCase):+ def setUp(self):+ # Create temporary directory+ self.test_dir = tempfile.mkdtemp()++ # File types and their corresponding directory+ self.file_types = {+ "test_image.png": "images",+ "test_doc.txt": "documents",+ "test_audio.mp3": "audio",+ }++ # Create test files+ for file_name in self.file_types.keys():+ open(os.path.join(self.test_dir, file_name), "a").close()++ def test_organize_files(self):+ # Call the organize_files.py script using subprocess+ subprocess.call(["python", "organize_files.py", "--directory_path=" + self.test_dir])++ # Check if the files have been moved to the correct directories+ for file_name, directory in self.file_types.items():+ self.assertTrue(os.path.isfile(os.path.join(self.test_dir, directory, file_name)))++ def tearDown(self):+ # Delete test directory and its contents+ for file_name, directory in self.file_types.items():+ os.remove(os.path.join(self.test_dir, directory, file_name))+ for directory in set(self.file_types.values()):+ os.rmdir(os.path.join(self.test_dir, directory))+ os.rmdir(self.test_dir)+++if __name__ == "__main__":+ unittest.main()diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/data.jsonnew file mode 100644index 000000000..031a789c2--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/11_file_organizer/data.json@@ -0,0 +1,29 @@+{+ "category": [+ "coding",+ "general"+ ],+ "cutoff": 90,+ "dependencies": [+ "TestPasswordGenerator"+ ],+ "eval_id": "029c1e6f-2b36-451e-bca6-60063b827d2e",+ "ground": {+ "answer": "The correct python file is written and organizes the files accordingly",+ "eval": {+ "type": "python"+ },+ "files": [+ "test_file_organize.py"+ ],+ "should_contain": [],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can create a file organizer.",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "FileOrganizer",+ "task": "Create a file organizer CLI tool in Python that sorts files in a directory based on their file types (e.g., images, documents, audio) and moves them into these corresponding folders: 'images', 'documents', 'audio'. The entry point will be a python file that can be run this way: python organize_files.py --directory_path=YOUR_DIRECTORY_PATH"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/custom_python/test_url_shorten.py b/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/custom_python/test_url_shorten.pynew file mode 100644index 000000000..c3daffa80--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/custom_python/test_url_shorten.py@@ -0,0 +1,22 @@+import unittest++from url_shortener import retrieve_url, shorten_url+++class TestURLShortener(unittest.TestCase):+ def test_url_retrieval(self):+ # Shorten the URL to get its shortened form+ shortened_url = shorten_url("https://www.example.com")++ # Retrieve the original URL using the shortened URL directly+ retrieved_url = retrieve_url(shortened_url)++ self.assertEqual(+ retrieved_url,+ "https://www.example.com",+ "Retrieved URL does not match the original!",+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/data.jsonnew file mode 100644index 000000000..8fb2b6fc6--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/12_url_shortener/data.json@@ -0,0 +1,28 @@+{+ "category": [+ "coding"+ ],+ "cutoff": 150,+ "dependencies": [+ "TestFileOrganizer"+ ],+ "eval_id": "8106fd7f-83fd-496e-9513-280f4a3f012c",+ "ground": {+ "answer": "The correct python file for a basic url shortener CLI",+ "eval": {+ "type": "python"+ },+ "files": [+ "test_url_shorten.py"+ ],+ "should_contain": [],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can create a URL shortener.",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "UrlShortener",+ "task": "Build a basic URL shortener using a python CLI. Here are the specifications.\n\nFunctionality: The program should have two primary functionalities.\n\nShorten a given URL.\nRetrieve the original URL from a shortened URL.\n\nCLI: The command-line interface should accept a URL as its first input. It should be able to determine if the url is a shortened url or not. If the url is not shortened, it will display ONLY the shortened url, otherwise, it will display ONLY the original unshortened URL. Afterwards, it should prompt the user for another URL to process.\n\nTechnical specifications:\nBuild a file called url_shortener.py. This file will be called through command lines. Do not write your own test cases or any unit test code.\n\nEdge cases:\nFor the sake of simplicity, there will be no edge cases, you can assume the input is always correct and the user immediately passes the shortened version of the url he just shortened.\n\nYou will be expected to create a python file called url_shortener.py that will function through imported as a module.\n\nThe url_shortener.py will be tested this way:\n```\nimport unittest\nfrom url_shortener import shorten_url, retrieve_url\n\nclass TestURLShortener(unittest.TestCase):\n def test_url_retrieval(self):\n # Shorten the URL to get its shortened form\n shortened_url = shorten_url('https://www.example.com')\n\n # Retrieve the original URL using the shortened URL directly\n retrieved_url = retrieve_url(shortened_url)\n\n self.assertEqual(retrieved_url, 'https://www.example.com', \"Retrieved URL does not match the original!\")\n\nif __name__ == \"__main__\":\n unittest.main()\n```"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/custom_python/test_tictactoe.py b/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/custom_python/test_tictactoe.pynew file mode 100644index 000000000..94b778208--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/custom_python/test_tictactoe.py@@ -0,0 +1,41 @@+import subprocess++import pytest+++def run_game_with_inputs(inputs):+ # Start the game process+ process = subprocess.Popen(+ ["python", "tic_tac_toe.py"],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ text=True,+ )++ # Send the input moves one by one+ output, errors = process.communicate("\n".join(inputs))++ # Print the inputs and outputs+ print("Inputs:\n", "\n".join(inputs))+ print("Output:\n", output)+ print("Errors:\n", errors)++ return output+++ "inputs, expected_output",+ [+ (["0,0", "1,0", "0,1", "1,1", "0,2"], "Player 1 won!"),+ (["1,0", "0,0", "1,1", "0,1", "2,0", "0,2"], "Player 2 won!"),+ (["0,0", "0,1", "0,2", "1,1", "1,0", "1,2", "2,1", "2,0", "2,2"], "Draw"),+ ],+)+def test_game(inputs, expected_output):+ output = run_game_with_inputs(inputs)+ assert expected_output in output+++if __name__ == "__main__":+ pytest.main()diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/data.jsonnew file mode 100644index 000000000..288f02742--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/13_tic_tac_toe/data.json@@ -0,0 +1,29 @@+{+ "category": [+ "coding",+ "general"+ ],+ "cutoff": 150,+ "dependencies": [+ "TestUrlShortener"+ ],+ "eval_id": "504b1648-e14a-4982-8b27-074598eb4fd0",+ "ground": {+ "answer": "The correct python file for a TicTacToe game is written",+ "eval": {+ "type": "python"+ },+ "files": [+ "test_tictactoe.py"+ ],+ "should_contain": [],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can create Tic-Tac-Toe game",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "TicTacToe",+ "task": "Build a Tic-Tac-Toe game using a python CLI. Here are the specifications.\n\nThe Grid: The game board is a 3x3 grid, consisting of 3 rows and 3 columns, creating a total of 9 squares.\n\nPlayers: There are two players. One player uses the number \"1\", and the other player uses the number \"2\".\n\nTaking Turns: Players take turns to put their respective numbers (\"1\" or \"2\") in an empty square of the grid. Once a player has placed their number in a square, it cannot be changed or removed.\n\nObjective: The goal is to get three of your numbers in a row, either horizontally, vertically, or diagonally.\n\nEnd of the Game: The game concludes in one of two ways: One player gets three of their numbers in a row (horizontally, vertically, or diagonally) and is declared the winner.\nAll squares on the grid are filled, and no player has three in a row. This situation is a \"draw\" or a \"tie\".\n\nTechnical specifications:\nBuild a file called tic_tac_toe.py. This file will be called through command lines. You will have to prompt users for their move. Player 1 will always start.\nPlayers will input their move in the following format: \"x,y\" where x and y represent the location in the grid (0,0 is top left, 2,2 is bottom right).\n\nYour primary requirement is to halt the game when appropriate and to print only one of these three exact sentences:\n\n\"Player 1 won!\"\n\"Player 2 won!\"\n\"Draw\"\n\nEdge cases: A player can send an incorrect location. Either the location is incorrect or the square is already filled. In this case, this counts as doing nothing, and the player gets prompted for new locations again.\n\n\nYou will be expected to create a python file called tic_tac_toe.py that will run through command lines by using ```python tic_tac_toe.py```.\n\nHere is an example of how your tic_tac_toe.py game will be tested.\n```\nprocess = subprocess.Popen(\n ['python', 'tic_tac_toe.py'],\n stdout=subprocess.PIPE,\n text=True\n)\n\noutput, _ = process.communicate('\\n'.join([\"0,0\", \"1,0\", \"0,1\", \"1,1\", \"0,2\"]))\n\nassert \"Player 1 won!\" in output\n```"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file1.csv b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file1.csvnew file mode 100644index 000000000..fe552d677--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file1.csv@@ -0,0 +1,4 @@+ID,Name,Age+101,John,28+102,Alice,34+103,Bob,45diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file2.csv b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file2.csvnew file mode 100644index 000000000..685e24f4b--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/artifacts_in/file2.csv@@ -0,0 +1,4 @@+ID,Occupation,Salary+101,Engineer,80000+102,Doctor,120000+103,Lawyer,95000diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/data.jsonnew file mode 100644index 000000000..1943803c7--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/2_combine_csv/data.json@@ -0,0 +1,32 @@+{+ "category": [+ "data",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestSortCsv"+ ],+ "eval_id": "52467beb-b951-4356-9776-9a0ae46bb33b",+ "ground": {+ "answer": "The csv data is combined",+ "eval": {+ "type": "file"+ },+ "files": [+ "output.csv"+ ],+ "should_contain": [+ "Age,ID,Name,Occupation,Salary\n28,101,John,Engineer,80000\n34,102,Alice,Doctor,120000\n45,103,Bob,Lawyer,95000"+ ]+ },+ "info": {+ "description": "Tests if the agent can combine data from a csv",+ "difficulty": "intermediate",+ "side_effects": [+ ""+ ]+ },+ "name": "CombineCsv",+ "task": "The csvs 'file1.csv' and 'file2.csv' both have a column 'ID'. Combine these 2 csvs using the 'ID' column. Then sort the rows by 'ID' in ascending order, sort the columns alphabetically. Write the output in 'output.csv'."+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/artifacts_in/file1.csv b/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/artifacts_in/file1.csvnew file mode 100644index 000000000..8f5826e9f--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/artifacts_in/file1.csv@@ -0,0 +1,12 @@+Date,Description,Amount,Category+2023-01-01,Grocery Store,52.3,Groceries+2023-01-02,Pharmacy,12.5,Healthcare+2023-01-03,Gas Station,29.1,Transportation+2023-01-04,Water,19,Utilities+2023-01-05,Grocery Store,60.25,Groceries+2023-01-06,Coffee Shop,4.5,Dining+2023-01-07,Cinema Tickets,20,Entertainment+2023-01-08,Book Store,30.4,Shopping+2023-01-09,Restaurant Dinner,55.8,Dining+2023-01-10,Electric Bill,65.35,Utilities+2023-01-11,Grocery Store,45.1,Groceriesdiff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/data.jsonnew file mode 100644index 000000000..639f7946a--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/3_qa_small_csv/data.json@@ -0,0 +1,32 @@+{+ "category": [+ "data",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestReadFile"+ ],+ "eval_id": "9df3f07a-5047-488f-b788-1e1f57eba970",+ "ground": {+ "answer": "The correct amount spent on utilities.",+ "eval": {+ "type": "file"+ },+ "files": [+ "output.txt"+ ],+ "should_contain": [+ "84"+ ]+ },+ "info": {+ "description": "Tests if the agent can answer a question from a small csv",+ "difficulty": "intermediate",+ "side_effects": [+ ""+ ]+ },+ "name": "AnswerQuestionSmallCsv",+ "task": "Read and analyze 'file1.csv', then answer the quetion: How much was spent on utilities in total? Write the answer in file 'output.txt'."+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/artifacts_in/file1.csv b/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/artifacts_in/file1.csvnew file mode 100644index 000000000..cb22e61c4--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/artifacts_in/file1.csv@@ -0,0 +1,305 @@+Date,Description,Amount,Category+2023-01-01,Grocery Store,52.3,Groceries+2023-01-02,Pharmacy,12.5,Healthcare+2023-01-03,Gas Station,29.1,Transportation+2023-01-04,Cinema Tickets,19,Entertainment+2023-01-05,Grocery Store,60.25,Groceries+2023-01-06,Coffee Shop,4.5,Dining+2023-01-07,Cinema Tickets,20,Entertainment+2023-01-08,Book Store,30.4,Shopping+2023-01-09,Restaurant Dinner,55.8,Dining+2023-01-10,Electric Bill,65.35,Utilities+2023-01-11,Grocery Store,45.1,Groceries+2023-01-12,Clothing Store,100.2,Shopping+2023-01-13,Pharmacy,20.3,Healthcare+2023-01-14,Coffee Shop,4.5,Dining+2023-01-15,Restaurant Dinner,50,Dining+2023-01-16,Gas Station,32.1,Transportation+2023-01-17,Online Shopping,80,Shopping+2023-01-18,Water Bill,20.35,Utilities+2023-01-19,Grocery Store,55.6,Groceries+2023-01-20,Gas Station,28,Transportation+2023-01-21,Pharmacy,15.4,Healthcare+2023-01-22,Phone Bill,40,Utilities+2023-01-23,Cinema Tickets,20,Entertainment+2023-01-24,Coffee Shop,5.5,Dining+2023-01-25,Book Purchase,14,Shopping+2023-01-26,Restaurant Lunch,30,Dining+2023-01-27,Public Transport,20,Transportation+2023-01-28,Grocery Store,58.25,Groceries+2023-01-29,Online Shopping,70,Shopping+2023-01-30,Grocery Store,62.1,Groceries+2023-01-31,Medical Prescription,10.4,Healthcare+2023-02-01,Gas Station,33,Transportation+2023-02-02,Coffee Shop,6,Dining+2023-02-03,Cinema Tickets,22,Entertainment+2023-02-04,Book Store,28.4,Shopping+2023-02-05,Internet Bill,50,Utilities+2023-02-06,Grocery Store,60.1,Groceries+2023-02-07,Clothing Store,120,Shopping+2023-02-08,Grocery Store,58.25,Groceries+2023-02-09,Coffee Shop,4.5,Dining+2023-02-10,Electric Bill,70,Utilities+2023-02-11,Grocery Store,50.1,Groceries+2023-02-12,Public Transport,18,Transportation+2023-02-13,Pharmacy,24,Healthcare+2023-02-14,Restaurant Dinner,60,Dining+2023-02-15,Medical Prescription,11.4,Healthcare+2023-02-16,Gas Station,30,Transportation+2023-02-17,Online Shopping,85,Shopping+2023-02-18,Water Bill,18,Utilities+2023-02-19,Grocery Store,53.6,Groceries+2023-02-20,Public Transport,22,Transportation+2023-02-21,Pharmacy,10,Healthcare+2023-02-22,Phone Bill,42,Utilities+2023-02-23,Cinema Tickets,24,Entertainment+2023-02-24,Coffee Shop,6,Dining+2023-02-25,Book Purchase,16,Shopping+2023-02-26,Restaurant Lunch,28,Dining+2023-02-27,Gas Station,34,Transportation+2023-02-28,Grocery Store,56,Groceries+2023-03-01,Online Shopping,90,Groceries+2023-03-02,Dentist Appointment,130,Healthcare+2023-03-03,Grocery Store,63.45,Groceries+2023-03-04,Cinema Tickets,21,Entertainment+2023-03-05,Coffee Shop,5.8,Dining+2023-03-06,Electric Bill,67.5,Utilities+2023-03-07,Gas Station,31.2,Transportation+2023-03-08,Restaurant Dinner,58,Dining+2023-03-09,Pharmacy,18.3,Healthcare+2023-03-10,Grocery Store,64.7,Groceries+2023-03-11,Book Store,25.4,Shopping+2023-03-12,Online Shopping,78,Shopping+2023-03-13,Coffee Shop,6.5,Dining+2023-03-14,Museum Tickets,15,Entertainment+2023-03-15,Internet Bill,52,Utilities+2023-03-16,Public Transport,19.5,Transportation+2023-03-17,Clothing Store,105.6,Shopping+2023-03-18,Phone Bill,41,Utilities+2023-03-19,Coffee Shop,5,Dining+2023-03-20,Grocery Store,59.2,Groceries+2023-03-21,Gas Station,29.8,Transportation+2023-03-22,Restaurant Lunch,32,Dining+2023-03-23,Pharmacy,16.5,Healthcare+2023-03-24,Concert Tickets,50,Entertainment+2023-03-25,Coffee Shop,5.5,Dining+2023-03-26,Grocery Store,61.8,Groceries+2023-03-27,Online Shopping,82,Shopping+2023-03-28,Water Bill,19.35,Utilities+2023-03-29,Public Transport,21,Transportation+2023-03-30,Book Purchase,17,Shopping+2023-03-31,Grocery Store,60,Groceries+2023-04-01,Cinema Tickets,23,Entertainment+2023-04-02,Pharmacy,17.4,Healthcare+2023-04-03,Gas Station,33.5,Transportation+2023-04-04,Restaurant Dinner,56.7,Dining+2023-04-05,Grocery Store,65.3,Groceries+2023-04-06,Coffee Shop,5.9,Dining+2023-04-07,Online Shopping,87,Shopping+2023-04-08,Electric Bill,69,Utilities+2023-04-09,Clothing Store,112.5,Shopping+2023-04-10,Grocery Store,57.4,Groceries+2023-04-11,Book Store,26.3,Shopping+2023-04-12,Gas Station,30.9,Transportation+2023-04-13,Coffee Shop,6.8,Dining+2023-04-14,Zoo Tickets,24,Entertainment+2023-04-15,Internet Bill,53,Utilities+2023-04-16,Public Transport,20.5,Transportation+2023-04-17,Restaurant Lunch,34,Dining+2023-04-18,Phone Bill,43,Utilities+2023-04-19,Coffee Shop,5.2,Dining+2023-04-20,Grocery Store,58.9,Groceries+2023-04-21,Pharmacy,14.7,Healthcare+2023-04-22,Cinema Tickets,25,Entertainment+2023-04-23,Online Shopping,90,Shopping+2023-04-24,Gas Station,31.4,Transportation+2023-04-25,Water Bill,21,Utilities+2023-04-26,Grocery Store,62.5,Groceries+2023-04-27,Coffee Shop,5.7,Dining+2023-04-28,Book Purchase,18.5,Shopping+2023-04-29,Public Transport,22,Transportation+2023-04-30,Grocery Store,63,Groceries+2023-05-01,Theater Tickets,45,Entertainment+2023-05-02,Dentist Appointment,135,Healthcare+2023-05-03,Gas Station,32.2,Transportation+2023-05-04,Restaurant Dinner,59,Dining+2023-05-05,Grocery Store,66.1,Groceries+2023-05-06,Coffee Shop,6,Dining+2023-05-07,Online Shopping,89,Shopping+2023-05-08,Electric Bill,70.5,Utilities+2023-05-09,Clothing Store,110,Shopping+2023-05-10,Grocery Store,59.7,Groceries+2023-05-11,Coffee Shop,6.1,Dining+2023-05-12,Book Store,29.2,Shopping+2023-05-13,Gas Station,29.9,Transportation+2023-05-14,Museum Tickets,16,Entertainment+2023-05-15,Internet Bill,52.5,Utilities+2023-05-16,Public Transport,21.3,Transportation+2023-05-17,Restaurant Lunch,35.4,Dining+2023-05-18,Phone Bill,43.5,Utilities+2023-05-19,Grocery Store,64.8,Groceries+2023-05-20,Pharmacy,15.2,Healthcare+2023-05-21,Cinema Tickets,26,Entertainment+2023-05-22,Coffee Shop,6.3,Dining+2023-05-23,Gas Station,30.8,Transportation+2023-05-24,Online Shopping,92.5,Shopping+2023-05-25,Water Bill,20.5,Utilities+2023-05-26,Grocery Store,61.9,Groceries+2023-05-27,Public Transport,23,Transportation+2023-05-28,Book Purchase,19,Shopping+2023-05-29,Coffee Shop,5.9,Dining+2023-05-30,Restaurant Dinner,57.8,Dining+2023-05-31,Grocery Store,66.7,Groceries+2023-06-01,Theater Tickets,47,Entertainment+2023-06-02,Dentist Appointment,140,Healthcare+2023-06-03,Gas Station,31.6,Transportation+2023-06-04,Coffee Shop,6.4,Dining+2023-06-05,Online Shopping,94,Shopping+2023-06-06,Electric Bill,72,Utilities+2023-06-07,Restaurant Lunch,36,Dining+2023-06-08,Grocery Store,65.3,Groceries+2023-06-09,Pharmacy,17,Healthcare+2023-06-10,Cinema Tickets,27.5,Entertainment+2023-06-11,Public Transport,21.5,Transportation+2023-06-12,Book Store,30,Shopping+2023-06-13,Gas Station,28.7,Transportation+2023-06-14,Coffee Shop,6.6,Dining+2023-06-15,Internet Bill,53.5,Utilities+2023-06-16,Zoo Tickets,28,Entertainment+2023-06-17,Grocery Store,67.4,Groceries+2023-06-18,Phone Bill,44,Utilities+2023-06-19,Restaurant Dinner,60,Dining+2023-06-20,Coffee Shop,6.7,Dining+2023-06-21,Public Transport,22.5,Transportation+2023-06-22,Online Shopping,96,Shopping+2023-06-23,Gas Station,32.4,Transportation+2023-06-24,Cinema Tickets,29,Entertainment+2023-06-25,Book Purchase,20,Shopping+2023-06-26,Grocery Store,68.3,Groceries+2023-06-27,Water Bill,22,Utilities+2023-06-28,Pharmacy,18.5,Healthcare+2023-06-29,Restaurant Lunch,37,Dining+2023-06-30,Coffee Shop,7,Dining+2023-07-01,Grocery Store,69.5,Groceries+2023-07-02,Theater Tickets,49,Entertainment+2023-07-03,Gas Station,33.2,Transportation+2023-07-04,Park Picnic,40,Dining+2023-07-05,Electric Bill,73.5,Utilities+2023-07-06,Clothing Store,120,Shopping+2023-07-07,Online Shopping,98,Shopping+2023-07-08,Grocery Store,70.6,Groceries+2023-07-09,Coffee Shop,7.1,Dining+2023-07-10,Internet Bill,54,Utilities+2023-07-11,Public Transport,23.5,Transportation+2023-07-12,Museum Tickets,18,Entertainment+2023-07-13,Book Store,31,Shopping+2023-07-14,Gas Station,29.9,Transportation+2023-07-15,Coffee Shop,7.2,Dining+2023-07-16,Restaurant Dinner,62,Dining+2023-07-17,Grocery Store,71.8,Groceries+2023-07-18,Phone Bill,45,Utilities+2023-07-19,Zoo Tickets,30,Entertainment+2023-07-20,Coffee Shop,7.3,Dining+2023-07-21,Public Transport,24,Transportation+2023-07-22,Online Shopping,99.5,Shopping+2023-07-23,Gas Station,34,Transportation+2023-07-24,Cinema Tickets,31,Entertainment+2023-07-25,Book Purchase,21.5,Shopping+2023-07-26,Grocery Store,72.9,Groceries+2023-07-27,Water Bill,23.5,Utilities+2023-07-28,Pharmacy,19.5,Healthcare+2023-07-29,Restaurant Lunch,38.5,Dining+2023-07-30,Coffee Shop,7.4,Dining+2023-07-31,Grocery Store,73.7,Groceries+2023-08-01,Theater Tickets,50,Entertainment+2023-08-02,Gas Station,34.5,Transportation+2023-08-03,Restaurant Dinner,63.5,Dining+2023-08-04,Online Shopping,101,Shopping+2023-08-05,Electric Bill,75,Utilities+2023-08-06,Grocery Store,74.6,Groceries+2023-08-07,Coffee Shop,7.5,Dining+2023-08-08,Phone Bill,46,Utilities+2023-08-09,Public Transport,24.5,Transportation+2023-08-10,Cinema Tickets,32.5,Entertainment+2023-08-11,Book Store,32,Shopping+2023-08-12,Gas Station,35,Transportation+2023-08-13,Coffee Shop,7.6,Dining+2023-08-14,Park Picnic,42,Dining+2023-08-15,Internet Bill,55,Utilities+2023-08-16,Grocery Store,76.3,Groceries+2023-08-17,Clothing Store,125,Shopping+2023-08-18,Pharmacy,20.5,Healthcare+2023-08-19,Restaurant Lunch,40,Dining+2023-08-20,Coffee Shop,7.7,Dining+2023-08-21,Museum Tickets,19,Entertainment+2023-08-22,Public Transport,25,Transportation+2023-08-23,Online Shopping,103,Shopping+2023-08-24,Grocery Store,77.8,Groceries+2023-08-25,Water Bill,24.5,Utilities+2023-08-26,Zoo Tickets,32,Entertainment+2023-08-27,Coffee Shop,7.8,Dining+2023-08-28,Gas Station,35.5,Transportation+2023-08-29,Book Purchase,23,Shopping+2023-08-30,Grocery Store,78.9,Groceries+2023-08-31,Cinema Tickets,34,Entertainment+2023-09-01,Theater Tickets,52,Entertainment+2023-09-02,Gas Station,36,Transportation+2023-09-03,Restaurant Dinner,65,Dining+2023-09-04,Online Shopping,105,Shopping+2023-09-05,Electric Bill,76.5,Utilities+2023-09-06,Grocery Store,79.6,Groceries+2023-09-07,Coffee Shop,8,Dining+2023-09-08,Phone Bill,47,Utilities+2023-09-09,Public Transport,26,Transportation+2023-09-10,Cinema Tickets,35.5,Entertainment+2023-09-11,Book Store,33,Shopping+2023-09-12,Gas Station,36.5,Transportation+2023-09-13,Coffee Shop,8.2,Dining+2023-09-14,Park Picnic,44,Dining+2023-09-15,Internet Bill,56,Utilities+2023-09-16,Grocery Store,80.4,Groceries+2023-09-17,Clothing Store,130,Shopping+2023-09-18,Pharmacy,21.5,Healthcare+2023-09-19,Restaurant Lunch,41.5,Dining+2023-09-20,Coffee Shop,8.4,Dining+2023-09-21,Museum Tickets,20,Entertainment+2023-09-22,Public Transport,26.5,Transportation+2023-09-23,Online Shopping,107,Shopping+2023-09-24,Grocery Store,81.3,Groceries+2023-09-25,Water Bill,25.5,Utilities+2023-09-26,Zoo Tickets,33.5,Entertainment+2023-09-27,Coffee Shop,8.6,Dining+2023-09-28,Gas Station,37.5,Transportation+2023-09-29,Book Purchase,24.5,Shopping+2023-09-30,Grocery Store,82.7,Groceries+2023-10-01,Cinema Tickets,36,Entertainment+2023-10-02,Theater Tickets,54,Entertainment+2023-10-03,Gas Station,38,Transportation+2023-10-04,Restaurant Dinner,66.5,Dining+2023-10-05,Online Shopping,109,Shopping+2023-10-06,Electric Bill,78,Utilities+2023-10-07,Grocery Store,83.9,Groceries+2023-10-08,Coffee Shop,8.8,Dining+2023-10-09,Phone Bill,48,Utilities+2023-10-10,Public Transport,27.5,Transportation+2023-10-11,Cinema Tickets,37.5,Entertainment+2023-10-12,Book Store,34.5,Shopping+2023-10-13,Gas Station,39.5,Transportation+2023-10-14,Coffee Shop,9,Dining+2023-10-15,Park Picnic,46,Dining+2023-10-16,Internet Bill,57.5,Utilities+2023-10-17,Grocery Store,85.2,Groceries+2023-10-18,Clothing Store,135,Shopping+2023-10-19,Pharmacy,22.5,Healthcare+2023-10-20,Restaurant Lunch,43,Dining+2023-10-21,Coffee Shop,9.2,Dining+2023-10-22,Museum Tickets,21.5,Entertainment+2023-10-23,Public Transport,28,Transportation+2023-10-24,Online Shopping,111,Shopping+2023-10-25,Grocery Store,86.5,Groceries+2023-10-26,Water Bill,26.5,Utilities+2023-10-27,Zoo Tickets,35,Entertainment+2023-10-28,Coffee Shop,9.4,Dining+2023-10-29,Gas Station,40.5,Transportation+2023-10-30,Book Purchase,26,Shopping+2023-10-31,Grocery Store,88,Groceriesdiff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/data.jsonnew file mode 100644index 000000000..6647a4397--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/4_qa_csv/data.json@@ -0,0 +1,31 @@+{+ "category": [+ "data"+ ],+ "cutoff": 90,+ "dependencies": [+ "TestAnswerQuestionSmallCsv"+ ],+ "eval_id": "bb6e0a4b-7faf-4aa6-a524-548cddbc2732",+ "ground": {+ "answer": "The correct amount spent on utilities.",+ "eval": {+ "type": "file"+ },+ "files": [+ "output.txt"+ ],+ "should_contain": [+ "1861"+ ]+ },+ "info": {+ "description": "Tests if the agent can answer a question from a csv",+ "difficulty": "intermediate",+ "side_effects": [+ ""+ ]+ },+ "name": "AnswerQuestionCsv",+ "task": "Read and analyze 'file1.txt', then answer the question: How much was spent on utilities in total? Write the answer in 'output.txt'."+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/5_search/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/5_search/data.jsonnew file mode 100644index 000000000..3a174bbe2--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/5_search/data.json@@ -0,0 +1,37 @@+{+ "category": [+ "general",+ "scrape_synthesize"+ ],+ "cutoff": 120,+ "dependencies": [+ "TestWriteFile"+ ],+ "eval_id": "0bb23182-b434-402b-a73e-9c226469b959",+ "ground": {+ "answer": "This is a Heading\nThis is a paragraph.",+ "eval": {+ "type": "file"+ },+ "files": [+ ".txt"+ ],+ "should_contain": [+ "Heading",+ "paragraph"+ ],+ "should_not_contain": [+ "The",+ "the"+ ]+ },+ "info": {+ "description": "Tests if the agent can search.",+ "difficulty": "interface",+ "side_effects": [+ ""+ ]+ },+ "name": "Search",+ "task": "Open 'https://silennaihin.com/random/plain.html' and paste all of the text on the page in a .txt file. Save it to the current directory: './' "+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/6_book_price/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/6_book_price/data.jsonnew file mode 100644index 000000000..86328ebd7--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/6_book_price/data.json@@ -0,0 +1,31 @@+{+ "category": [+ "scrape_synthesize",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestSearch"+ ],+ "eval_id": "cd96e6b2-779d-4a4a-8367-d520023e27ae",+ "ground": {+ "answer": "\u00a325.89",+ "eval": {+ "type": "file"+ },+ "files": [+ ".txt"+ ],+ "should_contain": [+ "25.89"+ ],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can retrieve a specific information from a website.",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "BasicRetrieval",+ "task": "Write the price of the book in this url 'http://books.toscrape.com/catalogue/meditations_33/index.html' to a .txt file. Save it to the current directory: './'"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/7_revenue/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/7_revenue/data.jsonnew file mode 100644index 000000000..923c1e36a--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/7_revenue/data.json@@ -0,0 +1,31 @@+{+ "category": [+ "scrape_synthesize",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestBasicRetrieval"+ ],+ "eval_id": "dc2114d7-1597-4c9b-bed0-a97937ad977f",+ "ground": {+ "answer": "It was $81.462 billion in 2022. In millions the answer is 81,462.",+ "eval": {+ "type": "file"+ },+ "files": [+ ".txt"+ ],+ "should_contain": [+ "81,462"+ ],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can retrieve Tesla's revenue in 2022.",+ "difficulty": "intermediate",+ "side_effects": []+ },+ "name": "RevenueRetrieval",+ "task": "Write tesla's exact revenue in 2022 into a .txt file. Use the US notation, with a precision rounded to the nearest million dollars (for instance, $31,578 million). Save the file to the current directory: './'"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/8_get_information/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/8_get_information/data.jsonnew file mode 100644index 000000000..f198d2e4e--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/8_get_information/data.json@@ -0,0 +1,35 @@+{+ "category": [+ "scrape_synthesize",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestRevenueRetrieval"+ ],+ "eval_id": "1758058c-f726-484f-96fa-f05e278e5ff5",+ "ground": {+ "answer": "The twitter handles of the two hosts of Latent Space.",+ "case_sensitive": false,+ "eval": {+ "type": "file"+ },+ "files": [+ "output.txt"+ ],+ "should_contain": [+ "swyx",+ "FanaHOVA"+ ],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can retrieve twitter handles given a vague description.",+ "difficulty": "intermediate",+ "side_effects": [+ ""+ ]+ },+ "name": "TestGetInformation",+ "task": "Write the twitter handle of the two hosts of Latent Space to a file called output.txt"+}diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/custom_python/test_three_sum.py b/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/custom_python/test_three_sum.pynew file mode 100644index 000000000..0cc4811f4--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/custom_python/test_three_sum.py@@ -0,0 +1,32 @@+# mypy: ignore-errors+from typing import List++from sample_code import three_sum+++def test_three_sum(nums: List[int], target: int, expected_result: List[int]) -> None:+ result = three_sum(nums, target)+ print(result)+ assert (+ result == expected_result+ ), f"AssertionError: Given input nums: {nums}, target: {target}\nExpected the output to be {expected_result}, got {result} instead."+++if __name__ == "__main__":+ # test the trivial case with the first three numbers+ nums = [2, 7, 11, 15]+ target = 20+ expected_result = [0, 1, 2]+ test_three_sum(nums, target, expected_result)++ # test for ability to use zero and the same number twice+ nums = [2, 7, 0, 15, 12, 0]+ target = 2+ expected_result = [0, 2, 5]+ test_three_sum(nums, target, expected_result)++ # test for first and last index usage and negative numbers+ nums = [-6, 7, 11, 4]+ target = 9+ expected_result = [0, 2, 3]+ test_three_sum(nums, target, expected_result)diff --git a/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/data.json b/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/data.jsonnew file mode 100644index 000000000..bf93a9e6b--- /dev/null+++ b/samples/tools/testbed/scenarios/AutoGPT/challenges/9_three_sum/data.json@@ -0,0 +1,33 @@+{+ "category": [+ "coding",+ "general"+ ],+ "cutoff": 60,+ "dependencies": [+ "TestWriteFile"+ ],+ "eval_id": "a1ff38a4-1032-4bf2-960a-3b927f9936f4",+ "ground": {+ "answer": "The three_sum function coded properly.",+ "eval": {+ "type": "python"+ },+ "files": [+ "test_three_sum.py"+ ],+ "should_contain": [+ "[0, 1, 2]",+ "[0, 2, 5]",+ "[0, 2, 3]"+ ],+ "should_not_contain": []+ },+ "info": {+ "description": "Tests if the agent can create the three_sum function.",+ "difficulty": "basic",+ "side_effects": []+ },+ "name": "ThreeSum",+ "task": "Create a three_sum function in a file called sample_code.py. Given an array of integers, return indices of the three numbers such that they add up to a specific target. You may assume that each input would have exactly one solution, and you should not use the same element twice. The input array is NOT sorted. Example: Given nums = [2, 7, 11, 15], target = 20, Because nums[0] + nums[1] + nums[2] = 2 + 7 + 11 = 20, return [0, 1, 2]."+}diff --git a/samples/tools/testbed/utils/collate_autogpt.py b/samples/tools/testbed/utils/collate_autogpt.pynew file mode 100644index 000000000..ddde5b793--- /dev/null+++ b/samples/tools/testbed/utils/collate_autogpt.py@@ -0,0 +1,108 @@+import argparse+import os+import re+import subprocess+import sys+++def collate(results_dir="results"):+ """+ Collate the results of running AutoGPT test.++ Args:+ results_dir (str, optional): The folder where results are saved. Defaults to "results".+ """++ all_results = list()+ max_instances = 0++ for test_name in os.listdir(results_dir):+ test_path = os.path.join(results_dir, test_name)++ # Collect the results vector+ results = [test_name]++ instance = 0+ instance_dir = os.path.join(test_path, str(instance))+ while os.path.isdir(instance_dir):+ console_log = os.path.join(instance_dir, "console_log.txt")+ if os.path.isfile(console_log):+ with open(console_log, "rt") as fh:+ content = fh.read()+ if "ALL TESTS PASSED!" in content:+ # Ideally we would have a more distinctive pattern.+ results.append(str(len(re.findall(r"\n(.*?) \(to (.*?)\)\:\n", content))))+ else:+ # Sometimes the task actually succeeds, but the check.py isn't properly called+ result = subprocess.run(+ [sys.executable, "../check.py"],+ cwd=os.path.join(instance_dir, "coding"),+ capture_output=True,+ text=True,+ )+ if "error" in result.stderr or result.returncode != 0:+ results.append("-1")+ else:+ # The task actually succeeds.+ if "ALL TESTS PASSED!" in result.stdout:+ results.append(str(len(re.findall(r"\n(.*?) \(to (.*?)\)\:\n", content))))+ else:+ results.append("-1")+ else:+ # Missing results will appear as blanks+ results.append("")++ instance += 1+ instance_dir = os.path.join(test_path, str(instance))++ max_instances = max(max_instances, instance)++ # Buffer the results+ all_results.append(results)++ # Create a header+ header = "TestName"+ for i in range(0, max_instances):+ header += ",Trial" + str(i)+ print(header)++ # Print a fully-populated table of results+ for r in all_results:+ while len(r) < max_instances + 1:+ r.append("")+ print(",".join(r))+++if __name__ == "__main__":+ script_path = os.path.realpath(__file__)+ script_name = os.path.basename(script_path)+ script_dir = os.path.dirname(script_path)++ # Path to the default results directory+ # (relative to this script, up on directory, then into the results folder)+ default_results_dir = os.path.realpath(os.path.join(script_dir, os.path.pardir, "results"))++ parser = argparse.ArgumentParser(+ description=f"""+{script_name} will collate the results of the AutoGPT scenarios and output them to a CSV. The CSV format is as follows:++TestName, Trial0, Trial1, ..., TrialN+Test_1, x_10, x_11, ..., X_1N+Test_2, x_20, x_21, ..., X_2N+...+Test_M, x_M0, x_M1, ..., X_MN+++Where x_ij is the number of AsssitantAgent conversation turns needed to pass all the tests for problem i, in Trial/repetition j. If the agent was not able to pass the tests by the end of the conversation, the value will be -1. If data for the trial is missing, the value will be an empty string "".+""".strip(),+ formatter_class=argparse.RawTextHelpFormatter,+ )++ parser.add_argument(+ "scenario",+ nargs="?",+ help="Path to the scenario results. (default: " + default_results_dir + ")",+ default=default_results_dir,+ )+ args = parser.parse_args()+ collate(args.scenario)diff --git a/samples/tools/testbed/utils/collate_human_eval.py b/samples/tools/testbed/utils/collate_human_eval.pyindex 19dcaac27..594051dc2 100644--- a/samples/tools/testbed/utils/collate_human_eval.py+++ b/samples/tools/testbed/utils/collate_human_eval.py@@ -10,7 +10,7 @@ def collate(results_dir):Collate the results of running human eval.Args:- results_dir (path): The folder were results were be saved.+ results_dir (path): The folder where results are saved."""all_results = list()@@ -19,7 +19,7 @@ def collate(results_dir):for test_id in os.listdir(results_dir):test_path = os.path.join(results_dir, test_id)- # Collect the reslts vector+ # Collect the results vectorresults = [test_id]instance = 0diff --git a/samples/tools/testbed/scenarios/AutoGPT/utils/prepare_data.py b/samples/tools/testbed/utils/prepare_autogpt.pysimilarity index 69%rename from samples/tools/testbed/scenarios/AutoGPT/utils/prepare_data.pyrename to samples/tools/testbed/utils/prepare_autogpt.pyindex f298b15e8..9da279735 100644--- a/samples/tools/testbed/scenarios/AutoGPT/utils/prepare_data.py+++ b/samples/tools/testbed/utils/prepare_autogpt.py@@ -4,7 +4,9 @@ import jsonimport osimport shutil-data_paths = glob.glob("challenges/*/data.json")+current_file_dir = os.path.dirname(os.path.abspath(__file__))+challenge_path = os.path.join(os.path.dirname(current_file_dir), "scenarios/AutoGPT/challenges")+data_paths = glob.glob(str(challenge_path) + "/*/data.json")for data_path in data_paths:print("Converting data path: ", data_path)@@ -29,21 +31,32 @@ for data_path in data_paths:should_not_contain_base64.append(encoded_word)# copy all the files needed to 'coding' directory+ # 1. 'artifacts_in' directory: all the files needed for QA+ save_path = os.path.join(os.path.dirname(current_file_dir), "scenarios/AutoGPT")artifacts_in = Falseif os.path.exists(os.path.join(workspace, "artifacts_in")):artifacts_in = True- target_folder = os.path.join("Templates/TwoAgents/coding/file", data["name"])+ target_folder = os.path.join(save_path, "Templates/TwoAgents/coding/file", data["name"])if os.path.exists(target_folder):shutil.rmtree(target_folder)shutil.copytree(os.path.join(workspace, "artifacts_in"), target_folder)# print(f"All the artifacts are copied from {os.path.join(workspace, 'artifacts_in')} to {target_folder}")+ # 2. 'custom_python' directory: all the files needed for testing python code+ if os.path.exists(os.path.join(workspace, "custom_python")):+ target_folder = os.path.join(save_path, "Templates/TwoAgents/custom_python")+ if not os.path.exists(target_folder):+ os.makedirs(target_folder)+ for filename in os.listdir(os.path.join(workspace, "custom_python")):+ shutil.copy(os.path.join(workspace, "custom_python", filename), os.path.join(target_folder, filename))+ # print(f"File copied from {os.path.join(workspace, 'custom_python', filename)} to {target_folder}")+record = {- "id": data["eval_id"],+ "id": data["name"],"template": "Templates/TwoAgents","substitutions": {"scenario.py": {- "__MODEL__": "gpt-3.5-turbo-16k",+ "__MODEL__": "gpt-35-turbo-16k","__TASK__": data["task"],"__TARGET_FOLDER__": f"file/{data['name']}" if artifacts_in else "",},@@ -60,11 +73,11 @@ for data_path in data_paths:},},}- with open(f"{data['name']}_gpt35.jsonl", "wt") as f:+ with open(os.path.join(save_path, "autogpt_twoagent_gpt35.jsonl"), "a") as f:f.write(json.dumps(record).strip() + "\n")record = {- "id": data["eval_id"],+ "id": data["name"],"template": "Templates/TwoAgents","substitutions": {"scenario.py": {@@ -85,5 +98,5 @@ for data_path in data_paths:},},}- with open(f"{data['name']}_gpt4.jsonl", "wt") as f:+ with open(os.path.join(save_path, "autogpt_twoagent_gpt4.jsonl"), "a") as f:f.write(json.dumps(record).strip() + "\n")