🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
1
kitb97fa40e0e26
diff --git a/examples/di/InfiAgent-DABench/DABench.py b/examples/di/InfiAgent-DABench/DABench.pyindex 4f907adab..50ec04b29 100644--- a/examples/di/InfiAgent-DABench/DABench.py+++ b/examples/di/InfiAgent-DABench/DABench.py@@ -8,6 +8,7 @@ import nest_asynciofrom examples.di.requirements_prompt import DABENCHfrom metagpt.const import DABENCH_PATH+from metagpt.logs import loggerfrom metagpt.utils.exceptions import handle_exception@@ -473,14 +474,14 @@ class DABench:if __name__ == "__main__":- DA = DABench()+ bench = DABench()id = 0prediction = "@mean_fare[34.65]"- print(DA.eval(id, prediction))+ logger.info(bench.eval(id, prediction))ids = [0, 5, 6]predictions = ["@mean_fare[34.89]","@correlation_coefficient[0.21]","@mean_fare_child[31.09], @mean_fare_teenager[31.98], @mean_fare_adult[35.17], @mean_fare_elderly[43.47]",]- print(DA.eval_all(ids, predictions))+ logger.info(bench.eval_all(ids, predictions))diff --git a/examples/di/InfiAgent-DABench/README.md b/examples/di/InfiAgent-DABench/README.mdindex db8842782..814a1937b 100644--- a/examples/di/InfiAgent-DABench/README.md+++ b/examples/di/InfiAgent-DABench/README.md@@ -9,7 +9,7 @@ mv InfiAgent/examples/DA-Agent/data ./```## How to run```-python run_InfiAgent-DABench_sigle.py --id x # run a task, x represents the id of the question you want to test+python run_InfiAgent-DABench_single.py --id x # run a task, x represents the id of the question you want to testpython run_InfiAgent-DABench_all.py # Run all tasks seriallypython run_InfiAgent-DABench.py --k x # Run all tasks in parallel, x represents the number of parallel tasks at a time```\ No newline at end of filediff --git a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.py b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.pyindex dd27ef7cd..7e1fbad8b 100644--- a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.py+++ b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.py@@ -3,6 +3,7 @@ import jsonfrom DABench import DABench+from metagpt.logs import loggerfrom metagpt.roles.di.data_interpreter import DataInterpreter@@ -30,7 +31,7 @@ async def get_prediction(agent, requirement):return prediction # Return the extracted predictionexcept Exception as e:# Log an error message if an exception occurs during processing- print(f"Error processing requirement: {requirement}. Error: {e}")+ logger.info(f"Error processing requirement: {requirement}. Error: {e}")return None # Return None in case of an error@@ -43,13 +44,13 @@ async def evaluate_all(agent, k):agent: The baseline agent used for making predictions.k (int): The number of tasks to process in each group concurrently."""- DA = DABench() # Create an instance of DABench to access its methods and data+ bench = DABench() # Create an instance of DABench to access its methods and dataid_list, predictions = [], [] # Initialize lists to store IDs and predictionstasks = [] # Initialize a list to hold the tasks# Iterate over the answers in DABench to generate tasks- for key, value in DA.answers.items():- requirement = DA.generate_formatted_prompt(key) # Generate a formatted prompt for the current key+ for key, value in bench.answers.items():+ requirement = bench.generate_formatted_prompt(key) # Generate a formatted prompt for the current keytasks.append(get_prediction(agent, requirement)) # Append the prediction task to the tasks listid_list.append(key) # Append the current key to the ID list@@ -62,8 +63,8 @@ async def evaluate_all(agent, k):# Filter out any None values from the predictions and extend the predictions listpredictions.extend(pred for pred in group_predictions if pred is not None)- # Evaluate the results using all valid predictions and print the evaluation- print(DA.eval_all(id_list, predictions))+ # Evaluate the results using all valid predictions and logger.info the evaluation+ logger.info(bench.eval_all(id_list, predictions))def main(k=5):diff --git a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.py b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.pyindex bbdbc64e7..5cd1ef4b0 100644--- a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.py+++ b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.py@@ -9,26 +9,26 @@ from metagpt.utils.recovery_util import save_historyasync def main():"""Evaluate all"""- DA = DABench()+ bench = DABench()id_list, predictions, labels, is_true = [], [], [], []- for key, value in DA.answers.items():+ for key, value in bench.answers.items():id_list.append(key)- labels.append(str(DA.get_answer(key)))+ labels.append(str(bench.get_answer(key)))try:- requirement = DA.generate_formatted_prompt(key)+ requirement = bench.generate_formatted_prompt(key)di = DataInterpreter()result = await di.run(requirement)logger.info(result)save_history(role=di)- temp_prediction, temp_istrue = DA.eval(key, str(result))+ temp_prediction, temp_istrue = bench.eval(key, str(result))is_true.append(str(temp_istrue))predictions.append(str(temp_prediction))except:- is_true.append(str(DA.eval(key, "")))+ is_true.append(str(bench.eval(key, "")))predictions.append(str(""))df = pd.DataFrame({"Label": labels, "Prediction": predictions, "T/F": is_true})df.to_excel("DABench_output.xlsx", index=False)- logger.info(DA.eval_all(id_list, predictions))+ logger.info(bench.eval_all(id_list, predictions))if __name__ == "__main__":diff --git a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.py b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_single.pysimilarity index 79%rename from examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.pyrename to examples/di/InfiAgent-DABench/run_InfiAgent-DABench_single.pyindex 22c0a3f45..470f12fc8 100644--- a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.py+++ b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_single.py@@ -8,13 +8,13 @@ from metagpt.utils.recovery_util import save_historyasync def main(id=0):"""Evaluate one task"""- DA = DABench()- requirement = DA.generate_formatted_prompt(id)+ bench = DABench()+ requirement = bench.generate_formatted_prompt(id)di = DataInterpreter()result = await di.run(requirement)logger.info(result)save_history(role=di)- _, is_correct = DA.eval(id, str(result))+ _, is_correct = bench.eval(id, str(result))logger.info(f"Prediction is {'correct' if is_correct else 'incorrect'}.")