🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
1
kit2a19c174d940
diff --git a/examples/di/InfiAgent-DABench/DABench.py b/examples/di/InfiAgent-DABench/DABench.pyindex 4994acb78..0e3b7ccb4 100644--- a/examples/di/InfiAgent-DABench/DABench.py+++ b/examples/di/InfiAgent-DABench/DABench.py@@ -1,18 +1,51 @@import jsonfrom pathlib import Path-from metagpt.const import DABENCH_PATH+from examples.di.requirements_prompt import DABENCH+from metagpt.const import DABENCH_PATH+++# This code is referenced from https://github.com/InfiAgent/InfiAgent/blob/main/examples/DA-Agent/eval_closed_form.py+def evaluate_accuracy_by_question(results):+ correct = sum("correctness" in result and all(result["correctness"].values()) for result in results)+ total = len(results)+ return round(correct / total, 4) if total > 0 else 0+++def evaluate_accuracy_by_sub_question(results):+ correct = sum(sum(result["correctness"].values()) for result in results if "correctness" in result)+ total = sum(len(result["correctness"]) for result in results if "correctness" in result)+ return round(correct / total, 4) if total > 0 else 0+++def evaluate_accuracy_proportional_by_sub_question_adjusted(results):+ total_score = 0+ for result in results:+ if "correctness" in result:+ sub_question_count = len(result["correctness"])+ score_per_sub_question = 1 / sub_question_count if sub_question_count > 0 else 0+ question_score = sum(result["correctness"].values()) * score_per_sub_question+ total_score += question_score+ return round(total_score / len(results), 4) if results else 0++class DABench:- def __init__(self, questions_file=Path(DABENCH_PATH) / 'da-dev-questions.jsonl', answers_file=Path(DABENCH_PATH) / 'da-dev-labels.jsonl', template = ''):+ def __init__(+ self,+ questions_file=Path(DABENCH_PATH) / "da-dev-questions.jsonl",+ answers_file=Path(DABENCH_PATH) / "da-dev-labels.jsonl",+ template="",+ ):# Read questions from a JSONL file- with open(questions_file, 'r') as file:- self.questions = {int(json.loads(line)['id']): json.loads(line) for line in file}+ with open(questions_file, "r") as file:+ self.questions = {int(json.loads(line)["id"]): json.loads(line) for line in file}# Read answers from a JSONL file- with open(answers_file, 'r') as file:- self.answers = {int(json.loads(line)['id']): json.loads(line) for line in file}+ with open(answers_file, "r") as file:+ self.answers = {int(json.loads(line)["id"]): json.loads(line) for line in file}self.template = template if template else DABENCH+def get_question(self, question_id):"""Retrieve the question by its id."""return self.questions.get(question_id, "Question not found.")@@ -20,7 +53,13 @@ class DABench:def get_prompt(self, question_id):"""Retrieve the question by its id."""temp = self.get_question(question_id)- return self.template.format(question=temp['question'], constraints=temp['constraints'], format=temp['format'], file_name= str(DABENCH_PATH) + '/da-dev-tables/' + temp['file_name'], level=temp['level'],)+ return self.template.format(+ question=temp["question"],+ constraints=temp["constraints"],+ format=temp["format"],+ file_name=str(DABENCH_PATH) + "/da-dev-tables/" + temp["file_name"],+ level=temp["level"],+ )def get_answer(self, answer_id):"""Retrieve the answer list by its id."""@@ -28,13 +67,13 @@ class DABench:def eval(self, id, prediction):"""Evaluate the prediction against the true label."""- true_label = self.get_answer(id)['common_answers']+ true_label = self.get_answer(id)["common_answers"]# Parse the prediction string into a dictionary of metric-value pairspred_dict = {}- for pred in prediction.split(','):- parts = pred.strip().split('[')- metric = parts[0].strip().replace('@', '')- value = float(parts[1].rstrip(']'))+ for pred in prediction.split(","):+ parts = pred.strip().split("[")+ metric = parts[0].strip().replace("@", "")+ value = float(parts[1].rstrip("]"))pred_dict[metric] = value# Sort the true labels to match the order of predictions@@ -49,9 +88,56 @@ class DABench:return correct+ def eval_all(self, id_list, predictions):+ """Evaluate all predictions and calculate accuracy rates."""++ def sigle_eval(id, prediction):+ """Evaluate the prediction against the true label for a single question and return a dictionary indicating the correctness of each metric."""+ true_label = self.get_answer(id)["common_answers"]+ pred_dict = {}++ # Parse the prediction string into a dictionary of metric-value pairs+ for pred in prediction.split(","):+ parts = pred.strip().split("[")+ metric = parts[0].strip().replace("@", "")+ value = float(parts[1].rstrip("]"))+ pred_dict[metric] = value++ # Initialize the correctness dictionary with False values+ correctness = {metric: False for metric, _ in true_label}++ # Check each metric's prediction against the true label+ for metric, true_value in true_label:+ if metric in pred_dict:+ # Consider the prediction correct if it's within a small tolerance+ if abs(pred_dict[metric] - float(true_value)) < 1e-6:+ correctness[metric] = True++ return correctness++ results = []+ for id, prediction in zip(id_list, predictions):+ correct = sigle_eval(id, prediction)+ results.append({"id": id, "correctness": correct})++ # Calculate the three accuracy rates+ accuracy_by_question = evaluate_accuracy_by_question(results)+ accuracy_by_sub_question = evaluate_accuracy_by_sub_question(results)+ proportional_accuracy_by_sub_question = evaluate_accuracy_proportional_by_sub_question_adjusted(results)++ return {+ "accuracy_by_question": accuracy_by_question,+ "accuracy_by_sub_question": accuracy_by_sub_question,+ "proportional_accuracy_by_sub_question": proportional_accuracy_by_sub_question,+ }++if __name__ == "__main__":DA = DABench()- id = 6- prediction = "@mean_fare_child[31.09], @mean_fare_teenager[31.98], @mean_fare_adult[35.17], @mean_fare_elderly[43.47]"- is_correct = DA.eval(id, prediction)- print(f"Prediction is {'correct' if is_correct else 'incorrect'}.")\ No newline at end of file+ id = [0, 5, 6]+ prediction = [+ "@mean_fare[34.89]",+ "@correlation_coefficient[0.21]",+ "@mean_fare_child[31.09], @mean_fare_teenager[31.98], @mean_fare_adult[35.17], @mean_fare_elderly[43.47]",+ ]+ print(DA.eval_all(id, prediction))diff --git a/examples/di/InfiAgent-DABench/README.md b/examples/di/InfiAgent-DABench/README.mdindex 23eb6504f..8263e34f5 100644--- a/examples/di/InfiAgent-DABench/README.md+++ b/examples/di/InfiAgent-DABench/README.md@@ -7,5 +7,6 @@ git clone https://github.com/InfiAgent/InfiAgent.git```## How to run```-python run_InfiAgent-DABench.py --id x+python run_InfiAgent-DABench_sigle.py --id x # Run a task+python run_InfiAgent-DABench_all.py # Run all tasks```\ No newline at end of filediff --git a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.py b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.pynew file mode 100644index 000000000..852761c6a--- /dev/null+++ b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_all.py@@ -0,0 +1,24 @@+import json++import fire+from DABench import DABench++from metagpt.roles.di.data_interpreter import DataInterpreter+++async def main():+ """Evaluate all"""+ DA = DABench()+ id_list, predictions = [], []+ for key, value in DA.answers.items():+ requirement = DA.get_prompt(key)+ di = DataInterpreter()+ result = await di.run(requirement)+ prediction = json.loads(str(result).split("Current Plan")[1].split("## Current Task")[0])[-1]["result"]+ id_list.append(key)+ predictions.append(prediction)+ print(DA.eval_all(id_list, predictions))+++if __name__ == "__main__":+ fire.Fire(main)diff --git a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.py b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.pysimilarity index 87%rename from examples/di/InfiAgent-DABench/run_InfiAgent-DABench.pyrename to examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.pyindex d68845b91..3dc4ad0e9 100644--- a/examples/di/InfiAgent-DABench/run_InfiAgent-DABench.py+++ b/examples/di/InfiAgent-DABench/run_InfiAgent-DABench_sigle.py@@ -1,17 +1,20 @@import json+import fire+from DABench import DABenchfrom metagpt.roles.di.data_interpreter import DataInterpreter-from DABench import DABench-async def main(id=0):++async def main(id=5):DA = DABench()requirement = DA.get_prompt(id)di = DataInterpreter()result = await di.run(requirement)- prediction = json.loads(str(result).split("Current Plan")[1].split("## Current Task")[0])[-1]['result']+ prediction = json.loads(str(result).split("Current Plan")[1].split("## Current Task")[0])[-1]["result"]is_correct = DA.eval(id, prediction)print(f"Prediction is {'correct' if is_correct else 'incorrect'}.")+if __name__ == "__main__":fire.Fire(main)diff --git a/examples/di/requirements_prompt.py b/examples/di/requirements_prompt.pyindex 4b9950611..1172e1fe5 100644--- a/examples/di/requirements_prompt.py+++ b/examples/di/requirements_prompt.py@@ -1,4 +1,4 @@-#InfiAgent-DABench requirements+# InfiAgent-DABench requirementsDABENCH = "You are required to {question} from a CSV file named {file_name}. {constraints}. The output format should be {format}. This task is categorized as {level}."# ML-Benchmark requirements