🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
Update llm.py & handle exception
didi66b523953d39
diff --git a/examples/aflow/benchmark/drop.py b/examples/aflow/benchmark/drop.pyindex 547356f0f..e8ee124d7 100644--- a/examples/aflow/benchmark/drop.py+++ b/examples/aflow/benchmark/drop.py@@ -10,6 +10,7 @@ from typing import Any, Callable, Dict, List, Tupleimport aiofilesimport pandas as pdfrom tqdm.asyncio import tqdm_asyncio+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_typefrom examples.aflow.benchmark.benchmark import BaseBenchmark@@ -52,40 +53,41 @@ class DROPBenchmark(BaseBenchmark):f1 = (2 * precision * recall) / (precision + recall)return f1, prediction+ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, input_text):+ return await graph(input_text)+async def evaluate_problem(self, problem: dict, graph: Callable) -> Tuple[str, str, str, float, float]:input_text = problem["context"]expected_output = problem["ref_text"]answers = expected_output.split("|")- max_retries = 5- retries = 0-- while retries < max_retries:- try:- output, cost = await graph(input_text)- f1_scores = []-- for answer in answers:- if answer.strip() != "":- output_parts = output.split("|")- for output_part in output_parts:- f1_score, _ = self.calculate_score(answer, output_part)- f1_scores.append(f1_score)+ try:+ output, cost = await self._generate_output(graph, input_text)+ f1_scores = []- uni_score = max(f1_scores)+ for answer in answers:+ if answer.strip() != "":+ output_parts = output.split("|")+ for output_part in output_parts:+ f1_score, _ = self.calculate_score(answer, output_part)+ f1_scores.append(f1_score)- if uni_score < 0.3:- self.log_mismatch(input_text, expected_output, output, output)+ uni_score = max(f1_scores)- return input_text, output, expected_output, uni_score, cost+ if uni_score < 0.3:+ self.log_mismatch(input_text, expected_output, output, output)- except Exception as e:- retries += 1- print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})")+ return input_text, output, expected_output, uni_score, cost- if retries == max_retries:- print("Maximum retries reached. Skipping this sample.")- return input_text, str(e), expected_output, 0.0, 0.0+ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, str(e), expected_output, 0.0, 0.0def get_result_columns(self) -> List[str]:return ["inputs", "prediction", "expected_output", "score", "cost"]diff --git a/examples/aflow/benchmark/gsm8k.py b/examples/aflow/benchmark/gsm8k.pyindex 1e38fc23d..f3b86644c 100644--- a/examples/aflow/benchmark/gsm8k.py+++ b/examples/aflow/benchmark/gsm8k.py@@ -14,6 +14,8 @@ from tqdm.asyncio import tqdm_asyncioimport osimport timefrom datetime import datetime+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type+from examples.aflow.benchmark.benchmark import BaseBenchmark@@ -36,31 +38,33 @@ class GSM8KBenchmark(BaseBenchmark):if prediction is None:return 0.0, predictionreturn 1.0 if abs(expected_output - prediction) <= 1e-6 else 0.0, prediction-- async def evaluate_problem(self, problem: dict, graph: Callable) -> Tuple[str, str, float, float, float]:- max_retries = 5- retries = 0- while retries < max_retries:- try:- prediction, cost = await graph(problem["question"])- predicted_number = self.extract_number(prediction)- expected_output = self.extract_number(problem["answer"])+ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, input_text):+ return await graph(input_text)- score, _ = self.calculate_score(expected_output, predicted_number)+ async def evaluate_problem(self, problem: dict, graph: Callable) -> Tuple[str, str, float, float, float]:+ input_text = problem["question"]+ expected_output = self.extract_number(problem["answer"])- if score == 0:- self.log_mismatch(problem["question"], expected_output, prediction, predicted_number)+ try:+ output, cost = await self._generate_output(graph, input_text)+ predicted_number = self.extract_number(output)+ score, extracted_output = self.calculate_score(expected_output, predicted_number)- return problem["question"], prediction, expected_output, score, cost+ if score == 0:+ self.log_mismatch(input_text, expected_output, output, extracted_output)- except Exception as e:- retries += 1- print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})")+ return input_text, output, expected_output, score, cost- if retries == max_retries:- print("Maximum retries reached. Skipping this sample.")- return problem["question"], str(e), self.extract_number(problem["answer"]), 0.0, 0.0+ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, str(e), expected_output, 0.0, 0.0def get_result_columns(self) -> List[str]:return ["question", "prediction", "expected_output", "score", "cost"]diff --git a/examples/aflow/benchmark/hotpotqa.py b/examples/aflow/benchmark/hotpotqa.pyindex fa6371aff..2a715c0ca 100644--- a/examples/aflow/benchmark/hotpotqa.py+++ b/examples/aflow/benchmark/hotpotqa.py@@ -7,6 +7,7 @@ import stringimport reimport osfrom collections import Counter+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_typefrom examples.aflow.benchmark.benchmark import BaseBenchmark@@ -42,6 +43,15 @@ class HotpotQABenchmark(BaseBenchmark):f1 = (2 * precision * recall) / (precision + recall)return f1, prediction+ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, input_text):+ return await graph(input_text)+async def evaluate_problem(self, problem: dict, graph: Callable) -> Tuple[str, str, str, str, float, float]:input_text = problem["question"]expected_output = problem["answer"]@@ -49,26 +59,18 @@ class HotpotQABenchmark(BaseBenchmark):context_str = "\n".join(" ".join(paragraph) for paragraph in paragraphs)inputs = f"Context: {context_str}\n\nQuestion: {input_text}\n\nAnswer:"- max_retries = 5- retries = 0-- while retries < max_retries:- try:- output, cost = await graph(inputs)- score, _ = self.calculate_score(expected_output, output)-- if score < 0.3:- self.log_mismatch(input_text, expected_output, output, output)+ try:+ output, cost = await self._generate_output(graph, inputs)+ score, extracted_output = self.calculate_score(expected_output, output)- return input_text, context_str, output, expected_output, score, cost+ if score < 0.3: # We set the threshold for collecting incorrect questions to 0.3, as F1 Score cannot be simply judged using 0-1+ self.log_mismatch(input_text, expected_output, output, extracted_output)- except Exception as e:- retries += 1- print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})")+ return input_text, context_str, output, expected_output, score, cost- if retries == max_retries:- print("Maximum retries reached. Skipping this sample.")- return input_text, context_str, str(e), expected_output, 0.0, 0.0+ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, context_str, str(e), expected_output, 0.0, 0.0def get_result_columns(self) -> List[str]:return ["question", "context", "prediction", "expected_output", "score", "cost"]diff --git a/examples/aflow/benchmark/humaneval.py b/examples/aflow/benchmark/humaneval.pyindex 797fe765c..75252626e 100644--- a/examples/aflow/benchmark/humaneval.py+++ b/examples/aflow/benchmark/humaneval.py@@ -5,6 +5,8 @@ import asyncioimport threadingfrom datetime import datetimefrom typing import List, Tuple, Callable, Dict, Any, Optional+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type+import pandas as pd@@ -93,44 +95,45 @@ class HumanEvalBenchmark(BaseBenchmark):return result- async def evaluate_problem(self, data: dict, graph: Callable) -> Tuple[str, str, str, float, float]:- max_retries = 5- retries = 0+ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, prompt, entry_point):+ # Generate output with a timeout of 60 seconds+ return await asyncio.wait_for(graph(prompt, entry_point), timeout=60)+ async def evaluate_problem(self, data: dict, graph: Callable) -> Tuple[str, str, str, float, float]:+ input_text = data["prompt"]expected_output = "\nCorrect Solution:\ndef " + data["entry_point"] + "(params you should put here):" + "\n\n" + data["canonical_solution"]- while retries < max_retries:- try:- prediction, cost = await asyncio.wait_for(graph(data["prompt"], data["entry_point"]), timeout=60)- ret = self.check_solution(prediction, data["test"], data["entry_point"])- test_case_details = ret[1]- expected_output = test_case_details + "\nCorrect Solution:\ndef " + data["entry_point"] + "(params you should put here):" + "\n\n" + data["canonical_solution"]- score = 1.0 if ret[0] == self.PASS else 0.0-- if score == 0:- self.log_mismatch(data["prompt"], expected_output, prediction, score)- break-- except asyncio.TimeoutError:- prediction = None- ret = (self.FAIL, ["Timeout"])- score = 0.0- cost = 0.0- break+ try:+ # Generate prediction using the graph function+ prediction, cost = await self._generate_output(graph, input_text, data["entry_point"])++ # Check the solution+ ret = self.check_solution(prediction, data["test"], data["entry_point"])+ test_case_details = ret[1]+ expected_output = test_case_details + expected_output++ # Calculate score based on the check result+ score = 1.0 if ret[0] == self.PASS else 0.0- except Exception as e:- retries += 1- print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})")-- if retries == max_retries:- print("Maximum retries reached. Skipping this sample.")- prediction = None- ret = (self.FAIL, [])- score = 0.0- cost = 0.0- break-- return data["prompt"], prediction, expected_output, score, cost+ # Log mismatch if the score is 0+ if score == 0:+ self.log_mismatch(input_text, expected_output, prediction, score)++ return input_text, prediction, expected_output, score, cost++ except asyncio.TimeoutError:+ print("Timeout error. Skipping this sample.")+ return input_text, "Timeout", expected_output, 0.0, 0.0++ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, str(e), expected_output, 0.0, 0.0def calculate_score(self, expected_output: str, prediction: str) -> Tuple[float, str]:# The scoring logic for HumanEval is already implemented in evaluate_problem, this is just to conform to the interfacediff --git a/examples/aflow/benchmark/math.py b/examples/aflow/benchmark/math.pyindex d98859484..1b91c1bde 100644--- a/examples/aflow/benchmark/math.py+++ b/examples/aflow/benchmark/math.py@@ -6,6 +6,8 @@ from sympy.parsing.sympy_parser import parse_exprfrom math import iscloseimport multiprocessingfrom typing import Any, Callable, Tuple, List+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type+from examples.aflow.benchmark.benchmark import BaseBenchmark@@ -95,22 +97,34 @@ class MATHBenchmark(BaseBenchmark):passreturn False++ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, input_text):+ return await graph(input_text)++async def evaluate_problem(self, problem: dict, graph: Callable) -> Tuple[str, str, str, int, float]:input_text = problem["problem"]expected_output = problem["solution"]- max_retries = 2- retries = 0- prediction = await graph(input_text)- cost = prediction[1]- output = prediction[0]+ try:+ output, cost = await self._generate_output(graph, input_text)+ uni_score, extracted_output = self.calculate_score(expected_output, output)++ if uni_score == 0:+ self.log_mismatch(input_text, expected_output, output, extracted_output)- uni_score, extracted_output = self.calculate_score(expected_output, output)+ return input_text, output, expected_output, uni_score, cost- if uni_score == 0:- self.log_mismatch(input_text, expected_output, output, extracted_output)+ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, str(e), expected_output, 0.0, 0.0- return input_text, output, expected_output, uni_score, costdef get_result_columns(self) -> List[str]:return ["question", "prediction", "expected_output", "score", "cost"]diff --git a/examples/aflow/benchmark/mbpp.py b/examples/aflow/benchmark/mbpp.pyindex afa66c934..2edede302 100644--- a/examples/aflow/benchmark/mbpp.py+++ b/examples/aflow/benchmark/mbpp.py@@ -5,6 +5,7 @@ import asyncioimport threadingfrom datetime import datetimefrom typing import List, Tuple, Callable, Any, Optional, Dict+from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_typefrom metagpt.actions.code_sanitize import sanitizefrom examples.aflow.benchmark.benchmark import BaseBenchmark@@ -83,37 +84,40 @@ class MBPPBenchmark(BaseBenchmark):return result- async def evaluate_problem(self, data: dict, graph: Callable) -> Tuple[str, str, str, float, float]:- max_retries = 5- retries = 0+ @retry(+ stop=stop_after_attempt(5),+ wait=wait_fixed(1),+ retry=retry_if_exception_type(Exception),+ reraise=True+ )+ async def _generate_output(self, graph, prompt, entry_point):+ return await graph(prompt, entry_point)+ async def evaluate_problem(self, data: dict, graph: Callable) -> Tuple[str, str, str, float, float]:+ input_text = data["prompt"]expected_output = "\nCorrect Solution:\ndef " + data["code"]- while retries < max_retries:- try:- prediction, cost = await graph(data["prompt"], data["entry_point"])- ret = self.check_solution(prediction, data["test"], data["entry_point"])- test_case_details = ret[1]- expected_output = test_case_details + "\nCorrect Solution:" + data["code"]- score = 1.0 if ret[0] == self.PASS else 0.0+ try:+ # Generate prediction using the graph function+ prediction, cost = await self._generate_output(graph, input_text, data["entry_point"])++ # Check the solution+ ret = self.check_solution(prediction, data["test"], data["entry_point"])+ test_case_details = ret[1]+ expected_output = test_case_details + "\nCorrect Solution:" + data["code"]++ # Calculate score based on the check result+ score = 1.0 if ret[0] == self.PASS else 0.0- if score == 0:- self.log_mismatch(data["prompt"], expected_output, prediction, score)- break+ # Log mismatch if the score is 0+ if score == 0:+ self.log_mismatch(input_text, expected_output, prediction, score)- except Exception as e:- retries += 1- print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})")-- if retries == max_retries:- print("Maximum retries reached. Skipping this sample.")- prediction = None- ret = (self.FAIL, [])- score = 0.0- cost = 0.0- break-- return data["prompt"], prediction, expected_output, score, cost+ return input_text, prediction, expected_output, score, cost++ except Exception as e:+ print(f"Maximum retries reached. Skipping this sample. Error: {e}")+ return input_text, str(e), expected_output, 0.0, 0.0def calculate_score(self, expected_output: str, prediction: str) -> Tuple[float, str]:# The scoring logic for MBPP is already implemented in evaluate_problem, this is just to conform to the interfacediff --git a/metagpt/llm.py b/metagpt/llm.pyindex a918edd2a..465e419a1 100644--- a/metagpt/llm.py+++ b/metagpt/llm.py@@ -10,12 +10,6 @@ from typing import Optionalfrom metagpt.configs.llm_config import LLMConfigfrom metagpt.context import Contextfrom metagpt.provider.base_llm import BaseLLM-from metagpt.utils.cost_manager import CostManager--global cost_manager--if not globals().get("cost_manager"):- cost_manager = CostManager()def LLM(llm_config: Optional[LLMConfig] = None, context: Context = None) -> BaseLLM:@@ -23,6 +17,4 @@ def LLM(llm_config: Optional[LLMConfig] = None, context: Context = None) -> Basectx = context or Context()if llm_config is not None:return ctx.llm_with_cost_manager_from_llm_config(llm_config)- llm = ctx.llm()- llm.cost_manager = cost_manager- return llm+ return ctx.llm()