🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
Update
didi344d87d2e603
diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.pyindex 6648b6e05..c286b2fdd 100644--- a/metagpt/actions/action_node.py+++ b/metagpt/actions/action_node.py@@ -18,8 +18,8 @@ from pydantic import BaseModel, Field, create_model, model_validatorfrom tenacity import retry, stop_after_attempt, wait_random_exponentialfrom metagpt.actions.action_outcls_registry import register_action_outcls-from metagpt.ext.aflow.benchmark.utils import sanitizefrom metagpt.const import USE_CONFIG_TIMEOUT+from metagpt.ext.aflow.scripts.utils import sanitizefrom metagpt.llm import BaseLLMfrom metagpt.logs import loggerfrom metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocessdiff --git a/metagpt/ext/aflow/benchmark/drop.py b/metagpt/ext/aflow/benchmark/drop.pyindex a3aa40740..3cec5795f 100644--- a/metagpt/ext/aflow/benchmark/drop.py+++ b/metagpt/ext/aflow/benchmark/drop.py@@ -8,6 +8,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fifrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmarkfrom metagpt.logs import logger+class DROPBenchmark(BaseBenchmark):def __init__(self, name: str, file_path: str, log_path: str):super().__init__(name, file_path, log_path)diff --git a/metagpt/ext/aflow/benchmark/gsm8k.py b/metagpt/ext/aflow/benchmark/gsm8k.pyindex 898b3cc4f..51979c0c5 100644--- a/metagpt/ext/aflow/benchmark/gsm8k.py+++ b/metagpt/ext/aflow/benchmark/gsm8k.py@@ -10,6 +10,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fifrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmarkfrom metagpt.logs import logger+class GSM8KBenchmark(BaseBenchmark):def __init__(self, name: str, file_path: str, log_path: str):super().__init__(name, file_path, log_path)diff --git a/metagpt/ext/aflow/benchmark/hotpotqa.py b/metagpt/ext/aflow/benchmark/hotpotqa.pyindex 0086fdd04..b3bafe22b 100644--- a/metagpt/ext/aflow/benchmark/hotpotqa.py+++ b/metagpt/ext/aflow/benchmark/hotpotqa.py@@ -8,6 +8,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fifrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmarkfrom metagpt.logs import logger+class HotpotQABenchmark(BaseBenchmark):def __init__(self, name: str, file_path: str, log_path: str):super().__init__(name, file_path, log_path)diff --git a/metagpt/ext/aflow/benchmark/humaneval.py b/metagpt/ext/aflow/benchmark/humaneval.pyindex abe44e6e4..36771ad7a 100644--- a/metagpt/ext/aflow/benchmark/humaneval.py+++ b/metagpt/ext/aflow/benchmark/humaneval.py@@ -5,9 +5,9 @@ from typing import Any, Callable, Dict, List, Optional, Tuplefrom tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed-from metagpt.actions.code_sanitize import sanitize-from metagpt.logs import loggerfrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmark+from metagpt.ext.aflow.scripts.utils import sanitize+from metagpt.logs import loggerclass HumanEvalBenchmark(BaseBenchmark):diff --git a/metagpt/ext/aflow/benchmark/math.py b/metagpt/ext/aflow/benchmark/math.pyindex 3db5e9596..61d994b69 100644--- a/metagpt/ext/aflow/benchmark/math.py+++ b/metagpt/ext/aflow/benchmark/math.py@@ -11,6 +11,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fifrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmarkfrom metagpt.logs import logger+class MATHBenchmark(BaseBenchmark):def __init__(self, name: str, file_path: str, log_path: str):super().__init__(name, file_path, log_path)diff --git a/metagpt/ext/aflow/benchmark/mbpp.py b/metagpt/ext/aflow/benchmark/mbpp.pyindex 2d9df3745..4e446404b 100644--- a/metagpt/ext/aflow/benchmark/mbpp.py+++ b/metagpt/ext/aflow/benchmark/mbpp.py@@ -5,8 +5,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuplefrom tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixedfrom metagpt.actions.code_sanitize import sanitize-from metagpt.logs import loggerfrom metagpt.ext.aflow.benchmark.benchmark import BaseBenchmark+from metagpt.logs import loggerclass MBPPBenchmark(BaseBenchmark):diff --git a/metagpt/ext/aflow/benchmark/utils.py b/metagpt/ext/aflow/benchmark/utils.pyindex 37d655d51..944fde6be 100644--- a/metagpt/ext/aflow/benchmark/utils.py+++ b/metagpt/ext/aflow/benchmark/utils.py@@ -9,14 +9,8 @@import jsonimport os-import ast-import traceback-import numpy as np-from enum import Enum-from typing import Dict, Generator, List, Optional, Set, Tuple-import tree_sitter_python-from tree_sitter import Language, Node, Parser+import numpy as npdef generate_random_indices(n, n_samples, test=False):@@ -82,172 +76,3 @@ def log_mismatch(problem, expected_output, prediction, predicted_number, path):# 将数据写回到log.json文件with open(log_file, "w", encoding="utf-8") as f:json.dump(data, f, indent=4, ensure_ascii=False)---class NodeType(Enum):- CLASS = "class_definition"- FUNCTION = "function_definition"- IMPORT = ["import_statement", "import_from_statement"]- IDENTIFIER = "identifier"- ATTRIBUTE = "attribute"- RETURN = "return_statement"- EXPRESSION = "expression_statement"- ASSIGNMENT = "assignment"---def traverse_tree(node: Node) -> Generator[Node, None, None]:- """- Traverse the tree structure starting from the given node.-- :param node: The root node to start the traversal from.- :return: A generator object that yields nodes in the tree.- """- cursor = node.walk()- depth = 0-- visited_children = False- while True:- if not visited_children:- yield cursor.node- if not cursor.goto_first_child():- depth += 1- visited_children = True- elif cursor.goto_next_sibling():- visited_children = False- elif not cursor.goto_parent() or depth == 0:- break- else:- depth -= 1---def syntax_check(code, verbose=False):- try:- ast.parse(code)- return True- except (SyntaxError, MemoryError):- if verbose:- traceback.print_exc()- return False---def code_extract(text: str) -> str:- lines = text.split("\n")- longest_line_pair = (0, 0)- longest_so_far = 0-- for i in range(len(lines)):- for j in range(i + 1, len(lines)):- current_lines = "\n".join(lines[i : j + 1])- if syntax_check(current_lines):- current_length = sum(1 for line in lines[i : j + 1] if line.strip())- if current_length > longest_so_far:- longest_so_far = current_length- longest_line_pair = (i, j)-- return "\n".join(lines[longest_line_pair[0] : longest_line_pair[1] + 1])---def get_definition_name(node: Node) -> str:- for child in node.children:- if child.type == NodeType.IDENTIFIER.value:- return child.text.decode("utf8")---def has_return_statement(node: Node) -> bool:- traverse_nodes = traverse_tree(node)- for node in traverse_nodes:- if node.type == NodeType.RETURN.value:- return True- return False---def get_deps(nodes: List[Tuple[str, Node]]) -> Dict[str, Set[str]]:- def dfs_get_deps(node: Node, deps: Set[str]) -> None:- for child in node.children:- if child.type == NodeType.IDENTIFIER.value:- deps.add(child.text.decode("utf8"))- else:- dfs_get_deps(child, deps)-- name2deps = {}- for name, node in nodes:- deps = set()- dfs_get_deps(node, deps)- name2deps[name] = deps- return name2deps---def get_function_dependency(entrypoint: str, call_graph: Dict[str, str]) -> Set[str]:- queue = [entrypoint]- visited = {entrypoint}- while queue:- current = queue.pop(0)- if current not in call_graph:- continue- for neighbour in call_graph[current]:- if neighbour not in visited:- visited.add(neighbour)- queue.append(neighbour)- return visited---def sanitize(code: str, entrypoint: Optional[str] = None) -> str:- """- Sanitize and extract relevant parts of the given Python code.- This function parses the input code, extracts import statements, class and function definitions,- and variable assignments. If an entrypoint is provided, it only includes definitions that are- reachable from the entrypoint in the call graph.-- :param code: The input Python code as a string.- :param entrypoint: Optional name of a function to use as the entrypoint for dependency analysis.- :return: A sanitized version of the input code, containing only relevant parts.- """- code = code_extract(code)- code_bytes = bytes(code, "utf8")- parser = Parser(Language(tree_sitter_python.language()))- tree = parser.parse(code_bytes)- class_names = set()- function_names = set()- variable_names = set()-- root_node = tree.root_node- import_nodes = []- definition_nodes = []-- for child in root_node.children:- if child.type in NodeType.IMPORT.value:- import_nodes.append(child)- elif child.type == NodeType.CLASS.value:- name = get_definition_name(child)- if not (name in class_names or name in variable_names or name in function_names):- definition_nodes.append((name, child))- class_names.add(name)- elif child.type == NodeType.FUNCTION.value:- name = get_definition_name(child)- if not (name in function_names or name in variable_names or name in class_names) and has_return_statement(- child- ):- definition_nodes.append((name, child))- function_names.add(get_definition_name(child))- elif child.type == NodeType.EXPRESSION.value and child.children[0].type == NodeType.ASSIGNMENT.value:- subchild = child.children[0]- name = get_definition_name(subchild)- if not (name in variable_names or name in function_names or name in class_names):- definition_nodes.append((name, subchild))- variable_names.add(name)-- if entrypoint:- name2deps = get_deps(definition_nodes)- reacheable = get_function_dependency(entrypoint, name2deps)-- sanitized_output = b""-- for node in import_nodes:- sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n"-- for pair in definition_nodes:- name, node = pair- if entrypoint and name not in reacheable:- continue- sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n"- return sanitized_output[:-1].decode("utf8")diff --git a/metagpt/ext/aflow/data/download_data.py b/metagpt/ext/aflow/data/download_data.pyindex 219e2fec7..198ef20c1 100644--- a/metagpt/ext/aflow/data/download_data.py+++ b/metagpt/ext/aflow/data/download_data.py@@ -12,6 +12,7 @@ from tqdm import tqdmfrom metagpt.logs import logger+def download_file(url: str, filename: str) -> None:"""Download a file from the given URL and show progress."""response = requests.get(url, stream=True)diff --git a/metagpt/ext/aflow/scripts/operator.py b/metagpt/ext/aflow/scripts/operator.pyindex 0d71f0a2f..9d27c7cd1 100644--- a/metagpt/ext/aflow/scripts/operator.py+++ b/metagpt/ext/aflow/scripts/operator.py@@ -13,7 +13,6 @@ from typing import Dict, List, Tuplefrom tenacity import retry, stop_after_attempt, wait_fixedfrom metagpt.actions.action_node import ActionNode-from metagpt.logs import loggerfrom metagpt.ext.aflow.scripts.operator_an import (AnswerGenerateOp,CodeGenerateOp,diff --git a/metagpt/ext/aflow/scripts/utils.py b/metagpt/ext/aflow/scripts/utils.pyindex f69eaf4ca..36332ffec 100644--- a/metagpt/ext/aflow/scripts/utils.py+++ b/metagpt/ext/aflow/scripts/utils.py@@ -1,12 +1,19 @@-# -*- coding: utf-8 -*--# @Date : 7/2/2024 17:36 PM-# @Author : didi-# @Desc : utils for experiment+"""+@Time : 2024/7/24 16:37+@Author : didi+@File : utils.py+@Acknowledgement https://github.com/evalplus/evalplus/blob/master/evalplus/sanitize.py+"""import astimport jsonimport re-from typing import Any, List, Tuple+import traceback+from enum import Enum+from typing import Any, Dict, Generator, List, Optional, Set, Tuple++import tree_sitter_python+from tree_sitter import Language, Node, Parserdef extract_task_id(task_id: str) -> int:@@ -161,3 +168,172 @@ def test_check():test_check()"""return tester_function+++class NodeType(Enum):+ CLASS = "class_definition"+ FUNCTION = "function_definition"+ IMPORT = ["import_statement", "import_from_statement"]+ IDENTIFIER = "identifier"+ ATTRIBUTE = "attribute"+ RETURN = "return_statement"+ EXPRESSION = "expression_statement"+ ASSIGNMENT = "assignment"+++def traverse_tree(node: Node) -> Generator[Node, None, None]:+ """+ Traverse the tree structure starting from the given node.++ :param node: The root node to start the traversal from.+ :return: A generator object that yields nodes in the tree.+ """+ cursor = node.walk()+ depth = 0++ visited_children = False+ while True:+ if not visited_children:+ yield cursor.node+ if not cursor.goto_first_child():+ depth += 1+ visited_children = True+ elif cursor.goto_next_sibling():+ visited_children = False+ elif not cursor.goto_parent() or depth == 0:+ break+ else:+ depth -= 1+++def syntax_check(code, verbose=False):+ try:+ ast.parse(code)+ return True+ except (SyntaxError, MemoryError):+ if verbose:+ traceback.print_exc()+ return False+++def code_extract(text: str) -> str:+ lines = text.split("\n")+ longest_line_pair = (0, 0)+ longest_so_far = 0++ for i in range(len(lines)):+ for j in range(i + 1, len(lines)):+ current_lines = "\n".join(lines[i : j + 1])+ if syntax_check(current_lines):+ current_length = sum(1 for line in lines[i : j + 1] if line.strip())+ if current_length > longest_so_far:+ longest_so_far = current_length+ longest_line_pair = (i, j)++ return "\n".join(lines[longest_line_pair[0] : longest_line_pair[1] + 1])+++def get_definition_name(node: Node) -> str:+ for child in node.children:+ if child.type == NodeType.IDENTIFIER.value:+ return child.text.decode("utf8")+++def has_return_statement(node: Node) -> bool:+ traverse_nodes = traverse_tree(node)+ for node in traverse_nodes:+ if node.type == NodeType.RETURN.value:+ return True+ return False+++def get_deps(nodes: List[Tuple[str, Node]]) -> Dict[str, Set[str]]:+ def dfs_get_deps(node: Node, deps: Set[str]) -> None:+ for child in node.children:+ if child.type == NodeType.IDENTIFIER.value:+ deps.add(child.text.decode("utf8"))+ else:+ dfs_get_deps(child, deps)++ name2deps = {}+ for name, node in nodes:+ deps = set()+ dfs_get_deps(node, deps)+ name2deps[name] = deps+ return name2deps+++def get_function_dependency(entrypoint: str, call_graph: Dict[str, str]) -> Set[str]:+ queue = [entrypoint]+ visited = {entrypoint}+ while queue:+ current = queue.pop(0)+ if current not in call_graph:+ continue+ for neighbour in call_graph[current]:+ if neighbour not in visited:+ visited.add(neighbour)+ queue.append(neighbour)+ return visited+++def sanitize(code: str, entrypoint: Optional[str] = None) -> str:+ """+ Sanitize and extract relevant parts of the given Python code.+ This function parses the input code, extracts import statements, class and function definitions,+ and variable assignments. If an entrypoint is provided, it only includes definitions that are+ reachable from the entrypoint in the call graph.++ :param code: The input Python code as a string.+ :param entrypoint: Optional name of a function to use as the entrypoint for dependency analysis.+ :return: A sanitized version of the input code, containing only relevant parts.+ """+ code = code_extract(code)+ code_bytes = bytes(code, "utf8")+ parser = Parser(Language(tree_sitter_python.language()))+ tree = parser.parse(code_bytes)+ class_names = set()+ function_names = set()+ variable_names = set()++ root_node = tree.root_node+ import_nodes = []+ definition_nodes = []++ for child in root_node.children:+ if child.type in NodeType.IMPORT.value:+ import_nodes.append(child)+ elif child.type == NodeType.CLASS.value:+ name = get_definition_name(child)+ if not (name in class_names or name in variable_names or name in function_names):+ definition_nodes.append((name, child))+ class_names.add(name)+ elif child.type == NodeType.FUNCTION.value:+ name = get_definition_name(child)+ if not (name in function_names or name in variable_names or name in class_names) and has_return_statement(+ child+ ):+ definition_nodes.append((name, child))+ function_names.add(get_definition_name(child))+ elif child.type == NodeType.EXPRESSION.value and child.children[0].type == NodeType.ASSIGNMENT.value:+ subchild = child.children[0]+ name = get_definition_name(subchild)+ if not (name in variable_names or name in function_names or name in class_names):+ definition_nodes.append((name, subchild))+ variable_names.add(name)++ if entrypoint:+ name2deps = get_deps(definition_nodes)+ reacheable = get_function_dependency(entrypoint, name2deps)++ sanitized_output = b""++ for node in import_nodes:+ sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n"++ for pair in definition_nodes:+ name, node = pair+ if entrypoint and name not in reacheable:+ continue+ sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n"+ return sanitized_output[:-1].decode("utf8")