Skip to content

🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming

Update HotpotQA's init round

didid8c7174fc0a5
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/processed_experience.json b/examples/aflow/scripts/optimized/HotpotQA/workflows/processed_experience.json
new file mode 100644
index 000000000..e69de29bb
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/results.json b/examples/aflow/scripts/optimized/HotpotQA/workflows/results.json
new file mode 100644
index 000000000..e69de29bb
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/__init__.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/graph.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/graph.py
new file mode 100644
index 000000000..65b49abb2
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/graph.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# @Date : 6/27/2024 22:07 PM
+# @Author : didi
+# @Desc : Basic Graph Class
+
+from typing import Literal
+import examples.aflow.scripts.optimized.HotpotQA.workflows.template.operator as operator
+import examples.aflow.scripts.optimized.HotpotQA.workflows.round_1.prompt as prompt_custom
+from metagpt.provider.llm_provider_registry import create_llm_instance
+from metagpt.utils.cost_manager import CostManager
+
+DatasetType = Literal["HumanEval", "MBPP", "GSM8K", "MATH", "HotpotQA", "DROP"]
+
+class Workflow:
+ def __init__(
+ self,
+ name: str,
+ llm_config,
+ dataset: DatasetType,
+ ) -> None:
+ self.name = name
+ self.dataset = dataset
+ self.llm = create_llm_instance(llm_config)
+ self.llm.cost_manager = CostManager()
+ self.custom = operator.Custom(self.llm)
+
+ async def __call__(self, problem: str):
+ """
+ Implementation of the workflow
+ """
+ solution = await self.custom(input=problem, instruction="")
+ return solution['response'], self.llm.cost_manager.total_cost
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/log.json b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/log.json
new file mode 100644
index 000000000..e69de29bb
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/prompt.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/prompt.py
new file mode 100644
index 000000000..d651ea494
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/round_1/prompt.py
@@ -0,0 +1,6 @@
+# XXX_PROMPT = """
+#
+# Solve it.
+#
+# """
+
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/template/op_prompt.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/op_prompt.py
new file mode 100644
index 000000000..2d776db1a
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/op_prompt.py
@@ -0,0 +1,15 @@
+SC_ENSEMBLE_PROMPT = """
+Several answers have been generated to a same question. They are as follows:
+{solutions}
+
+Identify the concise answer that appears most frequently across them. This consistency in answers is crucial for determining the most reliable solution.
+
+In the "thought" field, provide a detailed explanation of your thought process. In the "solution_letter" field, output only the single letter ID (A, B, C, etc.) corresponding to the most consistent solution. Do not include any additional text or explanation in the "solution_letter" field.
+"""
+
+ANSWER_GENERATION_PROMPT = """
+Think step by step and solve the problem.
+1. In the "thought" field, explain your thinking process in detail.
+2. In the "answer" field, provide the final answer concisely and clearly. The answer should be a direct response to the question, without including explanations or reasoning.
+Your task: {input}
+"""
\ No newline at end of file
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.json b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.json
new file mode 100644
index 000000000..5725c4ce5
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.json
@@ -0,0 +1,14 @@
+{
+ "Custom": {
+ "description": "Generates anything based on customized input and instruction.",
+ "interface": "custom(input: str, instruction: str) -> dict with key 'response' of type str"
+ },
+ "ScEnsemble": {
+ "description": "Uses self-consistency to select the solution that appears most frequently in the solution list, improve the selection to enhance the choice of the best solution.",
+ "interface": "sc_ensemble(solutions: List[str]) -> dict with key 'response' of type str"
+ },
+ "AnswerGenerate": {
+ "description": "Generate step by step based on the input. The step by step thought process is in the field of 'thought', and the final answer is in the field of 'answer'.",
+ "interface": "answer_generate(input: str) -> dict with key 'thought' of type str, 'answer' of type str"
+ }
+}
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.py
new file mode 100644
index 000000000..3473ffa47
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# @Date : 6/27/2024 17:36 PM
+# @Author : didi
+# @Desc : operator demo of ags
+import ast
+import random
+import sys
+import traceback
+from collections import Counter
+from typing import Dict, List, Tuple
+
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+from examples.aflow.scripts.optimized.HotpotQA.workflows.template.operator_an import *
+from examples.aflow.scripts.optimized.HotpotQA.workflows.template.op_prompt import *
+from metagpt.actions.action_node import ActionNode
+from metagpt.llm import LLM
+from metagpt.logs import logger
+import re
+
+
+class Operator:
+ def __init__(self, llm: LLM, name: str):
+ self.name = name
+ self.llm = llm
+
+ def __call__(self, *args, **kwargs):
+ raise NotImplementedError
+
+ async def _fill_node(self, op_class, prompt, mode=None, **extra_kwargs):
+ fill_kwargs = {"context": prompt, "llm": self.llm}
+ if mode:
+ fill_kwargs["mode"] = mode
+ fill_kwargs.update(extra_kwargs)
+ node = await ActionNode.from_pydantic(op_class).fill(**fill_kwargs)
+ return node.instruct_content.model_dump()
+
+
+class Custom(Operator):
+ def __init__(self, llm: LLM, name: str = "Custom"):
+ super().__init__(llm, name)
+
+ async def __call__(self, input, instruction):
+ prompt = instruction + input
+ response = await self._fill_node(GenerateOp, prompt, mode="single_fill")
+ return response
+
+class AnswerGenerate(Operator):
+ def __init__(self, llm: LLM, name: str = "AnswerGenerate"):
+ super().__init__(llm, name)
+
+ async def __call__(self, input: str, mode: str = None) -> Tuple[str, str]:
+ prompt = ANSWER_GENERATION_PROMPT.format(input=input)
+ response = await self._fill_node(AnswerGenerateOp, prompt, mode="context_fill")
+ return response
+
+class ScEnsemble(Operator):
+ """
+ Paper: Self-Consistency Improves Chain of Thought Reasoning in Language Models
+ Link: https://arxiv.org/abs/2203.11171
+ Paper: Universal Self-Consistency for Large Language Model Generation
+ Link: https://arxiv.org/abs/2311.17311
+ """
+
+ def __init__(self, llm: LLM, name: str = "ScEnsemble"):
+ super().__init__(llm, name)
+
+ async def __call__(self, solutions: List[str]):
+ answer_mapping = {}
+ solution_text = ""
+ for index, solution in enumerate(solutions):
+ answer_mapping[chr(65 + index)] = index
+ solution_text += f"{chr(65 + index)}: \n{str(solution)}\n\n\n"
+
+ prompt = SC_ENSEMBLE_PROMPT.format(solutions=solution_text)
+ response = await self._fill_node(ScEnsembleOp, prompt, mode="context_fill")
+
+ answer = response.get("solution_letter", "")
+ answer = answer.strip().upper()
+
+ return {"response": solutions[answer_mapping[answer]]}
\ No newline at end of file
diff --git a/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator_an.py b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator_an.py
new file mode 100644
index 000000000..4e3065bfb
--- /dev/null
+++ b/examples/aflow/scripts/optimized/HotpotQA/workflows/template/operator_an.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# @Date : 6/27/2024 19:46 PM
+# @Author : didi
+# @Desc : action nodes for operator
+
+from pydantic import BaseModel, Field
+
+
+class GenerateOp(BaseModel):
+ response: str = Field(default="", description="Your solution for this problem")
+
+class ScEnsembleOp(BaseModel):
+ thought: str = Field(default="", description="The thought of the most consistent solution.")
+ solution_letter: str = Field(default="", description="The letter of most consistent solution.")
+
+class AnswerGenerateOp(BaseModel):
+ thought: str = Field(default="", description="The step by step thinking process")
+ answer: str = Field(default="", description="The final answer to the question")
\ No newline at end of file