🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
adapt SPO to MetaGPT
isaacJinyua56b0e340aec
diff --git a/metagpt/ext/spo/optimize.py b/metagpt/ext/spo/optimize.pynew file mode 100644index 000000000..8f821d6c9--- /dev/null+++ b/metagpt/ext/spo/optimize.py@@ -0,0 +1,22 @@+from metagpt.ext.spo.scripts.optimizer import Optimizer+from metagpt.ext.spo.scripts.utils.llm_client import SPO_LLM+++if __name__ == "__main__":++ SPO_LLM.initialize(+ optimize_kwargs={"model": "claude-3-5-sonnet-20240620", "temperature": 0.7},+ evaluate_kwargs={"model": "gpt-4o-mini", "temperature": 0.3},+ execute_kwargs={"model": "gpt-4o-mini", "temperature": 0.3}+ )++ optimizer = Optimizer(+ optimized_path="workspace",+ initial_round=1,+ max_rounds=10,+ template="Poem.yaml",+ name="Poem",+ iteration=True,+ )++ optimizer.optimize()\ No newline at end of filediff --git a/metagpt/ext/spo/scripts/evaluator.py b/metagpt/ext/spo/scripts/evaluator.pyindex c6f63c04b..73c73c9f5 100644--- a/metagpt/ext/spo/scripts/evaluator.py+++ b/metagpt/ext/spo/scripts/evaluator.py@@ -5,10 +5,10 @@import asynciofrom typing import Dict, Literal, Tuple, List, Any-from utils import load-from utils.llm_client import responser, extract_content-from prompt.evaluate_prompt import EVALUATE_PROMPT+from metagpt.ext.spo.scripts.utils import load+from metagpt.ext.spo.prompts.evaluate_prompt import EVALUATE_PROMPTimport random+from metagpt.ext.spo.scripts.utils.llm_client import SPO_LLM, extract_contentclass QuickExecute:@@ -16,21 +16,20 @@ class QuickExecute:完成不同数据集的评估。"""- def __init__(self, prompt: str, k: int = 3, model=None):+ def __init__(self, prompt: str):self.prompt = prompt- self.k = k- self.model = model+ self.llm = SPO_LLM.get_instance()async def prompt_execute(self) -> tuple[Any]:- _, _, qa, _ = load.load_meta_data(k=self.k)+ _, _, qa, _ = load.load_meta_data()answers = []async def fetch_answer(q: str) -> Dict[str, Any]:messages = [{"role": "user", "content": f"{self.prompt}\n\n{q}"}]try:- answer = await responser(messages, model=self.model['name'], temperature=self.model['temperature'])- return {'question': q, 'answer': answer.content}+ answer = await self.llm.responser(role="execute", messages=messages)+ return {'question': q, 'answer': answer}except Exception as e:return {'question': q, 'answer': str(e)}@@ -45,11 +44,11 @@ class QuickEvaluate:Complete the evaluation for different datasets here."""- def __init__(self, k: int = 3):- self.k = k+ def __init__(self):+ self.llm = SPO_LLM.get_instance()- async def prompt_evaluate(self, sample: list, new_sample: list, model: dict) -> bool:- _, requirement, qa, _ = load.load_meta_data(k=self.k)+ async def prompt_evaluate(self, sample: list, new_sample: list) -> bool:+ _, requirement, qa, _ = load.load_meta_data()if random.random() < 0.5:sample, new_sample = new_sample, sample@@ -64,8 +63,8 @@ class QuickEvaluate:answers=str(qa))}]try:- response = await responser(messages, model=model['name'], temperature=model['temperature'])- choose = extract_content(response.content, 'choose')+ response = await self.llm.responser(role="evaluate", messages=messages)+ choose = extract_content(response, 'choose')if is_swapped:return choose == "A"@@ -78,7 +77,7 @@ class QuickEvaluate:if __name__ == "__main__":- execute = QuickExecute(prompt="Answer the Question,{question}", k=3)+ execute = QuickExecute(prompt="Answer the Question,{question}")# 使用asyncio.run来运行异步方法answers = asyncio.run(execute.prompt_evaluate())diff --git a/metagpt/ext/spo/scripts/optimizer.py b/metagpt/ext/spo/scripts/optimizer.pyindex 1363cbd23..f3ec3421c 100644--- a/metagpt/ext/spo/scripts/optimizer.py+++ b/metagpt/ext/spo/scripts/optimizer.py@@ -5,14 +5,14 @@import asyncioimport time-from optimizer_utils.data_utils import DataUtils-from optimizer_utils.evaluation_utils import EvaluationUtils-from optimizer_utils.prompt_utils import PromptUtils-from prompt.optimize_prompt import PROMPT_OPTIMIZE_PROMPT-from utils import load-from utils.logs import logger-from utils.llm_client import responser, extract_content-from utils.token_manager import get_token_tracker+from metagpt.ext.spo.scripts.utils.data_utils import DataUtils+from metagpt.ext.spo.scripts.utils.evaluation_utils import EvaluationUtils+from metagpt.ext.spo.scripts.utils.prompt_utils import PromptUtils+from metagpt.ext.spo.prompts.optimize_prompt import PROMPT_OPTIMIZE_PROMPT+from metagpt.ext.spo.scripts.utils import load+from metagpt.logs import logger+from metagpt.ext.spo.scripts.utils.llm_client import extract_content, SPO_LLM+class Optimizer:@@ -21,11 +21,8 @@ class Optimizer:optimized_path: str = None,initial_round: int = 1,max_rounds: int = 10,- name: str = "test",- template: str = "meta.yaml",- execute_model=None,- optimize_model=None,- evaluate_model=None,+ name: str = "",+ template: str = "",iteration: bool = True,) -> None:@@ -34,16 +31,13 @@ class Optimizer:self.top_scores = []self.round = initial_roundself.max_rounds = max_rounds- self.execute_model = execute_model- self.optimize_model = optimize_model- self.evaluate_model = evaluate_modelself.iteration = iterationself.template = templateself.prompt_utils = PromptUtils(self.root_path)self.data_utils = DataUtils(self.root_path)self.evaluation_utils = EvaluationUtils(self.root_path)- self.token_tracker = get_token_tracker()+ self.llm = SPO_LLM.get_instance()def optimize(self):if self.iteration is True:@@ -55,8 +49,6 @@ class Optimizer:self.round += 1logger.info(f"Score for round {self.round}: {score}")- time.sleep(5)-else:loop = asyncio.new_event_loop()asyncio.set_event_loop(loop)@@ -77,14 +69,12 @@ class Optimizer:prompt, _, _, _ = load.load_meta_data()self.prompt = promptself.prompt_utils.write_prompt(directory, prompt=self.prompt)- new_sample = await self.evaluation_utils.execute_prompt(self, directory, data, model=self.execute_model,- initial=True)- _, answers = await self.evaluation_utils.evaluate_prompt(self, None, new_sample, model=self.evaluate_model,- path=prompt_path, data=data, initial=True)+ new_sample = await self.evaluation_utils.execute_prompt(self, directory, initial=True)+ _, answers = await self.evaluation_utils.evaluate_prompt(self, None, new_sample, path=prompt_path, data=data, initial=True)self.prompt_utils.write_answers(directory, answers=answers)- _, requirements, qa, count = load.load_meta_data(3)+ _, requirements, qa, count = load.load_meta_data()directory = self.prompt_utils.create_round_directory(prompt_path, self.round + 1)@@ -105,11 +95,10 @@ class Optimizer:golden_answers=golden_answer,count=count)- response = await responser(messages=[{"role": "user", "content": optimize_prompt}],- model=self.optimize_model['name'], temperature=self.optimize_model['temperature'])+ response = await self.llm.responser(role="optimize", messages=[{"role": "user", "content": optimize_prompt}])- modification = extract_content(response.content, "modification")- prompt = extract_content(response.content, "prompt")+ modification = extract_content(response, "modification")+ prompt = extract_content(response, "prompt")if prompt:self.prompt = promptelse:@@ -119,11 +108,10 @@ class Optimizer:self.prompt_utils.write_prompt(directory, prompt=self.prompt)- new_sample = await self.evaluation_utils.execute_prompt(self, directory, data, model=self.execute_model,- initial=False)+ new_sample = await self.evaluation_utils.execute_prompt(self, directory, data)success, answers = await self.evaluation_utils.evaluate_prompt(self, sample, new_sample,- model=self.evaluate_model, path=prompt_path,+ path=prompt_path,data=data, initial=False)self.prompt_utils.write_answers(directory, answers=answers)@@ -133,11 +121,6 @@ class Optimizer:logger.info(f"now is {self.round + 1}")- self.token_tracker.print_usage_report()- usage = self.token_tracker.get_total_usage()-- self.data_utils.save_cost(directory, usage)-return promptasync def _test_prompt(self):@@ -150,8 +133,7 @@ class Optimizer:directory = self.prompt_utils.create_round_directory(prompt_path, self.round)# Load prompt using prompt_utils- new_sample = await self.evaluation_utils.execute_prompt(self, directory, data, model=self.execute_model,- initial=False, k=100)+ new_sample = await self.evaluation_utils.execute_prompt(self, directory, data)self.prompt_utils.write_answers(directory, answers=new_sample["answers"], name="test_answers.txt")logger.info(new_sample)diff --git a/metagpt/ext/spo/scripts/utils/data_utils.py b/metagpt/ext/spo/scripts/utils/data_utils.pyindex 26fb515d7..77532e5ee 100644--- a/metagpt/ext/spo/scripts/utils/data_utils.py+++ b/metagpt/ext/spo/scripts/utils/data_utils.py@@ -1,9 +1,13 @@import datetimeimport jsonimport os+import randomfrom typing import Union, List, Dict-import pandas as pd+import yaml++FILE_NAME = ''+SAMPLE_K = 3class DataUtils:@@ -23,7 +27,7 @@ class DataUtils:def get_best_round(self):- top_rounds = self._load_scores()+ self._load_scores()for entry in self.top_scores:if entry["succeed"]:@@ -66,6 +70,39 @@ class DataUtils:return self.top_scores+ def set_file_name(name):+ global FILE_NAME+ FILE_NAME = name++ def load_meta_data(k=SAMPLE_K):++ # 读取 YAML 文件+ config_path = os.path.join(os.path.dirname(__file__), '../settings', FILE_NAME)+ with open(config_path, 'r', encoding='utf-8') as file:+ data = yaml.safe_load(file)++ qa = []++ # 提取问题和答案+ for item in data['faq']:+ question = item['question']+ answer = item['answer']+ qa.append({'question': question, 'answer': answer})++ prompt = data['prompt']+ requirements = data['requirements']+ count = data['count']++ if isinstance(count, int):+ count = f", within {count} words"+ else:+ count = ""++ # 随机选择三组问答+ random_qa = random.sample(qa, min(k, len(qa))) # 确保不超过列表长度++ return prompt, requirements, random_qa, count+def list_to_markdown(self, questions_list):"""Convert a list of question-answer dictionaries to a formatted Markdown string.diff --git a/metagpt/ext/spo/scripts/utils/evaluation_utils.py b/metagpt/ext/spo/scripts/utils/evaluation_utils.pyindex 42c4395c6..146ae9eb9 100644--- a/metagpt/ext/spo/scripts/utils/evaluation_utils.py+++ b/metagpt/ext/spo/scripts/utils/evaluation_utils.py@@ -1,7 +1,7 @@import asyncio-from script.evaluator import QuickEvaluate, QuickExecute-from utils.logs import logger+from metagpt.ext.spo.scripts.evaluator import QuickEvaluate, QuickExecute+from metagpt.logs import loggerimport tiktoken@@ -16,10 +16,10 @@ class EvaluationUtils:def __init__(self, root_path: str):self.root_path = root_path- async def execute_prompt(self, optimizer, prompt_path, data, model, initial=False, k=3):+ async def execute_prompt(self, optimizer, prompt_path, initial=False):optimizer.prompt = optimizer.prompt_utils.load_prompt(optimizer.round, prompt_path)- evaluator = QuickExecute(prompt=optimizer.prompt, k=k, model=model)+ evaluator = QuickExecute(prompt=optimizer.prompt)answers = await evaluator.prompt_execute()@@ -29,10 +29,9 @@ class EvaluationUtils:return new_data- async def evaluate_prompt(self, optimizer, sample, new_sample, path, data, model, initial=False):+ async def evaluate_prompt(self, optimizer, sample, new_sample, path, data, initial=False):- evaluator = QuickEvaluate(k=3)- original_token = count_tokens(sample)+ evaluator = QuickEvaluate()new_token = count_tokens(new_sample)if initial is True:@@ -40,7 +39,7 @@ class EvaluationUtils:else:evaluation_results = []for _ in range(4):- result = await evaluator.prompt_evaluate(sample=sample, new_sample=new_sample, model=model)+ result = await evaluator.prompt_evaluate(sample=sample, new_sample=new_sample)evaluation_results.append(result)logger.info(evaluation_results)diff --git a/metagpt/ext/spo/scripts/utils/llm_client.py b/metagpt/ext/spo/scripts/utils/llm_client.pynew file mode 100644index 000000000..1f25d9723--- /dev/null+++ b/metagpt/ext/spo/scripts/utils/llm_client.py@@ -0,0 +1,81 @@+import re+from typing import Optional+from metagpt.configs.models_config import ModelsConfig+from metagpt.llm import LLM+import asyncio+++class SPO_LLM:+ _instance: Optional['SPO_LLM'] = None++ def __init__(self, optimize_kwargs=None, evaluate_kwargs=None, execute_kwargs=None):+ self.evaluate_llm = LLM(llm_config=self._load_llm_config(evaluate_kwargs))+ self.optimize_llm = LLM(llm_config=self._load_llm_config(optimize_kwargs))+ self.execute_llm = LLM(llm_config=self._load_llm_config(execute_kwargs))++ def _load_llm_config(self, kwargs: dict):+ model = kwargs.get('model')+ config = ModelsConfig.default().get(model).model_copy()++ for key, value in kwargs.items():+ if hasattr(config, key):+ setattr(config, key, value)++ return config++ async def responser(self, role: str, messages):+ if role == "optimize":+ response = await self.optimize_llm.acompletion(messages)+ elif role == "evaluate":+ response = await self.evaluate_llm.acompletion(messages)+ elif role == "execute":+ response = await self.execute_llm.acompletion(messages)+ else:+ raise ValueError("Please set the correct name: optimize, evaluate or execute")++ rsp = response.choices[0].message.content+ return rsp++ @classmethod+ def initialize(cls, optimize_kwargs, evaluate_kwargs, execute_kwargs):+ """Initialize the global instance"""+ cls._instance = cls(optimize_kwargs, evaluate_kwargs, execute_kwargs)++ @classmethod+ def get_instance(cls):+ """Get the global instance"""+ if cls._instance is None:+ raise RuntimeError("SPO_LLM not initialized. Call initialize() first.")+ return cls._instance++def extract_content(xml_string, tag):+ pattern = rf'<{tag}>(.*?)</{tag}>'+ match = re.search(pattern, xml_string, re.DOTALL)+ return match.group(1).strip() if match else None+++async def spo():+ # 在入口处初始化配置+ SPO_LLM.initialize(+ optimize_kwargs={"model": "gpt-4o-mini", "temperature": 0.7},+ evaluate_kwargs={"model": "gpt-4o-mini", "temperature": 0.3},+ execute_kwargs={"model": "gpt-4o-mini", "temperature": 0.3}+ )++ llm = SPO_LLM.get_instance()++ # 测试消息+ hello_msg = [{"role": "user", "content": "你是什么模型"}]+ response = await llm.responser(role='execute', messages=hello_msg)+ print(f"AI回复: {response}")+ response = await llm.responser(role='optimize', messages=hello_msg)+ print(f"AI回复: {response}")+ response = await llm.responser(role='evaluate', messages=hello_msg)+ print(f"AI回复: {response}")+++if __name__ == "__main__":+ asyncio.run(spo())+++diff --git a/metagpt/ext/spo/scripts/utils/load.py b/metagpt/ext/spo/scripts/utils/load.pyindex 22bc10e80..0d5112859 100644--- a/metagpt/ext/spo/scripts/utils/load.py+++ b/metagpt/ext/spo/scripts/utils/load.py@@ -2,7 +2,8 @@ import yamlimport randomimport os-FILE_NAME = 'meta.yaml' # 默认值+FILE_NAME = 'meta.yaml'+SAMPLE_K = 3def load_llm():@@ -19,11 +20,10 @@ def set_file_name(name):FILE_NAME = name-def load_meta_data(k=5):+def load_meta_data(k=SAMPLE_K):- k = 5# 读取 YAML 文件- config_path = os.path.join(os.path.dirname(__file__), '../settings', FILE_NAME)+ config_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'settings', FILE_NAME)with open(config_path, 'r', encoding='utf-8') as file:data = yaml.safe_load(file)@@ -44,7 +44,7 @@ def load_meta_data(k=5):else:count = ""- # 随机选择三组问答+ # 随机选择k组问答random_qa = random.sample(qa, min(k, len(qa))) # 确保不超过列表长度return prompt, requirements, random_qa, countdiff --git a/metagpt/ext/spo/scripts/utils/prompt_utils.py b/metagpt/ext/spo/scripts/utils/prompt_utils.pyindex 806423572..211f96572 100644--- a/metagpt/ext/spo/scripts/utils/prompt_utils.py+++ b/metagpt/ext/spo/scripts/utils/prompt_utils.py@@ -4,7 +4,7 @@ import reimport timeimport tracebackfrom typing import List-from utils.logs import logger+from metagpt.logs import loggerclass PromptUtils: