🐝 The First Self-Improving agents with RL / Prompting Optimization
Python agentaigptmulti-agentpythonreinforcement-learningself-improvementsociety-of-mindswarm-intelligence
Cost Calculation (#22)
Mingchen Zhuge13f22ae3a2af
diff --git a/datasets/gaia/README.md b/datasets/gaia/README.mdnew file mode 100644index 0000000..7efbc79--- /dev/null+++ b/datasets/gaia/README.md@@ -0,0 +1,5 @@+### GAIA Download++* Official Download: https://huggingface.co/datasets/gaia-benchmark/GAIA++* We have also re-constructed the datasets, found in Google Could: https://drive.google.com/file/d/1Mzcy3Z5S23FSWcQ2qjs8Ma8NbYl3wTkN/view?usp=sharingdiff --git a/experiments/run_gaia.py b/experiments/run_gaia.pyindex a22ad41..d05d44b 100644--- a/experiments/run_gaia.py+++ b/experiments/run_gaia.py@@ -19,9 +19,9 @@ from swarm.environment.agents.gaia.web_io import WebIOfrom swarm.environment.agents.gaia.tool_tot import ToolTOTfrom swarm.environment.operations import DirectAnswerfrom swarm.memory.memory import GlobalMemory-from swarm.utils.globals import Time+from swarm.utils.globals import Time, Cost, CompletionTokens, PromptTokensfrom swarm.utils.const import GPTSWARM_ROOT-from swarm.utils.log import logger+from swarm.utils.log import initialize_log_file, logger, swarmlogfrom swarm.environment.domain.gaia import question_scorerfrom swarm.environment.operations.final_decision import MergingStrategy@@ -48,6 +48,12 @@ async def main():result_path = GPTSWARM_ROOT / "result"os.makedirs(result_path, exist_ok=True)+ current_time = Time.instance().value or time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())+ Time.instance().value = current_time+++ log_file_path = initialize_log_file("GAIA", Time.instance().value)+if args.config:config_args = YAMLReader.parse(args.config, return_str=False)for key, value in config_args.items():@@ -97,6 +103,8 @@ async def main():ground_truth = item["Final answer"]inputs = {"task": task, "files": files, "GT": ground_truth}+ swarmlog("🐝GPTSWARM SYS", f"Finish {i} samples...", Cost.instance().value, PromptTokens.instance().value, CompletionTokens.instance().value, log_file_path)+# Swarm# answer = await swarm.composite_graph.run(inputs)# answer = answer[-1].split("FINAL ANSWER: ")[-1]@@ -129,8 +137,8 @@ async def main():print("-----")"""- current_time = Time.instance().value or time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())- Time.instance().value = current_time+ # current_time = Time.instance().value or time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())+ # Time.instance().value = current_timeresult_dir = Path(f"{GPTSWARM_ROOT}/result/eval")result_file = result_file or (result_dir / f"{'_'.join(experiment_name.split())}_{args.llm}_{current_time}.json")@@ -156,6 +164,7 @@ async def main():"Total executed": total_executed + 1,"Accuracy": (total_solved + is_solved) / (total_executed + 1),"Time": exe_time,+ "Total Cost": Cost.instance().value,}data.append(updated_item)diff --git a/swarm/llm/gpt_chat.py b/swarm/llm/gpt_chat.pyindex 7d78897..fb48b93 100644--- a/swarm/llm/gpt_chat.py+++ b/swarm/llm/gpt_chat.py@@ -33,6 +33,7 @@ def gpt_chat(max_tokens: int = 8192,temperature: float = 0.0,num_comps=1,+ return_cost=False,) -> Union[List[str], str]:if messages[0].content == '$skip$':return ''@@ -60,6 +61,7 @@ def gpt_chat(return response.choices[0].message.contentcost_count(response, model)+return [choice.message.content for choice in response.choices]@@ -70,6 +72,7 @@ async def gpt_achat(max_tokens: int = 8192,temperature: float = 0.0,num_comps=1,+ return_cost=False,) -> Union[List[str], str]:if messages[0].content == '$skip$':return ''@@ -101,6 +104,7 @@ async def gpt_achat(return response.choices[0].message.contentcost_count(response, model)+return [choice.message.content for choice in response.choices]diff --git a/swarm/llm/price.py b/swarm/llm/price.pyindex 0ddaacd..e05ddb8 100644--- a/swarm/llm/price.py+++ b/swarm/llm/price.py@@ -2,7 +2,7 @@# -*- coding: utf-8 -*-from swarm.utils.log import swarmlog-from swarm.utils.globals import Cost+from swarm.utils.globals import Cost, PromptTokens, CompletionTokens# GPT-4: https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo# GPT3.5: https://platform.openai.com/docs/models/gpt-3-5@@ -45,6 +45,8 @@ def cost_count(response, model_name):completion_len = response.usage.completion_tokensCost.instance().value += price+ PromptTokens.instance().value += prompt_len+ CompletionTokens.instance().value += completion_len# print(f"Prompt Tokens: {prompt_len}, Completion Tokens: {completion_len}")return price, prompt_len, completion_len@@ -52,6 +54,12 @@ def cost_count(response, model_name):OPENAI_MODEL_INFO ={"gpt-4": {"current_recommended": "gpt-4-1106-preview",+ "gpt-4-0125-preview": {+ "context window": 128000,+ "training": "Jan 2024",+ "input": 0.01,+ "output": 0.03+ },"gpt-4-1106-preview": {"context window": 128000,"training": "Apr 2023",@@ -97,6 +105,12 @@ OPENAI_MODEL_INFO ={},"gpt-3.5": {"current_recommended": "gpt-3.5-turbo-1106",+ "gpt-3.5-turbo-0125": {+ "context window": 16385,+ "training": "Jan 2024",+ "input": 0.0010,+ "output": 0.0020+ },"gpt-3.5-turbo-1106": {"context window": 16385,"training": "Sep 2021",diff --git a/swarm/utils/globals.py b/swarm/utils/globals.pyindex 8276f70..d6b766e 100644--- a/swarm/utils/globals.py+++ b/swarm/utils/globals.py@@ -1,6 +1,7 @@#!/usr/bin/env python# -*- coding: utf-8 -*-+import sysfrom typing import Optionalclass Singleton:@@ -11,11 +12,22 @@ class Singleton:if cls._instance is None:cls._instance = cls()return cls._instance++ def reset(self):+ self.value = 0.0class Cost(Singleton):def __init__(self):self.value = 0.0+class PromptTokens(Singleton):+ def __init__(self):+ self.value = 0.0++class CompletionTokens(Singleton):+ def __init__(self):+ self.value = 0.0+class Time(Singleton):def __init__(self):self.value = ""diff --git a/swarm/utils/log.py b/swarm/utils/log.pyindex d28c762..10f18bb 100644--- a/swarm/utils/log.py+++ b/swarm/utils/log.py@@ -5,6 +5,7 @@ import osimport sysfrom pathlib import Pathfrom loguru import logger+# from globals import CompletionTokens, PromptTokens, Costfrom swarm.utils.const import GPTSWARM_ROOTdef configure_logging(print_level: str = "INFO", logfile_level: str = "DEBUG") -> None:@@ -17,9 +18,9 @@ def configure_logging(print_level: str = "INFO", logfile_level: str = "DEBUG") -"""logger.remove()logger.add(sys.stderr, level=print_level)- logger.add(GPTSWARM_ROOT / 'logs/log.txt', level=logfile_level)+ logger.add(GPTSWARM_ROOT / 'logs/log.txt', level=logfile_level, rotation="10 MB")-def initialize_log_file(mode: str, time_stamp: str) -> Path:+def initialize_log_file(experiment_name: str, time_stamp: str) -> Path:"""Initialize the log file with a start message and return its path.@@ -31,7 +32,7 @@ def initialize_log_file(mode: str, time_stamp: str) -> Path:Path: The path to the initialized log file."""try:- log_file_path = GPTSWARM_ROOT / f'result/{mode}/logs/log_{time_stamp}.txt'+ log_file_path = GPTSWARM_ROOT / f'result/{experiment_name}/logs/log_{time_stamp}.txt'os.makedirs(log_file_path.parent, exist_ok=True)with open(log_file_path, 'w') as file:file.write("============ Start ============\n")@@ -40,9 +41,9 @@ def initialize_log_file(mode: str, time_stamp: str) -> Path:raisereturn log_file_path-def swarmlog(sender: str, text: str, cost: float, result_file: Path = None, solution: list = []) -> None:+def swarmlog(sender: str, text: str, cost: float, prompt_tokens: int, complete_tokens: int, log_file_path: str) -> None:"""- Custom log function for swarm operations.+ Custom log function for swarm operations. Includes dynamic global variables.Args:sender (str): The name of the sender.@@ -51,12 +52,28 @@ def swarmlog(sender: str, text: str, cost: float, result_file: Path = None, soluresult_file (Path, optional): Path to the result file. Default is None.solution (list, optional): Solution data to be logged. Default is an empty list."""- formatted_message = f"{sender} | 💵Total Cost: {cost:.5f}\n{text}"+ # Directly reference global variables for dynamic values+ formatted_message = (+ f"{sender} | 💵Total Cost: ${cost:.5f} | "+ f"Prompt Tokens: {prompt_tokens} | "+ f"Completion Tokens: {complete_tokens} | \n {text}"+ )logger.info(formatted_message)-# It's generally a good practice to have a main function to control the flow of your script+ try:+ os.makedirs(log_file_path.parent, exist_ok=True)+ with open(log_file_path, 'a') as file:+ file.write(f"{formatted_message}\n")+ except OSError as error:+ logger.error(f"Error initializing log file: {error}")+ raise++def main():configure_logging()+ # Example usage of swarmlog with dynamic values+ swarmlog("SenderName", "This is a test message.", 0.123)if __name__ == "__main__":main()+