π The First Self-Improving agents with RL / Prompting Optimization
Python agentaigptmulti-agentpythonreinforcement-learningself-improvementsociety-of-mindswarm-intelligence
upload experiments
Mingchen Zhugec8efe841fd54
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.ymlnew file mode 100644index 0000000..4e65025--- /dev/null+++ b/.github/workflows/pytest.yml@@ -0,0 +1,54 @@+# This workflow will install Python dependencies, run tests and lint with a single version of Python+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python++name: Pytest GPTSwarm++on:+ push:+ branches: [ "main" ]+ pull_request:+ branches: [ "main" ]++permissions:+ contents: read++jobs:+ pytest_test:+ runs-on: ubuntu-latest+ strategy:+ max-parallel: 5++ steps:+ - uses: actions/checkout@v3+ with:+ lfs: true++ - name: Checkout LFS objects+ run: git lfs pull++ - name: Set up Python environment and install dependencies+ uses: actions/setup-python@v3+ with:+ python-version: "3.10"++ - uses: actions/cache@v2+ with:+ path: ~/.cache/pip+ key: ${{ hashFiles('requirements_py310_linux.txt') }}++ - name: Add conda to system path+ run: |+ # $CONDA is an environment variable pointing to the root of the miniconda directory+ echo $CONDA/bin >> $GITHUB_PATH++ - name: Install dependencies+ run: |+ python -m pip install --upgrade pip+ pip install -r requirements_py310_linux.txt++ - name: Test with pytest+ env:+ GOOGLE_API_KEY: "${{ secrets.GOOGLE_API_KEY }}"+ GOOGLE_CSE_ID: "${{ secrets.GOOGLE_CSE_ID }}"+ run: |+ pytest -s -m mock_llm test/\ No newline at end of filediff --git a/experiments/README.md b/experiments/README.mdnew file mode 100644index 0000000..1abeba4--- /dev/null+++ b/experiments/README.md@@ -0,0 +1,64 @@+## Run the following commands to reproduce our experiments in the paper++### Download dataset++The datasets are stored in git LFS. If git LFS is already installed in your system, no action is required. To check if git LFS is installed, run:+```bash+git lfs+```+If you see an error message, install git LFS via this instruction](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage). Then run:+```bash+git lfs install+git lfs pull+```++### Include necessary submodules++If the project has already been cloned:+```bash+git submodule init+git submodule update+```+or a one-liner:+```bash+git submodule update --init --recursive+```++### **MMLU**+Run the baseline:+```bash+PYTHONPATH=. python experiments/run_mmlu.py --mode=DirectAnswer+```++Run fully-connected swarm ablation:+```bash+PYTHONPATH=. python experiments/run_mmlu.py --num-truthful-agents=3 --mode=FullConnectedSwarm+```++Run randomly-connected swarm ablation:+```bash+PYTHONPATH=. python experiments/run_mmlu.py --num-truthful-agents=3 --mode=RandomSwarm+```++Run the main experiment with optimization and eventual evaluation:+```bash+PYTHONPATH=. python experiments/run_mmlu.py --num-truthful-agents=3 --mode=OptimizedSwarm+```++### **Mini Crosswords**+Run the REINFORCE algorithm for edge optimization with three agents as described in the paper.+```bash+PYTHONPATH=. python experiments/run_crosswords.py+```++### **HumanEval**+Run node optimization that improves the demonstration examples of each node.+```bash+PYTHONPATH=. python experiments/run_humaneval.py --learn_demonstration True+```++### **GAIA**+Run the general assistant tasks.+```bash+PYTHONPATH=. python experiments/run_gaia.py+```\ No newline at end of filediff --git a/experiments/crosswords/evaluate.py b/experiments/crosswords/evaluate.pynew file mode 100644index 0000000..49ae8d4--- /dev/null+++ b/experiments/crosswords/evaluate.py@@ -0,0 +1,63 @@+import json+from tqdm import tqdm+import asyncio+import numpy as np+from copy import deepcopy+import pickle+import torch+import sys+import random++from swarm.environment.domain.crosswords.env import MiniCrosswordsEnv+from swarm.environment.agents.agent_registry import AgentRegistry+from swarm.graph.swarm import Swarm+from swarm.optimizer.edge_optimizer.optimization import optimize+from swarm.environment.domain.crosswords.evaluator import CrosswordsEvaluator+++def batched_evaluator(evaluator, batch_size, graph, loop):+ tasks = []+ for _ in range(batch_size):+ tasks.append(evaluator.evaluate(deepcopy(graph)))+ return loop.run_until_complete(asyncio.gather(*tasks))++if __name__ == "__main__":+ file_path = "datasets/crosswords/mini0505_0_100_5.json"+ with open(file_path, "r") as file:+ test_data = json.load(file)++ experiment_id = "experiment1"+ init_connection_probability = .1+ epochs = 1+ batch_size = 4+ use_learned_order = True+ num_batches = int(len(test_data) / batch_size)+ evaluator = CrosswordsEvaluator(test_data, batch_size=batch_size, metric="words", window_size=num_batches)+ swarm = Swarm(["CrosswordsReflection", "CrosswordsToT"], "crosswords", "gpt-4-1106-preview", #"gpt-3.5-turbo-1106",+ final_node_class="TakeBest", final_node_kwargs={}, edge_optimize=True,+ init_connection_probability=init_connection_probability, connect_output_nodes_to_final_node=True)+ swarm.connection_dist.load_state_dict(torch.load(f"result/crosswords_Jan15/{experiment_id}_edge_logits_{int(epochs * len(test_data) / batch_size) - 1}.pkl"))++ num_edges = []+ for _ in range(100):+ graph = swarm.connection_dist.realize(swarm.composite_graph, use_learned_order=use_learned_order)[0]+ num_edges.append(graph.num_edges)+ num_edges = int(np.array(num_edges).mean())+ print(f"Expected number of edges: {num_edges}")++ graphs = [+ swarm.connection_dist.random_sample_num_edges(swarm.composite_graph, num_edges),+ swarm.connection_dist.realize(swarm.composite_graph, threshold=init_connection_probability, use_learned_order=use_learned_order)[0],+ swarm.connection_dist.realize(swarm.composite_graph, use_learned_order=use_learned_order)[0],+ swarm.composite_graph,+ ]+ loop = asyncio.get_event_loop()+ for i, graph in tqdm(enumerate(graphs)):+ print(f"{graph.num_edges} edges")+ utilities = []+ evaluator.reset()+ for k in range(num_batches):+ utilities += batched_evaluator(evaluator, batch_size, graph, loop)+ print(f"avg. utility = {np.mean(utilities):.3f}")+ with open(f"result/crosswords/{experiment_id}_final_utilities_{i}.pkl", "wb") as file:+ pickle.dump(utilities, file)\ No newline at end of filediff --git a/experiments/crosswords/train.py b/experiments/crosswords/train.pynew file mode 100644index 0000000..c735154--- /dev/null+++ b/experiments/crosswords/train.py@@ -0,0 +1,44 @@+import json+from tqdm import tqdm+import asyncio+import numpy as np+from copy import deepcopy+import pickle+import torch+import sys+import random++from swarm.environment.domain.crosswords.env import MiniCrosswordsEnv+from swarm.environment.agents.agent_registry import AgentRegistry+from swarm.graph.swarm import Swarm+from swarm.optimizer.edge_optimizer.optimization import optimize+from swarm.environment.domain.crosswords.evaluator import CrosswordsEvaluator+++if __name__ == "__main__":+ if len(sys.argv) == 2:+ id = int(sys.argv[1])+ experiment_id = f"experiment{id}"+ torch.manual_seed(id)+ np.random.seed(id)+ random.seed(id)+ else:+ experiment_id = "experiment"++ file_path = "datasets/crosswords/mini0505_0_100_5.json"+ with open(file_path, "r") as file:+ test_data = json.load(file)++ init_connection_probability = .1+ epochs = 2+ batch_size = 4+ use_learned_order = True+ include_inner_agent_connections = True+ window_size = int(len(test_data) / batch_size)+ evaluator = CrosswordsEvaluator(test_data, batch_size=batch_size, metric="words", window_size=window_size)+ swarm = Swarm(["CrosswordsReflection", "CrosswordsToT"], "crosswords", "gpt-4-1106-preview", #"gpt-3.5-turbo-1106",+ final_node_class="TakeBest", final_node_kwargs={}, edge_optimize=True,+ init_connection_probability=init_connection_probability, connect_output_nodes_to_final_node=True,+ include_inner_agent_connections=include_inner_agent_connections)+ optimize(swarm, evaluator, batch_size=batch_size, num_iter=int(epochs * len(test_data) / batch_size), display_freq=1, record=True,+ experiment_id=experiment_id, lr=.25, use_learned_order=use_learned_order)\ No newline at end of filediff --git a/experiments/evaluator/accuracy.py b/experiments/evaluator/accuracy.pynew file mode 100644index 0000000..c9b5468--- /dev/null+++ b/experiments/evaluator/accuracy.py@@ -0,0 +1,18 @@++class Accuracy:+ def __init__(self):+ self._num_correct = 0+ self._num_total = 0++ def update(self, predicted: str, target: str) -> None:+ is_correct = predicted == target+ self._num_correct += int(is_correct)+ self._num_total += 1++ def get(self) -> float:+ return self._num_correct / self._num_total++ def print(self):+ accuracy = self.get()+ print(f"Accuracy: {accuracy*100:.1f}% "+ f"({self._num_correct}/{self._num_total})")diff --git a/experiments/evaluator/datasets/base_dataset.py b/experiments/evaluator/datasets/base_dataset.pynew file mode 100644index 0000000..fddd2d1--- /dev/null+++ b/experiments/evaluator/datasets/base_dataset.py@@ -0,0 +1,30 @@+import pandas as pd+from typing import Dict, Any, Union, List+from collections.abc import Sequence+from abc import ABC, abstractmethod+++SwarmInput = Dict[str, Any]+++class BaseDataset(ABC, Sequence[Any]):+ @staticmethod+ @abstractmethod+ def get_domain() -> str:+ """ To be overriden. """++ @abstractmethod+ def split(self) -> str:+ """ To be overriden. """++ @abstractmethod+ def record_to_swarm_input(self, record: pd.DataFrame) -> SwarmInput:+ """ To be overriden. """++ @abstractmethod+ def postprocess_answer(self, answer: Union[str, List[str]]) -> str:+ """ To be overriden. """++ @abstractmethod+ def record_to_target_answer(self, record: pd.DataFrame) -> str:+ """ To be overriden. """diff --git a/experiments/evaluator/datasets/mmlu_dataset.py b/experiments/evaluator/datasets/mmlu_dataset.pynew file mode 100644index 0000000..966a1db--- /dev/null+++ b/experiments/evaluator/datasets/mmlu_dataset.py@@ -0,0 +1,94 @@+import glob+import pandas as pd+from typing import Union, List, Literal+import numpy as np++from experiments.evaluator.datasets.base_dataset import BaseDataset, SwarmInput+++class MMLUDataset(BaseDataset):+ def __init__(self,+ split: Union[Literal['dev'], Literal['val'], Literal['test']],+ ) -> None:++ self._split = split++ data_path = f"datasets/MMLU/data/{self._split}/"+ self._total_df: pd.DataFrame = self._load_data(data_path)++ @staticmethod+ def get_domain() -> str:+ return 'mmlu'++ @staticmethod+ def _load_data(+ data_path: str,+ ) -> pd.DataFrame:++ rng = np.random.default_rng(888)++ csv_paths = glob.glob(data_path + "*.csv")+ csv_paths = sorted(csv_paths)+ print("Number of topics: ", len(csv_paths))++ names = ['question', 'A', 'B', 'C', 'D', 'correct_answer']++ total_df = pd.DataFrame(columns=names)+ for path in csv_paths:+ single_df = pd.read_csv(path, header=None,+ names=names)+ total_df = pd.concat([total_df, single_df])++ total_df = total_df.reset_index(drop=True)++ # Pseudorandom shuffle+ total_df = total_df.reindex(rng.permutation(total_df.index))++ print("Total number of questions: ", len(total_df))++ return total_df++ @property+ def split(self) -> str:+ return self._split++ def __len__(self) -> int:+ return len(self._total_df)++ def __getitem__(self, index: int) -> pd.DataFrame:+ record = self._total_df.iloc[index]+ assert isinstance(record, pd.DataFrame) or isinstance(record, pd.Series)+ return record++ @staticmethod+ def record_to_swarm_input(record: pd.DataFrame) -> SwarmInput:+ demo_question = (+ f"{record['question']}\n"+ f"Option A: {record['A']}\n"+ f"Option B: {record['B']}\n"+ f"Option C: {record['C']}\n"+ f"Option D: {record['D']}\n"+ )+ input_dict = {"task": demo_question}+ return input_dict++ def postprocess_answer(self, answer: Union[str, List[str]]) -> str:+ if isinstance(answer, list):+ if len(answer) > 0:+ answer = answer[0]+ else:+ answer = ""+ if not isinstance(answer, str):+ raise Exception("Expected string")+ if len(answer) > 0:+ answer = answer[0] # Try to format the answer by taking the first letter+ return answer++ @staticmethod+ def record_to_target_answer(record: pd.DataFrame) -> str:+ correct_answer = record['correct_answer']+ assert isinstance(correct_answer, str), (+ f"String expected but got {correct_answer} "+ f"of type {type(correct_answer)} (2)" \+ f" record={record}")+ return correct_answerdiff --git a/experiments/evaluator/evaluator.py b/experiments/evaluator/evaluator.pynew file mode 100644index 0000000..53d2733--- /dev/null+++ b/experiments/evaluator/evaluator.py@@ -0,0 +1,308 @@+import os+import asyncio+import pandas as pd+from typing import Iterable, Optional, Iterator, Union, Literal, List, Dict, Any+from tqdm import tqdm+import torch+import time+import datetime+from torch.utils.tensorboard.writer import SummaryWriter+import numpy as np+import json+import math++from swarm.graph import Graph+from swarm.environment.agents import IO+from swarm.graph.swarm import Swarm+from experiments.evaluator.datasets.base_dataset import BaseDataset+from experiments.evaluator.accuracy import Accuracy+++class Evaluator():+ def __init__(+ self,+ swarm: Optional[Swarm],+ train_dataset: BaseDataset,+ val_dataset: BaseDataset,+ model_name: Optional[str] = None,+ enable_tensorboard: bool = False,+ enable_artifacts: bool = False,+ tensorboard_tag: Optional[str] = None,+ ) -> None:++ self._swarm: Optional[Swarm] = swarm+ self._train_dataset: BaseDataset = train_dataset+ self._val_dataset: BaseDataset = val_dataset+ self._model_name: Optional[str] = model_name++ datetime_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+ art_dir_name = (f"{datetime_str}" ++ (f"_{tensorboard_tag}" if tensorboard_tag is not None else ""))++ if enable_artifacts or enable_tensorboard:+ self._art_dir_name = os.path.join("runs", art_dir_name)+ os.makedirs(self._art_dir_name, exist_ok=True)+ else:+ self._art_dir_name = None++ if enable_tensorboard:+ self._logger = SummaryWriter(log_dir=self._art_dir_name)+ else:+ self._logger = None++ async def evaluate_direct_answer(self,+ limit_questions: Optional[int] = None,+ ) -> float:++ dataset = self._val_dataset++ print(f"Evaluating DirectAnswer on {dataset.get_domain()} split {dataset.split}")++ io_agent = IO(dataset.get_domain(), self._model_name)++ accuracy = Accuracy()++ for i_question, record in tqdm(enumerate(dataset)):+ print(80*'-')+ if limit_questions is not None:+ if i_question >= limit_questions:+ break++ input_dict = dataset.record_to_swarm_input(record)+ print(input_dict)++ raw_answer = await io_agent.run(input_dict)++ print("Raw answer:", raw_answer)+ answer = dataset.postprocess_answer(raw_answer)+ print("Postprocessed answer:", answer)+ correct_answer = dataset.record_to_target_answer(record)+ accuracy.update(answer, correct_answer)+ accuracy.print()++ print("Final accuracy:")+ accuracy.print()++ self._dump_eval_results(dict(+ accuracy=accuracy.get(),+ limit_questions=limit_questions))++ print("Done!")+ return accuracy.get()++ async def evaluate_swarm(+ self,+ mode: Union[+ Literal['full_connected_swarm'],+ Literal['randomly_connected_swarm'],+ Literal['external_edge_probs'],+ ],+ edge_probs: Optional[torch.Tensor] = None,+ limit_questions: Optional[int] = None,+ eval_batch_size: int = 4,+ ) -> float:++ assert self._swarm is not None++ dataset = self._val_dataset++ print(f"Evaluating swarm on {dataset.__class__.__name__} split {dataset.split}")++ realized_graph: Optional[Graph]+ if mode == 'full_connected_swarm':+ realized_graph = self._swarm.connection_dist.realize_full(self._swarm.composite_graph)+ elif mode == 'external_edge_probs':+ assert edge_probs is not None+ edge_mask = edge_probs > 0.5+ realized_graph = self._swarm.connection_dist.realize_mask(self._swarm.composite_graph, edge_mask)+ realized_graph.display()+ else:+ realized_graph = None++ accuracy = Accuracy()++ def eval_loader(batch_size: int) -> Iterator[List[Any]]:+ records = []+ for i_record, record in enumerate(dataset):+ if limit_questions is not None:+ if i_record >= limit_questions:+ break+ records.append(record)+ if len(records) >= batch_size:+ yield records+ records = []+ if len(records) > 0:+ yield records+ return++ data_len = min(len(dataset), limit_questions) if limit_questions is not None else len(dataset)+ num_batches = int(math.ceil(data_len / eval_batch_size))++ for i_batch, record_batch in tqdm(enumerate(eval_loader(batch_size=eval_batch_size)), total=num_batches):+ print(80*'-')++ start_ts = time.time()++ future_answers = []+ for record in record_batch:+ if mode == 'randomly_connected_swarm':+ realized_graph, _ = self._swarm.connection_dist.realize(self._swarm.composite_graph)+ assert realized_graph is not None++ input_dict = dataset.record_to_swarm_input(record)+ print(input_dict)++ future_answer = self._swarm.arun(input_dict, realized_graph)+ future_answers.append(future_answer)++ raw_answers = await asyncio.gather(*future_answers)++ print(f"Batch time {time.time() - start_ts:.3f}")++ for raw_answer, record in zip(raw_answers, record_batch):+ print("Raw answer:", raw_answer)+ answer = dataset.postprocess_answer(raw_answer)+ print("Postprocessed answer:", answer)+ correct_answer = dataset.record_to_target_answer(record)+ print("Correct answer:", correct_answer)+ accuracy.update(answer, correct_answer)+ accuracy.print()++ accuracy.print()+ print("Done!")++ self._dump_eval_results(dict(+ accuracy=accuracy.get(),+ limit_questions=limit_questions))++ return accuracy.get()++ def _dump_eval_results(self, dct: Dict[str, Any]) -> None:+ if self._art_dir_name is not None:+ eval_json_name = os.path.join(self._art_dir_name, "evaluation.json")+ with open(eval_json_name, "w") as f:+ json.dump(dct, f)++ def _print_conns(self, edge_probs: torch.Tensor, save_to_file: bool = False):+ assert self._swarm is not None+ msgs = []+ for i_conn, (conn, prob) in enumerate(zip(+ self._swarm.connection_dist.potential_connections, edge_probs)):+ src_id, dst_id = conn+ src_node = self._swarm.composite_graph.find_node(src_id)+ dst_node = self._swarm.composite_graph.find_node(dst_id)+ msg = (f"{i_conn}: src={src_node.node_name}({src_node.id}), "+ f"dst={dst_node.node_name}({dst_node.id}), prob={prob.item():.3f}")+ msgs.append(msg+"\n")+ print(msg)+ if save_to_file:+ if self._art_dir_name is not None:+ txt_name = os.path.join(self._art_dir_name, "connections.txt")+ with open(txt_name, "w") as f:+ f.writelines(msgs)++ async def optimize_swarm(+ self,+ num_iters: int,+ lr: float,+ batch_size: int = 4,+ ) -> torch.Tensor:++ assert self._swarm is not None++ dataset = self._train_dataset++ print(f"Optimizing swarm on {dataset.__class__.__name__} split {dataset.split}")++ optimizer = torch.optim.Adam(self._swarm.connection_dist.parameters(), lr=lr)++ if self._art_dir_name is not None:+ hp_json_name = os.path.join(self._art_dir_name, "hp.json")+ with open(hp_json_name, "w") as f:+ json.dump(dict(lr=lr,+ batch_size=batch_size,+ num_iters=num_iters,+ model_name=self._model_name+ ), f)++ def infinite_data_loader() -> Iterator[pd.DataFrame]:+ perm = np.random.permutation(len(dataset))+ while True:+ for idx in perm:+ record = dataset[idx.item()]+ yield record++ loader = infinite_data_loader()++ edge_probs = None+ for i_iter in range(num_iters):+ print(f"Iter {i_iter}", 80*'-')++ start_ts = time.time()++ future_answers = []+ log_probs = []+ correct_answers = []+ for i_record, record in zip(range(batch_size), loader):++ realized_graph, log_prob = self._swarm.connection_dist.realize(+ self._swarm.composite_graph,+ # temperature=3.0, # DEBUG+ )++ input_dict = dataset.record_to_swarm_input(record)+ answer = self._swarm.arun(input_dict, realized_graph)+ future_answers.append(answer)+ log_probs.append(log_prob)+ correct_answer = dataset.record_to_target_answer(record)+ correct_answers.append(correct_answer)++ raw_answers = await asyncio.gather(*future_answers)++ print(f"Batch time {time.time() - start_ts:.3f}")++ loss_list: List[torch.Tensor] = []+ utilities: List[float] = []+ for raw_answer, log_prob, correct_answer in zip(raw_answers, log_probs, correct_answers):+ answer = dataset.postprocess_answer(raw_answer)+ assert isinstance(correct_answer, str), \+ f"String expected but got {correct_answer} of type {type(correct_answer)} (1)"+ accuracy = Accuracy()+ accuracy.update(answer, correct_answer)+ utility = accuracy.get()+ utilities.append(utility)+ single_loss = - log_prob * utility+ loss_list.append(single_loss)++ print("utilities:", utilities)+ mean_utility = np.mean(np.array(utilities))+ total_loss = torch.mean(torch.stack(loss_list))++ print("loss:", total_loss.item())+ optimizer.zero_grad()+ total_loss.backward()+ print("Grad:", self._swarm.connection_dist.edge_logits.grad)+ optimizer.step()++ print("edge_logits:", self._swarm.connection_dist.edge_logits)+ edge_probs = torch.sigmoid(self._swarm.connection_dist.edge_logits)+ print("edge_probs:", edge_probs)++ self._print_conns(edge_probs)++ if self._logger is not None:+ self._logger.add_scalar("train/loss", total_loss.item(), i_iter)+ self._logger.add_scalar("train/utility", mean_utility.item(), i_iter)+ if self._art_dir_name is not None:+ log_jsonl_name = os.path.join(self._art_dir_name, "training.jsonl")+ with open(log_jsonl_name, "a") as f:+ json.dump(dict(iter=i_iter, train_loss=total_loss.item(), train_utility=mean_utility.item()), f)+ f.write("\n")+ print("end of iteration")++ if edge_probs is not None:+ self._print_conns(edge_probs, save_to_file=True)++ print("Done!")+ edge_probs = torch.sigmoid(self._swarm.connection_dist.edge_logits)+ return edge_probsdiff --git a/experiments/run_crosswords.py b/experiments/run_crosswords.pynew file mode 100644index 0000000..04736ac--- /dev/null+++ b/experiments/run_crosswords.py@@ -0,0 +1,50 @@+import json+from tqdm import tqdm+import asyncio+import numpy as np+from copy import deepcopy+import pickle+import torch+import sys+import random++from swarm.environment.domain.crosswords.env import MiniCrosswordsEnv+from swarm.environment.agents.agent_registry import AgentRegistry+from swarm.graph.swarm import Swarm+from swarm.optimizer.edge_optimizer.optimization import optimize+from swarm.environment.domain.crosswords.evaluator import CrosswordsEvaluator+++if __name__ == "__main__":+ if len(sys.argv) == 2:+ id = int(sys.argv[1])+ experiment_id = f"experiment{id}"+ torch.manual_seed(id)+ np.random.seed(id)+ random.seed(id)+ else:+ experiment_id = "experiment"+ id = 0++ print(experiment_id)++ file_path = "datasets/crosswords/mini0505_0_100_5.json"+ with open(file_path, "r") as file:+ test_data = json.load(file)++ init_connection_probability = .1+ batch_size = 20+ use_learned_order = False+ include_inner_agent_connections = True+ connect_output_nodes_to_final_node = True+ window_size = 10+ evaluator = CrosswordsEvaluator(test_data, batch_size=batch_size, metric="words", window_size=window_size, init_socre=0.4, use_init_score=True)+ swarm = Swarm(["CrosswordsReflection", "CrosswordsToT", "CrosswordsBruteForceOpt"], "crosswords", "gpt-3.5-turbo-1106", #"gpt-4-1106-preview"+ final_node_class="ReturnAll",+ final_node_kwargs={},+ edge_optimize=True,+ init_connection_probability=init_connection_probability,+ connect_output_nodes_to_final_node=connect_output_nodes_to_final_node,+ include_inner_agent_connections=include_inner_agent_connections)+ optimize(swarm, evaluator, batch_size=batch_size, num_iter=11, display_freq=1, record=True,+ experiment_id=experiment_id, lr=.4, use_learned_order=use_learned_order)\ No newline at end of filediff --git a/experiments/run_gaia.py b/experiments/run_gaia.pynew file mode 100644index 0000000..a22ad41--- /dev/null+++ b/experiments/run_gaia.py@@ -0,0 +1,167 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-++import os+import sys+import argparse+import yaml+import json+import time+import asyncio+from pathlib import Path++from swarm.graph.swarm import Swarm+from swarm.environment.tools.reader.readers import JSONReader, YAMLReader+from swarm.environment.agents.io import IO+from swarm.environment.agents.gaia.normal_io import NormalIO+from swarm.environment.agents.gaia.tool_io import ToolIO+from swarm.environment.agents.gaia.web_io import WebIO+from swarm.environment.agents.gaia.tool_tot import ToolTOT+from swarm.environment.operations import DirectAnswer+from swarm.memory.memory import GlobalMemory+from swarm.utils.globals import Time+from swarm.utils.const import GPTSWARM_ROOT+from swarm.utils.log import logger+from swarm.environment.domain.gaia import question_scorer+from swarm.environment.operations.final_decision import MergingStrategy+++def dataloader(data_list):+ for data in data_list:+ yield data++def load_config(config_path):+ with open(config_path, 'r') as file:+ return yaml.safe_load(file)++async def main():+ parser = argparse.ArgumentParser(description="GPTSwarm Experiments on GAIA")+ parser.add_argument("--config", type=str, help="Path to configuration YAML file.")+ parser.add_argument("--domain", type=str, default="gaia")+ parser.add_argument("--agents", nargs='+', default=["IO"])+ parser.add_argument("--dataset_json", type=str, default="datasets/gaia/level_1_val.json") #level_1_val_solveable.json+ parser.add_argument("--dataset_files", type=str, default="datasets/gaia/val_files")+ parser.add_argument("--result_file", type=str, default=None)+ parser.add_argument("--llm", type=str, default="gpt-4-1106-preview") #gpt-4-1106-preview gpt-3.5-turbo-1106 gpt-3.5-turbo gpt-4+ args = parser.parse_args()++ result_path = GPTSWARM_ROOT / "result"+ os.makedirs(result_path, exist_ok=True)++ if args.config:+ config_args = YAMLReader.parse(args.config, return_str=False)+ for key, value in config_args.items():+ setattr(args, key, value)++ start_index = 0+ result_file = None++ dataset = JSONReader.parse_file(args.dataset_json)++ ####################################++ # strategy = MergingStrategy.SelfConsistency #MergingStrategy.SelectBest #MergingStrategy.SelfConsistency #MergingStrategy.SelectBest #MergingStrategy.SelectBest #MergingStrategy.SelfConsistency # MergingStrategy.MajorityVote MergingStrategy.RandomChoice++ experiment_name = "ToolTOT"++ # swarm = Swarm(["ToolTOT"]*7,+ # "gaia",+ # model_name="mock", #args.llm, #"mock", #args.llm,#args.llm,+ # final_node_class="FinalDecision",+ # final_node_kwargs=dict(strategy=strategy)+ # )+ # swarm.composite_graph.display()++ print(args.llm)++ #agent = IO(domain="gaia", model_name=args.llm)+ #agent = WebIO(domain="gaia", model_name=args.llm)+ #agent = ToolIO(domain="gaia", model_name=args.llm)+ agent = ToolTOT(domain="gaia", model_name=args.llm)++ #io = DirectAnswer(domain="gaia", model_name=args.llm)++ agent.display()++ ####################################++ for i, item in enumerate(dataloader(dataset)):++ if i < start_index:+ print(f"Skipping index {i}...")+ continue++ start_time = time.time()+ task = item["Question"]+ files = [os.path.join(args.dataset_files, item["file_name"])] if item["file_name"] else item["file_name"]+ ground_truth = item["Final answer"]+ inputs = {"task": task, "files": files, "GT": ground_truth}++ # Swarm+ # answer = await swarm.composite_graph.run(inputs)+ # answer = answer[-1].split("FINAL ANSWER: ")[-1]++ # end_time = time.time()+ # exe_time = end_time - start_time++ # print("-----")+ # print(f"SWARM ANSWER: {answer}")+ # print("-----")++ # Agent+ answer = await agent.run(inputs=inputs)+ answer = answer[-1].split("FINAL ANSWER: ")[-1]++ end_time = time.time()+ exe_time = end_time - start_time+++ print("-----")+ print(f"AGENT ANSWER: {answer}")+ print("-----")++ """+ answer = await io._execute(inputs=inputs)+ answer = answer[-1]["output"].split("FINAL ANSWER: ")[-1]++ print("-----")+ print(f"OPERATION ANSWER: {answer}")+ print("-----")+ """++ current_time = Time.instance().value or time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())+ Time.instance().value = current_time++ result_dir = Path(f"{GPTSWARM_ROOT}/result/eval")+ result_file = result_file or (result_dir / f"{'_'.join(experiment_name.split())}_{args.llm}_{current_time}.json")++ result_dir.mkdir(parents=True, exist_ok=True)++ if not result_file.exists():+ with open(result_file, 'w') as file:+ json.dump([], file)++ with open(result_file, 'r') as file:+ data = json.load(file)++ total_solved, total_executed = (0, 0) if not data else (data[-1]["Total solved"], data[-1]["Total executed"])+ is_solved = question_scorer(answer, item['Final answer'])++ updated_item = {+ "Question": item["Question"],+ "GT": item['Final answer'],+ "Attempt answer": answer,+ "Solved": is_solved,+ "Total solved": total_solved + is_solved,+ "Total executed": total_executed + 1,+ "Accuracy": (total_solved + is_solved) / (total_executed + 1),+ "Time": exe_time,+ }+ data.append(updated_item)++ with open(result_file, 'w') as file:+ json.dump(data, file, indent=4)+++if __name__ == '__main__':+ asyncio.run(main())diff --git a/experiments/run_humaneval.py b/experiments/run_humaneval.pynew file mode 100644index 0000000..48cd4c2--- /dev/null+++ b/experiments/run_humaneval.py@@ -0,0 +1,122 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-++import os+import argparse+import yaml+import json+import time+import asyncio+from pathlib import Path++from swarm.environment.tools.reader.readers import JSONLReader, YAMLReader+from swarm.environment.agents.humaneval.code_react import CodeReact+from swarm.environment.tools.coding.python_executor import PyExecutor+from swarm.memory.memory import GlobalMemory+from swarm.utils.globals import Time+from swarm.utils.const import GPTSWARM_ROOT+from swarm.utils.log import logger+from swarm.environment.operations.optimizable_operation import OptimizableOperation+from swarm.optimizer.node_optimizer.node_optimization import optimize+++def load_result(result_file):+ if not result_file.exists():+ with open(result_file, 'w') as file:+ json.dump([], file)++ with open(result_file, 'r') as file:+ data = json.load(file)+ return data++def dataloader(data_list):+ for data in data_list:+ yield data++def load_config(config_path):+ with open(config_path, 'r') as file:+ return yaml.safe_load(file)++def parse_args():+ parser = argparse.ArgumentParser(description="GPTSwarm Experiments on HumanEval")+ parser.add_argument("--config", type=str, help="Path to configuration YAML file.")+ parser.add_argument("--dataset_json", type=str, default="datasets/humaneval/humaneval-py.jsonl")+ parser.add_argument("--result_file", type=str, default=None)+ parser.add_argument("--llm", type=str, default="gpt-4-1106-preview")+ parser.add_argument("--learn_prompt", type=bool, default=False)+ parser.add_argument("--learn_demonstration", type=bool, default=False)++ args = parser.parse_args()+ result_path = GPTSWARM_ROOT / "result"+ os.makedirs(result_path, exist_ok=True)+ if args.config:+ config_args = YAMLReader.parse(args.config, return_str=False)+ for key, value in config_args.items():+ setattr(args, key, value)+ return args++async def main():+ args = parse_args()+ result_file = None++ dataset = JSONLReader.parse_file(args.dataset_json)++ ####################################+++ current_time = Time.instance().value or time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())+ Time.instance().value = current_time+ result_dir = Path(f"{GPTSWARM_ROOT}/result/eval")+ result_dir.mkdir(parents=True, exist_ok=True)+ result_file = result_dir / f"{'' if args.learn_prompt else 'not'}_learn_prompt_{'' if args.learn_demonstration else 'not'}_learn_demo_{args.llm}_{current_time}.json"+ agent = CodeReact(domain="humaneval",+ model_name=args.llm,+ )+ memory = GlobalMemory.instance()+ ####################################+ opt_frequency = 4+ for i, item in enumerate(dataloader(dataset)):+ task = item["prompt"]+ tests = item["test"]+ inputs = {"task": task, "tests": tests}++ # Agent+ answer = await agent.run(inputs=inputs)+ answer = answer[-1]+ # Evaluate the answer against the test cases here++ data = load_result(result_file)+ total_solved, total_executed = (0, 0) if not data else (data[-1]["Total solved"], data[-1]["Total executed"])++ is_solved, _, _ = PyExecutor().execute(answer, [tests], timeout=100)+ memory.add(task, is_solved)++ total_solved = total_solved + is_solved+ total_executed = total_executed + 1+ accuracy = total_solved/ total_executed++ logger.info(f"total_solved: \n{total_solved}")+ logger.info(f"total_executed: \n{total_executed}")+ logger.info(f"accuracy: \n{accuracy}")++ updated_item = {+ "Question": task,+ "Tests": tests,+ "Attempt answer": answer,+ "Solved": is_solved,+ "Solution": answer,+ "Total solved": total_solved,+ "Total executed": total_executed,+ "Accuracy": accuracy+ }+ data.append(updated_item)++ with open(result_file, 'w') as file:+ json.dump(data, file, indent=4)++ if i % opt_frequency == opt_frequency - 1 and (args.learn_prompt or args.learn_demonstration):+ tasks = [optimize(node, args.learn_demonstration, args.learn_prompt) for node in agent.nodes.values() if isinstance(node, OptimizableOperation)]+ await asyncio.gather(*tasks)++if __name__ == '__main__':+ asyncio.run(main())diff --git a/experiments/run_mmlu.py b/experiments/run_mmlu.pynew file mode 100644index 0000000..d70db1e--- /dev/null+++ b/experiments/run_mmlu.py@@ -0,0 +1,122 @@+import asyncio+from typing import Union, Literal, Optional+import argparse++from swarm.graph.swarm import Swarm+from swarm.environment.operations.final_decision import MergingStrategy+from experiments.evaluator.evaluator import Evaluator+from experiments.evaluator.datasets.mmlu_dataset import MMLUDataset+++def parse_args():+ parser = argparse.ArgumentParser(description="Process some parameters.")++ parser.add_argument('--mode', type=str, default='OptimizedSwarm',+ choices=['DirectAnswer', 'FullConnectedSwarm', 'RandomSwarm', 'OptimizedSwarm'],+ help="Mode of operation. Default is 'OptimizedSwarm'.")++ parser.add_argument('--num-truthful-agents', type=int, default=1,+ help="Number of truthful agents. The total will be N truthful and N adversarial.")++ parser.add_argument('--num-iterations', type=int, default=200,+ help="Number of optimization iterations. Default 200.")++ parser.add_argument('--model_name', type=str, default=None,+ help="Model name, None runs the default ChatGPT4.")++ parser.add_argument('--domain', type=str, default="mmlu",+ help="Domain (the same as dataset name), default 'MMLU'")++ parser.add_argument('--debug', action='store_true', default=False,+ help="Set for a quick debug cycle")++ args = parser.parse_args()+ return args+++async def main():++ args = parse_args()++ debug: bool = args.debug++ model_name: Optional[str] = args.model_name++ mode: Union[Literal['DirectAnswer'],+ Literal['FullConnectedSwarm'],+ Literal['RandomSwarm'],+ Literal['OptimizedSwarm']]++ mode = args.mode++ strategy = MergingStrategy.MajorityVote++ domain: str = args.domain++ if mode == 'DirectAnswer':+ swarm_name = None+ swarm = None+ else:+ N = args.num_truthful_agents+ M = N+ agent_name_list = N * ["IO"] + M * ["AdversarialAgent"]++ swarm_name = f"{N}true_{M}adv"++ swarm = Swarm(+ agent_name_list,+ domain,+ model_name=model_name,+ final_node_class="FinalDecision",+ final_node_kwargs=dict(strategy=strategy),+ edge_optimize=True,+ )++ tag = f"{domain}_{swarm_name}_{strategy.name}_{mode}"++ dataset_train = MMLUDataset('dev')+ dataset_val = MMLUDataset('val')++ evaluator = Evaluator(+ swarm,+ dataset_train,+ dataset_val,+ model_name=model_name,+ enable_tensorboard = mode=='OptimizedSwarm',+ enable_artifacts=True,+ tensorboard_tag=tag)++ limit_questions = 5 if debug else 153++ if mode == 'DirectAnswer':+ score = await evaluator.evaluate_direct_answer(+ limit_questions=limit_questions)+ elif mode == 'FullConnectedSwarm':+ score = await evaluator.evaluate_swarm(+ mode='full_connected_swarm',+ limit_questions=limit_questions)+ elif mode == 'RandomSwarm':+ score = await evaluator.evaluate_swarm(+ mode='randomly_connected_swarm',+ limit_questions=limit_questions)+ elif mode == 'OptimizedSwarm':++ num_iters = 5 if debug else args.num_iterations++ lr = 0.1++ edge_probs = await evaluator.optimize_swarm(num_iters=num_iters, lr=lr)++ score = await evaluator.evaluate_swarm(+ mode='external_edge_probs',+ edge_probs=edge_probs,+ limit_questions=limit_questions,+ )+ else:+ raise Exception(f"Unsupported mode {mode}")++ print(f"Score: {score}")+++if __name__ == "__main__":+ asyncio.run(main())