🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Data generation (#12)
hammoudhasan0c6e84cee119
diff --git a/README.md b/README.mddeleted file mode 100644index 6a0354285..000000000--- a/README.md+++ /dev/null@@ -1 +0,0 @@-# camel\ No newline at end of filediff --git a/camel/agent.py b/camel/agent.pyindex 920616536..7e8379fd1 100644--- a/camel/agent.py+++ b/camel/agent.py@@ -1,7 +1,7 @@from typing import Any, Dict, List, Optional, Tupleimport openai-from tenacity import retry, stop_after_attempt, wait_fixed+from tenacity import retry, stop_after_attempt, wait_exponentialfrom camel.configs import ChatGPTConfigfrom camel.generator import SystemMessageGenerator@@ -58,7 +58,7 @@ class ChatAgent:self.stored_messages.append(message)return self.stored_messages- @retry(wait=wait_fixed(60), stop=stop_after_attempt(5))+ @retry(wait=wait_exponential(min=5, max=60), stop=stop_after_attempt(5))def step(self,input_message: ChatMessage,diff --git a/camel/generator.py b/camel/generator.pyindex 806c4e625..9f2ce2789 100644--- a/camel/generator.py+++ b/camel/generator.py@@ -149,10 +149,16 @@ class RoleNameGenerator:user_role_names_path: str = "data/user_roles.txt") -> None:with open(assistant_role_names_path, "r") as f:- self.assistant_role_names: List[str] = f.read().splitlines()+ assistant_role_names: List[str] = f.read().splitlines()+ self.assistant_role_names = [+ " ".join(name.split(" ")[1:]) for name in assistant_role_names+ ]with open(user_role_names_path, "r") as f:- self.user_role_names: List[str] = f.read().splitlines()+ user_role_names: List[str] = f.read().splitlines()+ self.user_role_names = [+ " ".join(name.split(" ")[1:]) for name in user_role_names+ ]def from_role_files(self) -> Generator[Tuple, None, None]:for assistant_role_name in self.assistant_role_names:@@ -165,11 +171,14 @@ class TaskPromptGenerator:def __init__(self,generate_tasks_prompt_path: str = "prompts/generate_tasks.txt",+ num_tasks: int = 10,) -> None:with open(generate_tasks_prompt_path, "r") as f:self.generate_tasks_prompt: str = f.read()+ self.num_tasks = num_tasks+def from_role_files(self, assistant_role_names_path: str = "data/assistant_roles.txt",user_role_names_path: str = "data/user_roles.txt"@@ -177,9 +186,11 @@ class TaskPromptGenerator:roles_generator = RoleNameGenerator(assistant_role_names_path, user_role_names_path).from_role_files()for role_1, role_2 in roles_generator:- yield (self.generate_tasks_prompt.replace("<ROLE_1>",- role_1).replace(- "<ROLE_2>", role_2))+ yield (self.generate_tasks_prompt.replace(+ "<ROLE_1>",+ role_1).replace("<ROLE_2>",+ role_2).replace("<NUM_TASKS>",+ str(self.num_tasks)))def from_role_generator(self, role_generator: Generator[Tuple, None, None]diff --git a/examples/role_playing_multiprocess.py b/examples/role_playing_multiprocess.pynew file mode 100644index 000000000..11b8cf6d5--- /dev/null+++ b/examples/role_playing_multiprocess.py@@ -0,0 +1,186 @@+import json+import multiprocessing+import os++from camel.agent import ChatAgent, TaskSpecifyAgent+from camel.configs import ChatGPTConfig+from camel.generator import SystemMessageGenerator+from camel.message import (AssistantChatMessage, AssistantSystemMessage,+ UserChatMessage, UserSystemMessage)+from camel.typing import ModeType, RoleType+++def init_chat(+ assistant_agent: ChatAgent,+ user_agent: ChatAgent,+ user_sys_msg: UserSystemMessage,+ assistant_sys_msg: AssistantSystemMessage,+):+ assistant_agent.reset()+ user_agent.reset()++ # Send the system messages again to the agents using chat messages+ assistant_msg = AssistantChatMessage(+ "Computer Programer",+ content=(f"{user_sys_msg.content}. "+ "Now start to give me introductions one by one. "+ "Only reply with Instruction and Input."))+ assistant_msg.role = "user"++ user_msg = UserChatMessage(user_sys_msg.role_name,+ content=f"{assistant_sys_msg.content}")+ msgs, _, _ = assistant_agent.step(user_msg)++ return assistant_msg, msgs+++def generate_data(assistant_idx: int, assistant_role_name: str, user_idx: int,+ user_role_name: str, task_idx: int,+ task_prompt: str) -> None:++ max_num_messages = 20++ original_task_prompt = task_prompt+ task_specify_agent = TaskSpecifyAgent(ModeType.GPT_3_5_TURBO,+ ChatGPTConfig(temperature=1.4))+ specified_task_prompt = task_specify_agent.specify_task(+ original_task_prompt)+ print(f"Original Task: {original_task_prompt}")+ print(f"Specfied Task: {specified_task_prompt}")+ sys_msg_generator = SystemMessageGenerator(with_task=True)+ assistant_sys_msg, user_sys_msg = sys_msg_generator.from_roles(+ roles=[(assistant_role_name, RoleType.ASSISTANT),+ (user_role_name, RoleType.USER)],+ task_prompt=specified_task_prompt)+ assistant_agent = ChatAgent(assistant_sys_msg, ModeType.GPT_3_5_TURBO,+ message_window_size=max_num_messages)+ user_agent = ChatAgent(user_sys_msg, ModeType.GPT_3_5_TURBO,+ message_window_size=max_num_messages)++ assistant_msg, _ = init_chat(assistant_agent, user_agent, user_sys_msg,+ assistant_sys_msg)++ message_counter = 0+ message_dict = {}++ # Append roles to the dictionary+ # We start number from 1 not 0.+ message_dict[+ "role_1"] = f"{assistant_role_name}_{str(assistant_agent.role_type)}"+ message_dict["role_2"] = f"{user_role_name}_{str(user_agent.role_type)}"+ message_dict[+ "id"] = f"{(assistant_idx+1):03}_{(user_idx+1):03}_{(task_idx+1):03}"+ message_dict["original_task"] = task_prompt.replace(f"{task_idx+1}. ", "")+ message_dict["specified_task"] = specified_task_prompt.replace(+ f"{task_idx+1}. ", "")++ # Threshold to terminate the conversation if no end token appears+ repeat_word_counter = 0+ repeat_word_threshold = 3+ repeat_word_list = ["goodbye", "good bye", "thank", "bye", "welcome"]++ # Set maximum number of messages for the chat++ while message_counter < max_num_messages:+ user_msgs, user_terminated, user_info = user_agent.step(assistant_msg)+ if user_terminated:+ message_dict["termination_reason"] = (+ f"{str(user_agent.role_type)}: "+ f"{user_info['finish_reasons'][0]}")+ break++ user_msg = user_msgs[0]+ print(f"User:\n{user_msg.content}\n")+ user_msg.role = "user"++ message_counter += 1+ message_dict[f"message_{message_counter}"] = user_msg.to_dict()++ if "<CAMEL_TASK_DONE>" in user_msg.content:+ message_dict['termination_reason'] = "<CAMEL_TASK_DONE>"+ break++ assistant_msgs, assistant_terminated, assistant_info = (+ assistant_agent.step(user_msg))+ if assistant_terminated:+ message_dict["termination_reason"] = (+ f"{str(assistant_agent.role_type)}: "+ f"{assistant_info['finish_reasons'][0]}")+ break+ assistant_msg = assistant_msgs[0]+ print(f"Assistant:\n{assistant_msg.content}\n")+ assistant_msg.role = "user"++ message_counter += 1+ message_dict[f"message_{message_counter}"] = assistant_msg.to_dict()++ for repeat_word in repeat_word_list:+ if repeat_word in user_msg.content.lower():+ repeat_word_counter += 1+ if repeat_word_counter == repeat_word_threshold:+ break+ else:+ repeat_word_counter = 0++ message_dict["num_messages"] = message_counter++ with open(f"./camel_data/{message_dict['id']}.json", "w") as json_file:+ json.dump(message_dict, json_file)+++def main() -> None:++ # Chunk for parallel jobs+ # array_idx = int(os.environ.get('SLURM_ARRAY_TASK_ID'))+ array_idx = 1+ roles_per_chunk = 10++ # Parameters for filtering the generated task string+ start_token = "1."+ num_tasks = 40++ with open("./data/user_roles.txt", "r") as f:+ user_roles = f.read().splitlines()++ with open("./data/assistant_roles.txt", "r") as f:+ assistant_roles = f.read().splitlines()++ assert (array_idx + 1) * roles_per_chunk <= len(assistant_roles)+ assistant_roles = assistant_roles[array_idx *+ roles_per_chunk:(array_idx + 1) *+ roles_per_chunk]++ pool = multiprocessing.Pool()++ for assistant_idx, assistant_role_name in enumerate(assistant_roles):+ for user_idx, user_role_name in enumerate(user_roles):++ # Load the task list assigned for assistant and user roles+ with open(f"./tasks/{assistant_role_name}_{user_role_name}.txt",+ "r") as f:+ tasks = f.read().splitlines()++ # Filter out the generated response to include the tasks only+ for i, task in enumerate(tasks):+ if start_token in task:+ tasks = tasks[i:i + num_tasks]+ break++ # Ensure exact number of tasks is generated+ assert str(num_tasks) in tasks[-1], print(tasks)++ for task_idx, task_prompt in enumerate(tasks):+ id = (f"{(assistant_idx+1):03}_"+ f"{(user_idx+1):03}_{(task_idx+1):03}")+ if not os.path.exists(f"./camel_data/{id}.json"):+ pool.apply_async(+ generate_data,+ (assistant_idx, assistant_role_name, user_idx,+ user_role_name, task_idx, task_prompt))++ pool.close()+ pool.join()+++if __name__ == "__main__":+ main()diff --git a/examples/task_generation.py b/examples/task_generation.pyindex 9aa71a61b..de41d59ae 100644--- a/examples/task_generation.py+++ b/examples/task_generation.py@@ -7,8 +7,8 @@ from camel.generator import (RoleNameGenerator, SystemMessageGenerator,from camel.typing import ModeType, RoleType-def generate_tasks(role_name: str, task_generator_prompt: str,- start_token: str = "1.", num_tasks: int = 40) -> None:+def generate_tasks(role_names: str, task_generator_prompt: str,+ start_token: str = "1.", num_tasks: int = 10) -> None:sys_msg_generator = SystemMessageGenerator()assistant_sys_msg = sys_msg_generator.from_role(RoleType.DEFAULT)@@ -29,21 +29,26 @@ def generate_tasks(role_name: str, task_generator_prompt: str,# Ensure exact number of tasks is generatedassert str(num_tasks) in tasks[-1], print(tasks)- with open(f"./tasks/{'_'.join(role_name)}.txt", "w") as file:+ with open(f"./tasks/{'_'.join(role_names)}.txt", "w") as file:file.write("\n".join(tasks))def main() -> None:+ num_tasks = 10+ start_token = "1."+role_names_generator = RoleNameGenerator().from_role_files()- task_generator_prompt_generator = TaskPromptGenerator().from_role_files()+ task_generator_prompt_generator = TaskPromptGenerator(+ num_tasks=num_tasks).from_role_files()- pool = multiprocessing.Pool(processes=1)- for task_generator_prompt, role_name in zip(+ pool = multiprocessing.Pool()+ for task_generator_prompt, role_names in zip(task_generator_prompt_generator, role_names_generator):- if not os.path.exists(f"./tasks/{'_'.join(role_name)}.txt"):- print(f"Generating tasks for {role_name}")- pool.apply_async(generate_tasks,- (role_name, task_generator_prompt))+ if not os.path.exists(f"./tasks/{'_'.join(role_names)}.txt"):+ print(f"Generating tasks for {role_names}")+ pool.apply_async(+ generate_tasks,+ (role_names, task_generator_prompt, start_token, num_tasks))pool.close()pool.join()diff --git a/misc/logo.png b/misc/logo.pngnew file mode 100644index 000000000..aea587ba1Binary files /dev/null and b/misc/logo.png differdiff --git a/prompts/assistant_prompt_with_task.txt b/prompts/assistant_prompt_with_task.txtindex 234267b55..6a6fe3557 100644--- a/prompts/assistant_prompt_with_task.txt+++ b/prompts/assistant_prompt_with_task.txt@@ -1,5 +1,5 @@-Never forget you are a <ASSISTANT_ROLE> and I am a <USER_ROLE>.-We share a common interest in collaborating to successfully complete a task.+Never forget you are a <ASSISTANT_ROLE> and I am a <USER_ROLE>. Never flip roles!+We share a common interest in collaborating to successfully complete a task in the minimum number of steps.You must help me to complete the task.Here is the task: <TASK>. Never forget our task!I must instruct you based on your expertise and my needs to complete the task in the following to two ways:@@ -18,4 +18,4 @@ You must write a response that appropriately completes the requested instructionYou must decline my instruction politely and honestly if you cannot perform the instruction due to physical, moral, legal reasons or your capability and explain the reasons.Now I must start to instruct you using the two ways described above.Do not add anything else other than your response to my instruction.-Once the task is completed, I must reply with <CAMEL_TASK_DONE> to indicate.\ No newline at end of file+You are never supposed to ask me any questions you only answer questions.\ No newline at end of filediff --git a/prompts/generate_tasks.txt b/prompts/generate_tasks.txtindex 2b92e8599..c978fedac 100644--- a/prompts/generate_tasks.txt+++ b/prompts/generate_tasks.txt@@ -1 +1 @@-List 40 diverse tasks that <ROLE_1> can assist <ROLE_2> cooperatively to achieve together. Be concise. Be creative.+List <NUM_TASKS> diverse tasks that <ROLE_1> can assist <ROLE_2> cooperatively to achieve together. Be concise. Be creative.diff --git a/prompts/user_prompt_with_task.txt b/prompts/user_prompt_with_task.txtindex d9fe7c4b5..8fcbd1fcc 100644--- a/prompts/user_prompt_with_task.txt+++ b/prompts/user_prompt_with_task.txt@@ -1,5 +1,5 @@-Never forget you are a <USER_ROLE> and I am a <ASSISTANT_ROLE>.-We share a common interest in collaborating to successfully complete a task.+Never forget you are a <USER_ROLE> and I am a <ASSISTANT_ROLE>. Never flip roles!+We share a common interest in collaborating to successfully complete a task in the minimum number of steps.I must help you to complete the task.Here is the task: <TASK>. Never forget our task!You must instruct me based on my expertise and your needs to complete the task in the following to two ways:@@ -19,4 +19,5 @@ I must decline your instruction politely and honestly if I cannot perform the inNow you must start to instruct me using the two ways described above.Do not add anything else other than your instruction and the optional corresponding input!Keep giving me instructions and necessary inputs until you think the task is completed.-Once the task is completed, you must reply with <CAMEL_TASK_DONE> to indicate that.\ No newline at end of file+When the task is completed, you must only reply with a single word <CAMEL_TASK_DONE>.+Never say <CAMEL_TASK_DONE> unless my responses have solved your task.\ No newline at end of file