🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Evaluation Samples Generation (#110)
Hasan Abed Al Kader Hammoudeff01d4c65a8
diff --git a/camel/prompts/__init__.py b/camel/prompts/__init__.pyindex 006552be1..6dc72dec6 100644--- a/camel/prompts/__init__.py+++ b/camel/prompts/__init__.py@@ -3,6 +3,7 @@ from .ai_society import AISocietyPromptTemplateDictfrom .code import CodePromptTemplateDictfrom .misalignment import MisalignmentPromptTemplateDictfrom .translation import TranslationPromptTemplateDict+from .evaluation import EvaluationPromptTemplateDictfrom .task_prompt_template import TaskPromptTemplateDictfrom .prompt_templates import PromptTemplateGenerator@@ -13,6 +14,7 @@ __all__ = ['CodePromptTemplateDict','MisalignmentPromptTemplateDict','TranslationPromptTemplateDict',+ 'EvaluationPromptTemplateDict','TaskPromptTemplateDict','PromptTemplateGenerator',]diff --git a/camel/prompts/evaluation.py b/camel/prompts/evaluation.pynew file mode 100644index 000000000..ebf1c156c--- /dev/null+++ b/camel/prompts/evaluation.py@@ -0,0 +1,29 @@+from typing import Any++from camel.prompts import TextPrompt, TextPromptDict+from camel.typing import RoleType+++# flake8: noqa+class EvaluationPromptTemplateDict(TextPromptDict):+ r"""A dictionary containing :obj:`TextPrompt` used in the `Evaluation`+ task.++ Attributes:+ GENERATE_QUESTIONS (TextPrompt): A prompt to generate a set of questions+ to be used for evaluating emergence of knowledge based on a particular+ field of knowledge.+ """++ GENERATE_QUESTIONS = TextPrompt(+ """Generate {num_questions} {category} diverse questions.+Here are some example questions:+{examples}++Now generate {num_questions} questions of your own. Be creative""")++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self.update({+ "generate_questions": self.GENERATE_QUESTIONS,+ })diff --git a/camel/prompts/task_prompt_template.py b/camel/prompts/task_prompt_template.pyindex 8da01206c..9cdd30446 100644--- a/camel/prompts/task_prompt_template.py+++ b/camel/prompts/task_prompt_template.py@@ -3,6 +3,7 @@ from typing import Any, Dictfrom camel.prompts import (AISocietyPromptTemplateDict,CodePromptTemplateDict,+ EvaluationPromptTemplateDict,MisalignmentPromptTemplateDict,TextPromptDict,TranslationPromptTemplateDict,@@ -27,4 +28,5 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):TaskType.CODE: CodePromptTemplateDict(),TaskType.MISALIGNMENT: MisalignmentPromptTemplateDict(),TaskType.TRANSLATION: TranslationPromptTemplateDict(),+ TaskType.EVALUATION: EvaluationPromptTemplateDict(),})diff --git a/camel/typing.py b/camel/typing.pyindex 3ecc0bd78..b64cc92d6 100644--- a/camel/typing.py+++ b/camel/typing.py@@ -20,6 +20,7 @@ class TaskType(Enum):CODE = "code"MISALIGNMENT = "misalignment"TRANSLATION = "translation"+ EVALUATION = "evaluation"DEFAULT = "default"diff --git a/examples/evaluation/single_agent.py b/examples/evaluation/single_agent.pynew file mode 100644index 000000000..c28d327de--- /dev/null+++ b/examples/evaluation/single_agent.py@@ -0,0 +1,126 @@+import json+import os+import re+from typing import Any, Dict, List++from camel.agents import ChatAgent+from camel.messages import AssistantSystemMessage, UserChatMessage+from camel.prompts import PromptTemplateGenerator+from camel.typing import TaskType+++def parse_question_string(question_string: str,+ category: str) -> List[Dict[str, Any]]:+ pattern = r'^(\d+)\.\s+(.*?)\s*\n*$'+ questions = []+ for match in re.finditer(pattern, question_string, re.MULTILINE):+ question_id = int(match.group(1))+ text = match.group(2)+ questions.append({+ 'question_id': question_id,+ 'text': text,+ 'category': category+ })+ return questions+++def generate_questions(examples: str, category: str, save_file_name: str,+ key: str = 'generate_questions',+ num_questions: int = 20) -> None:+ prompt_template = PromptTemplateGenerator().get_prompt_from_key(+ TaskType.EVALUATION, key)++ evaluation_dict = {+ 'num_questions': num_questions,+ 'category': category,+ 'examples': examples,+ }++ prompt = prompt_template.format(**evaluation_dict)+ print(prompt)+ assistant_sys_msg = AssistantSystemMessage(+ role_name="Assistant",+ content="You are a helpful assistant.",+ )+ agent = ChatAgent(assistant_sys_msg)+ agent.reset()++ user_msg = UserChatMessage(role_name="User", content=prompt)+ assistant_msg, _, _ = agent.step(user_msg)++ if assistant_msg is not None:++ print(assistant_msg[0].content)++ parsed_assistant_msg = parse_question_string(assistant_msg[0].content,+ category)++ # save json file+ folder_path = './evaluation_data/questions'+ if not os.path.exists(folder_path):+ os.makedirs(folder_path)++ with open(f"{folder_path}/{save_file_name}.jsonl", "w") as f:+ for item in parsed_assistant_msg:+ json.dump(item, f)+ f.write('\n')+++def main() -> None:++ # generate ai society evaluation questions+ examples = ("1. What are the most effective ways to deal with stress?\n"+ "2. Explain the process of natural selection and how it "+ "contributes to the evolution and adaptation of species.\n"+ "3. Can you help me write a formal email to a potential"+ "business partner proposing a joint venture?")++ category = 'generic task Q&A'+ save_file_name = 'questions_ai_society'+ generate_questions(examples, category, save_file_name)++ # generate coding evaluation questions+ examples = (+ "1. Develop a C++ program that reads a text file line by line and"+ "counts the number of occurrences of a specific word in the file.\n"+ "2. Implement a Jave function to find the longest common"+ " subsequence of two input strings using dynamic programming.\n"+ "3. Implement a machine learning-based chatbot system in Python.")++ category = 'coding task'+ save_file_name = 'questions_code'+ generate_questions(examples, category, save_file_name)++ # generate math evaluation questions+ examples = (+ "1. Given that f(x) = 5x^3 - 2x + 3, find the value of f(2).\n"+ "2. If the endpoints of a line segment are (2, -2) and (10, 4), "+ "what is the length of the segment?\n"+ "3. Solve for x in the equation 3x + 10 = 5(x - 2).")++ category = 'math'+ save_file_name = 'questions_math'+ generate_questions(examples, category, save_file_name)++ # generate science evaluation questions+ examples = (+ "1. What is the probability of finding a particle with a given energy"+ " in a one-dimensional infinite square well potential when the"+ " potential width is 2 nm and the particle has a mass of 5x10^-26"+ " kg? Use the Schrödinger equation to solve for the allowed energy"+ " states and their wave functions.\n"+ "2. How does habitat loss and fragmentation affect the migratory"+ " patterns and survival of a specific species, such as the monarch"+ " butterfly, over time?\n"+ "3. What is the systematic name of the organic compound with the"+ " molecular formula C6H12O and a ketone functional group located"+ " on the second carbon atom from the left end?")++ category = 'science'++ save_file_name = 'questions_science'+ generate_questions(examples, category, save_file_name, num_questions=60)+++if __name__ == "__main__":+ main()