Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Evaluation Samples Generation (#110)

Hasan Abed Al Kader Hammoudeff01d4c65a8
diff --git a/camel/prompts/__init__.py b/camel/prompts/__init__.py
index 006552be1..6dc72dec6 100644
--- a/camel/prompts/__init__.py
+++ b/camel/prompts/__init__.py
@@ -3,6 +3,7 @@ from .ai_society import AISocietyPromptTemplateDict
from .code import CodePromptTemplateDict
from .misalignment import MisalignmentPromptTemplateDict
from .translation import TranslationPromptTemplateDict
+from .evaluation import EvaluationPromptTemplateDict
from .task_prompt_template import TaskPromptTemplateDict
from .prompt_templates import PromptTemplateGenerator
@@ -13,6 +14,7 @@ __all__ = [
'CodePromptTemplateDict',
'MisalignmentPromptTemplateDict',
'TranslationPromptTemplateDict',
+ 'EvaluationPromptTemplateDict',
'TaskPromptTemplateDict',
'PromptTemplateGenerator',
]
diff --git a/camel/prompts/evaluation.py b/camel/prompts/evaluation.py
new file mode 100644
index 000000000..ebf1c156c
--- /dev/null
+++ b/camel/prompts/evaluation.py
@@ -0,0 +1,29 @@
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.typing import RoleType
+
+
+# flake8: noqa
+class EvaluationPromptTemplateDict(TextPromptDict):
+ r"""A dictionary containing :obj:`TextPrompt` used in the `Evaluation`
+ task.
+
+ Attributes:
+ GENERATE_QUESTIONS (TextPrompt): A prompt to generate a set of questions
+ to be used for evaluating emergence of knowledge based on a particular
+ field of knowledge.
+ """
+
+ GENERATE_QUESTIONS = TextPrompt(
+ """Generate {num_questions} {category} diverse questions.
+Here are some example questions:
+{examples}
+
+Now generate {num_questions} questions of your own. Be creative""")
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.update({
+ "generate_questions": self.GENERATE_QUESTIONS,
+ })
diff --git a/camel/prompts/task_prompt_template.py b/camel/prompts/task_prompt_template.py
index 8da01206c..9cdd30446 100644
--- a/camel/prompts/task_prompt_template.py
+++ b/camel/prompts/task_prompt_template.py
@@ -3,6 +3,7 @@ from typing import Any, Dict
from camel.prompts import (
AISocietyPromptTemplateDict,
CodePromptTemplateDict,
+ EvaluationPromptTemplateDict,
MisalignmentPromptTemplateDict,
TextPromptDict,
TranslationPromptTemplateDict,
@@ -27,4 +28,5 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):
TaskType.CODE: CodePromptTemplateDict(),
TaskType.MISALIGNMENT: MisalignmentPromptTemplateDict(),
TaskType.TRANSLATION: TranslationPromptTemplateDict(),
+ TaskType.EVALUATION: EvaluationPromptTemplateDict(),
})
diff --git a/camel/typing.py b/camel/typing.py
index 3ecc0bd78..b64cc92d6 100644
--- a/camel/typing.py
+++ b/camel/typing.py
@@ -20,6 +20,7 @@ class TaskType(Enum):
CODE = "code"
MISALIGNMENT = "misalignment"
TRANSLATION = "translation"
+ EVALUATION = "evaluation"
DEFAULT = "default"
diff --git a/examples/evaluation/single_agent.py b/examples/evaluation/single_agent.py
new file mode 100644
index 000000000..c28d327de
--- /dev/null
+++ b/examples/evaluation/single_agent.py
@@ -0,0 +1,126 @@
+import json
+import os
+import re
+from typing import Any, Dict, List
+
+from camel.agents import ChatAgent
+from camel.messages import AssistantSystemMessage, UserChatMessage
+from camel.prompts import PromptTemplateGenerator
+from camel.typing import TaskType
+
+
+def parse_question_string(question_string: str,
+ category: str) -> List[Dict[str, Any]]:
+ pattern = r'^(\d+)\.\s+(.*?)\s*\n*$'
+ questions = []
+ for match in re.finditer(pattern, question_string, re.MULTILINE):
+ question_id = int(match.group(1))
+ text = match.group(2)
+ questions.append({
+ 'question_id': question_id,
+ 'text': text,
+ 'category': category
+ })
+ return questions
+
+
+def generate_questions(examples: str, category: str, save_file_name: str,
+ key: str = 'generate_questions',
+ num_questions: int = 20) -> None:
+ prompt_template = PromptTemplateGenerator().get_prompt_from_key(
+ TaskType.EVALUATION, key)
+
+ evaluation_dict = {
+ 'num_questions': num_questions,
+ 'category': category,
+ 'examples': examples,
+ }
+
+ prompt = prompt_template.format(**evaluation_dict)
+ print(prompt)
+ assistant_sys_msg = AssistantSystemMessage(
+ role_name="Assistant",
+ content="You are a helpful assistant.",
+ )
+ agent = ChatAgent(assistant_sys_msg)
+ agent.reset()
+
+ user_msg = UserChatMessage(role_name="User", content=prompt)
+ assistant_msg, _, _ = agent.step(user_msg)
+
+ if assistant_msg is not None:
+
+ print(assistant_msg[0].content)
+
+ parsed_assistant_msg = parse_question_string(assistant_msg[0].content,
+ category)
+
+ # save json file
+ folder_path = './evaluation_data/questions'
+ if not os.path.exists(folder_path):
+ os.makedirs(folder_path)
+
+ with open(f"{folder_path}/{save_file_name}.jsonl", "w") as f:
+ for item in parsed_assistant_msg:
+ json.dump(item, f)
+ f.write('\n')
+
+
+def main() -> None:
+
+ # generate ai society evaluation questions
+ examples = ("1. What are the most effective ways to deal with stress?\n"
+ "2. Explain the process of natural selection and how it "
+ "contributes to the evolution and adaptation of species.\n"
+ "3. Can you help me write a formal email to a potential"
+ "business partner proposing a joint venture?")
+
+ category = 'generic task Q&A'
+ save_file_name = 'questions_ai_society'
+ generate_questions(examples, category, save_file_name)
+
+ # generate coding evaluation questions
+ examples = (
+ "1. Develop a C++ program that reads a text file line by line and"
+ "counts the number of occurrences of a specific word in the file.\n"
+ "2. Implement a Jave function to find the longest common"
+ " subsequence of two input strings using dynamic programming.\n"
+ "3. Implement a machine learning-based chatbot system in Python.")
+
+ category = 'coding task'
+ save_file_name = 'questions_code'
+ generate_questions(examples, category, save_file_name)
+
+ # generate math evaluation questions
+ examples = (
+ "1. Given that f(x) = 5x^3 - 2x + 3, find the value of f(2).\n"
+ "2. If the endpoints of a line segment are (2, -2) and (10, 4), "
+ "what is the length of the segment?\n"
+ "3. Solve for x in the equation 3x + 10 = 5(x - 2).")
+
+ category = 'math'
+ save_file_name = 'questions_math'
+ generate_questions(examples, category, save_file_name)
+
+ # generate science evaluation questions
+ examples = (
+ "1. What is the probability of finding a particle with a given energy"
+ " in a one-dimensional infinite square well potential when the"
+ " potential width is 2 nm and the particle has a mass of 5x10^-26"
+ " kg? Use the Schrödinger equation to solve for the allowed energy"
+ " states and their wave functions.\n"
+ "2. How does habitat loss and fragmentation affect the migratory"
+ " patterns and survival of a specific species, such as the monarch"
+ " butterfly, over time?\n"
+ "3. What is the systematic name of the organic compound with the"
+ " molecular formula C6H12O and a ketone functional group located"
+ " on the second carbon atom from the left end?")
+
+ category = 'science'
+
+ save_file_name = 'questions_science'
+ generate_questions(examples, category, save_file_name, num_questions=60)
+
+
+if __name__ == "__main__":
+ main()