🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: o1datagen core (#1274)
正经人王同学62659c9cc799
diff --git a/camel/datagen/__init__.py b/camel/datagen/__init__.pynew file mode 100644index 000000000..843819c80--- /dev/null+++ b/camel/datagen/__init__.py@@ -0,0 +1,17 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from .o1datagen import O1DataGenerator++__all__ = ['O1DataGenerator']diff --git a/camel/datagen/o1datagen.py b/camel/datagen/o1datagen.pynew file mode 100644index 000000000..79b28ebf4--- /dev/null+++ b/camel/datagen/o1datagen.py@@ -0,0 +1,435 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import json+from datetime import datetime+from typing import Annotated, Dict, Optional, Union++from pydantic import BaseModel, Field, confloat++from camel.agents import ChatAgent+from camel.logger import get_logger++# Get a logger for this module+logger = get_logger('o1datagenerator')+++class AgentResponse(BaseModel):+ r"""Model for structured agent responses.++ A Pydantic model class that represents structured responses from agents,+ including a similarity score that measures the quality of the response.++ Args:+ score (float): A similarity score between 0 and 1 that compares the+ current answer to the correct answer. Must be within the range+ [0, 1].+ """++ score: Annotated[float, confloat(ge=0, le=1)] = Field(+ ...,+ description="""Similarity score between 0 and 1+ comparing current answer to correct answer""",+ )+++class VerificationResponse(BaseModel):+ r"""Model for structured verification responses.++ A Pydantic model class that represents verification results from agents,+ indicating whether an answer is correct or not.++ Args:+ is_correct (bool): Boolean indicating if the answer is correct.+ """++ is_correct: bool = Field(+ ...,+ description="Boolean indicating if the answer is correct",+ )+++class O1DataGenerator:+ r"""Class for generating and managing data through chat agent interactions.++ handling the generation of data by a chat agent, managing golden answers,+ and maintaining a solution tree for correct solution steps.++ Args:+ chat_agent (Optional[ChatAgent]): Optional single agent+ for both tasks (legacy mode). (default::obj:`None`)+ generator_agent (Optional[ChatAgent]): Optional specialized agent for+ answer generation. (default::obj:`None`)+ verifier_agent (Optional[ChatAgent]): Optional specialized agent for+ answer verification. (default::obj:`None`)+ golden_answers (Dict[str, str]): Dictionary containing pre-defined+ correct answers for validation and comparison. Required for answer+ verification.+ search_limit (int): Maximum number of search iterations allowed.+ (default::obj:`100`)+ """++ def __init__(+ self,+ chat_agent: Optional[ChatAgent] = None,+ *,+ generator_agent: Optional[ChatAgent] = None,+ verifier_agent: Optional[ChatAgent] = None,+ golden_answers: Dict[str, str],+ search_limit: int = 100,+ ):+ r"""Initialize the O1DataGenerator.++ This constructor supports both single-agent and dual-agent modes:+ 1. Single-agent mode (legacy): Pass a single chat_agent that will be+ used for both generation and verification.+ 2. Dual-agent mode: Pass separate generator_agent and verifier_agent+ for specialized tasks.++ Args:+ chat_agent (Optional[ChatAgent]): Optional single agent for both+ tasks (legacy mode). (default::obj:`None`)+ generator_agent (Optional[ChatAgent]): Optional specialized agent+ for answer generation. (default::obj:`None`)+ verifier_agent (Optional[ChatAgent]): Optional specialized agent+ for answer verification. (default::obj:`None`)+ golden_answers (Dict[str, str]): Dictionary containing pre-defined+ correct answers for validation and comparison. Required for+ answer verification.+ search_limit (int): Maximum number of search iterations allowed.+ (default::obj:`100`)+ """+ if chat_agent is not None:+ if generator_agent is not None or verifier_agent is not None:+ raise ValueError(+ "Cannot specify both chat_agent \+ and generator/verifier agents"+ )+ self.generator_agent = chat_agent+ self.verifier_agent = chat_agent+ else:+ if generator_agent is None or verifier_agent is None:+ raise ValueError(+ "Must specify either chat_agent or both generator and "+ "verifier agents"+ )+ self.generator_agent = generator_agent+ self.verifier_agent = verifier_agent++ self.golden_answers = golden_answers+ self.search_limit = search_limit+ self.solution_tree: Dict[str, Dict[str, Union[str, int]]] = {}+ logger.info(+ "O1DataGenerator initialized with search_limit=%d", search_limit+ )++ def get_answer(self, question: str, context: str = "") -> str:+ r"""Get an answer from the chat agent for a given question.++ Args:+ question (str): The question to ask.+ context (str): Additional context for the question.+ (default::obj:`""`)++ Returns:+ str: The generated answer.+ """+ prompt = f"""+ Please think step by step and solve this problem: {question}+ Existing content: {context}+ Requirements:+ 1. Analyze the problem requirements+ 2. List the steps to solve the problem+ 3. Execute the solution process+ 4. Provide the final answer+ Please explain the thought process of each step in detail.+ """+ self.generator_agent.reset()+ response = self.generator_agent.step(prompt)+ answer = response.msgs[0].content+ logger.info("AI thought process:\n%s", answer)+ return answer++ def verify_answer(self, question: str, answer: str) -> bool:+ r"""Verify if a generated answer is semantically equivalent to+ the golden answer for a given question.++ Args:+ question (str): The question being answered.+ answer (str): The answer to verify.++ Returns:+ bool: True if the answer matches the golden answer based on+ semantic equivalence (meaning the core content and meaning are+ the same, even if the exact wording differs).+ False in the following cases:+ - If the provided question doesn't exist in the golden answers+ - If the answer's meaning differs from the golden answer+ """+ golden_answer = self.golden_answers.get(question)+ if not golden_answer:+ raise ValueError(+ f"No golden answer found for question: {question}"+ )++ prompt = (+ f"Question: {question}\n"+ f"Student Answer: {answer}\n"+ f"Correct Answer: {golden_answer}\n"+ "Is the student's answer correct? Please respond with 'true' or "+ "'false' only."+ )+ self.verifier_agent.reset()+ response = self.verifier_agent.step(+ prompt, response_format=VerificationResponse+ )+ is_correct = response.msgs[0].parsed.is_correct # type:ignore [union-attr]+ logger.info("Answer verification result: %s", is_correct)+ return is_correct++ def monte_carlo_tree_search(+ self, question: str, partial_solution: str = ""+ ) -> float:+ r"""Perform Monte Carlo Tree Search to find the best solution.++ Args:+ question (str): The question to solve.+ partial_solution (str): The current partial solution.+ (default::obj:`""`)++ Returns:+ float: The similarity score between the current+ solution and golden answer.+ """+ if question not in self.golden_answers:+ raise ValueError(+ f"No golden answer found for question: {question}"+ )++ golden_answer = self.golden_answers[question]++ prompt = (+ f"Please evaluate this solution and "+ f"give a score between 0-1:\n"+ f"Question: {question}\n"+ f"Solution: {partial_solution}\n"+ f"Correct answer: {golden_answer}\n"+ f"Return a JSON object with a single field 'score' containing "+ f"a float between 0 and 1, like this: {{'score': 0.85}}\n"+ )+ self.generator_agent.reset()+ response = self.generator_agent.step(+ prompt, response_format=AgentResponse+ )+ agent_response = response.msgs[0].parsed.score # type: ignore [union-attr]++ return agent_response++ def binary_search_error(self, question: str, solution: str) -> int:+ r"""Use binary search to locate the first error in the solution.+ This method splits the solution into sentences using both English and+ Chinese sentence delimiters and performs binary search to find the+ first error.++ Args:+ question (str): The question being solved.+ solution (str): The complete solution to analyze.++ Returns:+ int: The position of the first error found in the solution.+ Returns -1. If no errors are found (all sentences are correct).+ """+ logger.info("Starting binary search for error location")+ # Split by both English period and Chinese period+ sentences = [+ s.strip()+ for s in solution.replace('。', '.').split('.')+ if s.strip()+ ]++ # First check if the entire solution is correct+ if self.verify_answer(question, solution):+ return -1++ left, right = 0, len(sentences)+ while left < right:+ mid = (left + right) // 2+ partial_solution = '. '.join(sentences[:mid]) + '.'+ logger.info("Checking solution fragment:\n%s", partial_solution)+ # Verify if the current part is correct+ is_correct = self.verify_answer(question, partial_solution)+ if is_correct:+ left = mid + 1+ else:+ right = mid+ logger.info("First error position found: sentence %d", left)+ return left++ def solve(self, question: str) -> str:+ r"""Solve a question using a multi-step approach.++ The solution process follows these steps:+ 1. Try to solve directly - if correct, return the solution+ 2. If not correct, use Monte Carlo Tree Search to find a good solution+ 3. If the solution isn't perfect, use binary search to locate errors+ 4. Generate a new solution based on the correct part++ Args:+ question (str): The question to solve.++ Returns:+ str: The best solution found.+ """+ # 1. Try direct solution first+ solution = self.get_answer(question)+ if self.verify_answer(question, solution):+ logger.info("Initial solution is correct")+ return solution++ # 2. If direct solution fails, try Monte Carlo Tree Search+ # to find a solution with high similarity score+ best_solution = ""+ best_score: float = 0.0+ for i in range(self.search_limit):+ # Generate new answer+ current_solution = self.get_answer(question, best_solution)++ # Evaluate solution similarity score+ prompt = (+ f"Please evaluate this solution and "+ f"give a score between 0-1:\n"+ f"Question: {question}\n"+ f"Solution: {current_solution}\n"+ f"Correct answer: {self.golden_answers.get(question, '')}\n"+ f"Return a JSON object with a single field 'score' containing "+ f"a float between 0 and 1, like this: {{'score': 0.85}}\n"+ )+ self.generator_agent.reset()+ response = self.generator_agent.step(prompt)+ try:+ response = self.generator_agent.step(+ prompt, response_format=AgentResponse+ )+ agent_response = response.msgs[0].parsed.score # type: ignore [union-attr]+ score = agent_response++ # Exit early if we find a very good solution (score > 0.9)+ if score > 0.9:+ logger.info(+ "Found excellent solution with score %.2f. "+ "Stopping search early.",+ score,+ )+ return current_solution++ if score > best_score:+ best_score = score+ best_solution = current_solution++ logger.info(+ "Current search progress: %d/%d, best score: %.2f",+ i + 1,+ self.search_limit,+ best_score,+ )+ except Exception as e:+ logger.error("Error parsing agent response: %s", str(e))+ continue++ # 3. If the answer is not completely correct,+ # use binary search to locate the error+ error_pos = self.binary_search_error(question, best_solution)++ # If no errors found (error_pos == -1), return the current solution+ if error_pos == -1:+ logger.info("No specific errors found in the solution")+ return best_solution++ # 4. Generate new solution based on correct part+ correct_part = '. '.join(best_solution.split('. ')[:error_pos]) + '.'+ final_solution = self.get_answer(question, correct_part)+ self.solution_tree[question] = {+ "solution": final_solution,+ "error_position": error_pos,+ }+ return final_solution++ def import_qa_from_json(self, data: Union[str, Dict[str, str]]) -> bool:+ r"""Import question and answer data from either a JSON file or a+ dictionary.++ Args:+ data (Union[str, Dict[str, str]]): Either a path to a JSON file+ containing QA pairs or a dictionary of question-answer pairs.+ If a string is provided, it's treated as a file path.+ The expected format is:+ {"question1": "answer1",+ "question2": "answer2",+ ...}++ Returns:+ bool: True if import was successful, False otherwise.+ """+ try:+ if isinstance(data, str):+ logger.info("Loading QA pairs from file: %s", data)+ with open(data, 'r', encoding='utf-8') as f:+ qa_data = json.load(f)+ else:+ logger.info("Loading QA pairs from provided dictionary")+ qa_data = data++ # Validate the data format+ if not isinstance(qa_data, dict):+ logger.error("Invalid data format: expected dictionary")+ return False++ # Update golden answers+ self.golden_answers.update(qa_data)+ logger.info("Successfully imported %d QA pairs", len(qa_data))+ return True++ except Exception as e:+ logger.error("Error importing QA data: %s", str(e))+ return False++ def export_solutions(self, filepath: str = 'solutions.json') -> None:+ r"""Export the solution process and results to a JSON file.+ Exports the solution tree, golden answers,+ and export timestamp to a JSON file.+ The exported data includes:+ - solutions: The solution tree+ with intermediate steps+ - golden_answers: The reference answers used for verification+ - export_time: ISO format timestamp of the export++ Args:+ filepath (str, optional): Path where the JSON file will be saved.+ (default::obj:`'solutions.json'`)++ Returns:+ None: The method writes to a file and logs the result but does not+ return any value.+ """+ export_data = {+ "solutions": self.solution_tree,+ "golden_answers": self.golden_answers,+ "export_time": datetime.now().isoformat(),+ }+ try:+ with open(filepath, 'w', encoding='utf-8') as f:+ json.dump(export_data, f, ensure_ascii=False, indent=2)+ logger.info(f"Solutions exported successfully to {filepath}")+ except Exception as e:+ logger.error(f"Error exporting solutions: {e!s}")diff --git a/camel/datahubs/models.py b/camel/datahubs/models.pyindex cd63de8f2..44aba278d 100644--- a/camel/datahubs/models.py+++ b/camel/datahubs/models.py@@ -19,4 +19,7 @@ from pydantic import BaseModelclass Record(BaseModel):id: Optional[str] = Nonemetadata: Optional[Dict[str, Any]] = None- content: Dict[str, Any]+ content: Optional[Dict[str, Any]] = None++ class Config:+ extra = "allow" # Allow any additional fields