🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add self instruct generator (#2013)
Zeyu Zhang12647ab78e6f
diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.ymlindex 27fcb5305..2f16302a4 100644--- a/.github/workflows/build_package.yml+++ b/.github/workflows/build_package.yml@@ -99,6 +99,7 @@ jobs:NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}"VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"run: |source test_venv/bin/activatepytest --fast-test-mode ./testdiff --git a/.github/workflows/pytest_package.yml b/.github/workflows/pytest_package.ymlindex 65897cc80..c394c8e7f 100644--- a/.github/workflows/pytest_package.yml+++ b/.github/workflows/pytest_package.yml@@ -64,6 +64,7 @@ jobs:AIML_API_KEY: "${{ secrets.AIML_API_KEY }}"VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"run: |source .venv/bin/activateuv pip install -e ".[all, dev, docs]"@@ -120,6 +121,7 @@ jobs:AIML_API_KEY: "${{ secrets.AIML_API_KEY }}"VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"run: |source .venv/bin/activateuv pip install -e ".[all, dev, docs]"@@ -174,6 +176,7 @@ jobs:MOONSHOT_API_KEY: "${{ secrets.MOONSHOT_API_KEY }}"VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"run: |source .venv/bin/activateuv pip install -e ".[all, dev, docs]"diff --git a/camel/datasets/__init__.py b/camel/datasets/__init__.pyindex 481452410..dcd6e2bf8 100644--- a/camel/datasets/__init__.py+++ b/camel/datasets/__init__.py@@ -14,6 +14,7 @@from .base_generator import BaseGeneratorfrom .few_shot_generator import FewShotGeneratorfrom .models import DataPoint+from .self_instruct_generator import SelfInstructGeneratorfrom .static_dataset import StaticDataset__all__ = [@@ -21,4 +22,5 @@ __all__ = ["DataPoint","FewShotGenerator","StaticDataset",+ "SelfInstructGenerator",]diff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.pynew file mode 100644index 000000000..becc3bde6--- /dev/null+++ b/camel/datasets/self_instruct_generator.py@@ -0,0 +1,426 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+import random+from datetime import datetime+from typing import Iterable, List, Optional, cast++from pydantic import BaseModel, Field, ValidationError++from camel.agents import ChatAgent+from camel.logger import get_logger+from camel.models import ModelFactory+from camel.types import ModelPlatformType, ModelType+from camel.verifiers import BaseVerifier++from .base_generator import BaseGenerator+from .models import DataPoint+from .static_dataset import StaticDataset++logger = get_logger(__name__)++DEFAULT_INSTRUCTION_SYSTEM_PROMPT = """+You are a high-capacity instruction generation assistant.++Your task is to generate a **new, creative, and challenging question** based on+several examples.+These examples may cover different domains or styles, but your goal is to:+- **Understand their specific patterns** in structure, and complexity;+- **Combine and synthesize** ideas from multiple examples, rather than copying+ or lightly editing any single one;+- **Intelligently integrate** multiple reasoning steps, constraints, or+ concepts into a single, coherent question;+- Ensure the new question is **non-trivial** and requires deep thinking or+ multi-step reasoning.++**Guidelines:**+- Use the examples as inspiration for format, depth, and tone.+- Your new question should be self-contained, logically sound, and answerable.+- Do not repeat exact phrasings or create shallow combinations; instead,+ produce something meaningfully new.+- Avoid open-ended or subjective questions that depend on personal opinions or+ discussion.+- The generated question must have a **clear, objective, and verifiable+ answer**.+- Aim for increased depth or novelty through subtle combination or+ transformation.+- Keep the final output to a **single unified question** with one clear answer,+ not a multi-part task.++**Output Format (strict):**+```+Question: [Generated question]+```+"""++DEFAULT_RATIONALE_SYSTEM_PROMPT = """You are an advanced Python code assistant.++Your task is to **solve the given question by writing Python code only**,+without any explanation or natural language output.+The code must compute the answer **programmatically**, not by hardcoding or+guessing the result.++**Rules:**+- Use Python code to perform the actual computation.+- Use {package_list} to solve the problem. Do not import any other libraries.+- **Do not hardcode the final answer** (e.g., avoid writing `print(1/2)` unless+ that value is computed).+- The result must be obtained through valid computation logic in code.+- Do not include explanations. Output code only.+- The entire code must be wrapped in triple backticks:+```+[Your Python code here]+```++Now, solve the following question using Python. Only output the code:+"""+++class SelfInstructGenerator(BaseGenerator):+ r"""A generator for creating synthetic datapoints using self-instruct.++ It utilizes both a human-provided dataset (seed_dataset) and generated+ machine instructions (machine_instructions) to produce new, synthetic+ datapoints that include a question, a computed rationale (code), and a+ final answer (from a verifier).+ """++ class QuestionSchema(BaseModel):+ r"""Schema for the generated question.++ Attributes:+ question (str): The question generated by the model.+ """++ question: str = Field(description="The question generated")++ class RationaleSchema(BaseModel):+ r"""Schema for the generated rationale code.++ Attributes:+ code (str): The generated code without any formatting.+ """++ code: str = Field(+ description="The generated code without any formatting"+ )++ def __init__(+ self,+ seed_dataset: StaticDataset,+ verifier: BaseVerifier,+ instruction_agent: Optional[ChatAgent] = None,+ rationale_agent: Optional[ChatAgent] = None,+ seed: int = 42,+ **kwargs,+ ):+ r"""Initialize the self-instruct generator.++ Args:+ seed_dataset (StaticDataset): Dataset containing seed instructions.+ verifier (BaseVerifier): Verifier instance to validate generated+ solutions.+ instruction_agent (Optional[ChatAgent]): Agent for generating+ instructions. If not provided, a default agent will be created.+ rationale_agent (Optional[ChatAgent]): Agent for generating+ rationales. If not provided, a default agent will be created.+ seed (int): Random seed for reproducibility. (default: :obj:`42`)+ **kwargs: Additional keyword arguments passed to the BaseGenerator.+ """+ super().__init__(seed=seed, **kwargs)+ self.seed_dataset = seed_dataset+ self.verifier = verifier+ # extract packages from verifier+ self.packages: List[str] = getattr(+ self.verifier, "required_packages", []+ )+ # create default agents if not provided+ self.instruction_agent = (+ instruction_agent or self.default_instruction_agent()+ )+ self.rationale_agent = (+ rationale_agent or self.default_rationale_agent()+ )++ # Extract questions from the seed dataset as human_instructions+ self.human_instructions: List[str] = [+ dp.question+ for dp in list(cast(Iterable[DataPoint], self.seed_dataset))+ ]+ self.machine_instructions: List[DataPoint] = []+ # Create an instance-level lock for thread-safe updates to _data+ self._lock = asyncio.Lock()+ self._data = [] # Storage for generated DataPoint instances++ def default_instruction_agent(self) -> ChatAgent:+ r"""Create the default instruction generation agent.++ This agent is configured with a moderate temperature setting to+ encourage creative and diverse instruction generation behavior.++ Returns:+ ChatAgent: An agent with the default instruction prompt.+ """+ model = ModelFactory.create(+ model_platform=ModelPlatformType.DEFAULT,+ model_type=ModelType.DEFAULT,+ model_config_dict={"temperature": 0.7},+ )+ return ChatAgent(+ DEFAULT_INSTRUCTION_SYSTEM_PROMPT,+ model=model,+ )++ def default_rationale_agent(self) -> ChatAgent:+ r"""Create the default rationale generation agent.++ This agent is configured with a deterministic (zero temperature)+ setting to ensure consistent and precise rationale generation based on+ a given instruction and package list.++ Returns:+ ChatAgent: An agent with the rationale prompt+ """+ model = ModelFactory.create(+ model_platform=ModelPlatformType.DEFAULT,+ model_type=ModelType.DEFAULT,+ model_config_dict={"temperature": 0.0},+ )+ return ChatAgent(+ DEFAULT_RATIONALE_SYSTEM_PROMPT.format(package_list=self.packages),+ model=model,+ )++ @staticmethod+ def format_support_block(dp: DataPoint) -> str:+ r"""Format a DataPoint into a few-shot example block.++ Args:+ dp (DataPoint): A data point.++ Returns:+ str: A formatted string containing the question and its+ corresponding code block in Markdown-style Python format.+ """+ support_q = dp.question.strip()+ support_code = dp.rationale.strip() if dp.rationale else ""+ return (+ f"Question:\n{support_q}\n\n"+ "Code:\n"+ "```python\n"+ f"{support_code}\n"+ "```"+ )++ def generate_new_instruction(+ self,+ agent: ChatAgent,+ support_human_dps: list[DataPoint],+ support_machine_dps: list[DataPoint],+ ) -> str:+ r"""Generate a new instruction using self-instruct prompting.++ Args:+ agent (ChatAgent): The agent to use for generating the instruction.+ support_human_dps (list[DataPoint]): List of human examples to+ sample.+ support_machine_dps (list[DataPoint]): List of machine examples to+ sample.++ Returns:+ str: The newly generated question.+ """+ human_sample = [dp.question for dp in list(support_human_dps)]+ machine_sample = [dp.question for dp in list(support_machine_dps)]++ few_shot_examples = human_sample + machine_sample++ # Build the prompt using the few-shot examples+ prompt = "Below are some question examples:\n\n"+ for idx, instr in enumerate(few_shot_examples, start=1):+ prompt += f"Question {idx}: {instr}\n"+ prompt += f"Question {len(few_shot_examples) + 1}:\n"+ prompt += "Now generate a new question based on the given examples.\n"++ question_template = f"Question: {prompt}"+ response = cast(+ SelfInstructGenerator.QuestionSchema,+ agent.step(question_template, response_format=self.QuestionSchema)+ .msgs[0]+ .parsed,+ )+ return response.question++ def generate_rationale(+ self,+ question: str,+ agent: Optional[ChatAgent] = None,+ support_human_dps: Optional[list[DataPoint]] = None,+ ) -> str:+ r"""Generate rationale code (solution) for the given question.++ Args:+ question (str): The question to be solved.+ agent (Optional[ChatAgent]): The agent to use for generating the+ rationale. If None is provided, the default rationale agent+ will be used. (default: :obj:`None`)+ support_human_dps (Optional[list[DataPoint]]): List of human+ examples to sample. (default: :obj:`None`)++ Returns:+ str: The generated code solution as a string.+ """++ # Build few-shot example prompt+ few_shot_prompt = ""+ if support_human_dps:+ few_shot_examples = [+ self.format_support_block(dp) for dp in support_human_dps+ ]+ few_shot_prompt += "Below are example questions and solutions:\n\n"+ few_shot_prompt += "\n\n".join(few_shot_examples)++ few_shot_prompt += f"\n\nWrite code to solve the question:\n{question}"++ response = cast(+ SelfInstructGenerator.RationaleSchema,+ (agent or self.default_rationale_agent())+ .step(few_shot_prompt, response_format=self.RationaleSchema)+ .msgs[0]+ .parsed,+ )+ return response.code++ async def generate_new(+ self,+ n: int,+ max_retries: int = 10,+ human_sample_count: int = 3,+ machine_sample_count: int = 1,+ **kwargs,+ ) -> list[DataPoint]:+ r"""Generates and validates `n` new datapoints through+ self-instruct prompting, with a retry limit.++ Args:+ n (int): The number of valid datapoints to generate.+ max_retries (int): Maximum number of retries before stopping.+ (default: :obj:`10`)+ human_sample_count (int): Number of human examples to sample.+ (default: :obj:`3`)+ machine_sample_count (int): Number of machine examples to sample.+ (default: :obj:`1`)+ **kwargs: Additional keyword arguments.++ Returns:+ List[DataPoint]: A list of newly generated valid datapoints.++ Raises:+ TypeError: If the agent's output is not a dictionary (or does not+ match the expected format).+ KeyError: If required keys are missing from the response.+ AttributeError: If the verifier response lacks attributes.+ ValidationError: If a datapoint fails schema validation.+ RuntimeError: If retries are exhausted before `n` valid datapoints+ are generated.++ Notes:+ - Retries on validation failures until `n` valid datapoints exist+ or `max_retries` is reached, whichever comes first.+ - If retries are exhausted before reaching `n`, a `RuntimeError`+ is raised.+ - Metadata includes a timestamp for tracking datapoint creation.+ """+ valid_data_points: list[DataPoint] = []+ retries = 0++ while len(valid_data_points) < n and retries < max_retries:+ try:+ human_dps_list = list(cast(List[DataPoint], self.seed_dataset))+ support_human_dps = random.sample(+ human_dps_list,+ min(human_sample_count, len(human_dps_list)),+ )++ machine_dps_list = list(self.machine_instructions)+ support_machine_dps = random.sample(+ machine_dps_list,+ min(machine_sample_count, len(machine_dps_list)),+ )+ question = self.generate_new_instruction(+ self.instruction_agent,+ support_human_dps,+ support_machine_dps,+ )+ rationale = self.generate_rationale(+ question, self.rationale_agent, support_human_dps+ )+ if not isinstance(rationale, str):+ raise TypeError(f"Rationale {rationale} is not a string.")++ try:+ verifier_response = await self.verifier.verify(+ solution=rationale,+ ground_truth=None,+ )+ if not verifier_response or not verifier_response.result:+ raise ValueError(+ "Verifier unsuccessful, response: "+ f"{verifier_response}"+ )+ except (ValueError, AttributeError) as e:+ logger.warning(+ f"Verifier issue: {e}, "+ f"retrying... ({retries + 1}/{max_retries})"+ )+ retries += 1+ continue+ try:+ new_datapoint = DataPoint(+ question=question,+ rationale=rationale,+ final_answer=verifier_response.result,+ metadata={+ "synthetic": str(True),+ "created": datetime.now().isoformat(),+ "generator": "self_instruct",+ },+ )+ except ValidationError as e:+ logger.warning(+ f"Datapoint validation failed: {e}, "+ f"retrying... ({retries + 1}/{max_retries})"+ )+ retries += 1+ continue++ valid_data_points.append(new_datapoint)++ except Exception as e:+ logger.warning(+ f"Unexpected error: {e}, retrying..."+ f" ({retries + 1}/{max_retries})"+ )+ retries += 1++ if len(valid_data_points) < n:+ raise RuntimeError(+ f"Failed to generate {n} valid datapoints "+ f"after {max_retries} retries."+ )++ async with self._lock:+ self._data.extend(valid_data_points)+ return valid_data_pointsdiff --git a/examples/dataset/self_instruct_generator.py b/examples/dataset/self_instruct_generator.pynew file mode 100644index 000000000..7a0dc2d48--- /dev/null+++ b/examples/dataset/self_instruct_generator.py@@ -0,0 +1,120 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+from pathlib import Path++from dotenv import load_dotenv++from camel.agents import ChatAgent+from camel.configs import ChatGPTConfig+from camel.datasets import SelfInstructGenerator, StaticDataset+from camel.logger import get_logger+from camel.models import ModelFactory+from camel.types import ModelPlatformType, ModelType+from camel.verifiers import PythonVerifier++logger = get_logger(__name__)++verifier = PythonVerifier(required_packages=["sympy"], timeout=60)+asyncio.run(verifier.setup())++raw_data2 = [+ {+ "question": "Evaluate the limit as x approaches 0 of (sin(3*x) - 3*x) / x**3.", # noqa: E501+ "final_answer": "-9/2",+ "rationale": '''from sympy import symbols, limit, sin+x = symbols('x')+expr = (sin(3*x) - 3*x) / x**3+result = limit(expr, x, 0)+print(result)''',+ },+ {+ "question": "Evaluate the definite integral of (1 - x**2)**3 from x = 0 to x = 1.", # noqa: E501+ "final_answer": "16/35",+ "rationale": '''from sympy import symbols, integrate+x = symbols('x')+expr = (1 - x**2)**3+result = integrate(expr, (x, 0, 1))+print(result)''',+ },+ {+ "question": "Evaluate the limit as n approaches infinity of n*(sqrt(n**2 + 1) - n).", # noqa: E501+ "final_answer": "1/2",+ "rationale": '''from sympy import symbols, limit, sqrt+n = symbols('n', positive=True)+expr = n*(sqrt(n**2 + 1) - n)+result = limit(expr, n, float("inf"))+print(result)''',+ },+ {+ "question": "Compute the sum of the series sum from n = 1 to 50 of 1/(n*(n+1)).", # noqa: E501+ "final_answer": "50/51",+ "rationale": '''from sympy import symbols, summation+n = symbols('n', positive=True, integer=True)+expr = 1/(n*(n+1))+result = summation(expr, (n, 1, 50))+print(result)''',+ },+]++seed_dataset = StaticDataset(raw_data2)++load_dotenv()++model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ model_config_dict=ChatGPTConfig().as_dict(),+)++RATIONALE_SYSTEM_PROMPT = """You are an advanced Python code assistant.++Your task is to **solve the given question by writing Python code only**,+without any explanation or natural language output.+The code must compute the answer **programmatically**, not by hardcoding or+guessing the result.++**Rules:**+- Use Python code to perform the actual computation.+- Use sympy to solve the problem. Do not import any other libraries.+- **Do not hardcode the final answer** (e.g., avoid writing `print(1/2)` unless+ that value is computed).+- The result must be obtained through valid computation logic in code.+- Do not include explanations. Output code only.+- The entire code must be wrapped in triple backticks:+```+[Your Python code here]+```++Now, solve the following question using Python. Only output the code:+"""++rationale_agent = ChatAgent(RATIONALE_SYSTEM_PROMPT, model=model)++generator = SelfInstructGenerator(+ seed_dataset=seed_dataset,+ verifier=verifier,+ instruction_agent=None, # use default instruction agent+ rationale_agent=rationale_agent,+)++new_data = asyncio.run(generator.generate_new(n=3, max_retries=5))++for dp in new_data:+ print(dp)++generator.save_to_jsonl(Path("generated_data.jsonl"))++asyncio.run(verifier.cleanup())diff --git a/tests/datasets/test_self_instruct_generator.py b/tests/datasets/test_self_instruct_generator.pynew file mode 100644index 000000000..e19bc8e28--- /dev/null+++ b/tests/datasets/test_self_instruct_generator.py@@ -0,0 +1,372 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from pydantic import ValidationError++from camel.agents import ChatAgent+from camel.datasets import DataPoint, SelfInstructGenerator+from camel.verifiers import BaseVerifier+++def mock_verifier():+ r"""Create a mock verifier fixture for testing."""+ verifier = MagicMock(spec=BaseVerifier)+ verifier.required_packages = ["math", "numpy"]+ verifier.verify = AsyncMock()+ return verifier+++def mock_datapoint():+ r"""Create a mock datapoint fixture for testing."""+ return DataPoint(+ question="Sample question?",+ rationale="print('Sample code')",+ final_answer="Sample code output",+ metadata={"synthetic": "False"},+ )+++def mock_dataset(mock_datapoint):+ r"""Create a mock dataset fixture for testing."""+ dataset = MagicMock()+ dataset.__iter__ = MagicMock(return_value=iter([mock_datapoint]))+ return dataset+++def mock_instruction_agent():+ r"""Create a mock instruction agent fixture for testing."""+ agent = MagicMock(spec=ChatAgent)+ response = MagicMock()+ response.msgs = [MagicMock()]+ response.msgs[0].parsed = SelfInstructGenerator.QuestionSchema(+ question="Generated question?"+ )+ agent.step.return_value = response+ return agent+++def mock_rationale_agent():+ r"""Create a mock rationale agent fixture for testing."""+ agent = MagicMock(spec=ChatAgent)+ response = MagicMock()+ response.msgs = [MagicMock()]+ response.msgs[0].parsed = SelfInstructGenerator.RationaleSchema(+ code="print('Generated code')"+ )+ agent.step.return_value = response+ return agent+++def generator(+ mock_dataset, mock_verifier, mock_instruction_agent, mock_rationale_agent+):+ r"""Create a SelfInstructGenerator instance with mock dependencies."""+ return SelfInstructGenerator(+ seed_dataset=mock_dataset,+ verifier=mock_verifier,+ instruction_agent=mock_instruction_agent,+ rationale_agent=mock_rationale_agent,+ seed=42,+ )+++class TestSelfInstructGeneratorInitialization:+ r"""Tests for the initialization of SelfInstructGenerator."""++ def test_init_with_provided_agents(+ self,+ mock_dataset,+ mock_verifier,+ mock_instruction_agent,+ mock_rationale_agent,+ ):+ r"""Test initialization with provided agents."""+ generator = SelfInstructGenerator(+ seed_dataset=mock_dataset,+ verifier=mock_verifier,+ instruction_agent=mock_instruction_agent,+ rationale_agent=mock_rationale_agent,+ )++ assert generator.seed_dataset == mock_dataset+ assert generator.verifier == mock_verifier+ assert generator.instruction_agent == mock_instruction_agent+ assert generator.rationale_agent == mock_rationale_agent+ assert generator.packages == mock_verifier.required_packages+ assert generator.human_instructions == ["Sample question?"]+ assert generator.machine_instructions == []+ assert generator._data == []+++class TestFormatSupportBlock:+ r"""Tests for the format_support_block static method."""++ def test_format_support_block_with_rationale(self, mock_datapoint):+ r"""Test format_support_block with a datapoint that has a rationale."""+ result = SelfInstructGenerator.format_support_block(mock_datapoint)++ expected = (+ f"Question:\n{mock_datapoint.question}\n\n"+ "Code:\n"+ "```python\n"+ f"{mock_datapoint.rationale}\n"+ "```"+ )++ assert result == expected++ def test_format_support_block_without_rationale(self):+ r"""Test format_support_block with a datapoint has no rationale."""+ datapoint = DataPoint(+ question="Sample question without rationale?",+ final_answer="Sample answer",+ metadata={"synthetic": "False"},+ )++ result = SelfInstructGenerator.format_support_block(datapoint)++ expected = (+ f"Question:\n{datapoint.question}\n\n"+ "Code:\n"+ "```python\n"+ "\n"+ "```"+ )++ assert result == expected+++class TestGenerateNewInstruction:+ r"""Tests for the generate_new_instruction method."""++ def test_generate_new_instruction(self, generator, mock_datapoint):+ r"""Test generate_new_instruction method."""+ # Create sample datapoints+ support_human_dps = [mock_datapoint]+ support_machine_dps = [+ DataPoint(+ question="Machine generated question?",+ rationale="print('Machine code')",+ final_answer="Machine output",+ metadata={"synthetic": "True"},+ )+ ]++ result = generator.generate_new_instruction(+ generator.instruction_agent, support_human_dps, support_machine_dps+ )++ # Check if the agent was called with the correct prompt+ generator.instruction_agent.step.assert_called_once()+ args, kwargs = generator.instruction_agent.step.call_args++ # Verify the prompt contains the examples+ prompt = args[0]+ assert "Sample question?" in prompt+ assert "Machine generated question?" in prompt++ # Verify the response format is correct+ assert kwargs["response_format"] == generator.QuestionSchema++ # Verify the result+ assert result == "Generated question?"+++class TestGenerateRationale:+ r"""Tests for the generate_rationale method."""++ def test_generate_rationale_with_agent(self, generator, mock_datapoint):+ r"""Test generate_rationale with a provided agent."""+ question = "Test question?"+ support_human_dps = [mock_datapoint]++ result = generator.generate_rationale(+ question, generator.rationale_agent, support_human_dps+ )++ # Check if the agent was called with the correct prompt+ generator.rationale_agent.step.assert_called_once()+ args, kwargs = generator.rationale_agent.step.call_args++ # Verify the prompt contains the examples and question+ prompt = args[0]+ assert "Sample question?" in prompt+ assert "Test question?" in prompt++ # Verify the response format is correct+ assert kwargs["response_format"] == generator.RationaleSchema++ # Verify the result+ assert result == "print('Generated code')"++ @patch(+ 'camel.datasets.self_instruct_generator.SelfInstructGenerator.default_rationale_agent'+ )+ def test_generate_rationale_with_default_agent(+ self, mock_default_agent, generator, mock_datapoint+ ):+ r"""Test generate_rationale with the default agent."""+ # Setup mock default agent+ mock_agent = MagicMock()+ response = MagicMock()+ response.msgs = [MagicMock()]+ response.msgs[0].parsed = SelfInstructGenerator.RationaleSchema(+ code="print('Default agent code')"+ )+ mock_agent.step.return_value = response+ mock_default_agent.return_value = mock_agent++ question = "Test question?"+ support_human_dps = [mock_datapoint]++ result = generator.generate_rationale(+ question, support_human_dps=support_human_dps+ )++ # Verify default agent was created+ mock_default_agent.assert_called_once()++ # Verify the result+ assert result == "print('Default agent code')"+++class TestGenerateNew:+ r"""Tests for the generate_new async method."""++ @pytest.mark.asyncio+ async def test_generate_new_success(self, generator, mock_datapoint):+ r"""Test successful generation of new datapoints."""+ # Setup mock verifier response+ verifier_response = MagicMock()+ verifier_response.result = "Verified output"+ generator.verifier.verify.return_value = verifier_response++ # Mock the generate methods+ generator.generate_new_instruction = MagicMock(+ return_value="New question?"+ )+ generator.generate_rationale = MagicMock(+ return_value="print('New code')"+ )++ result = await generator.generate_new(n=1)++ # Verify generated datapoint+ assert len(result) == 1+ assert result[0].question == "New question?"+ assert result[0].rationale == "print('New code')"+ assert result[0].final_answer == "Verified output"+ assert result[0].metadata["synthetic"] == "True"+ assert result[0].metadata["generator"] == "self_instruct"++ @pytest.mark.asyncio+ async def test_generate_new_verification_failure(self, generator):+ r"""Test handling of verification failures during generation."""+ # Create proper mock responses+ none_response = None+ success_response = MagicMock()+ success_response.result = "Verified output"++ # Setup the verify method to first return None, then return a+ # successful response+ generator.verifier.verify = AsyncMock(+ side_effect=[+ none_response, # First call returns None (failure)+ success_response, # Second call returns success+ ]+ )++ # Mock the generate methods+ generator.generate_new_instruction = MagicMock(+ side_effect=["Failed question?", "Successful question?"]+ )+ generator.generate_rationale = MagicMock(+ side_effect=["print('Failed code')", "print('Successful code')"]+ )++ result = await generator.generate_new(+ n=1, max_retries=3+ ) # Reduce retries for test speed++ # Verify generated datapoint+ assert len(result) == 1+ assert result[0].question == "Successful question?"+ assert result[0].rationale == "print('Successful code')"+ assert result[0].final_answer == "Verified output"++ @pytest.mark.asyncio+ async def test_generate_new_validation_error(self, generator):+ r"""Test handling of validation errors during generation."""+ # Setup mock verifier response+ verifier_response = MagicMock()+ verifier_response.result = "Verified output"+ generator.verifier.verify.return_value = verifier_response++ # Mock the generate methods+ generator.generate_new_instruction = MagicMock(+ side_effect=["Failed question?", "Successful question?"]+ )+ generator.generate_rationale = MagicMock(+ side_effect=["print('Failed code')", "print('Successful code')"]+ )++ # Use a custom DataPoint creation function to simulate validation+ # error then success+ original_datapoint = DataPoint++ datapoint_calls = 0++ def mock_datapoint_factory(*args, **kwargs):+ nonlocal datapoint_calls+ datapoint_calls += 1++ if datapoint_calls == 1:+ # First call - raise validation error+ raise ValidationError.from_exception_data(+ title="ValidationError",+ line_errors=[+ {+ "type": "string_type",+ "loc": ("final_answer",),+ "msg": "Input should be a valid string",+ }+ ],+ model=DataPoint,+ )+ else:+ # Second call - succeed+ return original_datapoint(*args, **kwargs)++ with patch(+ 'camel.datasets.self_instruct_generator.DataPoint',+ side_effect=mock_datapoint_factory,+ ):+ result = await generator.generate_new(+ n=1, max_retries=3+ ) # Reduce retries for test speed++ # Verify generated datapoint+ assert len(result) == 1+ assert result[0].question == "Successful question?"+ assert result[0].rationale == "print('Successful code')"+ assert result[0].final_answer == "Verified output"