Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add self instruct generator (#2013)

Zeyu Zhang12647ab78e6f
diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml
index 27fcb5305..2f16302a4 100644
--- a/.github/workflows/build_package.yml
+++ b/.github/workflows/build_package.yml
@@ -99,6 +99,7 @@ jobs:
NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}"
VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"
OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"
+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"
run: |
source test_venv/bin/activate
pytest --fast-test-mode ./test
diff --git a/.github/workflows/pytest_package.yml b/.github/workflows/pytest_package.yml
index 65897cc80..c394c8e7f 100644
--- a/.github/workflows/pytest_package.yml
+++ b/.github/workflows/pytest_package.yml
@@ -64,6 +64,7 @@ jobs:
AIML_API_KEY: "${{ secrets.AIML_API_KEY }}"
VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"
OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"
+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"
run: |
source .venv/bin/activate
uv pip install -e ".[all, dev, docs]"
@@ -120,6 +121,7 @@ jobs:
AIML_API_KEY: "${{ secrets.AIML_API_KEY }}"
VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"
OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"
+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"
run: |
source .venv/bin/activate
uv pip install -e ".[all, dev, docs]"
@@ -174,6 +176,7 @@ jobs:
MOONSHOT_API_KEY: "${{ secrets.MOONSHOT_API_KEY }}"
VOLCANO_API_KEY: "${{ secrets.VOLCANO_API_KEY }}"
OPENROUTER_API_KEY: "${{ secrets.OPENROUTER_API_KEY }}"
+ MODELSCOPE_SDK_TOKEN: "${{ secrets.MODELSCOPE_SDK_TOKEN }}"
run: |
source .venv/bin/activate
uv pip install -e ".[all, dev, docs]"
diff --git a/camel/datasets/__init__.py b/camel/datasets/__init__.py
index 481452410..dcd6e2bf8 100644
--- a/camel/datasets/__init__.py
+++ b/camel/datasets/__init__.py
@@ -14,6 +14,7 @@
from .base_generator import BaseGenerator
from .few_shot_generator import FewShotGenerator
from .models import DataPoint
+from .self_instruct_generator import SelfInstructGenerator
from .static_dataset import StaticDataset
__all__ = [
@@ -21,4 +22,5 @@ __all__ = [
"DataPoint",
"FewShotGenerator",
"StaticDataset",
+ "SelfInstructGenerator",
]
diff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.py
new file mode 100644
index 000000000..becc3bde6
--- /dev/null
+++ b/camel/datasets/self_instruct_generator.py
@@ -0,0 +1,426 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+import random
+from datetime import datetime
+from typing import Iterable, List, Optional, cast
+
+from pydantic import BaseModel, Field, ValidationError
+
+from camel.agents import ChatAgent
+from camel.logger import get_logger
+from camel.models import ModelFactory
+from camel.types import ModelPlatformType, ModelType
+from camel.verifiers import BaseVerifier
+
+from .base_generator import BaseGenerator
+from .models import DataPoint
+from .static_dataset import StaticDataset
+
+logger = get_logger(__name__)
+
+DEFAULT_INSTRUCTION_SYSTEM_PROMPT = """
+You are a high-capacity instruction generation assistant.
+
+Your task is to generate a **new, creative, and challenging question** based on
+several examples.
+These examples may cover different domains or styles, but your goal is to:
+- **Understand their specific patterns** in structure, and complexity;
+- **Combine and synthesize** ideas from multiple examples, rather than copying
+ or lightly editing any single one;
+- **Intelligently integrate** multiple reasoning steps, constraints, or
+ concepts into a single, coherent question;
+- Ensure the new question is **non-trivial** and requires deep thinking or
+ multi-step reasoning.
+
+**Guidelines:**
+- Use the examples as inspiration for format, depth, and tone.
+- Your new question should be self-contained, logically sound, and answerable.
+- Do not repeat exact phrasings or create shallow combinations; instead,
+ produce something meaningfully new.
+- Avoid open-ended or subjective questions that depend on personal opinions or
+ discussion.
+- The generated question must have a **clear, objective, and verifiable
+ answer**.
+- Aim for increased depth or novelty through subtle combination or
+ transformation.
+- Keep the final output to a **single unified question** with one clear answer,
+ not a multi-part task.
+
+**Output Format (strict):**
+```
+Question: [Generated question]
+```
+"""
+
+DEFAULT_RATIONALE_SYSTEM_PROMPT = """You are an advanced Python code assistant.
+
+Your task is to **solve the given question by writing Python code only**,
+without any explanation or natural language output.
+The code must compute the answer **programmatically**, not by hardcoding or
+guessing the result.
+
+**Rules:**
+- Use Python code to perform the actual computation.
+- Use {package_list} to solve the problem. Do not import any other libraries.
+- **Do not hardcode the final answer** (e.g., avoid writing `print(1/2)` unless
+ that value is computed).
+- The result must be obtained through valid computation logic in code.
+- Do not include explanations. Output code only.
+- The entire code must be wrapped in triple backticks:
+```
+[Your Python code here]
+```
+
+Now, solve the following question using Python. Only output the code:
+"""
+
+
+class SelfInstructGenerator(BaseGenerator):
+ r"""A generator for creating synthetic datapoints using self-instruct.
+
+ It utilizes both a human-provided dataset (seed_dataset) and generated
+ machine instructions (machine_instructions) to produce new, synthetic
+ datapoints that include a question, a computed rationale (code), and a
+ final answer (from a verifier).
+ """
+
+ class QuestionSchema(BaseModel):
+ r"""Schema for the generated question.
+
+ Attributes:
+ question (str): The question generated by the model.
+ """
+
+ question: str = Field(description="The question generated")
+
+ class RationaleSchema(BaseModel):
+ r"""Schema for the generated rationale code.
+
+ Attributes:
+ code (str): The generated code without any formatting.
+ """
+
+ code: str = Field(
+ description="The generated code without any formatting"
+ )
+
+ def __init__(
+ self,
+ seed_dataset: StaticDataset,
+ verifier: BaseVerifier,
+ instruction_agent: Optional[ChatAgent] = None,
+ rationale_agent: Optional[ChatAgent] = None,
+ seed: int = 42,
+ **kwargs,
+ ):
+ r"""Initialize the self-instruct generator.
+
+ Args:
+ seed_dataset (StaticDataset): Dataset containing seed instructions.
+ verifier (BaseVerifier): Verifier instance to validate generated
+ solutions.
+ instruction_agent (Optional[ChatAgent]): Agent for generating
+ instructions. If not provided, a default agent will be created.
+ rationale_agent (Optional[ChatAgent]): Agent for generating
+ rationales. If not provided, a default agent will be created.
+ seed (int): Random seed for reproducibility. (default: :obj:`42`)
+ **kwargs: Additional keyword arguments passed to the BaseGenerator.
+ """
+ super().__init__(seed=seed, **kwargs)
+ self.seed_dataset = seed_dataset
+ self.verifier = verifier
+ # extract packages from verifier
+ self.packages: List[str] = getattr(
+ self.verifier, "required_packages", []
+ )
+ # create default agents if not provided
+ self.instruction_agent = (
+ instruction_agent or self.default_instruction_agent()
+ )
+ self.rationale_agent = (
+ rationale_agent or self.default_rationale_agent()
+ )
+
+ # Extract questions from the seed dataset as human_instructions
+ self.human_instructions: List[str] = [
+ dp.question
+ for dp in list(cast(Iterable[DataPoint], self.seed_dataset))
+ ]
+ self.machine_instructions: List[DataPoint] = []
+ # Create an instance-level lock for thread-safe updates to _data
+ self._lock = asyncio.Lock()
+ self._data = [] # Storage for generated DataPoint instances
+
+ def default_instruction_agent(self) -> ChatAgent:
+ r"""Create the default instruction generation agent.
+
+ This agent is configured with a moderate temperature setting to
+ encourage creative and diverse instruction generation behavior.
+
+ Returns:
+ ChatAgent: An agent with the default instruction prompt.
+ """
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.DEFAULT,
+ model_type=ModelType.DEFAULT,
+ model_config_dict={"temperature": 0.7},
+ )
+ return ChatAgent(
+ DEFAULT_INSTRUCTION_SYSTEM_PROMPT,
+ model=model,
+ )
+
+ def default_rationale_agent(self) -> ChatAgent:
+ r"""Create the default rationale generation agent.
+
+ This agent is configured with a deterministic (zero temperature)
+ setting to ensure consistent and precise rationale generation based on
+ a given instruction and package list.
+
+ Returns:
+ ChatAgent: An agent with the rationale prompt
+ """
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.DEFAULT,
+ model_type=ModelType.DEFAULT,
+ model_config_dict={"temperature": 0.0},
+ )
+ return ChatAgent(
+ DEFAULT_RATIONALE_SYSTEM_PROMPT.format(package_list=self.packages),
+ model=model,
+ )
+
+ @staticmethod
+ def format_support_block(dp: DataPoint) -> str:
+ r"""Format a DataPoint into a few-shot example block.
+
+ Args:
+ dp (DataPoint): A data point.
+
+ Returns:
+ str: A formatted string containing the question and its
+ corresponding code block in Markdown-style Python format.
+ """
+ support_q = dp.question.strip()
+ support_code = dp.rationale.strip() if dp.rationale else ""
+ return (
+ f"Question:\n{support_q}\n\n"
+ "Code:\n"
+ "```python\n"
+ f"{support_code}\n"
+ "```"
+ )
+
+ def generate_new_instruction(
+ self,
+ agent: ChatAgent,
+ support_human_dps: list[DataPoint],
+ support_machine_dps: list[DataPoint],
+ ) -> str:
+ r"""Generate a new instruction using self-instruct prompting.
+
+ Args:
+ agent (ChatAgent): The agent to use for generating the instruction.
+ support_human_dps (list[DataPoint]): List of human examples to
+ sample.
+ support_machine_dps (list[DataPoint]): List of machine examples to
+ sample.
+
+ Returns:
+ str: The newly generated question.
+ """
+ human_sample = [dp.question for dp in list(support_human_dps)]
+ machine_sample = [dp.question for dp in list(support_machine_dps)]
+
+ few_shot_examples = human_sample + machine_sample
+
+ # Build the prompt using the few-shot examples
+ prompt = "Below are some question examples:\n\n"
+ for idx, instr in enumerate(few_shot_examples, start=1):
+ prompt += f"Question {idx}: {instr}\n"
+ prompt += f"Question {len(few_shot_examples) + 1}:\n"
+ prompt += "Now generate a new question based on the given examples.\n"
+
+ question_template = f"Question: {prompt}"
+ response = cast(
+ SelfInstructGenerator.QuestionSchema,
+ agent.step(question_template, response_format=self.QuestionSchema)
+ .msgs[0]
+ .parsed,
+ )
+ return response.question
+
+ def generate_rationale(
+ self,
+ question: str,
+ agent: Optional[ChatAgent] = None,
+ support_human_dps: Optional[list[DataPoint]] = None,
+ ) -> str:
+ r"""Generate rationale code (solution) for the given question.
+
+ Args:
+ question (str): The question to be solved.
+ agent (Optional[ChatAgent]): The agent to use for generating the
+ rationale. If None is provided, the default rationale agent
+ will be used. (default: :obj:`None`)
+ support_human_dps (Optional[list[DataPoint]]): List of human
+ examples to sample. (default: :obj:`None`)
+
+ Returns:
+ str: The generated code solution as a string.
+ """
+
+ # Build few-shot example prompt
+ few_shot_prompt = ""
+ if support_human_dps:
+ few_shot_examples = [
+ self.format_support_block(dp) for dp in support_human_dps
+ ]
+ few_shot_prompt += "Below are example questions and solutions:\n\n"
+ few_shot_prompt += "\n\n".join(few_shot_examples)
+
+ few_shot_prompt += f"\n\nWrite code to solve the question:\n{question}"
+
+ response = cast(
+ SelfInstructGenerator.RationaleSchema,
+ (agent or self.default_rationale_agent())
+ .step(few_shot_prompt, response_format=self.RationaleSchema)
+ .msgs[0]
+ .parsed,
+ )
+ return response.code
+
+ async def generate_new(
+ self,
+ n: int,
+ max_retries: int = 10,
+ human_sample_count: int = 3,
+ machine_sample_count: int = 1,
+ **kwargs,
+ ) -> list[DataPoint]:
+ r"""Generates and validates `n` new datapoints through
+ self-instruct prompting, with a retry limit.
+
+ Args:
+ n (int): The number of valid datapoints to generate.
+ max_retries (int): Maximum number of retries before stopping.
+ (default: :obj:`10`)
+ human_sample_count (int): Number of human examples to sample.
+ (default: :obj:`3`)
+ machine_sample_count (int): Number of machine examples to sample.
+ (default: :obj:`1`)
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ List[DataPoint]: A list of newly generated valid datapoints.
+
+ Raises:
+ TypeError: If the agent's output is not a dictionary (or does not
+ match the expected format).
+ KeyError: If required keys are missing from the response.
+ AttributeError: If the verifier response lacks attributes.
+ ValidationError: If a datapoint fails schema validation.
+ RuntimeError: If retries are exhausted before `n` valid datapoints
+ are generated.
+
+ Notes:
+ - Retries on validation failures until `n` valid datapoints exist
+ or `max_retries` is reached, whichever comes first.
+ - If retries are exhausted before reaching `n`, a `RuntimeError`
+ is raised.
+ - Metadata includes a timestamp for tracking datapoint creation.
+ """
+ valid_data_points: list[DataPoint] = []
+ retries = 0
+
+ while len(valid_data_points) < n and retries < max_retries:
+ try:
+ human_dps_list = list(cast(List[DataPoint], self.seed_dataset))
+ support_human_dps = random.sample(
+ human_dps_list,
+ min(human_sample_count, len(human_dps_list)),
+ )
+
+ machine_dps_list = list(self.machine_instructions)
+ support_machine_dps = random.sample(
+ machine_dps_list,
+ min(machine_sample_count, len(machine_dps_list)),
+ )
+ question = self.generate_new_instruction(
+ self.instruction_agent,
+ support_human_dps,
+ support_machine_dps,
+ )
+ rationale = self.generate_rationale(
+ question, self.rationale_agent, support_human_dps
+ )
+ if not isinstance(rationale, str):
+ raise TypeError(f"Rationale {rationale} is not a string.")
+
+ try:
+ verifier_response = await self.verifier.verify(
+ solution=rationale,
+ ground_truth=None,
+ )
+ if not verifier_response or not verifier_response.result:
+ raise ValueError(
+ "Verifier unsuccessful, response: "
+ f"{verifier_response}"
+ )
+ except (ValueError, AttributeError) as e:
+ logger.warning(
+ f"Verifier issue: {e}, "
+ f"retrying... ({retries + 1}/{max_retries})"
+ )
+ retries += 1
+ continue
+ try:
+ new_datapoint = DataPoint(
+ question=question,
+ rationale=rationale,
+ final_answer=verifier_response.result,
+ metadata={
+ "synthetic": str(True),
+ "created": datetime.now().isoformat(),
+ "generator": "self_instruct",
+ },
+ )
+ except ValidationError as e:
+ logger.warning(
+ f"Datapoint validation failed: {e}, "
+ f"retrying... ({retries + 1}/{max_retries})"
+ )
+ retries += 1
+ continue
+
+ valid_data_points.append(new_datapoint)
+
+ except Exception as e:
+ logger.warning(
+ f"Unexpected error: {e}, retrying..."
+ f" ({retries + 1}/{max_retries})"
+ )
+ retries += 1
+
+ if len(valid_data_points) < n:
+ raise RuntimeError(
+ f"Failed to generate {n} valid datapoints "
+ f"after {max_retries} retries."
+ )
+
+ async with self._lock:
+ self._data.extend(valid_data_points)
+ return valid_data_points
diff --git a/examples/dataset/self_instruct_generator.py b/examples/dataset/self_instruct_generator.py
new file mode 100644
index 000000000..7a0dc2d48
--- /dev/null
+++ b/examples/dataset/self_instruct_generator.py
@@ -0,0 +1,120 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from camel.agents import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.datasets import SelfInstructGenerator, StaticDataset
+from camel.logger import get_logger
+from camel.models import ModelFactory
+from camel.types import ModelPlatformType, ModelType
+from camel.verifiers import PythonVerifier
+
+logger = get_logger(__name__)
+
+verifier = PythonVerifier(required_packages=["sympy"], timeout=60)
+asyncio.run(verifier.setup())
+
+raw_data2 = [
+ {
+ "question": "Evaluate the limit as x approaches 0 of (sin(3*x) - 3*x) / x**3.", # noqa: E501
+ "final_answer": "-9/2",
+ "rationale": '''from sympy import symbols, limit, sin
+x = symbols('x')
+expr = (sin(3*x) - 3*x) / x**3
+result = limit(expr, x, 0)
+print(result)''',
+ },
+ {
+ "question": "Evaluate the definite integral of (1 - x**2)**3 from x = 0 to x = 1.", # noqa: E501
+ "final_answer": "16/35",
+ "rationale": '''from sympy import symbols, integrate
+x = symbols('x')
+expr = (1 - x**2)**3
+result = integrate(expr, (x, 0, 1))
+print(result)''',
+ },
+ {
+ "question": "Evaluate the limit as n approaches infinity of n*(sqrt(n**2 + 1) - n).", # noqa: E501
+ "final_answer": "1/2",
+ "rationale": '''from sympy import symbols, limit, sqrt
+n = symbols('n', positive=True)
+expr = n*(sqrt(n**2 + 1) - n)
+result = limit(expr, n, float("inf"))
+print(result)''',
+ },
+ {
+ "question": "Compute the sum of the series sum from n = 1 to 50 of 1/(n*(n+1)).", # noqa: E501
+ "final_answer": "50/51",
+ "rationale": '''from sympy import symbols, summation
+n = symbols('n', positive=True, integer=True)
+expr = 1/(n*(n+1))
+result = summation(expr, (n, 1, 50))
+print(result)''',
+ },
+]
+
+seed_dataset = StaticDataset(raw_data2)
+
+load_dotenv()
+
+model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=ChatGPTConfig().as_dict(),
+)
+
+RATIONALE_SYSTEM_PROMPT = """You are an advanced Python code assistant.
+
+Your task is to **solve the given question by writing Python code only**,
+without any explanation or natural language output.
+The code must compute the answer **programmatically**, not by hardcoding or
+guessing the result.
+
+**Rules:**
+- Use Python code to perform the actual computation.
+- Use sympy to solve the problem. Do not import any other libraries.
+- **Do not hardcode the final answer** (e.g., avoid writing `print(1/2)` unless
+ that value is computed).
+- The result must be obtained through valid computation logic in code.
+- Do not include explanations. Output code only.
+- The entire code must be wrapped in triple backticks:
+```
+[Your Python code here]
+```
+
+Now, solve the following question using Python. Only output the code:
+"""
+
+rationale_agent = ChatAgent(RATIONALE_SYSTEM_PROMPT, model=model)
+
+generator = SelfInstructGenerator(
+ seed_dataset=seed_dataset,
+ verifier=verifier,
+ instruction_agent=None, # use default instruction agent
+ rationale_agent=rationale_agent,
+)
+
+new_data = asyncio.run(generator.generate_new(n=3, max_retries=5))
+
+for dp in new_data:
+ print(dp)
+
+generator.save_to_jsonl(Path("generated_data.jsonl"))
+
+asyncio.run(verifier.cleanup())
diff --git a/tests/datasets/test_self_instruct_generator.py b/tests/datasets/test_self_instruct_generator.py
new file mode 100644
index 000000000..e19bc8e28
--- /dev/null
+++ b/tests/datasets/test_self_instruct_generator.py
@@ -0,0 +1,372 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from pydantic import ValidationError
+
+from camel.agents import ChatAgent
+from camel.datasets import DataPoint, SelfInstructGenerator
+from camel.verifiers import BaseVerifier
+
+
+def mock_verifier():
+ r"""Create a mock verifier fixture for testing."""
+ verifier = MagicMock(spec=BaseVerifier)
+ verifier.required_packages = ["math", "numpy"]
+ verifier.verify = AsyncMock()
+ return verifier
+
+
+def mock_datapoint():
+ r"""Create a mock datapoint fixture for testing."""
+ return DataPoint(
+ question="Sample question?",
+ rationale="print('Sample code')",
+ final_answer="Sample code output",
+ metadata={"synthetic": "False"},
+ )
+
+
+def mock_dataset(mock_datapoint):
+ r"""Create a mock dataset fixture for testing."""
+ dataset = MagicMock()
+ dataset.__iter__ = MagicMock(return_value=iter([mock_datapoint]))
+ return dataset
+
+
+def mock_instruction_agent():
+ r"""Create a mock instruction agent fixture for testing."""
+ agent = MagicMock(spec=ChatAgent)
+ response = MagicMock()
+ response.msgs = [MagicMock()]
+ response.msgs[0].parsed = SelfInstructGenerator.QuestionSchema(
+ question="Generated question?"
+ )
+ agent.step.return_value = response
+ return agent
+
+
+def mock_rationale_agent():
+ r"""Create a mock rationale agent fixture for testing."""
+ agent = MagicMock(spec=ChatAgent)
+ response = MagicMock()
+ response.msgs = [MagicMock()]
+ response.msgs[0].parsed = SelfInstructGenerator.RationaleSchema(
+ code="print('Generated code')"
+ )
+ agent.step.return_value = response
+ return agent
+
+
+def generator(
+ mock_dataset, mock_verifier, mock_instruction_agent, mock_rationale_agent
+):
+ r"""Create a SelfInstructGenerator instance with mock dependencies."""
+ return SelfInstructGenerator(
+ seed_dataset=mock_dataset,
+ verifier=mock_verifier,
+ instruction_agent=mock_instruction_agent,
+ rationale_agent=mock_rationale_agent,
+ seed=42,
+ )
+
+
+class TestSelfInstructGeneratorInitialization:
+ r"""Tests for the initialization of SelfInstructGenerator."""
+
+ def test_init_with_provided_agents(
+ self,
+ mock_dataset,
+ mock_verifier,
+ mock_instruction_agent,
+ mock_rationale_agent,
+ ):
+ r"""Test initialization with provided agents."""
+ generator = SelfInstructGenerator(
+ seed_dataset=mock_dataset,
+ verifier=mock_verifier,
+ instruction_agent=mock_instruction_agent,
+ rationale_agent=mock_rationale_agent,
+ )
+
+ assert generator.seed_dataset == mock_dataset
+ assert generator.verifier == mock_verifier
+ assert generator.instruction_agent == mock_instruction_agent
+ assert generator.rationale_agent == mock_rationale_agent
+ assert generator.packages == mock_verifier.required_packages
+ assert generator.human_instructions == ["Sample question?"]
+ assert generator.machine_instructions == []
+ assert generator._data == []
+
+
+class TestFormatSupportBlock:
+ r"""Tests for the format_support_block static method."""
+
+ def test_format_support_block_with_rationale(self, mock_datapoint):
+ r"""Test format_support_block with a datapoint that has a rationale."""
+ result = SelfInstructGenerator.format_support_block(mock_datapoint)
+
+ expected = (
+ f"Question:\n{mock_datapoint.question}\n\n"
+ "Code:\n"
+ "```python\n"
+ f"{mock_datapoint.rationale}\n"
+ "```"
+ )
+
+ assert result == expected
+
+ def test_format_support_block_without_rationale(self):
+ r"""Test format_support_block with a datapoint has no rationale."""
+ datapoint = DataPoint(
+ question="Sample question without rationale?",
+ final_answer="Sample answer",
+ metadata={"synthetic": "False"},
+ )
+
+ result = SelfInstructGenerator.format_support_block(datapoint)
+
+ expected = (
+ f"Question:\n{datapoint.question}\n\n"
+ "Code:\n"
+ "```python\n"
+ "\n"
+ "```"
+ )
+
+ assert result == expected
+
+
+class TestGenerateNewInstruction:
+ r"""Tests for the generate_new_instruction method."""
+
+ def test_generate_new_instruction(self, generator, mock_datapoint):
+ r"""Test generate_new_instruction method."""
+ # Create sample datapoints
+ support_human_dps = [mock_datapoint]
+ support_machine_dps = [
+ DataPoint(
+ question="Machine generated question?",
+ rationale="print('Machine code')",
+ final_answer="Machine output",
+ metadata={"synthetic": "True"},
+ )
+ ]
+
+ result = generator.generate_new_instruction(
+ generator.instruction_agent, support_human_dps, support_machine_dps
+ )
+
+ # Check if the agent was called with the correct prompt
+ generator.instruction_agent.step.assert_called_once()
+ args, kwargs = generator.instruction_agent.step.call_args
+
+ # Verify the prompt contains the examples
+ prompt = args[0]
+ assert "Sample question?" in prompt
+ assert "Machine generated question?" in prompt
+
+ # Verify the response format is correct
+ assert kwargs["response_format"] == generator.QuestionSchema
+
+ # Verify the result
+ assert result == "Generated question?"
+
+
+class TestGenerateRationale:
+ r"""Tests for the generate_rationale method."""
+
+ def test_generate_rationale_with_agent(self, generator, mock_datapoint):
+ r"""Test generate_rationale with a provided agent."""
+ question = "Test question?"
+ support_human_dps = [mock_datapoint]
+
+ result = generator.generate_rationale(
+ question, generator.rationale_agent, support_human_dps
+ )
+
+ # Check if the agent was called with the correct prompt
+ generator.rationale_agent.step.assert_called_once()
+ args, kwargs = generator.rationale_agent.step.call_args
+
+ # Verify the prompt contains the examples and question
+ prompt = args[0]
+ assert "Sample question?" in prompt
+ assert "Test question?" in prompt
+
+ # Verify the response format is correct
+ assert kwargs["response_format"] == generator.RationaleSchema
+
+ # Verify the result
+ assert result == "print('Generated code')"
+
+ @patch(
+ 'camel.datasets.self_instruct_generator.SelfInstructGenerator.default_rationale_agent'
+ )
+ def test_generate_rationale_with_default_agent(
+ self, mock_default_agent, generator, mock_datapoint
+ ):
+ r"""Test generate_rationale with the default agent."""
+ # Setup mock default agent
+ mock_agent = MagicMock()
+ response = MagicMock()
+ response.msgs = [MagicMock()]
+ response.msgs[0].parsed = SelfInstructGenerator.RationaleSchema(
+ code="print('Default agent code')"
+ )
+ mock_agent.step.return_value = response
+ mock_default_agent.return_value = mock_agent
+
+ question = "Test question?"
+ support_human_dps = [mock_datapoint]
+
+ result = generator.generate_rationale(
+ question, support_human_dps=support_human_dps
+ )
+
+ # Verify default agent was created
+ mock_default_agent.assert_called_once()
+
+ # Verify the result
+ assert result == "print('Default agent code')"
+
+
+class TestGenerateNew:
+ r"""Tests for the generate_new async method."""
+
+ @pytest.mark.asyncio
+ async def test_generate_new_success(self, generator, mock_datapoint):
+ r"""Test successful generation of new datapoints."""
+ # Setup mock verifier response
+ verifier_response = MagicMock()
+ verifier_response.result = "Verified output"
+ generator.verifier.verify.return_value = verifier_response
+
+ # Mock the generate methods
+ generator.generate_new_instruction = MagicMock(
+ return_value="New question?"
+ )
+ generator.generate_rationale = MagicMock(
+ return_value="print('New code')"
+ )
+
+ result = await generator.generate_new(n=1)
+
+ # Verify generated datapoint
+ assert len(result) == 1
+ assert result[0].question == "New question?"
+ assert result[0].rationale == "print('New code')"
+ assert result[0].final_answer == "Verified output"
+ assert result[0].metadata["synthetic"] == "True"
+ assert result[0].metadata["generator"] == "self_instruct"
+
+ @pytest.mark.asyncio
+ async def test_generate_new_verification_failure(self, generator):
+ r"""Test handling of verification failures during generation."""
+ # Create proper mock responses
+ none_response = None
+ success_response = MagicMock()
+ success_response.result = "Verified output"
+
+ # Setup the verify method to first return None, then return a
+ # successful response
+ generator.verifier.verify = AsyncMock(
+ side_effect=[
+ none_response, # First call returns None (failure)
+ success_response, # Second call returns success
+ ]
+ )
+
+ # Mock the generate methods
+ generator.generate_new_instruction = MagicMock(
+ side_effect=["Failed question?", "Successful question?"]
+ )
+ generator.generate_rationale = MagicMock(
+ side_effect=["print('Failed code')", "print('Successful code')"]
+ )
+
+ result = await generator.generate_new(
+ n=1, max_retries=3
+ ) # Reduce retries for test speed
+
+ # Verify generated datapoint
+ assert len(result) == 1
+ assert result[0].question == "Successful question?"
+ assert result[0].rationale == "print('Successful code')"
+ assert result[0].final_answer == "Verified output"
+
+ @pytest.mark.asyncio
+ async def test_generate_new_validation_error(self, generator):
+ r"""Test handling of validation errors during generation."""
+ # Setup mock verifier response
+ verifier_response = MagicMock()
+ verifier_response.result = "Verified output"
+ generator.verifier.verify.return_value = verifier_response
+
+ # Mock the generate methods
+ generator.generate_new_instruction = MagicMock(
+ side_effect=["Failed question?", "Successful question?"]
+ )
+ generator.generate_rationale = MagicMock(
+ side_effect=["print('Failed code')", "print('Successful code')"]
+ )
+
+ # Use a custom DataPoint creation function to simulate validation
+ # error then success
+ original_datapoint = DataPoint
+
+ datapoint_calls = 0
+
+ def mock_datapoint_factory(*args, **kwargs):
+ nonlocal datapoint_calls
+ datapoint_calls += 1
+
+ if datapoint_calls == 1:
+ # First call - raise validation error
+ raise ValidationError.from_exception_data(
+ title="ValidationError",
+ line_errors=[
+ {
+ "type": "string_type",
+ "loc": ("final_answer",),
+ "msg": "Input should be a valid string",
+ }
+ ],
+ model=DataPoint,
+ )
+ else:
+ # Second call - succeed
+ return original_datapoint(*args, **kwargs)
+
+ with patch(
+ 'camel.datasets.self_instruct_generator.DataPoint',
+ side_effect=mock_datapoint_factory,
+ ):
+ result = await generator.generate_new(
+ n=1, max_retries=3
+ ) # Reduce retries for test speed
+
+ # Verify generated datapoint
+ assert len(result) == 1
+ assert result[0].question == "Successful question?"
+ assert result[0].rationale == "print('Successful code')"
+ assert result[0].final_answer == "Verified output"