🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
chore: enhance base dataset for loong (#1837)
Wendong-Fanf841d7b869ea
diff --git a/camel/datasets/__init__.py b/camel/datasets/__init__.pyindex 1f92d8b49..96d9783e7 100644--- a/camel/datasets/__init__.py+++ b/camel/datasets/__init__.py@@ -11,14 +11,16 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from .base import (- DataPoint,- GenerativeDataset,- StaticDataset,+from .base_generator import (+ BaseGenerator,+ FewShotGenerator,)+from .models import DataPoint+from .static_dataset import StaticDataset__all__ = [+ "BaseGenerator","DataPoint",+ "FewShotGenerator","StaticDataset",- "GenerativeDataset",]diff --git a/camel/datasets/base.py b/camel/datasets/base.pydeleted file mode 100644index f65fb2582..000000000--- a/camel/datasets/base.py+++ /dev/null@@ -1,644 +0,0 @@-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========--import json-import random-from datetime import datetime-from pathlib import Path-from typing import (- Any,- Dict,- List,- Optional,- Sized,- Union,-)--from datasets import Dataset as HFDataset-from pydantic import BaseModel, Field, ValidationError-from torch.utils.data import Dataset--from camel.agents import ChatAgent-from camel.logger import get_logger-from camel.verifiers import BaseVerifier-from camel.verifiers.models import VerifierInput--logger = get_logger(__name__)---class DataPoint(BaseModel):- r"""A single data point in the dataset.-- Attributes:- question (str): The primary question or issue to be addressed.- rationale (Optional[str]): Logical reasoning or explanation behind the- answer. (default: :obj:`None`)- final_answer (str): The final answer.- metadata Optional[Dict[str, Any]]: Additional metadata about the data- point. (default: :obj:`None`)- """-- question: str = Field(- ..., description="The primary question or issue to be addressed."- )- rationale: Optional[str] = Field(- default=None,- description="Logical reasoning or explanation behind the answer.",- )- final_answer: str = Field(..., description="The final answer.")-- metadata: Optional[Dict[str, Any]] = Field(- default=None, description="Additional metadata about the data point."- )-- def to_dict(self) -> Dict[str, Any]:- r"""Convert DataPoint to a dictionary.-- Returns:- Dict[str, Any]: Dictionary representation of the DataPoint.- """- return self.dict()-- @classmethod- def from_dict(cls, data: Dict[str, Any]) -> 'DataPoint':- r"""Create a DataPoint from a dictionary.-- Args:- data (Dict[str, Any]): Dictionary containing DataPoint fields.-- Returns:- DataPoint: New DataPoint instance.- """- return cls(**data)---class StaticDataset(Dataset):- r"""A static dataset containing a list of datapoints.- Ensures that all items adhere to the DataPoint schema.- This dataset extends :obj:`Dataset` from PyTorch and should- be used when its size is fixed at runtime.-- This class can initialize from Hugging Face Datasets,- PyTorch Datasets, JSON file paths, or lists of dictionaries,- converting them into a consistent internal format.- """-- def __init__(- self,- data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]],- seed: int = 42,- min_samples: int = 1,- strict: bool = False,- **kwargs,- ):- r"""Initialize the static dataset and validate integrity.-- Args:- data (:obj:`Union[HFDataset, Dataset,- Path, List[Dict[str, Any]]]`):- Input data, which can be one of the following:- - A Hugging Face Dataset (:obj:`HFDataset`).- - A PyTorch Dataset (:obj:`torch.utils.data.Dataset`).- - A :obj:`Path` object representing a JSON file.- - A list of dictionaries with :obj:`DataPoint`-compatible- fields.- seed (:obj:`int`): Random seed for reproducibility.- Default is :obj:`42`.- min_samples (:obj:`int`): Minimum required number of samples.- Default is :obj:`1`.- strict (:obj:`bool`): Whether to raise an error on invalid- datapoints (:obj:`True`) or skip/filter them (:obj:`False`).- Default is :obj:`False`.- **kwargs: Additional dataset parameters.-- Raises:- TypeError: If the input data type is unsupported.- ValueError: If the dataset contains fewer than :obj:`min_samples`- datapoints or if validation fails.- FileNotFoundError: If the specified JSON file path does not exist.- json.JSONDecodeError: If the JSON file contains invalid formatting.- """-- # Store all parameters in metadata dict for compatibility- self._metadata = {- **kwargs,- }- self._rng = random.Random(seed)- self._strict = strict-- self.data: List[DataPoint] = self._init_data(data)- self._length = len(self.data)-- if self._length < min_samples:- raise ValueError(- "The dataset does not contain enough samples. "- f"Need {max(0, min_samples)}, got {self._length}"- )-- def _init_data(- self, data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]- ) -> List[DataPoint]:- r"""Convert input data from various formats into a list of- :obj:`DataPoint` instances.-- Args:- data (:obj:`Union[- HFDataset,- Dataset,- Path,- List[Dict[str, Any]]- ]`):- Input dataset in one of the supported formats.-- Returns:- :obj:`List[DataPoint]`: A list of validated :obj:`DataPoint`- instances.-- Raises:- TypeError: If the input data type is unsupported.- """-- if isinstance(data, HFDataset):- raw_data = self._init_from_hf_dataset(data)- elif isinstance(data, Dataset):- raw_data = self._init_from_pytorch_dataset(data)- elif isinstance(data, Path):- raw_data = self._init_from_json_path(data)- elif isinstance(data, list):- raw_data = self._init_from_list(data)- else:- raise TypeError("Unsupported data type")-- def create_datapoint(- item: Dict[str, Any], idx: int- ) -> Optional[DataPoint]:- # Add type checks for required fields to make mypy happy- question = item.get('question')- if not isinstance(question, str):- if self._strict:- raise ValueError(- f"Sample at index {idx} has invalid 'question': "- f"expected str, got {type(question)}"- )- else:- logger.warning(- f"Skipping sample at index {idx}: invalid 'question'"- )- return None-- rationale = item.get('rationale')- if not isinstance(rationale, str):- if self._strict:- raise ValueError(- f"Sample at index {idx} has invalid 'rationale': "- f"expected str, got {type(rationale)}"- )- else:- logger.warning(- f"Skipping sample at index {idx}: invalid 'rationale'"- )- return None-- final_answer = item.get('final_answer')- if not isinstance(final_answer, str):- if self._strict:- raise ValueError(- f"Sample at index {idx} has invalid 'final_answer': "- f"expected str, got {type(final_answer)}"- )- else:- logger.warning(- f"Skipping sample at index {idx}: "- "invalid 'final_answer'"- )- return None-- try:- return DataPoint(- question=question,- rationale=rationale,- final_answer=final_answer,- metadata=item.get('metadata'),- )- except ValidationError as e:- if self._strict:- raise ValueError(- f"Sample at index {idx} validation error: {e}"- )- else:- logger.warning(- f"Skipping invalid sample at index {idx} "- f"due to validation error: {e}"- )- return None-- unfiltered_data = [- create_datapoint(item, i) for i, item in enumerate(raw_data)- ]- return [dp for dp in unfiltered_data if dp is not None]-- def __len__(self) -> int:- r"""Return the size of the dataset."""- return self._length-- def __getitem__(self, idx: int) -> DataPoint:- r"""Retrieve a datapoint by index.-- Args:- idx (:obj:`int`): Index of the datapoint.-- Returns:- :obj:`DataPoint`: The datapoint corresponding to the given index.-- Raises:- IndexError: If :obj:`idx` is out of bounds (negative or greater- than dataset length - 1).- """-- if idx < 0 or idx >= self._length:- raise IndexError(- f"Index {idx} out of bounds for dataset of size {self._length}"- )- return self.data[idx]-- def sample(self) -> DataPoint:- r"""Sample a random datapoint from the dataset.-- Returns:- :obj:`DataPoint`: A randomly sampled :obj:`DataPoint`.-- Raises:- RuntimeError: If the dataset is empty and no samples can be drawn.- """-- if self._length == 0:- raise RuntimeError("Dataset is empty, cannot sample.")- idx = self._rng.randint(0, self._length - 1)- return self[idx]-- @property- def metadata(self) -> Dict[str, Any]:- r"""Retrieve dataset metadata.-- Returns:- :obj:`Dict[str, Any]`: A copy of the dataset metadata dictionary.- """-- return self._metadata.copy()-- def _init_from_hf_dataset(self, data: HFDataset) -> List[Dict[str, Any]]:- r"""Convert a Hugging Face dataset into a list of dictionaries.-- Args:- data (:obj:`HFDataset`): A Hugging Face dataset.-- Returns:- :obj:`List[Dict[str, Any]]`: A list of dictionaries representing- the dataset, where each dictionary corresponds to a datapoint.- """- return [dict(item) for item in data]-- def _init_from_pytorch_dataset(- self, data: Dataset- ) -> List[Dict[str, Any]]:- r"""Convert a PyTorch dataset into a list of dictionaries.-- Args:- data (:obj:`Dataset`): A PyTorch dataset.-- Returns:- :obj:`List[Dict[str, Any]]`: A list of dictionaries representing- the dataset.-- Raises:- TypeError: If the dataset does not implement :obj:`__len__()`- or contains non-dictionary elements.- """- if not isinstance(data, Sized):- raise TypeError(- f"{type(data).__name__} does not implement `__len__()`."- )- raw_data = []-- for i in range(len(data)):- item = data[i]- if not isinstance(item, dict):- raise TypeError(- f"Item at index {i} is not a dict: "- f"got {type(item).__name__}"- )- raw_data.append(dict(item))- return raw_data-- def _init_from_json_path(self, data: Path) -> List[Dict[str, Any]]:- r"""Load and parse a dataset from a JSON file.-- Args:- data (:obj:`Path`): Path to the JSON file.-- Returns:- :obj:`List[Dict[str, Any]]`: A list of datapoint dictionaries.-- Raises:- FileNotFoundError: If the specified JSON file does not exist.- ValueError: If the JSON content is not a list of dictionaries.- json.JSONDecodeError: If the JSON file has invalid formatting.- """-- if not data.exists():- raise FileNotFoundError(f"JSON file not found: {data}")- try:- logger.debug(f"Loading JSON from {data}")- with data.open('r', encoding='utf-8') as f:- loaded_data = json.load(f)- logger.info(- f"Successfully loaded {len(loaded_data)} items from {data}"- )- except json.JSONDecodeError as e:- raise ValueError(f"Invalid JSON in file {data}: {e}")- if not isinstance(loaded_data, list):- raise ValueError("JSON file must contain a list of dictionaries")- for i, item in enumerate(loaded_data):- if not isinstance(item, dict):- raise ValueError(- f"Expected a dictionary at index {i}, "- f"got {type(item).__name__}"- )- return loaded_data-- def _init_from_list(- self, data: List[Dict[str, Any]]- ) -> List[Dict[str, Any]]:- r"""Validate and convert a list of dictionaries into a dataset.-- Args:- data (:obj:`List[Dict[str, Any]]`): A list of dictionaries where- each dictionary must be a valid :obj:`DataPoint`.-- Returns:- :obj:`List[Dict[str, Any]]`: The validated list of dictionaries.-- Raises:- ValueError: If any item in the list is not a dictionary.- """- for i, item in enumerate(data):- if not isinstance(item, dict):- raise ValueError(- f"Expected a dictionary at index {i}, "- f"got {type(item).__name__}"- )- return data---class GenerativeDataset(Dataset):- r"""A dataset for generating synthetic datapoints using external agents and- verifiers.-- This class leverages a seed dataset and external components to generate- new synthetic datapoints on demand.- """-- def __init__(- self,- seed_dataset: StaticDataset,- verifier: BaseVerifier,- agent: ChatAgent,- seed: int = 42,- **kwargs,- ):- r"""Initialize the generative dataset.-- Args:- seed_dataset (StaticDataset): Validated static dataset to- use for examples.- verifier (BaseVerifier): Verifier to validate generated content.- agent (ChatAgent): Agent to generate new datapoints.- seed (int): Random seed for reproducibility. (default: :obj:`42`)- **kwargs: Additional dataset parameters.- """-- self.seed_dataset = seed_dataset- self.verifier = verifier- self.agent = agent-- self._rng = random.Random(seed)-- self._data: List[DataPoint] = []-- def _construct_prompt(self, examples: List[DataPoint]) -> str:- r"""Construct a prompt for generating new datapoints- using a fixed sample of 3 examples from the seed dataset.-- Args:- examples (List[DataPoint]): Examples to include in the prompt.-- Returns:- str: Formatted prompt with examples.- """- prompt = (- "Generate a new datapoint similar to the following examples:\n\n"- )- for i, example in enumerate(examples, 1):- prompt += f"Example {i}:\n"- prompt += f"Question: {example.question}\n"- prompt += f"Rationale: {example.rationale}\n"- prompt += f"Final Answer: {example.final_answer}\n\n"- prompt += "New datapoint:"- return prompt-- async def generate_new(- self, n: int, max_retries: int = 10- ) -> List[DataPoint]:- r"""Generates and validates `n` new datapoints through- few-shot prompting, with a retry limit.-- Steps:- 1. Samples 3 examples from the seed dataset.- 2. Constructs a prompt using the selected examples.- 3. Uses an agent to generate a new datapoint.- 4. Verifies the datapoint using a verifier.- 5. Stores valid datapoints in memory.-- Args:- n (int): Number of valid datapoints to generate.- max_retries (int): Maximum number of retries before stopping.-- Returns:- List[DataPoint]: A list of newly generated valid datapoints.-- Raises:- TypeError: If the agent's output is not a dictionary (or does not- match the expected format).- KeyError: If required keys are missing from the response.- AttributeError: If the verifier response lacks attributes.- ValidationError: If a datapoint fails schema validation.- RuntimeError: If retries are exhausted before `n` valid datapoints- are generated.-- Notes:- - Retries on validation failures until `n` valid datapoints exist- or `max_retries` is reached, whichever comes first.- - If retries are exhausted before reaching `n`, a `RuntimeError`- is raised.- - Metadata includes a timestamp for tracking datapoint creation.- - This method can be overridden to implement custom generation.- """- valid_data_points: List[DataPoint] = []- retries = 0-- while len(valid_data_points) < n and retries < max_retries:- try:- examples = [self.seed_dataset.sample() for _ in range(3)]- prompt = self._construct_prompt(examples)-- try:- agent_output = (- self.agent.step(prompt, response_format=DataPoint)- .msgs[0]- .parsed- )- if not isinstance(agent_output, dict):- raise TypeError("Agent output must be a dictionary")- if (- "question" not in agent_output- or "rationale" not in agent_output- ):- raise KeyError(- "Missing 'question' or 'rationale' in agent output"- )- except (TypeError, KeyError) as e:- logger.warning(- f"Agent output issue: {e}, retrying... "- f"({retries + 1}/{max_retries})"- )- retries += 1- continue-- rationale = agent_output["rationale"]-- try:- verifier_response = await self.verifier.verify(- VerifierInput(- llm_response=rationale, ground_truth=None- )- )- if not verifier_response or not verifier_response.result:- raise ValueError(- "Verifier unsuccessful, response: "- f"{verifier_response}"- )- except (ValueError, AttributeError) as e:- logger.warning(- f"Verifier issue: {e}, "- f"retrying... ({retries + 1}/{max_retries})"- )- retries += 1- continue-- try:- new_datapoint = DataPoint(- question=agent_output["question"],- rationale=rationale,- final_answer=verifier_response.result,- metadata={- "synthetic": str(True),- "created": datetime.now().isoformat(),- },- )- except ValidationError as e:- logger.warning(- f"Datapoint validation failed: {e}, "- f"retrying... ({retries + 1}/{max_retries})"- )- retries += 1- continue-- valid_data_points.append(new_datapoint)-- except Exception as e:- logger.warning(- f"Unexpected error: {e}, retrying..."- f" ({retries + 1}/{max_retries})"- )- retries += 1-- if len(valid_data_points) < n:- raise RuntimeError(- f"Failed to generate {n} valid datapoints "- f"after {max_retries} retries."- )-- self._data.extend(valid_data_points)- return valid_data_points-- def __len__(self) -> int:- r"""Return the size of the dataset."""- return len(self._data)-- def __getitem__(self, idx: int) -> DataPoint:- r"""Retrieve a datapoint by index.-- Args:- idx (int): Index of the datapoint.-- Returns:- DataPoint: The datapoint corresponding to the given index.-- Raises:- IndexError: If idx is out of bounds.- """- if idx < 0 or idx >= len(self._data):- raise IndexError(- f"Index {idx} out of bounds for dataset of "- f"size {len(self._data)}"- )- return self._data[idx]-- def sample(self) -> DataPoint:- if len(self._data) == 0:- raise RuntimeError("Dataset is empty, cannot sample.")- idx = self._rng.randint(0, len(self._data) - 1)- return self[idx]-- def save_to_jsonl(self, file_path: Union[str, Path]) -> None:- r"""Saves the dataset to a JSONL (JSON Lines) file.-- Each datapoint is stored as a separate JSON object on a new line.-- Args:- file_path (Union[str, Path]): Path to save the JSONL file.-- Raises:- ValueError: If the dataset is empty.- IOError: If there is an issue writing to the file.-- Notes:- - Uses `self._data`, which contains the generated datapoints.- - Overwrites the file if it already exists.- - Ensures compatibility with large datasets by using JSONL format.- """- if not self._data:- raise ValueError("Dataset is empty. No data to save.")-- file_path = Path(file_path)-- try:- with file_path.open("w", encoding="utf-8") as f:- for datapoint in self._data:- json.dump(datapoint.to_dict(), f)- f.write("\n") # Ensure each entry is on a new line- logger.info(f"Dataset saved successfully to {file_path}")- except IOError as e:- logger.error(f"Error writing to file {file_path}: {e}")- raisediff --git a/camel/datasets/base_generator.py b/camel/datasets/base_generator.pynew file mode 100644index 000000000..e1f6713cd--- /dev/null+++ b/camel/datasets/base_generator.py@@ -0,0 +1,335 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import abc+import json+import random+from datetime import datetime+from pathlib import Path+from typing import (+ List,+ Union,+)++from pydantic import ValidationError++from camel.agents import ChatAgent+from camel.logger import get_logger+from camel.verifiers import BaseVerifier+from camel.verifiers.models import VerifierInput++from .models import DataPoint+from .static_dataset import StaticDataset++logger = get_logger(__name__)+++class BaseGenerator(abc.ABC):+ r"""Abstract base class for data generators.++ This class defines the interface for generating synthetic datapoints.+ Concrete implementations should provide specific generation strategies.+ """++ def __init__(self, seed: int = 42, **kwargs):+ r"""Initialize the base generator.++ Args:+ seed (int): Random seed for reproducibility. (default: :obj:`42`)+ **kwargs: Additional generator parameters.+ """+ self._rng = random.Random(seed)++ self._data: List[DataPoint] = []++ @abc.abstractmethod+ async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:+ r"""Generate n new datapoints.++ Args:+ n (int): Number of datapoints to generate.+ **kwargs: Additional generation parameters.++ Returns:+ List[DataPoint]: A list of newly generated datapoints.+ """+ pass++ def __len__(self) -> int:+ r"""Return the size of the generated dataset."""+ return len(self._data)++ def __getitem__(self, idx: int) -> DataPoint:+ r"""Retrieve a datapoint by index.++ Args:+ idx (int): Index of the datapoint.++ Returns:+ DataPoint: The datapoint corresponding to the given index.++ Raises:+ IndexError: If idx is out of bounds.+ """+ if idx < 0 or idx >= len(self._data):+ raise IndexError(+ f"Index {idx} out of bounds for dataset of "+ f"size {len(self._data)}"+ )+ return self._data[idx]++ def sample(self) -> DataPoint:+ if len(self._data) == 0:+ raise RuntimeError("Dataset is empty, cannot sample.")+ idx = self._rng.randint(0, len(self._data) - 1)+ return self[idx]++ def save_to_jsonl(self, file_path: Union[str, Path]) -> None:+ r"""Saves the generated datapoints to a JSONL (JSON Lines) file.++ Each datapoint is stored as a separate JSON object on a new line.++ Args:+ file_path (Union[str, Path]): Path to save the JSONL file.++ Raises:+ ValueError: If no datapoints have been generated.+ IOError: If there is an issue writing to the file.++ Notes:+ - Uses `self._data`, which contains the generated datapoints.+ - Overwrites the file if it already exists.+ - Ensures compatibility with large datasets by using JSONL format.+ """+ if not self._data:+ raise ValueError("Dataset is empty. No data to save.")++ file_path = Path(file_path)++ try:+ with file_path.open("w", encoding="utf-8") as f:+ for datapoint in self._data:+ json.dump(datapoint.to_dict(), f)+ f.write("\n") # Ensure each entry is on a new line+ logger.info(f"Dataset saved successfully to {file_path}")+ except IOError as e:+ logger.error(f"Error writing to file {file_path}: {e}")+ raise+++class FewShotGenerator(BaseGenerator):+ r"""A generator for creating synthetic datapoints using few-shot learning.++ This class leverages a seed dataset, an agent, and a verifier to generate+ new synthetic datapoints on demand through few-shot prompting.+ """++ def __init__(+ self,+ seed_dataset: StaticDataset,+ verifier: BaseVerifier,+ agent: ChatAgent,+ seed: int = 42,+ **kwargs,+ ):+ r"""Initialize the few-shot generator.++ Args:+ seed_dataset (StaticDataset): Validated static dataset to+ use for examples.+ verifier (BaseVerifier): Verifier to validate generated content.+ agent (ChatAgent): Agent to generate new datapoints.+ seed (int): Random seed for reproducibility. (default: :obj:`42`)+ **kwargs: Additional generator parameters.+ """+ super().__init__(seed=seed, **kwargs)+ self.seed_dataset = seed_dataset+ try:+ self._validate_seed_dataset()+ except Exception:+ raise RuntimeError("Seed Data does not follow Datapoint format")+ self.verifier = verifier+ self.agent = agent++ # TODO: Validate that seed dataset contains rationale+ def _validate_seed_dataset(self) -> None:+ pass++ def _construct_prompt(self, examples: List[DataPoint]) -> str:+ r"""Construct a prompt for generating new datapoints+ using a fixed sample of examples from the seed dataset.++ Args:+ examples (List[DataPoint]): Examples to include in the prompt.++ Returns:+ str: Formatted prompt with examples.+ """+ prompt = (+ "Generate a new datapoint similar to the following examples:\n\n"+ )+ for i, example in enumerate(examples, 1):+ prompt += f"Example {i}:\n"+ prompt += f"Question: {example.question}\n"+ if example.rationale is not None:+ prompt += f"Rationale: {example.rationale}\n"+ else:+ prompt += "Rationale: None\n"+ prompt += f"Final Answer: {example.final_answer}\n\n"+ prompt += "New datapoint:"+ return prompt++ async def generate_new(+ self,+ n: int,+ max_retries: int = 10,+ num_examples: int = 3,+ **kwargs,+ ) -> List[DataPoint]:+ r"""Generates and validates `n` new datapoints through+ few-shot prompting, with a retry limit.++ Steps:+ 1. Samples examples from the seed dataset.+ 2. Constructs a prompt using the selected examples.+ 3. Uses an agent to generate a new datapoint,+ consisting of a question and code to solve the question.+ 4. Executes code using a verifier to get pseudo ground truth.+ 5. Stores valid datapoints in memory.++ Args:+ n (int): Number of valid datapoints to generate.+ max_retries (int): Maximum number of retries before stopping.+ (default: :obj:`10`)+ num_examples (int): Number of examples to sample from the+ seed dataset for few shot prompting.+ (default: :obj:`3`)+ **kwargs: Additional generation parameters.++ Returns:+ List[DataPoint]: A list of newly generated valid datapoints.++ Raises:+ TypeError: If the agent's output is not a dictionary (or does not+ match the expected format).+ KeyError: If required keys are missing from the response.+ AttributeError: If the verifier response lacks attributes.+ ValidationError: If a datapoint fails schema validation.+ RuntimeError: If retries are exhausted before `n` valid datapoints+ are generated.++ Notes:+ - Retries on validation failures until `n` valid datapoints exist+ or `max_retries` is reached, whichever comes first.+ - If retries are exhausted before reaching `n`, a `RuntimeError`+ is raised.+ - Metadata includes a timestamp for tracking datapoint creation.+ """+ valid_data_points: List[DataPoint] = []+ retries = 0++ while len(valid_data_points) < n and retries < max_retries:+ try:+ examples = [+ self.seed_dataset.sample() for _ in range(num_examples)+ ]+ prompt = self._construct_prompt(examples)++ try:+ agent_output = (+ self.agent.step(prompt, response_format=DataPoint)+ .msgs[0]+ .parsed+ )+ if not isinstance(agent_output, dict):+ raise TypeError("Agent output must be a dictionary")+ if "question" not in agent_output:+ raise KeyError(+ "Missing 'question' in agent"+ f"output {agent_output}"+ )+ if "rationale" not in agent_output:+ raise KeyError(+ "Missing 'rationale' in agent"+ f"output {agent_output}"+ )+ except (TypeError, KeyError) as e:+ logger.warning(+ f"Agent output issue: {e}, retrying... "+ f"({retries + 1}/{max_retries})"+ )+ retries += 1+ continue++ rationale = agent_output.get("rationale")++ if not isinstance(rationale, str):+ raise TypeError(f"Rationale {rationale} is not a string.")++ try:+ verifier_response = await self.verifier.verify(+ VerifierInput(+ llm_response=rationale,+ ground_truth=None,+ )+ )+ if not verifier_response or not verifier_response.result:+ raise ValueError(+ "Verifier unsuccessful, response: "+ f"{verifier_response}"+ )+ except (ValueError, AttributeError) as e:+ logger.warning(+ f"Verifier issue: {e}, "+ f"retrying... ({retries + 1}/{max_retries})"+ )+ retries += 1+ continue++ try:+ new_datapoint = DataPoint(+ question=agent_output["question"],+ rationale=rationale,+ final_answer=verifier_response.result,+ metadata={+ "synthetic": str(True),+ "created": datetime.now().isoformat(),+ "generator": "few_shot",+ },+ )+ except ValidationError as e:+ logger.warning(+ f"Datapoint validation failed: {e}, "+ f"retrying... ({retries + 1}/{max_retries})"+ )+ retries += 1+ continue++ valid_data_points.append(new_datapoint)++ except Exception as e:+ logger.warning(+ f"Unexpected error: {e}, retrying..."+ f" ({retries + 1}/{max_retries})"+ )+ retries += 1++ if len(valid_data_points) < n:+ raise RuntimeError(+ f"Failed to generate {n} valid datapoints "+ f"after {max_retries} retries."+ )++ self._data.extend(valid_data_points)+ return valid_data_pointsdiff --git a/camel/datasets/models.py b/camel/datasets/models.pynew file mode 100644index 000000000..a70c4109b--- /dev/null+++ b/camel/datasets/models.py@@ -0,0 +1,61 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import Any, Dict, Optional++from pydantic import BaseModel, Field+++class DataPoint(BaseModel):+ r"""A single data point in the dataset.++ Attributes:+ question (str): The primary question or issue to be addressed.+ final_answer (str): The final answer.+ rationale (Optional[str]): Logical reasoning or explanation behind the+ answer. (default: :obj:`None`)+ metadata Optional[Dict[str, Any]]: Additional metadata about the data+ point. (default: :obj:`None`)+ """++ question: str = Field(+ ..., description="The primary question or issue to be addressed."+ )+ final_answer: str = Field(..., description="The final answer.")+ rationale: Optional[str] = Field(+ default=None,+ description="Logical reasoning or explanation behind the answer.",+ )+ metadata: Optional[Dict[str, Any]] = Field(+ default=None, description="Additional metadata about the data point."+ )++ def to_dict(self) -> Dict[str, Any]:+ r"""Convert DataPoint to a dictionary.++ Returns:+ Dict[str, Any]: Dictionary representation of the DataPoint.+ """+ return self.dict()++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> 'DataPoint':+ r"""Create a DataPoint from a dictionary.++ Args:+ data (Dict[str, Any]): Dictionary containing DataPoint fields.++ Returns:+ DataPoint: New DataPoint instance.+ """+ return cls(**data)diff --git a/camel/datasets/static_dataset.py b/camel/datasets/static_dataset.pynew file mode 100644index 000000000..c49818a0c--- /dev/null+++ b/camel/datasets/static_dataset.py@@ -0,0 +1,346 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import json+import random+from pathlib import Path+from typing import (+ Any,+ Dict,+ List,+ Optional,+ Sized,+ Union,+)++from datasets import Dataset as HFDataset+from pydantic import ValidationError+from torch.utils.data import Dataset++from camel.logger import get_logger++from .models import DataPoint++logger = get_logger(__name__)+++class StaticDataset(Dataset):+ r"""A static dataset containing a list of datapoints.+ Ensures that all items adhere to the DataPoint schema.+ This dataset extends :obj:`Dataset` from PyTorch and should+ be used when its size is fixed at runtime.++ This class can initialize from Hugging Face Datasets,+ PyTorch Datasets, JSON file paths, or lists of dictionaries,+ converting them into a consistent internal format.+ """++ def __init__(+ self,+ data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]],+ seed: int = 42,+ min_samples: int = 1,+ strict: bool = False,+ **kwargs,+ ):+ r"""Initialize the static dataset and validate integrity.++ Args:+ data (Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]):+ Input data, which can be one of the following:+ - A Hugging Face Dataset (:obj:`HFDataset`).+ - A PyTorch Dataset (:obj:`torch.utils.data.Dataset`).+ - A :obj:`Path` object representing a JSON file.+ - A list of dictionaries with :obj:`DataPoint`-compatible+ fields.+ seed (int): Random seed for reproducibility.+ (default: :obj:`42`)+ min_samples (int): Minimum required number of samples.+ (default: :obj:`1`)+ strict (bool): Whether to raise an error on invalid+ datapoints (:obj:`True`) or skip/filter them (:obj:`False`).+ (default: :obj:`False`)+ **kwargs: Additional dataset parameters.++ Raises:+ TypeError: If the input data type is unsupported.+ ValueError: If the dataset contains fewer than :obj:`min_samples`+ datapoints or if validation fails.+ FileNotFoundError: If the specified JSON file path does not exist.+ json.JSONDecodeError: If the JSON file contains invalid formatting.+ """++ # Store all parameters in metadata dict for compatibility+ self._metadata = {+ **kwargs,+ }+ self._rng = random.Random(seed)+ self._strict = strict++ self.data: List[DataPoint] = self._init_data(data)+ self._length = len(self.data)++ if self._length < min_samples:+ raise ValueError(+ "The dataset does not contain enough samples. "+ f"Need {max(0, min_samples)}, got {self._length}"+ )++ def _init_data(+ self, data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]+ ) -> List[DataPoint]:+ r"""Convert input data from various formats into a list of+ :obj:`DataPoint` instances.++ Args:+ data (Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]): Input+ dataset in one of the supported formats.++ Returns:+ List[DataPoint]: A list of validated :obj:`DataPoint`+ instances.++ Raises:+ TypeError: If the input data type is unsupported.+ """++ if isinstance(data, HFDataset):+ raw_data = self._init_from_hf_dataset(data)+ elif isinstance(data, Dataset):+ raw_data = self._init_from_pytorch_dataset(data)+ elif isinstance(data, Path):+ raw_data = self._init_from_json_path(data)+ elif isinstance(data, list):+ raw_data = self._init_from_list(data)+ else:+ raise TypeError("Unsupported data type")++ def create_datapoint(+ item: Dict[str, Any], idx: int+ ) -> Optional[DataPoint]:+ # Add type checks for required fields to make mypy happy+ question = item.get('question')+ if not isinstance(question, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'question': "+ f"expected str, got {type(question)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: invalid 'question'"+ )+ return None++ rationale = item.get('rationale')+ if not isinstance(rationale, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'rationale': "+ f"expected str, got {type(rationale)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: invalid 'rationale'"+ )+ return None++ final_answer = item.get('final_answer')+ if not isinstance(final_answer, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'final_answer': "+ f"expected str, got {type(final_answer)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: "+ "invalid 'final_answer'"+ )+ return None++ try:+ return DataPoint(+ question=question,+ rationale=rationale,+ final_answer=final_answer,+ metadata=item.get('metadata'),+ )+ except ValidationError as e:+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} validation error: {e}"+ )+ else:+ logger.warning(+ f"Skipping invalid sample at index {idx} "+ f"due to validation error: {e}"+ )+ return None++ unfiltered_data = [+ create_datapoint(item, i) for i, item in enumerate(raw_data)+ ]+ return [dp for dp in unfiltered_data if dp is not None]++ def __len__(self) -> int:+ r"""Return the size of the dataset."""+ return self._length++ def __getitem__(self, idx: int) -> DataPoint:+ r"""Retrieve a datapoint by index.++ Args:+ idx (int): Index of the datapoint.++ Returns:+ DataPoint: The datapoint corresponding to the given index.++ Raises:+ IndexError: If :obj:`idx` is out of bounds (negative or greater+ than dataset length - 1).+ """++ if idx < 0 or idx >= self._length:+ raise IndexError(+ f"Index {idx} out of bounds for dataset of size {self._length}"+ )+ return self.data[idx]++ def sample(self) -> DataPoint:+ r"""Sample a random datapoint from the dataset.++ Returns:+ DataPoint: A randomly sampled :obj:`DataPoint`.++ Raises:+ RuntimeError: If the dataset is empty and no samples can be drawn.+ """++ if self._length == 0:+ raise RuntimeError("Dataset is empty, cannot sample.")+ idx = self._rng.randint(0, self._length - 1)+ return self[idx]++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Retrieve dataset metadata.++ Returns:+ Dict[str, Any]: A copy of the dataset metadata dictionary.+ """++ return self._metadata.copy()++ def _init_from_hf_dataset(self, data: HFDataset) -> List[Dict[str, Any]]:+ r"""Convert a Hugging Face dataset into a list of dictionaries.++ Args:+ data (HFDataset): A Hugging Face dataset.++ Returns:+ List[Dict[str, Any]]: A list of dictionaries representing+ the dataset, where each dictionary corresponds to a datapoint.+ """+ return [dict(item) for item in data]++ def _init_from_pytorch_dataset(+ self, data: Dataset+ ) -> List[Dict[str, Any]]:+ r"""Convert a PyTorch dataset into a list of dictionaries.++ Args:+ data (Dataset): A PyTorch dataset.++ Returns:+ List[Dict[str, Any]]: A list of dictionaries representing+ the dataset.++ Raises:+ TypeError: If the dataset does not implement :obj:`__len__()`+ or contains non-dictionary elements.+ """+ if not isinstance(data, Sized):+ raise TypeError(+ f"{type(data).__name__} does not implement `__len__()`."+ )+ raw_data = []++ for i in range(len(data)):+ item = data[i]+ if not isinstance(item, dict):+ raise TypeError(+ f"Item at index {i} is not a dict: "+ f"got {type(item).__name__}"+ )+ raw_data.append(dict(item))+ return raw_data++ def _init_from_json_path(self, data: Path) -> List[Dict[str, Any]]:+ r"""Load and parse a dataset from a JSON file.++ Args:+ data (Path): Path to the JSON file.++ Returns:+ List[Dict[str, Any]]: A list of datapoint dictionaries.++ Raises:+ FileNotFoundError: If the specified JSON file does not exist.+ ValueError: If the JSON content is not a list of dictionaries.+ json.JSONDecodeError: If the JSON file has invalid formatting.+ """++ if not data.exists():+ raise FileNotFoundError(f"JSON file not found: {data}")+ try:+ logger.debug(f"Loading JSON from {data}")+ with data.open('r', encoding='utf-8') as f:+ loaded_data = json.load(f)+ logger.info(+ f"Successfully loaded {len(loaded_data)} items from {data}"+ )+ except json.JSONDecodeError as e:+ raise ValueError(f"Invalid JSON in file {data}: {e}")+ if not isinstance(loaded_data, list):+ raise ValueError("JSON file must contain a list of dictionaries")+ for i, item in enumerate(loaded_data):+ if not isinstance(item, dict):+ raise ValueError(+ f"Expected a dictionary at index {i}, "+ f"got {type(item).__name__}"+ )+ return loaded_data++ def _init_from_list(+ self, data: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Validate and convert a list of dictionaries into a dataset.++ Args:+ data (List[Dict[str, Any]]): A list of dictionaries where+ each dictionary must be a valid :obj:`DataPoint`.++ Returns:+ List[Dict[str, Any]]: The validated list of dictionaries.++ Raises:+ ValueError: If any item in the list is not a dictionary.+ """+ for i, item in enumerate(data):+ if not isinstance(item, dict):+ raise ValueError(+ f"Expected a dictionary at index {i}, "+ f"got {type(item).__name__}"+ )+ return datadiff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex 6eadf5630..5e7390771 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -14,9 +14,9 @@from abc import abstractmethod-from typing import Any, Dict, Optional, Tuple+from typing import Any, Dict, Optional, Tuple, Union-from camel.datasets.base import DataPoint, GenerativeDataset, StaticDataset+from camel.datasets import BaseGenerator, DataPoint, StaticDatasetfrom camel.extractors.base import BaseExtractorfrom camel.logger import get_loggerfrom camel.verifiers.base import (@@ -55,7 +55,7 @@ class SingleStepEnv:def __init__(self,- dataset: StaticDataset | GenerativeDataset,+ dataset: Union[StaticDataset, BaseGenerator],verifier: BaseVerifier,extractor: BaseExtractor,**kwargs,diff --git a/examples/dataset/generative_dataset.py b/examples/dataset/generative_dataset.pyindex 999582e89..b306f70a2 100644--- a/examples/dataset/generative_dataset.py+++ b/examples/dataset/generative_dataset.py@@ -16,7 +16,7 @@ import asynciofrom pathlib import Pathfrom typing import ClassVar-from camel.datasets import GenerativeDataset, StaticDataset+from camel.datasets import FewShotGenerator, StaticDatasetfrom camel.logger import get_loggerfrom camel.verifiers import PythonVerifier@@ -66,16 +66,16 @@ async def main():agent = MockChatAgent()- generative_dataset = GenerativeDataset(+ generator = FewShotGenerator(seed_dataset=seed_dataset, verifier=verifier, agent=agent)- new_data = await generative_dataset.generate_new(n=2)+ new_data = await generator.generate_new(n=2)for dp in new_data:print(dp)- generative_dataset.save_to_jsonl(Path("generated_data.jsonl"))+ generator.save_to_jsonl(Path("generated_data.jsonl"))await verifier.cleanup()diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pyindex 5c4f865b9..1675ac09c 100644--- a/test/datasets/test_base_dataset.py+++ b/test/datasets/test_base_dataset.py@@ -22,9 +22,9 @@ from datasets import Dataset as HFDatasetfrom pydantic import ValidationErrorfrom torch.utils.data import Dataset-from camel.datasets.base import (+from camel.datasets import (DataPoint,- GenerativeDataset,+ FewShotGenerator,StaticDataset,)@@ -51,12 +51,18 @@ def test_datapoint_validation():with pytest.raises(ValidationError):DataPoint(question='What is 2+2?')- # This should now work since rationale is optionaldatapoint = DataPoint(question='What is 2+2?', final_answer='4')assert datapoint.question == 'What is 2+2?'assert datapoint.rationale is Noneassert datapoint.final_answer == '4'+ datapoint = DataPoint(+ question='What is 2+2?', rationale='Addition', final_answer='4'+ )+ assert datapoint.question == 'What is 2+2?'+ assert datapoint.rationale == 'Addition'+ assert datapoint.final_answer == '4'+def test_datapoint_to_dict():r"""Test converting DataPoint to dictionary."""@@ -813,8 +819,8 @@ def test_static_dataset_methods():@pytest.mark.asyncio-async def test_generative_dataset():- r"""Test GenerativeDataset with mocked components."""+async def test_few_shot_generator():+ r"""Test FewShotGenerator with mocked components."""mock_static_dataset = MagicMock(spec=StaticDataset)mock_static_dataset.__len__.return_value = 5mock_static_dataset.__getitem__.side_effect = lambda i: DataPoint(@@ -839,8 +845,8 @@ async def test_generative_dataset():])- # Create GenerativeDataset with mocks- dataset = GenerativeDataset(+ # Create FewShotGenerator with mocks+ dataset = FewShotGenerator(seed_dataset=mock_static_dataset,verifier=mock_verifier,agent=mock_agent,@@ -861,7 +867,7 @@ async def test_generative_dataset():@pytest.mark.asyncioasync def test_generate_new():- r"""Test GenerativeDataset with mocked components."""+ r"""Test FewShotGenerator with mocked components."""mock_static_dataset = MagicMock(spec=StaticDataset)mock_static_dataset.__len__.return_value = 5mock_static_dataset.__getitem__.side_effect = lambda i: DataPoint(@@ -925,8 +931,8 @@ async def test_generate_new():),]- # Create GenerativeDataset with mocks- dataset = GenerativeDataset(+ # Create FewShotGenerator with mocks+ dataset = FewShotGenerator(seed_dataset=mock_static_dataset,verifier=mock_verifier,agent=mock_agent,@@ -965,7 +971,7 @@ async def test_generate_new():@pytest.mark.asyncioasync def test_generate_new_with_max_retries():- r"""Test GenerativeDataset retry mechanism with+ r"""Test FewShotGenerator retry mechanism withmax_retries=2 and a sequence of one correct, three wrong,and one correct output, expecting failure."""@@ -1049,7 +1055,7 @@ async def test_generate_new_with_max_retries():),]- dataset = GenerativeDataset(+ dataset = FewShotGenerator(seed_dataset=mock_static_dataset,verifier=mock_verifier,agent=mock_agent,@@ -1075,8 +1081,8 @@ async def test_generate_new_with_max_retries():@pytest.mark.asyncio-async def test_generative_dataset_save_to_jsonl(tmp_path):- r"""Test GenerativeDataset's save_to_jsonl method with mocked data.+async def test_few_shot_generator_save_to_jsonl(tmp_path):+ r"""Test FewShotGenerator's save_to_jsonl method with mocked data.This test verifies that:1. Data can be successfully saved to a JSONL file.@@ -1113,8 +1119,8 @@ async def test_generative_dataset_save_to_jsonl(tmp_path):])- # Create GenerativeDataset with mocks- dataset = GenerativeDataset(+ # Create FewShotGenerator with mocks+ dataset = FewShotGenerator(seed_dataset=mock_static_dataset,verifier=mock_verifier,agent=mock_agent,