Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

refactor Seed Dataset to improve compatibility and simplify usage (#1734)

Apokryphosx578dae28e291
diff --git a/camel/datasets/base.py b/camel/datasets/base.py
index e085eeb46..bcacd0e34 100644
--- a/camel/datasets/base.py
+++ b/camel/datasets/base.py
@@ -12,14 +12,17 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
import os
import random
+from pathlib import Path
from typing import (
Any,
Callable,
Dict,
List,
Optional,
+ Sized,
TypeVar,
Union,
)
@@ -326,42 +329,241 @@ class SeedDataset(BaseDataset):
r"""A dataset containing validated seed examples for data generation.
Ensures that all items adhere to the DataPoint schema.
- This class is used to initialize a dataset from a list of dictionary items,
- validating each against the DataPoint schema.
+ This class can initialize from Hugging Face Datasets,
+ PyTorch Datasets, JSON file paths, or lists of dictionaries,
+ converting them into a consistent internal format.
"""
def __init__(
self,
- data: List[Dict[str, str]],
+ data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]],
cache_dir: Optional[str] = None,
+ seed: Optional[int] = None,
min_samples: int = 1,
+ strict: bool = False,
**kwargs,
):
- r"""Initialize the seed dataset.
+ r"""Initialize the seed dataset and validate integrity.
Args:
- data (List[Dict[str, str]]): List of dictionary items to create the
- dataset from.
- cache_dir (Optional[str]): Directory to cache dataset files.
- (default: :obj:`None`)
+ data (Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]):
+ Input data, which can be:
+ - A Hugging Face Dataset (HFDataset)
+ - A PyTorch Dataset (torch.utils.data.Dataset)
+ - A Path object representing the path to a JSON file
+ - A list of dictionaries with DataPoint-compatible fields
+ seed (Optional[int]): Seed for reproducibility.
+ (default: :obj:`1`)
min_samples (int): Minimum number of samples required.
(default: :obj:`1`)
+ strict (bool): Whether to raise an error on invalid datapoints
+ (True) or skip/filter them (False). (default: False)
**kwargs: Additional dataset parameters.
Raises:
- ValueError: If dataset size is less than min_samples or if sample
- validation fails.
+ TypeError: If the data type is not supported.
+ ValueError: If dataset size is less than min_samples or
+ if sample validation fails.
+ FileNotFoundError: If the JSON file path doesn't exist.
+ json.JSONDecodeError: If the JSON file is invalid.
"""
- if len(data) < min_samples:
+ # Initialize BaseDataset with empty data, we'll populate it ourselves
+ super().__init__(data=[], cache_dir=cache_dir, **kwargs)
+
+ self._rng = random.Random(seed)
+ self._strict = strict
+
+ # Type checking and conversion into list of dicts to have a
+ # consistent internal format. Since Seed Dataset should be
+ # small, we can load it entirely into memory
+
+ self.data: List[DataPoint] = self._init_data(data)
+ self._length = len(self.data)
+
+ if self._length < min_samples:
raise ValueError(
- f"Seed dataset must contain at least {min_samples} samples."
+ "The dataset does not contain enough samples. "
+ f"Need {max(0, min_samples)}, got {self._length}"
)
- super().__init__(
- data=data,
- cache_dir=cache_dir,
- **kwargs,
- )
+ def _init_data(
+ self, data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]
+ ) -> List[DataPoint]:
+ if isinstance(data, HFDataset):
+ raw_data = self._init_from_hf_dataset(data)
+ elif isinstance(data, Dataset):
+ raw_data = self._init_from_pytorch_dataset(data)
+ elif isinstance(data, Path):
+ raw_data = self._init_from_json_path(data)
+ elif isinstance(data, list):
+ raw_data = self._init_from_list(data)
+ else:
+ raise TypeError("Unsupported data type")
+
+ def create_datapoint(
+ item: Dict[str, Any], idx: int
+ ) -> Optional[DataPoint]:
+ # Add type checks for required fields to make mypy happy
+ question = item.get('question')
+ if not isinstance(question, str):
+ if self._strict:
+ raise ValueError(
+ f"Sample at index {idx} has invalid 'question': "
+ f"expected str, got {type(question)}"
+ )
+ else:
+ logger.warning(
+ f"Skipping sample at index {idx}: invalid 'question'"
+ )
+ return None
+
+ rationale = item.get('rationale')
+ if not isinstance(rationale, str):
+ if self._strict:
+ raise ValueError(
+ f"Sample at index {idx} has invalid 'rationale': "
+ f"expected str, got {type(rationale)}"
+ )
+ else:
+ logger.warning(
+ f"Skipping sample at index {idx}: invalid 'rationale'"
+ )
+ return None
+
+ final_answer = item.get('final_answer')
+ if not isinstance(final_answer, str):
+ if self._strict:
+ raise ValueError(
+ f"Sample at index {idx} has invalid 'final_answer': "
+ f"expected str, got {type(final_answer)}"
+ )
+ else:
+ logger.warning(
+ f"Skipping sample at index {idx}: "
+ "invalid 'final_answer'"
+ )
+ return None
+
+ try:
+ return DataPoint(
+ question=question,
+ rationale=rationale,
+ final_answer=final_answer,
+ metadata=item.get('metadata'),
+ difficulty=item.get('difficulty'),
+ )
+ except ValidationError as e:
+ if self._strict:
+ raise ValueError(
+ f"Sample at index {idx} validation error: {e}"
+ )
+ else:
+ logger.warning(
+ f"Skipping invalid sample at index {idx} "
+ f"due to validation error: {e}"
+ )
+ return None
+
+ unfiltered_data = [
+ create_datapoint(item, i) for i, item in enumerate(raw_data)
+ ]
+ return [dp for dp in unfiltered_data if dp is not None]
+
+ def __len__(self) -> int:
+ r"""Return the size of the dataset."""
+ return self._length
+
+ def __getitem__(self, idx: int) -> DataPoint:
+ r"""Get an item from the dataset.
+
+ Args:
+ idx (int): Index of the item to get.
+
+ Returns:
+ DataPoint: DataPoint from the dataset with the given index.
+
+ Raises:
+ IndexError: If idx is out of bounds.
+ """
+ if idx < 0 or idx >= self._length:
+ raise IndexError(
+ f"Index {idx} out of bounds for dataset of size {self._length}"
+ )
+ return self.data[idx]
+
+ def sample(self) -> DataPoint:
+ r"""Sample a random datapoint from the dataset.
+
+ Returns:
+ DataPoint: A randomly sampled DataPoint.
+
+ Raises:
+ RuntimeError: If the dataset is empty.
+ """
+ if self._length == 0:
+ raise RuntimeError("Dataset is empty, cannot sample.")
+ idx = self._rng.randint(0, self._length - 1)
+ return self[idx]
+
+ @property
+ def metadata(self) -> Dict[str, Any]:
+ r"""Get dataset metadata."""
+ return self._metadata.copy()
+
+ def _init_from_hf_dataset(self, data: HFDataset) -> List[Dict[str, Any]]:
+ return [dict(item) for item in data]
+
+ def _init_from_pytorch_dataset(
+ self, data: Dataset
+ ) -> List[Dict[str, Any]]:
+ if not isinstance(data, Sized):
+ raise TypeError(
+ f"{type(data).__name__} does not implement `__len__()`."
+ )
+ raw_data = []
+
+ for i in range(len(data)):
+ item = data[i]
+ if not isinstance(item, dict):
+ raise TypeError(
+ f"Item at index {i} is not a dict: "
+ f"got {type(item).__name__}"
+ )
+ raw_data.append(dict(item))
+ return raw_data
+
+ def _init_from_json_path(self, data: Path) -> List[Dict[str, Any]]:
+ if not data.exists():
+ raise FileNotFoundError(f"JSON file not found: {data}")
+ try:
+ logger.debug(f"Loading JSON from {data}")
+ with data.open('r', encoding='utf-8') as f:
+ loaded_data = json.load(f)
+ logger.info(
+ f"Successfully loaded {len(loaded_data)} items from {data}"
+ )
+ except json.JSONDecodeError as e:
+ raise ValueError(f"Invalid JSON in file {data}: {e}")
+ if not isinstance(loaded_data, list):
+ raise ValueError("JSON file must contain a list of dictionaries")
+ for i, item in enumerate(loaded_data):
+ if not isinstance(item, dict):
+ raise ValueError(
+ f"Expected a dictionary at index {i}, "
+ f"got {type(item).__name__}"
+ )
+ return loaded_data
+
+ def _init_from_list(
+ self, data: List[Dict[str, Any]]
+ ) -> List[Dict[str, Any]]:
+ for i, item in enumerate(data):
+ if not isinstance(item, dict):
+ raise ValueError(
+ f"Expected a dictionary at index {i}, "
+ f"got {type(item).__name__}"
+ )
+ return data
class SyntheticDataset(BaseDataset):
diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.py
index 83db9ad60..3b4f611da 100644
--- a/test/datasets/test_base_dataset.py
+++ b/test/datasets/test_base_dataset.py
@@ -12,13 +12,16 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
import tempfile
+from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import pytest
import torch
from datasets import Dataset as HFDataset
from pydantic import ValidationError
+from torch.utils.data import Dataset
from camel.datasets.base import (
BaseDataset,
@@ -191,13 +194,731 @@ def test_base_dataset_metadata():
assert metadata['custom_param'] == 'value'
-def test_seed_dataset_init(sample_data):
- r"""Test SeedDataset initialization."""
- dataset = SeedDataset(data=sample_data, min_samples=1)
- assert dataset._raw_data == sample_data
+def test_seed_dataset_init_from_hf_dataset():
+ r"""
+ Test SeedDataset initialization from a
+ Hugging Face Dataset with various scenarios.
+ """
- with pytest.raises(ValueError):
- SeedDataset(data=sample_data, min_samples=3)
+ # **Test 1: Initialization with proper mock data (only required fields)**
+ valid_data = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Addition of 2 and 2.",
+ "final_answer": "4",
+ },
+ {
+ "question": "Is the sky blue?",
+ "rationale": "Observation of clear weather.",
+ "final_answer": "yes",
+ },
+ ]
+ hf_valid = HFDataset.from_list(valid_data)
+
+ dataset = SeedDataset(data=hf_valid, min_samples=1, strict=True)
+
+ # Verify the dataset initialized correctly
+ assert len(dataset) == 2, "Dataset should contain 2 items."
+ assert isinstance(
+ dataset[0], DataPoint
+ ), "Items should be DataPoint instances."
+ assert (
+ dataset[0].question == "What is 2 + 2?"
+ ), "Question should match input."
+ assert (
+ dataset[0].rationale == "Addition of 2 and 2."
+ ), "Rationale should match input."
+ assert dataset[0].final_answer == "4", "Final answer should match input."
+ assert (
+ dataset[1].question == "Is the sky blue?"
+ ), "Second question should match input."
+ assert (
+ dataset[1].rationale == "Observation of clear weather."
+ ), "Second rationale should match input."
+ assert (
+ dataset[1].final_answer == "yes"
+ ), "Second final answer should match input."
+ assert (
+ dataset[0].difficulty is None
+ ), "Difficulty should be None when not provided."
+ assert (
+ dataset[0].metadata is None
+ ), "Metadata should be None when not provided."
+
+ # **Test 2: Initialization with invalid data**
+ # Sub-test 2a: Missing required field with strict=True
+ invalid_data_missing = [
+ {"question": "What is 3 + 3?", "final_answer": "6"}
+ ]
+ hf_invalid_missing = HFDataset.from_list(invalid_data_missing)
+ with pytest.raises(
+ ValueError,
+ match="Sample at index 0 has invalid 'rationale': "
+ "expected str, got <class 'NoneType'>",
+ ):
+ SeedDataset(data=hf_invalid_missing, min_samples=1, strict=True)
+
+ # Sub-test 2b: Missing required field with strict=False
+ dataset_non_strict = SeedDataset(
+ data=hf_invalid_missing, min_samples=0, strict=False
+ )
+ assert (
+ len(dataset_non_strict) == 0
+ ), "Invalid items should be filtered out in non-strict mode."
+
+ # Sub-test 2c: Empty dataset with min_samples=1
+ hf_empty = HFDataset.from_list([])
+ with pytest.raises(
+ ValueError,
+ match="The dataset does not contain enough samples. Need 1, got 0",
+ ):
+ SeedDataset(data=hf_empty, min_samples=1, strict=True)
+
+ # Sub-test 2d: Empty dataset with min_samples=0
+ dataset_empty = SeedDataset(data=hf_empty, min_samples=0, strict=True)
+ assert (
+ len(dataset_empty) == 0
+ ), "Empty dataset should have zero length when min_samples=0."
+
+ # **Test 3: Initialization with optional fields and additional fields**
+ data_with_optional = [
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France is a country in Europe.",
+ "final_answer": "Paris",
+ "difficulty": "easy",
+ "metadata": {"source": "geography", "id": 123},
+ "extra_field": "this should be ignored", # Not in DataPoint
+ }
+ ]
+ hf_optional = HFDataset.from_list(data_with_optional)
+ dataset_optional = SeedDataset(
+ data=hf_optional, min_samples=1, strict=True
+ )
+
+ # Verify the dataset initialized correctly
+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."
+ assert isinstance(
+ dataset_optional[0], DataPoint
+ ), "Item should be a DataPoint instance."
+ assert (
+ dataset_optional[0].question == "What is the capital of France?"
+ ), "Question should match."
+ assert (
+ dataset_optional[0].rationale == "France is a country in Europe."
+ ), "Rationale should match."
+ assert (
+ dataset_optional[0].final_answer == "Paris"
+ ), "Final answer should match."
+ assert (
+ dataset_optional[0].difficulty == "easy"
+ ), "Difficulty should be set correctly."
+ assert dataset_optional[0].metadata == {
+ "source": "geography",
+ "id": 123,
+ }, "Metadata should match."
+ assert (
+ "extra_field" not in dataset_optional[0].__dict__
+ ), "Extra field should not be present in DataPoint."
+
+
+def test_seed_dataset_init_from_pytorch_dataset():
+ r"""
+ Test SeedDataset initialization from a
+ PyTorch Dataset with various scenarios.
+ """
+
+ class MockPyTorchDataset(Dataset):
+ def __init__(self, data_list):
+ self.data = data_list
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, idx):
+ return self.data[idx]
+
+ # **Test 1: Initialization with proper mock data (only required fields)**
+ valid_data = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Addition of 2 and 2.",
+ "final_answer": "4",
+ },
+ {
+ "question": "Is the sky blue?",
+ "rationale": "Observation of clear weather.",
+ "final_answer": "yes",
+ },
+ ]
+ pytorch_valid = MockPyTorchDataset(valid_data)
+ dataset = SeedDataset(data=pytorch_valid, min_samples=1, strict=True)
+
+ assert len(dataset) == 2, "Dataset should contain 2 items."
+ assert isinstance(
+ dataset[0], DataPoint
+ ), "Items should be DataPoint instances."
+ assert (
+ dataset[0].question == "What is 2 + 2?"
+ ), "Question should match input."
+ assert (
+ dataset[0].rationale == "Addition of 2 and 2."
+ ), "Rationale should match input."
+ assert dataset[0].final_answer == "4", "Final answer should match input."
+ assert (
+ dataset[1].question == "Is the sky blue?"
+ ), "Second question should match input."
+ assert (
+ dataset[1].rationale == "Observation of clear weather."
+ ), "Second rationale should match input."
+ assert (
+ dataset[1].final_answer == "yes"
+ ), "Second final answer should match input."
+ assert (
+ dataset[0].difficulty is None
+ ), "Difficulty should be None when not provided."
+ assert (
+ dataset[0].metadata is None
+ ), "Metadata should be None when not provided."
+
+ # **Test 2: Initialization with invalid data**
+ # Sub-test 2a: Missing required field with strict=True
+ invalid_data_missing = [
+ {
+ "question": "What is 3 + 3?",
+ "final_answer": "6",
+ } # Missing rationale
+ ]
+ pytorch_invalid_missing = MockPyTorchDataset(invalid_data_missing)
+ with pytest.raises(
+ ValueError,
+ match="Sample at index 0 has invalid 'rationale': "
+ "expected str, got <class 'NoneType'>",
+ ):
+ SeedDataset(data=pytorch_invalid_missing, min_samples=1, strict=True)
+
+ # Sub-test 2b: Missing required field with strict=False
+ dataset_non_strict = SeedDataset(
+ data=pytorch_invalid_missing, min_samples=0, strict=False
+ )
+ assert (
+ len(dataset_non_strict) == 0
+ ), "Invalid items should be filtered out in non-strict mode."
+
+ # Sub-test 2c: Empty dataset with min_samples=1
+ pytorch_empty = MockPyTorchDataset([])
+ with pytest.raises(
+ ValueError,
+ match="The dataset does not contain enough samples. Need 1, got 0",
+ ):
+ SeedDataset(data=pytorch_empty, min_samples=1, strict=True)
+
+ # Sub-test 2d: Empty dataset with min_samples=0
+ dataset_empty = SeedDataset(data=pytorch_empty, min_samples=0, strict=True)
+ assert (
+ len(dataset_empty) == 0
+ ), "Empty dataset should have zero length when min_samples=0."
+
+ # **Test 3: Initialization with optional fields and additional fields**
+ data_with_optional = [
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France is a country in Europe.",
+ "final_answer": "Paris",
+ "difficulty": "easy",
+ "metadata": {"source": "geography", "id": 123},
+ "extra_field": "this should be ignored", # Not in DataPoint
+ }
+ ]
+ pytorch_optional = MockPyTorchDataset(data_with_optional)
+ dataset_optional = SeedDataset(
+ data=pytorch_optional, min_samples=1, strict=True
+ )
+
+ # Verify the dataset initialized correctly
+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."
+ assert isinstance(
+ dataset_optional[0], DataPoint
+ ), "Item should be a DataPoint instance."
+ assert (
+ dataset_optional[0].question == "What is the capital of France?"
+ ), "Question should match."
+ assert (
+ dataset_optional[0].rationale == "France is a country in Europe."
+ ), "Rationale should match."
+ assert (
+ dataset_optional[0].final_answer == "Paris"
+ ), "Final answer should match."
+ assert (
+ dataset_optional[0].difficulty == "easy"
+ ), "Difficulty should be set correctly."
+ assert dataset_optional[0].metadata == {
+ "source": "geography",
+ "id": 123,
+ }, "Metadata should match."
+ assert (
+ "extra_field" not in dataset_optional[0].__dict__
+ ), "Extra field should not be present in DataPoint."
+
+ # **Test 4: PyTorch-specific edge cases**
+ # Sub-test 4a: Dataset without __len__
+ class NoLenDataset(Dataset):
+ def __getitem__(self, idx):
+ return {
+ "question": "Test",
+ "rationale": "Test",
+ "final_answer": "Test",
+ }
+
+ with pytest.raises(TypeError) as excinfo:
+ SeedDataset(NoLenDataset(), min_samples=1, strict=True)
+ assert "does not implement `__len__()`." in str(excinfo.value)
+
+ # Sub-test 4b: Dataset with non-dict items
+ class NonDictDataset(Dataset):
+ def __len__(self):
+ return 1
+
+ def __getitem__(self, idx):
+ return "not a dict"
+
+ with pytest.raises(
+ TypeError, match="Item at index 0 is not a dict: got str"
+ ):
+ SeedDataset(data=NonDictDataset(), min_samples=1, strict=True)
+
+
+def test_seed_dataset_init_from_json_file():
+ r"""
+ Test SeedDataset initialization from a JSON file with various scenarios.
+ """
+
+ # **Test 1: Initialization with a valid JSON file (only required fields)**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ valid_data = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Addition of 2 and 2.",
+ "final_answer": "4",
+ },
+ {
+ "question": "Is the sky blue?",
+ "rationale": "Observation of clear weather.",
+ "final_answer": "yes",
+ },
+ ]
+ json.dump(valid_data, tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ dataset = SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+ assert len(dataset) == 2, "Dataset should contain 2 items."
+ assert isinstance(
+ dataset[0], DataPoint
+ ), "Items should be DataPoint instances."
+ assert (
+ dataset[0].question == "What is 2 + 2?"
+ ), "Question should match input."
+ assert (
+ dataset[0].rationale == "Addition of 2 and 2."
+ ), "Rationale should match input."
+ assert dataset[0].final_answer == "4", "Final answer should match input."
+ assert (
+ dataset[1].question == "Is the sky blue?"
+ ), "Second question should match input."
+ assert (
+ dataset[1].rationale == "Observation of clear weather."
+ ), "Second rationale should match input."
+ assert (
+ dataset[1].final_answer == "yes"
+ ), "Second final answer should match input."
+ assert (
+ dataset[0].difficulty is None
+ ), "Difficulty should be None when not provided."
+ assert (
+ dataset[0].metadata is None
+ ), "Metadata should be None when not provided."
+
+ # **Test 2: Initialization with invalid JSON file (malformed JSON)**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ tmp_file.write(
+ '[{"question": "Test", "rationale": "Test", "final_answer": "Test"'
+ ) # Missing closing bracket
+ tmp_file_path = Path(tmp_file.name)
+
+ with pytest.raises(ValueError, match="Invalid JSON in file"):
+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+
+ # **Test 3: Initialization with JSON file containing non-list data**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ json.dump(
+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},
+ tmp_file,
+ )
+ tmp_file_path = Path(tmp_file.name)
+
+ with pytest.raises(
+ ValueError, match="JSON file must contain a list of dictionaries"
+ ):
+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+
+ # **Test 4: Initialization with JSON file containing non-dict items**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ invalid_data = [
+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},
+ "not a dict",
+ ]
+ json.dump(invalid_data, tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ with pytest.raises(
+ ValueError, match="Expected a dictionary at index 1, got str"
+ ):
+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+
+ # **Test 5: Initialization with a missing JSON file**
+ missing_file_path = Path("non_existent_file.json")
+ with pytest.raises(FileNotFoundError, match="JSON file not found:"):
+ SeedDataset(data=missing_file_path, min_samples=1, strict=True)
+
+ # **Test 6: Initialization with an empty JSON file**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ json.dump([], tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ # Sub-test 6a: Empty dataset with min_samples=1
+ with pytest.raises(
+ ValueError,
+ match="The dataset does not contain enough samples. Need 1, got 0",
+ ):
+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+
+ # Sub-test 6b: Empty dataset with min_samples=0
+ dataset_empty = SeedDataset(data=tmp_file_path, min_samples=0, strict=True)
+ assert (
+ len(dataset_empty) == 0
+ ), "Empty dataset should have zero length when min_samples=0."
+
+ # **Test 7: Initialization with optional fields and additional fields**
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ data_with_optional = [
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France is a country in Europe.",
+ "final_answer": "Paris",
+ "difficulty": "easy",
+ "metadata": {"source": "geography", "id": 123},
+ "extra_field": "this should be ignored",
+ }
+ ]
+ json.dump(data_with_optional, tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ dataset_optional = SeedDataset(
+ data=tmp_file_path, min_samples=1, strict=True
+ )
+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."
+ assert isinstance(
+ dataset_optional[0], DataPoint
+ ), "Item should be a DataPoint instance."
+ assert (
+ dataset_optional[0].question == "What is the capital of France?"
+ ), "Question should match."
+ assert (
+ dataset_optional[0].rationale == "France is a country in Europe."
+ ), "Rationale should match."
+ assert (
+ dataset_optional[0].final_answer == "Paris"
+ ), "Final answer should match."
+ assert (
+ dataset_optional[0].difficulty == "easy"
+ ), "Difficulty should be set correctly."
+ assert dataset_optional[0].metadata == {
+ "source": "geography",
+ "id": 123,
+ }, "Metadata should match."
+ assert (
+ "extra_field" not in dataset_optional[0].__dict__
+ ), "Extra field should not be present in DataPoint."
+
+ # **Test 8: JSON-specific edge cases**
+ # Sub-test 8a: Invalid data with strict=True
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ invalid_data = [
+ {
+ "question": "What is 3 + 3?",
+ "final_answer": "6",
+ } # Missing rationale
+ ]
+ json.dump(invalid_data, tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ with pytest.raises(
+ ValueError,
+ match="Sample at index 0 has invalid 'rationale': "
+ "expected str, got <class 'NoneType'>",
+ ):
+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)
+
+ # Sub-test 8b: Invalid data with strict=False
+ with tempfile.NamedTemporaryFile(
+ mode='w', suffix='.json', delete=False
+ ) as tmp_file:
+ invalid_data = [
+ {
+ "question": "What is 3 + 3?",
+ "final_answer": "6",
+ } # Missing rationale
+ ]
+ json.dump(invalid_data, tmp_file)
+ tmp_file_path = Path(tmp_file.name)
+
+ dataset_non_strict = SeedDataset(
+ data=tmp_file_path, min_samples=0, strict=False
+ )
+ assert (
+ len(dataset_non_strict) == 0
+ ), "Invalid items should be filtered out in non-strict mode."
+
+
+def test_seed_dataset_init_from_list():
+ r"""
+ Test SeedDataset initialization from a list of
+ dictionaries with various scenarios.
+ """
+
+ # **Test 1: Initialization with a valid list (only required fields)**
+ valid_data = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Addition of 2 and 2.",
+ "final_answer": "4",
+ },
+ {
+ "question": "Is the sky blue?",
+ "rationale": "Observation of clear weather.",
+ "final_answer": "yes",
+ },
+ ]
+ dataset = SeedDataset(data=valid_data, min_samples=1, strict=True)
+ assert len(dataset) == 2, "Dataset should contain 2 items."
+ assert isinstance(
+ dataset[0], DataPoint
+ ), "Items should be DataPoint instances."
+ assert (
+ dataset[0].question == "What is 2 + 2?"
+ ), "Question should match input."
+ assert (
+ dataset[0].rationale == "Addition of 2 and 2."
+ ), "Rationale should match input."
+ assert dataset[0].final_answer == "4", "Final answer should match input."
+ assert (
+ dataset[1].question == "Is the sky blue?"
+ ), "Second question should match input."
+ assert (
+ dataset[1].rationale == "Observation of clear weather."
+ ), "Second rationale should match input."
+ assert (
+ dataset[1].final_answer == "yes"
+ ), "Second final answer should match input."
+ assert (
+ dataset[0].difficulty is None
+ ), "Difficulty should be None when not provided."
+ assert (
+ dataset[0].metadata is None
+ ), "Metadata should be None when not provided."
+
+ # **Test 2: Initialization with a list containing non-dictionary items**
+ invalid_list_data = [
+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},
+ "not a dict",
+ ]
+ with pytest.raises(
+ ValueError, match="Expected a dictionary at index 1, got str"
+ ):
+ SeedDataset(data=invalid_list_data, min_samples=1, strict=True)
+
+ # **Test 3: Initialization with an empty list**
+ empty_data = []
+ # Sub-test 3a: Empty list with min_samples=1
+ with pytest.raises(
+ ValueError,
+ match="The dataset does not contain enough samples. Need 1, got 0",
+ ):
+ SeedDataset(data=empty_data, min_samples=1, strict=True)
+ # Sub-test 3b: Empty list with min_samples=0
+ dataset_empty = SeedDataset(data=empty_data, min_samples=0, strict=True)
+ assert (
+ len(dataset_empty) == 0
+ ), "Empty dataset should have zero length when min_samples=0."
+
+ # **Test 4: Initialization with dictionaries missing required fields**
+ # Sub-test 4a: Missing required field with strict=True
+ invalid_data_missing = [
+ {
+ "question": "What is 3 + 3?",
+ "final_answer": "6",
+ } # Missing rationale
+ ]
+ with pytest.raises(
+ ValueError,
+ match="Sample at index 0 has invalid 'rationale': "
+ "expected str, got <class 'NoneType'>",
+ ):
+ SeedDataset(data=invalid_data_missing, min_samples=1, strict=True)
+ # Sub-test 4b: Missing required field with strict=False
+ dataset_non_strict = SeedDataset(
+ data=invalid_data_missing, min_samples=0, strict=False
+ )
+ assert (
+ len(dataset_non_strict) == 0
+ ), "Invalid items should be filtered out in non-strict mode."
+
+ # **Test 5: Initialization with optional fields and additional fields**
+ data_with_optional = [
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France is a country in Europe.",
+ "final_answer": "Paris",
+ "difficulty": "easy",
+ "metadata": {"source": "geography", "id": 123},
+ "extra_field": "this should be ignored",
+ }
+ ]
+ dataset_optional = SeedDataset(
+ data=data_with_optional, min_samples=1, strict=True
+ )
+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."
+ assert isinstance(
+ dataset_optional[0], DataPoint
+ ), "Item should be a DataPoint instance."
+ assert (
+ dataset_optional[0].question == "What is the capital of France?"
+ ), "Question should match."
+ assert (
+ dataset_optional[0].rationale == "France is a country in Europe."
+ ), "Rationale should match."
+ assert (
+ dataset_optional[0].final_answer == "Paris"
+ ), "Final answer should match."
+ assert (
+ dataset_optional[0].difficulty == "easy"
+ ), "Difficulty should be set correctly."
+ assert dataset_optional[0].metadata == {
+ "source": "geography",
+ "id": 123,
+ }, "Metadata should match."
+ assert (
+ "extra_field" not in dataset_optional[0].__dict__
+ ), "Extra field should not be present in DataPoint."
+
+ # **Test 6: Mixed valid and invalid items with strict=False**
+ mixed_data = [
+ {
+ "question": "Valid question",
+ "rationale": "Valid rationale",
+ "final_answer": "Valid",
+ },
+ {
+ "question": "Invalid question",
+ "final_answer": "Invalid",
+ }, # Missing rationale
+ ]
+ dataset_mixed = SeedDataset(data=mixed_data, min_samples=1, strict=False)
+ assert (
+ len(dataset_mixed) == 1
+ ), "Only valid items should be included in non-strict mode."
+ assert (
+ dataset_mixed[0].question == "Valid question"
+ ), "Only the valid item should be present."
+
+
+def test_seed_dataset_methods():
+ r"""
+ Test the __len__, __getitem__, and sample methods
+ of SeedDataset with a mock dataset.
+ """
+ mock_data = [
+ {
+ "question": "What is 1+1?",
+ "rationale": "Basic addition.",
+ "final_answer": "2",
+ },
+ {
+ "question": "Is the Earth round?",
+ "rationale": "Scientific consensus.",
+ "final_answer": "yes",
+ },
+ {
+ "question": "What is the capital of Japan?",
+ "rationale": "Geography knowledge.",
+ "final_answer": "Tokyo",
+ },
+ {
+ "question": "How many sides does a triangle have?",
+ "rationale": "Definition of a triangle.",
+ "final_answer": "3",
+ },
+ ]
+ dataset = SeedDataset(data=mock_data, min_samples=1, strict=True)
+
+ assert len(dataset) == 4, "Dataset should have 4 items."
+ assert (
+ dataset[0].question == "What is 1+1?"
+ ), "First item question should match."
+ assert (
+ dataset[0].rationale == "Basic addition."
+ ), "First item rationale should match."
+ assert (
+ dataset[0].final_answer == "2"
+ ), "First item final_answer should match."
+ assert (
+ dataset[3].question == "How many sides does a triangle have?"
+ ), "Last item question should match."
+ assert (
+ dataset[3].rationale == "Definition of a triangle."
+ ), "Last item rationale should match."
+ assert (
+ dataset[3].final_answer == "3"
+ ), "Last item final_answer should match."
+ assert isinstance(
+ dataset[0], DataPoint
+ ), "Item should be a DataPoint instance."
+
+ # Test __getitem__ with out-of-bounds indices
+ with pytest.raises(
+ IndexError, match="Index -1 out of bounds for dataset of size 4"
+ ):
+ dataset[-1]
+ with pytest.raises(
+ IndexError, match="Index 4 out of bounds for dataset of size 4"
+ ):
+ dataset[4]
+
+ # Test sample from non-empty dataset
+ sampled_item = dataset.sample()
+ assert (
+ sampled_item in dataset.data
+ ), "Sampled item should be in the dataset."
+
+ # Test __len__ and sample with empty dataset
+ empty_dataset = SeedDataset(data=[], min_samples=0, strict=True)
+ assert len(empty_dataset) == 0, "Empty dataset should have length 0."
+ with pytest.raises(RuntimeError, match="Dataset is empty, cannot sample."):
+ empty_dataset.sample()
def test_synthetic_dataset_init():