🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor Seed Dataset to improve compatibility and simplify usage (#1734)
Apokryphosx578dae28e291
diff --git a/camel/datasets/base.py b/camel/datasets/base.pyindex e085eeb46..bcacd0e34 100644--- a/camel/datasets/base.py+++ b/camel/datasets/base.py@@ -12,14 +12,17 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import jsonimport osimport random+from pathlib import Pathfrom typing import (Any,Callable,Dict,List,Optional,+ Sized,TypeVar,Union,)@@ -326,42 +329,241 @@ class SeedDataset(BaseDataset):r"""A dataset containing validated seed examples for data generation.Ensures that all items adhere to the DataPoint schema.- This class is used to initialize a dataset from a list of dictionary items,- validating each against the DataPoint schema.+ This class can initialize from Hugging Face Datasets,+ PyTorch Datasets, JSON file paths, or lists of dictionaries,+ converting them into a consistent internal format."""def __init__(self,- data: List[Dict[str, str]],+ data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]],cache_dir: Optional[str] = None,+ seed: Optional[int] = None,min_samples: int = 1,+ strict: bool = False,**kwargs,):- r"""Initialize the seed dataset.+ r"""Initialize the seed dataset and validate integrity.Args:- data (List[Dict[str, str]]): List of dictionary items to create the- dataset from.- cache_dir (Optional[str]): Directory to cache dataset files.- (default: :obj:`None`)+ data (Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]):+ Input data, which can be:+ - A Hugging Face Dataset (HFDataset)+ - A PyTorch Dataset (torch.utils.data.Dataset)+ - A Path object representing the path to a JSON file+ - A list of dictionaries with DataPoint-compatible fields+ seed (Optional[int]): Seed for reproducibility.+ (default: :obj:`1`)min_samples (int): Minimum number of samples required.(default: :obj:`1`)+ strict (bool): Whether to raise an error on invalid datapoints+ (True) or skip/filter them (False). (default: False)**kwargs: Additional dataset parameters.Raises:- ValueError: If dataset size is less than min_samples or if sample- validation fails.+ TypeError: If the data type is not supported.+ ValueError: If dataset size is less than min_samples or+ if sample validation fails.+ FileNotFoundError: If the JSON file path doesn't exist.+ json.JSONDecodeError: If the JSON file is invalid."""- if len(data) < min_samples:+ # Initialize BaseDataset with empty data, we'll populate it ourselves+ super().__init__(data=[], cache_dir=cache_dir, **kwargs)++ self._rng = random.Random(seed)+ self._strict = strict++ # Type checking and conversion into list of dicts to have a+ # consistent internal format. Since Seed Dataset should be+ # small, we can load it entirely into memory++ self.data: List[DataPoint] = self._init_data(data)+ self._length = len(self.data)++ if self._length < min_samples:raise ValueError(- f"Seed dataset must contain at least {min_samples} samples."+ "The dataset does not contain enough samples. "+ f"Need {max(0, min_samples)}, got {self._length}")- super().__init__(- data=data,- cache_dir=cache_dir,- **kwargs,- )+ def _init_data(+ self, data: Union[HFDataset, Dataset, Path, List[Dict[str, Any]]]+ ) -> List[DataPoint]:+ if isinstance(data, HFDataset):+ raw_data = self._init_from_hf_dataset(data)+ elif isinstance(data, Dataset):+ raw_data = self._init_from_pytorch_dataset(data)+ elif isinstance(data, Path):+ raw_data = self._init_from_json_path(data)+ elif isinstance(data, list):+ raw_data = self._init_from_list(data)+ else:+ raise TypeError("Unsupported data type")++ def create_datapoint(+ item: Dict[str, Any], idx: int+ ) -> Optional[DataPoint]:+ # Add type checks for required fields to make mypy happy+ question = item.get('question')+ if not isinstance(question, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'question': "+ f"expected str, got {type(question)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: invalid 'question'"+ )+ return None++ rationale = item.get('rationale')+ if not isinstance(rationale, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'rationale': "+ f"expected str, got {type(rationale)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: invalid 'rationale'"+ )+ return None++ final_answer = item.get('final_answer')+ if not isinstance(final_answer, str):+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} has invalid 'final_answer': "+ f"expected str, got {type(final_answer)}"+ )+ else:+ logger.warning(+ f"Skipping sample at index {idx}: "+ "invalid 'final_answer'"+ )+ return None++ try:+ return DataPoint(+ question=question,+ rationale=rationale,+ final_answer=final_answer,+ metadata=item.get('metadata'),+ difficulty=item.get('difficulty'),+ )+ except ValidationError as e:+ if self._strict:+ raise ValueError(+ f"Sample at index {idx} validation error: {e}"+ )+ else:+ logger.warning(+ f"Skipping invalid sample at index {idx} "+ f"due to validation error: {e}"+ )+ return None++ unfiltered_data = [+ create_datapoint(item, i) for i, item in enumerate(raw_data)+ ]+ return [dp for dp in unfiltered_data if dp is not None]++ def __len__(self) -> int:+ r"""Return the size of the dataset."""+ return self._length++ def __getitem__(self, idx: int) -> DataPoint:+ r"""Get an item from the dataset.++ Args:+ idx (int): Index of the item to get.++ Returns:+ DataPoint: DataPoint from the dataset with the given index.++ Raises:+ IndexError: If idx is out of bounds.+ """+ if idx < 0 or idx >= self._length:+ raise IndexError(+ f"Index {idx} out of bounds for dataset of size {self._length}"+ )+ return self.data[idx]++ def sample(self) -> DataPoint:+ r"""Sample a random datapoint from the dataset.++ Returns:+ DataPoint: A randomly sampled DataPoint.++ Raises:+ RuntimeError: If the dataset is empty.+ """+ if self._length == 0:+ raise RuntimeError("Dataset is empty, cannot sample.")+ idx = self._rng.randint(0, self._length - 1)+ return self[idx]++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Get dataset metadata."""+ return self._metadata.copy()++ def _init_from_hf_dataset(self, data: HFDataset) -> List[Dict[str, Any]]:+ return [dict(item) for item in data]++ def _init_from_pytorch_dataset(+ self, data: Dataset+ ) -> List[Dict[str, Any]]:+ if not isinstance(data, Sized):+ raise TypeError(+ f"{type(data).__name__} does not implement `__len__()`."+ )+ raw_data = []++ for i in range(len(data)):+ item = data[i]+ if not isinstance(item, dict):+ raise TypeError(+ f"Item at index {i} is not a dict: "+ f"got {type(item).__name__}"+ )+ raw_data.append(dict(item))+ return raw_data++ def _init_from_json_path(self, data: Path) -> List[Dict[str, Any]]:+ if not data.exists():+ raise FileNotFoundError(f"JSON file not found: {data}")+ try:+ logger.debug(f"Loading JSON from {data}")+ with data.open('r', encoding='utf-8') as f:+ loaded_data = json.load(f)+ logger.info(+ f"Successfully loaded {len(loaded_data)} items from {data}"+ )+ except json.JSONDecodeError as e:+ raise ValueError(f"Invalid JSON in file {data}: {e}")+ if not isinstance(loaded_data, list):+ raise ValueError("JSON file must contain a list of dictionaries")+ for i, item in enumerate(loaded_data):+ if not isinstance(item, dict):+ raise ValueError(+ f"Expected a dictionary at index {i}, "+ f"got {type(item).__name__}"+ )+ return loaded_data++ def _init_from_list(+ self, data: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ for i, item in enumerate(data):+ if not isinstance(item, dict):+ raise ValueError(+ f"Expected a dictionary at index {i}, "+ f"got {type(item).__name__}"+ )+ return dataclass SyntheticDataset(BaseDataset):diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pyindex 83db9ad60..3b4f611da 100644--- a/test/datasets/test_base_dataset.py+++ b/test/datasets/test_base_dataset.py@@ -12,13 +12,16 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import jsonimport tempfile+from pathlib import Pathfrom unittest.mock import AsyncMock, MagicMockimport pytestimport torchfrom datasets import Dataset as HFDatasetfrom pydantic import ValidationError+from torch.utils.data import Datasetfrom camel.datasets.base import (BaseDataset,@@ -191,13 +194,731 @@ def test_base_dataset_metadata():assert metadata['custom_param'] == 'value'-def test_seed_dataset_init(sample_data):- r"""Test SeedDataset initialization."""- dataset = SeedDataset(data=sample_data, min_samples=1)- assert dataset._raw_data == sample_data+def test_seed_dataset_init_from_hf_dataset():+ r"""+ Test SeedDataset initialization from a+ Hugging Face Dataset with various scenarios.+ """- with pytest.raises(ValueError):- SeedDataset(data=sample_data, min_samples=3)+ # **Test 1: Initialization with proper mock data (only required fields)**+ valid_data = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Addition of 2 and 2.",+ "final_answer": "4",+ },+ {+ "question": "Is the sky blue?",+ "rationale": "Observation of clear weather.",+ "final_answer": "yes",+ },+ ]+ hf_valid = HFDataset.from_list(valid_data)++ dataset = SeedDataset(data=hf_valid, min_samples=1, strict=True)++ # Verify the dataset initialized correctly+ assert len(dataset) == 2, "Dataset should contain 2 items."+ assert isinstance(+ dataset[0], DataPoint+ ), "Items should be DataPoint instances."+ assert (+ dataset[0].question == "What is 2 + 2?"+ ), "Question should match input."+ assert (+ dataset[0].rationale == "Addition of 2 and 2."+ ), "Rationale should match input."+ assert dataset[0].final_answer == "4", "Final answer should match input."+ assert (+ dataset[1].question == "Is the sky blue?"+ ), "Second question should match input."+ assert (+ dataset[1].rationale == "Observation of clear weather."+ ), "Second rationale should match input."+ assert (+ dataset[1].final_answer == "yes"+ ), "Second final answer should match input."+ assert (+ dataset[0].difficulty is None+ ), "Difficulty should be None when not provided."+ assert (+ dataset[0].metadata is None+ ), "Metadata should be None when not provided."++ # **Test 2: Initialization with invalid data**+ # Sub-test 2a: Missing required field with strict=True+ invalid_data_missing = [+ {"question": "What is 3 + 3?", "final_answer": "6"}+ ]+ hf_invalid_missing = HFDataset.from_list(invalid_data_missing)+ with pytest.raises(+ ValueError,+ match="Sample at index 0 has invalid 'rationale': "+ "expected str, got <class 'NoneType'>",+ ):+ SeedDataset(data=hf_invalid_missing, min_samples=1, strict=True)++ # Sub-test 2b: Missing required field with strict=False+ dataset_non_strict = SeedDataset(+ data=hf_invalid_missing, min_samples=0, strict=False+ )+ assert (+ len(dataset_non_strict) == 0+ ), "Invalid items should be filtered out in non-strict mode."++ # Sub-test 2c: Empty dataset with min_samples=1+ hf_empty = HFDataset.from_list([])+ with pytest.raises(+ ValueError,+ match="The dataset does not contain enough samples. Need 1, got 0",+ ):+ SeedDataset(data=hf_empty, min_samples=1, strict=True)++ # Sub-test 2d: Empty dataset with min_samples=0+ dataset_empty = SeedDataset(data=hf_empty, min_samples=0, strict=True)+ assert (+ len(dataset_empty) == 0+ ), "Empty dataset should have zero length when min_samples=0."++ # **Test 3: Initialization with optional fields and additional fields**+ data_with_optional = [+ {+ "question": "What is the capital of France?",+ "rationale": "France is a country in Europe.",+ "final_answer": "Paris",+ "difficulty": "easy",+ "metadata": {"source": "geography", "id": 123},+ "extra_field": "this should be ignored", # Not in DataPoint+ }+ ]+ hf_optional = HFDataset.from_list(data_with_optional)+ dataset_optional = SeedDataset(+ data=hf_optional, min_samples=1, strict=True+ )++ # Verify the dataset initialized correctly+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."+ assert isinstance(+ dataset_optional[0], DataPoint+ ), "Item should be a DataPoint instance."+ assert (+ dataset_optional[0].question == "What is the capital of France?"+ ), "Question should match."+ assert (+ dataset_optional[0].rationale == "France is a country in Europe."+ ), "Rationale should match."+ assert (+ dataset_optional[0].final_answer == "Paris"+ ), "Final answer should match."+ assert (+ dataset_optional[0].difficulty == "easy"+ ), "Difficulty should be set correctly."+ assert dataset_optional[0].metadata == {+ "source": "geography",+ "id": 123,+ }, "Metadata should match."+ assert (+ "extra_field" not in dataset_optional[0].__dict__+ ), "Extra field should not be present in DataPoint."+++def test_seed_dataset_init_from_pytorch_dataset():+ r"""+ Test SeedDataset initialization from a+ PyTorch Dataset with various scenarios.+ """++ class MockPyTorchDataset(Dataset):+ def __init__(self, data_list):+ self.data = data_list++ def __len__(self):+ return len(self.data)++ def __getitem__(self, idx):+ return self.data[idx]++ # **Test 1: Initialization with proper mock data (only required fields)**+ valid_data = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Addition of 2 and 2.",+ "final_answer": "4",+ },+ {+ "question": "Is the sky blue?",+ "rationale": "Observation of clear weather.",+ "final_answer": "yes",+ },+ ]+ pytorch_valid = MockPyTorchDataset(valid_data)+ dataset = SeedDataset(data=pytorch_valid, min_samples=1, strict=True)++ assert len(dataset) == 2, "Dataset should contain 2 items."+ assert isinstance(+ dataset[0], DataPoint+ ), "Items should be DataPoint instances."+ assert (+ dataset[0].question == "What is 2 + 2?"+ ), "Question should match input."+ assert (+ dataset[0].rationale == "Addition of 2 and 2."+ ), "Rationale should match input."+ assert dataset[0].final_answer == "4", "Final answer should match input."+ assert (+ dataset[1].question == "Is the sky blue?"+ ), "Second question should match input."+ assert (+ dataset[1].rationale == "Observation of clear weather."+ ), "Second rationale should match input."+ assert (+ dataset[1].final_answer == "yes"+ ), "Second final answer should match input."+ assert (+ dataset[0].difficulty is None+ ), "Difficulty should be None when not provided."+ assert (+ dataset[0].metadata is None+ ), "Metadata should be None when not provided."++ # **Test 2: Initialization with invalid data**+ # Sub-test 2a: Missing required field with strict=True+ invalid_data_missing = [+ {+ "question": "What is 3 + 3?",+ "final_answer": "6",+ } # Missing rationale+ ]+ pytorch_invalid_missing = MockPyTorchDataset(invalid_data_missing)+ with pytest.raises(+ ValueError,+ match="Sample at index 0 has invalid 'rationale': "+ "expected str, got <class 'NoneType'>",+ ):+ SeedDataset(data=pytorch_invalid_missing, min_samples=1, strict=True)++ # Sub-test 2b: Missing required field with strict=False+ dataset_non_strict = SeedDataset(+ data=pytorch_invalid_missing, min_samples=0, strict=False+ )+ assert (+ len(dataset_non_strict) == 0+ ), "Invalid items should be filtered out in non-strict mode."++ # Sub-test 2c: Empty dataset with min_samples=1+ pytorch_empty = MockPyTorchDataset([])+ with pytest.raises(+ ValueError,+ match="The dataset does not contain enough samples. Need 1, got 0",+ ):+ SeedDataset(data=pytorch_empty, min_samples=1, strict=True)++ # Sub-test 2d: Empty dataset with min_samples=0+ dataset_empty = SeedDataset(data=pytorch_empty, min_samples=0, strict=True)+ assert (+ len(dataset_empty) == 0+ ), "Empty dataset should have zero length when min_samples=0."++ # **Test 3: Initialization with optional fields and additional fields**+ data_with_optional = [+ {+ "question": "What is the capital of France?",+ "rationale": "France is a country in Europe.",+ "final_answer": "Paris",+ "difficulty": "easy",+ "metadata": {"source": "geography", "id": 123},+ "extra_field": "this should be ignored", # Not in DataPoint+ }+ ]+ pytorch_optional = MockPyTorchDataset(data_with_optional)+ dataset_optional = SeedDataset(+ data=pytorch_optional, min_samples=1, strict=True+ )++ # Verify the dataset initialized correctly+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."+ assert isinstance(+ dataset_optional[0], DataPoint+ ), "Item should be a DataPoint instance."+ assert (+ dataset_optional[0].question == "What is the capital of France?"+ ), "Question should match."+ assert (+ dataset_optional[0].rationale == "France is a country in Europe."+ ), "Rationale should match."+ assert (+ dataset_optional[0].final_answer == "Paris"+ ), "Final answer should match."+ assert (+ dataset_optional[0].difficulty == "easy"+ ), "Difficulty should be set correctly."+ assert dataset_optional[0].metadata == {+ "source": "geography",+ "id": 123,+ }, "Metadata should match."+ assert (+ "extra_field" not in dataset_optional[0].__dict__+ ), "Extra field should not be present in DataPoint."++ # **Test 4: PyTorch-specific edge cases**+ # Sub-test 4a: Dataset without __len__+ class NoLenDataset(Dataset):+ def __getitem__(self, idx):+ return {+ "question": "Test",+ "rationale": "Test",+ "final_answer": "Test",+ }++ with pytest.raises(TypeError) as excinfo:+ SeedDataset(NoLenDataset(), min_samples=1, strict=True)+ assert "does not implement `__len__()`." in str(excinfo.value)++ # Sub-test 4b: Dataset with non-dict items+ class NonDictDataset(Dataset):+ def __len__(self):+ return 1++ def __getitem__(self, idx):+ return "not a dict"++ with pytest.raises(+ TypeError, match="Item at index 0 is not a dict: got str"+ ):+ SeedDataset(data=NonDictDataset(), min_samples=1, strict=True)+++def test_seed_dataset_init_from_json_file():+ r"""+ Test SeedDataset initialization from a JSON file with various scenarios.+ """++ # **Test 1: Initialization with a valid JSON file (only required fields)**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ valid_data = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Addition of 2 and 2.",+ "final_answer": "4",+ },+ {+ "question": "Is the sky blue?",+ "rationale": "Observation of clear weather.",+ "final_answer": "yes",+ },+ ]+ json.dump(valid_data, tmp_file)+ tmp_file_path = Path(tmp_file.name)++ dataset = SeedDataset(data=tmp_file_path, min_samples=1, strict=True)+ assert len(dataset) == 2, "Dataset should contain 2 items."+ assert isinstance(+ dataset[0], DataPoint+ ), "Items should be DataPoint instances."+ assert (+ dataset[0].question == "What is 2 + 2?"+ ), "Question should match input."+ assert (+ dataset[0].rationale == "Addition of 2 and 2."+ ), "Rationale should match input."+ assert dataset[0].final_answer == "4", "Final answer should match input."+ assert (+ dataset[1].question == "Is the sky blue?"+ ), "Second question should match input."+ assert (+ dataset[1].rationale == "Observation of clear weather."+ ), "Second rationale should match input."+ assert (+ dataset[1].final_answer == "yes"+ ), "Second final answer should match input."+ assert (+ dataset[0].difficulty is None+ ), "Difficulty should be None when not provided."+ assert (+ dataset[0].metadata is None+ ), "Metadata should be None when not provided."++ # **Test 2: Initialization with invalid JSON file (malformed JSON)**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ tmp_file.write(+ '[{"question": "Test", "rationale": "Test", "final_answer": "Test"'+ ) # Missing closing bracket+ tmp_file_path = Path(tmp_file.name)++ with pytest.raises(ValueError, match="Invalid JSON in file"):+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)++ # **Test 3: Initialization with JSON file containing non-list data**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ json.dump(+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},+ tmp_file,+ )+ tmp_file_path = Path(tmp_file.name)++ with pytest.raises(+ ValueError, match="JSON file must contain a list of dictionaries"+ ):+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)++ # **Test 4: Initialization with JSON file containing non-dict items**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ invalid_data = [+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},+ "not a dict",+ ]+ json.dump(invalid_data, tmp_file)+ tmp_file_path = Path(tmp_file.name)++ with pytest.raises(+ ValueError, match="Expected a dictionary at index 1, got str"+ ):+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)++ # **Test 5: Initialization with a missing JSON file**+ missing_file_path = Path("non_existent_file.json")+ with pytest.raises(FileNotFoundError, match="JSON file not found:"):+ SeedDataset(data=missing_file_path, min_samples=1, strict=True)++ # **Test 6: Initialization with an empty JSON file**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ json.dump([], tmp_file)+ tmp_file_path = Path(tmp_file.name)++ # Sub-test 6a: Empty dataset with min_samples=1+ with pytest.raises(+ ValueError,+ match="The dataset does not contain enough samples. Need 1, got 0",+ ):+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)++ # Sub-test 6b: Empty dataset with min_samples=0+ dataset_empty = SeedDataset(data=tmp_file_path, min_samples=0, strict=True)+ assert (+ len(dataset_empty) == 0+ ), "Empty dataset should have zero length when min_samples=0."++ # **Test 7: Initialization with optional fields and additional fields**+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ data_with_optional = [+ {+ "question": "What is the capital of France?",+ "rationale": "France is a country in Europe.",+ "final_answer": "Paris",+ "difficulty": "easy",+ "metadata": {"source": "geography", "id": 123},+ "extra_field": "this should be ignored",+ }+ ]+ json.dump(data_with_optional, tmp_file)+ tmp_file_path = Path(tmp_file.name)++ dataset_optional = SeedDataset(+ data=tmp_file_path, min_samples=1, strict=True+ )+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."+ assert isinstance(+ dataset_optional[0], DataPoint+ ), "Item should be a DataPoint instance."+ assert (+ dataset_optional[0].question == "What is the capital of France?"+ ), "Question should match."+ assert (+ dataset_optional[0].rationale == "France is a country in Europe."+ ), "Rationale should match."+ assert (+ dataset_optional[0].final_answer == "Paris"+ ), "Final answer should match."+ assert (+ dataset_optional[0].difficulty == "easy"+ ), "Difficulty should be set correctly."+ assert dataset_optional[0].metadata == {+ "source": "geography",+ "id": 123,+ }, "Metadata should match."+ assert (+ "extra_field" not in dataset_optional[0].__dict__+ ), "Extra field should not be present in DataPoint."++ # **Test 8: JSON-specific edge cases**+ # Sub-test 8a: Invalid data with strict=True+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ invalid_data = [+ {+ "question": "What is 3 + 3?",+ "final_answer": "6",+ } # Missing rationale+ ]+ json.dump(invalid_data, tmp_file)+ tmp_file_path = Path(tmp_file.name)++ with pytest.raises(+ ValueError,+ match="Sample at index 0 has invalid 'rationale': "+ "expected str, got <class 'NoneType'>",+ ):+ SeedDataset(data=tmp_file_path, min_samples=1, strict=True)++ # Sub-test 8b: Invalid data with strict=False+ with tempfile.NamedTemporaryFile(+ mode='w', suffix='.json', delete=False+ ) as tmp_file:+ invalid_data = [+ {+ "question": "What is 3 + 3?",+ "final_answer": "6",+ } # Missing rationale+ ]+ json.dump(invalid_data, tmp_file)+ tmp_file_path = Path(tmp_file.name)++ dataset_non_strict = SeedDataset(+ data=tmp_file_path, min_samples=0, strict=False+ )+ assert (+ len(dataset_non_strict) == 0+ ), "Invalid items should be filtered out in non-strict mode."+++def test_seed_dataset_init_from_list():+ r"""+ Test SeedDataset initialization from a list of+ dictionaries with various scenarios.+ """++ # **Test 1: Initialization with a valid list (only required fields)**+ valid_data = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Addition of 2 and 2.",+ "final_answer": "4",+ },+ {+ "question": "Is the sky blue?",+ "rationale": "Observation of clear weather.",+ "final_answer": "yes",+ },+ ]+ dataset = SeedDataset(data=valid_data, min_samples=1, strict=True)+ assert len(dataset) == 2, "Dataset should contain 2 items."+ assert isinstance(+ dataset[0], DataPoint+ ), "Items should be DataPoint instances."+ assert (+ dataset[0].question == "What is 2 + 2?"+ ), "Question should match input."+ assert (+ dataset[0].rationale == "Addition of 2 and 2."+ ), "Rationale should match input."+ assert dataset[0].final_answer == "4", "Final answer should match input."+ assert (+ dataset[1].question == "Is the sky blue?"+ ), "Second question should match input."+ assert (+ dataset[1].rationale == "Observation of clear weather."+ ), "Second rationale should match input."+ assert (+ dataset[1].final_answer == "yes"+ ), "Second final answer should match input."+ assert (+ dataset[0].difficulty is None+ ), "Difficulty should be None when not provided."+ assert (+ dataset[0].metadata is None+ ), "Metadata should be None when not provided."++ # **Test 2: Initialization with a list containing non-dictionary items**+ invalid_list_data = [+ {"question": "Test", "rationale": "Test", "final_answer": "Test"},+ "not a dict",+ ]+ with pytest.raises(+ ValueError, match="Expected a dictionary at index 1, got str"+ ):+ SeedDataset(data=invalid_list_data, min_samples=1, strict=True)++ # **Test 3: Initialization with an empty list**+ empty_data = []+ # Sub-test 3a: Empty list with min_samples=1+ with pytest.raises(+ ValueError,+ match="The dataset does not contain enough samples. Need 1, got 0",+ ):+ SeedDataset(data=empty_data, min_samples=1, strict=True)+ # Sub-test 3b: Empty list with min_samples=0+ dataset_empty = SeedDataset(data=empty_data, min_samples=0, strict=True)+ assert (+ len(dataset_empty) == 0+ ), "Empty dataset should have zero length when min_samples=0."++ # **Test 4: Initialization with dictionaries missing required fields**+ # Sub-test 4a: Missing required field with strict=True+ invalid_data_missing = [+ {+ "question": "What is 3 + 3?",+ "final_answer": "6",+ } # Missing rationale+ ]+ with pytest.raises(+ ValueError,+ match="Sample at index 0 has invalid 'rationale': "+ "expected str, got <class 'NoneType'>",+ ):+ SeedDataset(data=invalid_data_missing, min_samples=1, strict=True)+ # Sub-test 4b: Missing required field with strict=False+ dataset_non_strict = SeedDataset(+ data=invalid_data_missing, min_samples=0, strict=False+ )+ assert (+ len(dataset_non_strict) == 0+ ), "Invalid items should be filtered out in non-strict mode."++ # **Test 5: Initialization with optional fields and additional fields**+ data_with_optional = [+ {+ "question": "What is the capital of France?",+ "rationale": "France is a country in Europe.",+ "final_answer": "Paris",+ "difficulty": "easy",+ "metadata": {"source": "geography", "id": 123},+ "extra_field": "this should be ignored",+ }+ ]+ dataset_optional = SeedDataset(+ data=data_with_optional, min_samples=1, strict=True+ )+ assert len(dataset_optional) == 1, "Dataset should contain 1 item."+ assert isinstance(+ dataset_optional[0], DataPoint+ ), "Item should be a DataPoint instance."+ assert (+ dataset_optional[0].question == "What is the capital of France?"+ ), "Question should match."+ assert (+ dataset_optional[0].rationale == "France is a country in Europe."+ ), "Rationale should match."+ assert (+ dataset_optional[0].final_answer == "Paris"+ ), "Final answer should match."+ assert (+ dataset_optional[0].difficulty == "easy"+ ), "Difficulty should be set correctly."+ assert dataset_optional[0].metadata == {+ "source": "geography",+ "id": 123,+ }, "Metadata should match."+ assert (+ "extra_field" not in dataset_optional[0].__dict__+ ), "Extra field should not be present in DataPoint."++ # **Test 6: Mixed valid and invalid items with strict=False**+ mixed_data = [+ {+ "question": "Valid question",+ "rationale": "Valid rationale",+ "final_answer": "Valid",+ },+ {+ "question": "Invalid question",+ "final_answer": "Invalid",+ }, # Missing rationale+ ]+ dataset_mixed = SeedDataset(data=mixed_data, min_samples=1, strict=False)+ assert (+ len(dataset_mixed) == 1+ ), "Only valid items should be included in non-strict mode."+ assert (+ dataset_mixed[0].question == "Valid question"+ ), "Only the valid item should be present."+++def test_seed_dataset_methods():+ r"""+ Test the __len__, __getitem__, and sample methods+ of SeedDataset with a mock dataset.+ """+ mock_data = [+ {+ "question": "What is 1+1?",+ "rationale": "Basic addition.",+ "final_answer": "2",+ },+ {+ "question": "Is the Earth round?",+ "rationale": "Scientific consensus.",+ "final_answer": "yes",+ },+ {+ "question": "What is the capital of Japan?",+ "rationale": "Geography knowledge.",+ "final_answer": "Tokyo",+ },+ {+ "question": "How many sides does a triangle have?",+ "rationale": "Definition of a triangle.",+ "final_answer": "3",+ },+ ]+ dataset = SeedDataset(data=mock_data, min_samples=1, strict=True)++ assert len(dataset) == 4, "Dataset should have 4 items."+ assert (+ dataset[0].question == "What is 1+1?"+ ), "First item question should match."+ assert (+ dataset[0].rationale == "Basic addition."+ ), "First item rationale should match."+ assert (+ dataset[0].final_answer == "2"+ ), "First item final_answer should match."+ assert (+ dataset[3].question == "How many sides does a triangle have?"+ ), "Last item question should match."+ assert (+ dataset[3].rationale == "Definition of a triangle."+ ), "Last item rationale should match."+ assert (+ dataset[3].final_answer == "3"+ ), "Last item final_answer should match."+ assert isinstance(+ dataset[0], DataPoint+ ), "Item should be a DataPoint instance."++ # Test __getitem__ with out-of-bounds indices+ with pytest.raises(+ IndexError, match="Index -1 out of bounds for dataset of size 4"+ ):+ dataset[-1]+ with pytest.raises(+ IndexError, match="Index 4 out of bounds for dataset of size 4"+ ):+ dataset[4]++ # Test sample from non-empty dataset+ sampled_item = dataset.sample()+ assert (+ sampled_item in dataset.data+ ), "Sampled item should be in the dataset."++ # Test __len__ and sample with empty dataset+ empty_dataset = SeedDataset(data=[], min_samples=0, strict=True)+ assert len(empty_dataset) == 0, "Empty dataset should have length 0."+ with pytest.raises(RuntimeError, match="Dataset is empty, cannot sample."):+ empty_dataset.sample()def test_synthetic_dataset_init():