DSPy: The framework for programming—not prompting—language models
Python
feat(dspy): support for pandas dataframe in dataloader and ruff fixes
Anindyadeepe70d5b952fd7
diff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.pyindex 48a8fc879..edea8d7a3 100644--- a/dspy/datasets/dataloader.py+++ b/dspy/datasets/dataloader.py@@ -1,7 +1,8 @@import randomfrom collections.abc import Mapping-from typing import List, Tuple, Union+from typing import Any, Union+import pandas as pdfrom datasets import load_datasetimport dspy@@ -9,17 +10,19 @@ from dspy.datasets.dataset import Datasetclass DataLoader(Dataset):- def __init__(self,):+ def __init__(+ self,+ ):passdef from_huggingface(self,dataset_name: str,- *args,- input_keys: Tuple[str] = (),- fields: Tuple[str] = None,+ *args: list[Any],+ input_keys: tuple[str] = (),+ fields: tuple[str] = None,**kwargs,- ) -> Union[Mapping[str, List[dspy.Example]], List[dspy.Example]]:+ ) -> Union[Mapping[str, list[dspy.Example]], list[dspy.Example]]:if fields and not isinstance(fields, tuple):raise ValueError("Invalid fields provided. Please provide a tuple of fields.")@@ -27,42 +30,63 @@ class DataLoader(Dataset):raise ValueError("Invalid input keys provided. Please provide a tuple of input keys.")dataset = load_dataset(dataset_name, *args, **kwargs)-+if isinstance(dataset, list) and isinstance(kwargs["split"], list):- dataset = {split_name:dataset[idx] for idx, split_name in enumerate(kwargs["split"])}+ dataset = {split_name: dataset[idx] for idx, split_name in enumerate(kwargs["split"])}try:returned_split = {}- for split_name in dataset.keys():+ for split_name in dataset:if fields:- returned_split[split_name] = [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset[split_name]]+ returned_split[split_name] = [+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys)+ for row in dataset[split_name]+ ]else:- returned_split[split_name] = [dspy.Example({field:row[field] for field in row.keys()}).with_inputs(*input_keys) for row in dataset[split_name]]+ returned_split[split_name] = [+ dspy.Example({field: row[field] for field in row}).with_inputs(*input_keys)+ for row in dataset[split_name]+ ]return returned_splitexcept AttributeError:if fields:- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]- else:- return [dspy.Example({field:row[field] for field in row.keys()}).with_inputs(*input_keys) for row in dataset]+ return [+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset+ ]- def from_csv(self, file_path:str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ return [dspy.Example({field: row[field] for field in row}).with_inputs(*input_keys) for row in dataset]++ def from_csv(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:dataset = load_dataset("csv", data_files=file_path)["train"]-+if not fields:fields = list(dataset.features)-- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]- def from_json(self, file_path:str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ return [dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]++ def from_pandas(+ self,+ df: pd.DataFrame,+ fields: list[str] = None,+ input_keys: tuple[str] = (),+ ) -> list[dspy.Example]:+ if fields is None:+ fields = list(df.columns)++ return [+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for _, row in df.iterrows()+ ]++ def from_json(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:dataset = load_dataset("json", data_files=file_path)["train"]-+if not fields:fields = list(dataset.features)-- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]- def from_parquet(self, file_path: str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ return [dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]++ def from_parquet(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:dataset = load_dataset("parquet", data_files=file_path)["train"]if not fields:@@ -72,23 +96,23 @@ class DataLoader(Dataset):def sample(self,- dataset: List[dspy.Example],+ dataset: list[dspy.Example],n: int,- *args,+ *args: list[Any],**kwargs,- ) -> List[dspy.Example]:+ ) -> list[dspy.Example]:if not isinstance(dataset, list):raise ValueError(f"Invalid dataset provided of type {type(dataset)}. Please provide a list of examples.")-+return random.sample(dataset, n, *args, **kwargs)def train_test_split(self,- dataset: List[dspy.Example],+ dataset: list[dspy.Example],train_size: Union[int, float] = 0.75,test_size: Union[int, float] = None,random_state: int = None,- ) -> Mapping[str, List[dspy.Example]]:+ ) -> Mapping[str, list[dspy.Example]]:if random_state is not None:random.seed(random_state)@@ -115,6 +139,6 @@ class DataLoader(Dataset):test_end = len(dataset_shuffled) - train_endtrain_dataset = dataset_shuffled[:train_end]- test_dataset = dataset_shuffled[train_end:train_end + test_end]+ test_dataset = dataset_shuffled[train_end : train_end + test_end]- return {'train': train_dataset, 'test': test_dataset}+ return {"train": train_dataset, "test": test_dataset}