Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): support for pandas dataframe in dataloader and ruff fixes

Anindyadeepe70d5b952fd7
diff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.py
index 48a8fc879..edea8d7a3 100644
--- a/dspy/datasets/dataloader.py
+++ b/dspy/datasets/dataloader.py
@@ -1,7 +1,8 @@
import random
from collections.abc import Mapping
-from typing import List, Tuple, Union
+from typing import Any, Union
+import pandas as pd
from datasets import load_dataset
import dspy
@@ -9,17 +10,19 @@ from dspy.datasets.dataset import Dataset
class DataLoader(Dataset):
- def __init__(self,):
+ def __init__(
+ self,
+ ):
pass
def from_huggingface(
self,
dataset_name: str,
- *args,
- input_keys: Tuple[str] = (),
- fields: Tuple[str] = None,
+ *args: list[Any],
+ input_keys: tuple[str] = (),
+ fields: tuple[str] = None,
**kwargs,
- ) -> Union[Mapping[str, List[dspy.Example]], List[dspy.Example]]:
+ ) -> Union[Mapping[str, list[dspy.Example]], list[dspy.Example]]:
if fields and not isinstance(fields, tuple):
raise ValueError("Invalid fields provided. Please provide a tuple of fields.")
@@ -27,42 +30,63 @@ class DataLoader(Dataset):
raise ValueError("Invalid input keys provided. Please provide a tuple of input keys.")
dataset = load_dataset(dataset_name, *args, **kwargs)
-
+
if isinstance(dataset, list) and isinstance(kwargs["split"], list):
- dataset = {split_name:dataset[idx] for idx, split_name in enumerate(kwargs["split"])}
+ dataset = {split_name: dataset[idx] for idx, split_name in enumerate(kwargs["split"])}
try:
returned_split = {}
- for split_name in dataset.keys():
+ for split_name in dataset:
if fields:
- returned_split[split_name] = [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset[split_name]]
+ returned_split[split_name] = [
+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys)
+ for row in dataset[split_name]
+ ]
else:
- returned_split[split_name] = [dspy.Example({field:row[field] for field in row.keys()}).with_inputs(*input_keys) for row in dataset[split_name]]
+ returned_split[split_name] = [
+ dspy.Example({field: row[field] for field in row}).with_inputs(*input_keys)
+ for row in dataset[split_name]
+ ]
return returned_split
except AttributeError:
if fields:
- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]
- else:
- return [dspy.Example({field:row[field] for field in row.keys()}).with_inputs(*input_keys) for row in dataset]
+ return [
+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset
+ ]
- def from_csv(self, file_path:str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:
+ return [dspy.Example({field: row[field] for field in row}).with_inputs(*input_keys) for row in dataset]
+
+ def from_csv(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:
dataset = load_dataset("csv", data_files=file_path)["train"]
-
+
if not fields:
fields = list(dataset.features)
-
- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]
- def from_json(self, file_path:str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:
+ return [dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]
+
+ def from_pandas(
+ self,
+ df: pd.DataFrame,
+ fields: list[str] = None,
+ input_keys: tuple[str] = (),
+ ) -> list[dspy.Example]:
+ if fields is None:
+ fields = list(df.columns)
+
+ return [
+ dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for _, row in df.iterrows()
+ ]
+
+ def from_json(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:
dataset = load_dataset("json", data_files=file_path)["train"]
-
+
if not fields:
fields = list(dataset.features)
-
- return [dspy.Example({field:row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]
- def from_parquet(self, file_path: str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:
+ return [dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]
+
+ def from_parquet(self, file_path: str, fields: list[str] = None, input_keys: tuple[str] = ()) -> list[dspy.Example]:
dataset = load_dataset("parquet", data_files=file_path)["train"]
if not fields:
@@ -72,23 +96,23 @@ class DataLoader(Dataset):
def sample(
self,
- dataset: List[dspy.Example],
+ dataset: list[dspy.Example],
n: int,
- *args,
+ *args: list[Any],
**kwargs,
- ) -> List[dspy.Example]:
+ ) -> list[dspy.Example]:
if not isinstance(dataset, list):
raise ValueError(f"Invalid dataset provided of type {type(dataset)}. Please provide a list of examples.")
-
+
return random.sample(dataset, n, *args, **kwargs)
def train_test_split(
self,
- dataset: List[dspy.Example],
+ dataset: list[dspy.Example],
train_size: Union[int, float] = 0.75,
test_size: Union[int, float] = None,
random_state: int = None,
- ) -> Mapping[str, List[dspy.Example]]:
+ ) -> Mapping[str, list[dspy.Example]]:
if random_state is not None:
random.seed(random_state)
@@ -115,6 +139,6 @@ class DataLoader(Dataset):
test_end = len(dataset_shuffled) - train_end
train_dataset = dataset_shuffled[:train_end]
- test_dataset = dataset_shuffled[train_end:train_end + test_end]
+ test_dataset = dataset_shuffled[train_end : train_end + test_end]
- return {'train': train_dataset, 'test': test_dataset}
+ return {"train": train_dataset, "test": test_dataset}