DSPy: The framework for programming—not prompting—language models
Python
Style fix for dspy/datasets (#8175)
Chen Qian1ea510a4c606
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex d69d4533a..ba48d9034 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -298,7 +298,8 @@ def _get_stream_completion_fn(return async_stream_completion-def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):+def litellm_completion(request: Dict[str, Any], num_retries: int, cache: Optional[Dict[str, Any]] = None):+ cache = cache or {"no-cache": True, "no-store": True}stream_completion = _get_stream_completion_fn(request, cache, sync=True)if stream_completion is None:return litellm.completion(@@ -311,7 +312,8 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cacreturn stream_completion()-def litellm_text_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):+def litellm_text_completion(request: Dict[str, Any], num_retries: int, cache: Optional[Dict[str, Any]] = None):+ cache = cache or {"no-cache": True, "no-store": True}# Extract the provider and model from the model string.# TODO: Not all the models are in the format of "provider/model"model = request.pop("model").split("/", 1)@@ -336,7 +338,8 @@ def litellm_text_completion(request: Dict[str, Any], num_retries: int, cache={"n)-async def alitellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):+async def alitellm_completion(request: Dict[str, Any], num_retries: int, cache: Optional[Dict[str, Any]] = None):+ cache = cache or {"no-cache": True, "no-store": True}stream_completion = _get_stream_completion_fn(request, cache, sync=False)if stream_completion is None:return await litellm.acompletion(@@ -349,9 +352,8 @@ async def alitellm_completion(request: Dict[str, Any], num_retries: int, cache={return await stream_completion()-async def alitellm_text_completion(- request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}-):+async def alitellm_text_completion(request: Dict[str, Any], num_retries: int, cache: Optional[Dict[str, Any]] = None):+ cache = cache or {"no-cache": True, "no-store": True}model = request.pop("model").split("/", 1)provider, model = model[0] if len(model) > 1 else "openai", model[-1]diff --git a/dspy/datasets/__init__.py b/dspy/datasets/__init__.pyindex d9d8c510f..731284aa5 100644--- a/dspy/datasets/__init__.py+++ b/dspy/datasets/__init__.py@@ -1,9 +1,9 @@+from dspy.datasets.alfworld import AlfWorldfrom dspy.datasets.colors import Colorsfrom dspy.datasets.dataloader import DataLoaderfrom dspy.datasets.dataset import Datasetfrom dspy.datasets.hotpotqa import HotPotQAfrom dspy.datasets.math import MATH-from dspy.datasets.alfworld import AlfWorld__all__ = ["Colors",diff --git a/dspy/datasets/alfworld/__init__.py b/dspy/datasets/alfworld/__init__.pyindex 9a1bc42d5..7670a9999 100644--- a/dspy/datasets/alfworld/__init__.py+++ b/dspy/datasets/alfworld/__init__.py@@ -1 +1 @@-from dspy.datasets.alfworld.alfworld import AlfWorld\ No newline at end of file+from dspy.datasets.alfworld.alfworld import AlfWorlddiff --git a/dspy/datasets/alfworld/alfworld.py b/dspy/datasets/alfworld/alfworld.pyindex 8a78a3e01..a15727a47 100644--- a/dspy/datasets/alfworld/alfworld.py+++ b/dspy/datasets/alfworld/alfworld.py@@ -2,6 +2,7 @@ import osimport queueimport random+def env_worker(inq, outq):"""Worker process: creates a single AlfredTWEnv instance,@@ -10,16 +11,19 @@ def env_worker(inq, outq):try:import io- import yaml+ from contextlib import redirect_stderr, redirect_stdout+import alfworld.agents.environment as environment- from contextlib import redirect_stdout, redirect_stderr+ import yamlexcept ImportError:- raise ImportError("alfworld is not installed. " \- "Please install it via `pip install alfworld==0.3.5` then run `alfworld-download`.")+ raise ImportError(+ "alfworld is not installed. "+ "Please install it via `pip install alfworld==0.3.5` then run `alfworld-download`."+ )buf = io.StringIO()base_dir = os.path.dirname(os.path.abspath(__file__))- config_path = os.path.join(base_dir, 'base_config.yml')+ config_path = os.path.join(base_dir, "base_config.yml")with open(config_path) as f:config = yaml.safe_load(f)@@ -30,19 +34,19 @@ def env_worker(inq, outq):env = Nonewhile True:cmd, data = inq.get()- if cmd == 'init':+ if cmd == "init":env = base_env.init_env(batch_size=1)env.skip(data)task_def, info = env.reset()outq.put((task_def[0], info))- elif cmd == 'step':+ elif cmd == "step":obs, rew, done, info = env.step([data])outq.put((obs, rew, done, info))- elif cmd == 'close':- outq.put('CLOSED')+ elif cmd == "close":+ outq.put("CLOSED")breakelse:- outq.put('UNKNOWN_CMD')+ outq.put("UNKNOWN_CMD")class EnvPool:@@ -54,6 +58,7 @@ class EnvPool:obs, rew, done, info = sess.step("go north")..."""+def __init__(self, size=2):self.size = sizeself.workers = []@@ -62,8 +67,7 @@ class EnvPool:try:import multiprocess as mpexcept ImportError:- raise ImportError("multiprocess is not installed. " \- "Please install it via `pip install multiprocess`.")+ raise ImportError("multiprocess is not installed. " "Please install it via `pip install multiprocess`.")# Must call set_start_method('spawn') here, before creating any processestry:@@ -93,7 +97,7 @@ class EnvPool:while not self.available.empty():wid = self.available.get()inq, outq, proc = self.workers[wid]- inq.put(('close', None))+ inq.put(("close", None))outq.get() # Wait 'CLOSED'inq.close()outq.close()@@ -109,6 +113,7 @@ class _EnvSession:A context manager that acquires a worker from the pool,provides .init(idx) and .step(action), then releases the worker."""+def __init__(self, pool: EnvPool):self.pool = poolself.wid = None@@ -123,11 +128,11 @@ class _EnvSession:self.pool._release(self.wid)def init(self, idx):- self.inq.put(('init', idx))+ self.inq.put(("init", idx))return self.outq.get() # (task_def, info)def step(self, action):- self.inq.put(('step', action))+ self.inq.put(("step", action))return self.outq.get() # (obs, rew, done, info)@@ -136,7 +141,8 @@ class AlfWorld:self.POOL = EnvPool(size=max_threads)import dspy- dataset = [dspy.Example(idx=idx).with_inputs('idx') for idx in range(3500)]++ dataset = [dspy.Example(idx=idx).with_inputs("idx") for idx in range(3500)]random.Random(0).shuffle(dataset)trainset, devset = dataset[:3000], dataset[-500:]diff --git a/dspy/datasets/colors.py b/dspy/datasets/colors.pyindex edf2e4169..a2203e9a2 100644--- a/dspy/datasets/colors.py+++ b/dspy/datasets/colors.py@@ -3,7 +3,146 @@ import randomfrom dspy.datasets.dataset import Dataset### A bunch of colors, originally from matplotlib-all_colors = ['alice blue', 'dodger blue', 'light sky blue', 'deep sky blue', 'sky blue', 'steel blue', 'light steel blue', 'medium blue', 'navy blue', 'blue', 'royal blue', 'cadet blue', 'cornflower blue', 'medium slate blue', 'slate blue', 'dark slate blue', 'powder blue', 'turquoise', 'dark turquoise', 'medium turquoise', 'pale turquoise', 'light sea green', 'medium sea green', 'sea green', 'forest green', 'green yellow', 'lime green', 'dark green', 'green', 'lime', 'chartreuse', 'lawn green', 'yellow green', 'olive green', 'dark olive green', 'medium spring green', 'spring green', 'medium aquamarine', 'aquamarine', 'aqua', 'cyan', 'dark cyan', 'teal', 'medium orchid', 'dark orchid', 'orchid', 'blue violet', 'violet', 'dark violet', 'plum', 'thistle', 'magenta', 'fuchsia', 'dark magenta', 'medium purple', 'purple', 'rebecca purple', 'dark red', 'fire brick', 'indian red', 'light coral', 'dark salmon', 'light salmon', 'salmon', 'red', 'crimson', 'tomato', 'coral', 'orange red', 'dark orange', 'orange', 'yellow', 'gold', 'light goldenrod yellow', 'pale goldenrod', 'goldenrod', 'dark goldenrod', 'beige', 'moccasin', 'blanched almond', 'navajo white', 'antique white', 'bisque', 'burlywood', 'dark khaki', 'khaki', 'tan', 'wheat', 'snow', 'floral white', 'old lace', 'ivory', 'linen', 'seashell', 'honeydew', 'mint cream', 'azure', 'lavender', 'ghost white', 'white smoke', 'gainsboro', 'light gray', 'silver', 'dark gray', 'gray', 'dim gray', 'slate gray', 'light slate gray', 'dark slate gray', 'black', 'medium violet red', 'pale violet red', 'deep pink', 'hot pink', 'light pink', 'pink', 'peach puff', 'rosy brown', 'saddle brown', 'sandy brown', 'chocolate', 'peru', 'sienna', 'brown', 'maroon', 'white', 'misty rose', 'lavender blush', 'papaya whip', 'lemon chiffon', 'light yellow', 'corn silk', 'pale green', 'light green', 'olive drab', 'olive', 'dark sea green']+all_colors = [+ "alice blue",+ "dodger blue",+ "light sky blue",+ "deep sky blue",+ "sky blue",+ "steel blue",+ "light steel blue",+ "medium blue",+ "navy blue",+ "blue",+ "royal blue",+ "cadet blue",+ "cornflower blue",+ "medium slate blue",+ "slate blue",+ "dark slate blue",+ "powder blue",+ "turquoise",+ "dark turquoise",+ "medium turquoise",+ "pale turquoise",+ "light sea green",+ "medium sea green",+ "sea green",+ "forest green",+ "green yellow",+ "lime green",+ "dark green",+ "green",+ "lime",+ "chartreuse",+ "lawn green",+ "yellow green",+ "olive green",+ "dark olive green",+ "medium spring green",+ "spring green",+ "medium aquamarine",+ "aquamarine",+ "aqua",+ "cyan",+ "dark cyan",+ "teal",+ "medium orchid",+ "dark orchid",+ "orchid",+ "blue violet",+ "violet",+ "dark violet",+ "plum",+ "thistle",+ "magenta",+ "fuchsia",+ "dark magenta",+ "medium purple",+ "purple",+ "rebecca purple",+ "dark red",+ "fire brick",+ "indian red",+ "light coral",+ "dark salmon",+ "light salmon",+ "salmon",+ "red",+ "crimson",+ "tomato",+ "coral",+ "orange red",+ "dark orange",+ "orange",+ "yellow",+ "gold",+ "light goldenrod yellow",+ "pale goldenrod",+ "goldenrod",+ "dark goldenrod",+ "beige",+ "moccasin",+ "blanched almond",+ "navajo white",+ "antique white",+ "bisque",+ "burlywood",+ "dark khaki",+ "khaki",+ "tan",+ "wheat",+ "snow",+ "floral white",+ "old lace",+ "ivory",+ "linen",+ "seashell",+ "honeydew",+ "mint cream",+ "azure",+ "lavender",+ "ghost white",+ "white smoke",+ "gainsboro",+ "light gray",+ "silver",+ "dark gray",+ "gray",+ "dim gray",+ "slate gray",+ "light slate gray",+ "dark slate gray",+ "black",+ "medium violet red",+ "pale violet red",+ "deep pink",+ "hot pink",+ "light pink",+ "pink",+ "peach puff",+ "rosy brown",+ "saddle brown",+ "sandy brown",+ "chocolate",+ "peru",+ "sienna",+ "brown",+ "maroon",+ "white",+ "misty rose",+ "lavender blush",+ "papaya whip",+ "lemon chiffon",+ "light yellow",+ "corn silk",+ "pale green",+ "light green",+ "olive drab",+ "olive",+ "dark sea green",+]+class Colors(Dataset):def __init__(self, sort_by_suffix=True, *args, **kwargs) -> None:@@ -12,15 +151,17 @@ class Colors(Dataset):self.sort_by_suffix = sort_by_suffixcolors = self.sorted_by_suffix(all_colors)- train_size = int(len(colors) * 0.6) # chosen to ensure that similar colors aren't repeated between train and dev+ train_size = int(+ len(colors) * 0.6+ ) # chosen to ensure that similar colors aren't repeated between train and devtrain_colors, dev_colors = colors[:train_size], colors[train_size:]- self._train = [dict(color=color) for color in train_colors]- self._dev = [dict(color=color) for color in dev_colors]+ self._train = [{"color": color} for color in train_colors]+ self._dev = [{"color": color} for color in dev_colors]random.Random(0).shuffle(self._train)random.Random(0).shuffle(self._dev)-+def sorted_by_suffix(self, colors):if not self.sort_by_suffix:return colors@@ -28,6 +169,6 @@ class Colors(Dataset):if isinstance(colors[0], str):sorted_colors = sorted(colors, key=lambda x: x[::-1])else:- sorted_colors = sorted(colors, key=lambda x: x['color'][::-1])+ sorted_colors = sorted(colors, key=lambda x: x["color"][::-1])return sorted_colorsdiff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.pyindex a45d2c057..47fa51e13 100644--- a/dspy/datasets/dataloader.py+++ b/dspy/datasets/dataloader.py@@ -1,6 +1,6 @@import randomfrom collections.abc import Mapping-from typing import TYPE_CHECKING, List, Tuple, Union+from typing import TYPE_CHECKING, List, Optional, Tuple, Unionimport dspyfrom dspy.datasets.dataset import Dataset@@ -18,7 +18,7 @@ class DataLoader(Dataset):dataset_name: str,*args,input_keys: Tuple[str] = (),- fields: Tuple[str] = None,+ fields: Optional[Tuple[str]] = None,**kwargs,) -> Union[Mapping[str, List[dspy.Example]], List[dspy.Example]]:if fields and not isinstance(fields, tuple):@@ -60,7 +60,12 @@ class DataLoader(Dataset):for row in dataset]- def from_csv(self, file_path: str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ def from_csv(+ self,+ file_path: str,+ fields: Optional[List[str]] = None,+ input_keys: Tuple[str] = (),+ ) -> List[dspy.Example]:from datasets import load_datasetdataset = load_dataset("csv", data_files=file_path)["train"]@@ -73,7 +78,7 @@ class DataLoader(Dataset):def from_pandas(self,df: "pd.DataFrame",- fields: list[str] = None,+ fields: Optional[List[str]] = None,input_keys: tuple[str] = (),) -> list[dspy.Example]:if fields is None:@@ -83,7 +88,12 @@ class DataLoader(Dataset):dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for _, row in df.iterrows()]- def from_json(self, file_path: str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ def from_json(+ self,+ file_path: str,+ fields: Optional[List[str]] = None,+ input_keys: Tuple[str] = (),+ ) -> List[dspy.Example]:from datasets import load_datasetdataset = load_dataset("json", data_files=file_path)["train"]@@ -93,7 +103,12 @@ class DataLoader(Dataset):return [dspy.Example({field: row[field] for field in fields}).with_inputs(*input_keys) for row in dataset]- def from_parquet(self, file_path: str, fields: List[str] = None, input_keys: Tuple[str] = ()) -> List[dspy.Example]:+ def from_parquet(+ self,+ file_path: str,+ fields: Optional[List[str]] = None,+ input_keys: Tuple[str] = (),+ ) -> List[dspy.Example]:from datasets import load_datasetdataset = load_dataset("parquet", data_files=file_path)["train"]@@ -138,8 +153,8 @@ class DataLoader(Dataset):self,dataset: List[dspy.Example],train_size: Union[int, float] = 0.75,- test_size: Union[int, float] = None,- random_state: int = None,+ test_size: Optional[Union[int, float]] = None,+ random_state: Optional[int] = None,) -> Mapping[str, List[dspy.Example]]:if random_state is not None:random.seed(random_state)diff --git a/dspy/datasets/dataset.py b/dspy/datasets/dataset.pyindex 14ede3121..f31fda4aa 100644--- a/dspy/datasets/dataset.py+++ b/dspy/datasets/dataset.py@@ -1,19 +1,19 @@import randomimport uuid-from dspy.dsp.utils import dotdictfrom dspy import Example+from dspy.dsp.utils import dotdictclass Dataset:- def __init__(self, train_seed=0, train_size=None, eval_seed=0, dev_size=None, test_size=None, input_keys=[]):+ def __init__(self, train_seed=0, train_size=None, eval_seed=0, dev_size=None, test_size=None, input_keys=None):self.train_size = train_sizeself.train_seed = train_seedself.dev_size = dev_sizeself.dev_seed = eval_seedself.test_size = test_sizeself.test_seed = eval_seed- self.input_keys = input_keys+ self.input_keys = input_keys or []self.do_shuffle = True@@ -84,13 +84,14 @@ class Dataset:@classmethoddef prepare_by_seed(cls,- train_seeds=[1, 2, 3, 4, 5],+ train_seeds=None,train_size=16,dev_size=1000,divide_eval_per_seed=True,eval_seed=2023,**kwargs,):+ train_seeds = train_seeds or [1, 2, 3, 4, 5]data_args = dotdict(train_size=train_size, eval_seed=eval_seed, dev_size=dev_size, test_size=0, **kwargs)dataset = cls(**data_args)diff --git a/dspy/datasets/gsm8k.py b/dspy/datasets/gsm8k.pyindex d2bb7ac08..3c0041993 100644--- a/dspy/datasets/gsm8k.py+++ b/dspy/datasets/gsm8k.py@@ -25,7 +25,7 @@ class GSM8K:gold_reasoning = " ".join(answer[:-2])answer = str(int(answer[-1].replace(",", "")))- official_train.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))+ official_train.append({"question": question, "gold_reasoning": gold_reasoning, "answer": answer})for example in tqdm.tqdm(hf_official_test):question = example["question"]@@ -36,7 +36,7 @@ class GSM8K:gold_reasoning = " ".join(answer[:-2])answer = str(int(answer[-1].replace(",", "")))- official_test.append(dict(question=question, gold_reasoning=gold_reasoning, answer=answer))+ official_test.append({"question": question, "gold_reasoning": gold_reasoning, "answer": answer})rng = random.Random(0)rng.shuffle(official_train)diff --git a/dspy/datasets/hotpotqa.py b/dspy/datasets/hotpotqa.pyindex 995418388..167065f73 100644--- a/dspy/datasets/hotpotqa.py+++ b/dspy/datasets/hotpotqa.py@@ -85,9 +85,9 @@ if __name__ == "__main__":"""What was the population of the city where Woodward Avenue ends in 2010?-Where did the star , who is also an executive producer, of the Mick begin her carrer?+Where did the star , who is also an executive producer, of the Mick begin her carrer?16 1000 0Both London and German have seen attacks during war, there was one specific type of attack that Germany called the blitz, what did London call a similar attack?Pre-Madonna was a collection of demos by the singer who was a leading presence during the emergence of what network?-Alan Mills composed the classic folk song that tells the story of what?+Alan Mills composed the classic folk song that tells the story of what?"""