DSPy: The framework for programming—not prompting—language models
Python
ci(dspy): Add Ruff linting workflow (#538)
Isaac Millera637bf47b527
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.ymlindex 4a31b8645..52ac21e3c 100644--- a/.github/workflows/run_tests.yml+++ b/.github/workflows/run_tests.yml@@ -1,4 +1,4 @@-name: Build and Run Tests+name: Fix, Test, and Buildon:push:@@ -10,24 +10,54 @@ env:POETRY_VERSION: "1.6.1"jobs:+ fix:+ name: Apply Ruff Fix+ runs-on: ubuntu-latest+ permissions:+ contents: write+ steps:+ - uses: actions/checkout@v3+ - uses: actions/setup-python@v5+ - uses: chartboost/ruff-action@v1+ with:+ args: --fix-only+ - uses: stefanzweifel/git-auto-commit-action@v5+ with:+ commit_message: "Automatic Style fixes"+test:- name: Test+ name: Run Testsruns-on: ubuntu-lateststrategy:matrix:python-version: ["3.9"]steps:- - uses: actions/checkout@v3+ - uses: actions/checkout@v4+ with:+ ref: ${{ github.head_ref }}+ - name: Load cached Poetry installation+ id: cached-poetry+ uses: actions/cache@v3+ with:+ path: ~/.local+ key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}+ - name: Install Poetry+ if: steps.cached-poetry.outputs.cache-hit != 'true'+ uses: snok/install-poetry@v1- name: Set up python ${{ matrix.python-version }}uses: actions/setup-python@v5with:python-version: ${{ matrix.python-version }}- - name: Install Poetry- uses: snok/install-poetry@v1+ cache: "poetry"- name: Install dependencies- run: poetry install+ run: poetry install --no-interaction --no-root+ - name: Run lint with tests+ uses: chartboost/ruff-action@v1+ with:+ args: --fix-only- name: Run tests with pytestrun: poetry run pytest tests/+build_poetry:name: Build Poetryruns-on: ubuntu-latest@@ -35,13 +65,23 @@ jobs:matrix:python-version: ["3.9"]steps:- - uses: actions/checkout@v3+ - uses: actions/checkout@v4+ with:+ ref: ${{ github.head_ref }}+ - name: Load cached Poetry installation+ id: cached-poetry+ uses: actions/cache@v3+ with:+ path: ~/.local+ key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}+ - name: Install Poetry+ if: steps.cached-poetry.outputs.cache-hit != 'true'+ uses: snok/install-poetry@v1- name: Set up python ${{ matrix.python-version }}uses: actions/setup-python@v5with:python-version: ${{ matrix.python-version }}- - name: Install Poetry- uses: snok/install-poetry@v1+ cache: "poetry"- name: Buildrun: poetry build- name: Install built package@@ -50,6 +90,7 @@ jobs:run: python -c "import dspy"- name: Test import dsprun: python -c "import dsp"+build_setup:name: Build Setupruns-on: ubuntu-latest@@ -57,10 +98,22 @@ jobs:matrix:python-version: ["3.9"]steps:- - uses: actions/checkout@v3+ - uses: actions/checkout@v4+ with:+ ref: ${{ github.head_ref }}+ - name: Load cached Poetry installation+ id: cached-poetry+ uses: actions/cache@v3+ with:+ path: ~/.local+ key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}+ - name: Install Poetry+ if: steps.cached-poetry.outputs.cache-hit != 'true'+ uses: snok/install-poetry@v1- name: Set up python ${{ matrix.python-version }}uses: actions/setup-python@v5with:python-version: ${{ matrix.python-version }}+ cache: "poetry"- name: Run setup.py buildrun: python setup.py builddiff --git a/dsp/__init__.py b/dsp/__init__.pyindex fc8a74241..4f98315f1 100644--- a/dsp/__init__.py+++ b/dsp/__init__.py@@ -3,7 +3,6 @@ from .primitives import *from .templates import *from .utils import settings-"""TODO:diff --git a/dsp/evaluation/utils.py b/dsp/evaluation/utils.pyindex 1bfd740e4..7b6a47780 100644--- a/dsp/evaluation/utils.py+++ b/dsp/evaluation/utils.py@@ -1,7 +1,8 @@-import dsp-import tqdmimport pandas as pd+import tqdm++import dsptry:from IPython.display import display as ipython_displaydiff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex d824d3fa5..241a349c5 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -1,19 +1,15 @@-from .cache_utils import *from .azure_openai import AzureOpenAI-from .gpt3 import *+from .bedrock import *+from .cache_utils import *+from .clarifai import *+from .cohere import *+from .colbertv2 import ColBERTv2from .databricks import *+from .google import *+from .gpt3 import *from .hf import HFModel-from .colbertv2 import ColBERTv2-from .sentence_vectorizer import *-from .cohere import *-from .sbert import *-from .pyserini import *+from .hf_client import Anyscale, HFClientTGI, Togetherfrom .ollama import *-from .clarifai import *-from .bedrock import *-from .google import *---from .hf_client import HFClientTGI-from .hf_client import Anyscale-from .hf_client import Together+from .pyserini import *+from .sbert import *+from .sentence_vectorizer import *diff --git a/dsp/modules/aws_lm.py b/dsp/modules/aws_lm.pyindex 83674db10..a44e30a49 100644--- a/dsp/modules/aws_lm.py+++ b/dsp/modules/aws_lm.py@@ -4,10 +4,11 @@ A generalized AWS LLM.from __future__ import annotations-from abc import abstractmethod+import jsonimport logging+from abc import abstractmethodfrom typing import Any, Literal-import json+from dsp.modules.lm import LM# Heuristic translating number of chars to tokensdiff --git a/dsp/modules/azurecognitivesearch.py b/dsp/modules/azurecognitivesearch.pyindex e4ff2e996..fedc59b01 100644--- a/dsp/modules/azurecognitivesearch.py+++ b/dsp/modules/azurecognitivesearch.py@@ -1,6 +1,7 @@-from typing import Union, Any+from typing import Any, Unionfrom dsp.utils import dotdict+try:from azure.core.credentials import AzureKeyCredentialfrom azure.search.documents import SearchClientdiff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.pyindex c52ae6197..ac8b42d2d 100644--- a/dsp/modules/bedrock.py+++ b/dsp/modules/bedrock.py@@ -1,7 +1,8 @@from __future__ import annotations-from typing import Anyimport json+from typing import Any+from dsp.modules.aws_lm import AWSLMdiff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.pyindex 78270c879..e6c09fe71 100644--- a/dsp/modules/cache_utils.py+++ b/dsp/modules/cache_utils.py@@ -1,12 +1,11 @@import os-+from functools import wrapsfrom pathlib import Path+from joblib import Memory-from functools import wrapsfrom dsp.utils import dotdict-cache_turn_on = Truediff --git a/dsp/modules/clarifai.py b/dsp/modules/clarifai.pyindex 8a691a4be..2d5839c62 100644--- a/dsp/modules/clarifai.py+++ b/dsp/modules/clarifai.py@@ -3,6 +3,7 @@ from typing import Any, Optionalfrom dsp.modules.lm import LM+class ClarifaiLLM(LM):"""Integration to call models hosted in clarifai platform.diff --git a/dsp/modules/cohere.py b/dsp/modules/cohere.pyindex 405eaea48..7308d5935 100644--- a/dsp/modules/cohere.py+++ b/dsp/modules/cohere.py@@ -1,5 +1,6 @@import mathfrom typing import Any, Optional+import backofffrom dsp.modules.lm import LMdiff --git a/dsp/modules/colbertv2.py b/dsp/modules/colbertv2.pyindex 1b33d077a..8ff3c1622 100644--- a/dsp/modules/colbertv2.py+++ b/dsp/modules/colbertv2.py@@ -1,11 +1,11 @@import functools-from typing import Optional, Union, Any+from typing import Any, Optional, Union+import requestsfrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemoryfrom dsp.utils import dotdict-# TODO: Ideally, this takes the name of the index and looks up its port.diff --git a/dsp/modules/databricks.py b/dsp/modules/databricks.pyindex 8f7d7d6e8..7d90e4fc2 100644--- a/dsp/modules/databricks.py+++ b/dsp/modules/databricks.py@@ -19,8 +19,8 @@ from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turnfrom dsp.modules.gpt3 import GPT3try:- from openai.openai_object import OpenAIObjectimport openai.error+ from openai.openai_object import OpenAIObjectERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)except Exception:ERRORS = (openai.RateLimitError, openai.APIError)diff --git a/dsp/modules/finetuning/finetune_hf.py b/dsp/modules/finetuning/finetune_hf.pyindex 0ade213ba..a22ab10fc 100644--- a/dsp/modules/finetuning/finetune_hf.py+++ b/dsp/modules/finetuning/finetune_hf.py@@ -1,28 +1,30 @@# Adapted from: https://www.philschmid.de/fine-tune-flan-t5#3-fine-tune-and-evaluate-flan-t5-import os-import jsonimport copyimport glob-import torch+import json+import osimport warnings+from dataclasses import dataclass+import evaluateimport numpy as np+import torchfrom datasets import Dataset-from dataclasses import dataclassfrom transformers import (- set_seed,AutoConfig,- AutoModelForSeq2SeqLM,AutoModelForCausalLM,+ AutoModelForSeq2SeqLM,AutoTokenizer,+ DataCollatorForSeq2Seq,PreTrainedTokenizer,- Trainer,Seq2SeqTrainer,- TrainingArguments,Seq2SeqTrainingArguments,- DataCollatorForSeq2Seq,+ Trainer,+ TrainingArguments,+ set_seed,)+# from peft import get_peft_model, LoraConfig, TaskTypefrom transformers.trainer_callback import TrainerCallbackdiff --git a/dsp/modules/google.py b/dsp/modules/google.pyindex 9a1c3937c..5d97534ea 100644--- a/dsp/modules/google.py+++ b/dsp/modules/google.py@@ -1,6 +1,7 @@import os-from typing import Any, Optionalfrom collections.abc import Iterable+from typing import Any, Optional+import backofffrom dsp.modules.lm import LMdiff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex ad440f234..d0b343af3 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -11,10 +11,10 @@ import functoolsimport jsonfrom typing import Any, Literal, Optional, cast-import dspimport backoffimport openai+import dspfrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_onfrom dsp.modules.lm import LM@@ -24,8 +24,8 @@ except Exception:OPENAI_LEGACY = Truetry:- from openai.openai_object import OpenAIObjectimport openai.error+ from openai.openai_object import OpenAIObjectERRORS = (openai.error.RateLimitError,diff --git a/dsp/modules/hf.py b/dsp/modules/hf.pyindex 3c9306a8f..aad0c0e36 100644--- a/dsp/modules/hf.py+++ b/dsp/modules/hf.py@@ -1,8 +1,9 @@# from peft import PeftConfig, PeftModel# from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig-from typing import Optional, Literal+from typing import Literal, Optionalfrom dsp.modules.lm import LM+# from dsp.modules.finetuning.finetune_hf import preprocess_promptdef openai_to_hf(**kwargs):@@ -43,8 +44,8 @@ class HFModel(LM):self.device_map = hf_device_mapif not self.is_client:try:- from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfigimport torch+ from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizerexcept ImportError as exc:raise ModuleNotFoundError("You need to install Hugging Face transformers library to use HF models.",diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex 4def62819..fb92c48d5 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -1,15 +1,15 @@import osimport random-import requests-from dsp.modules.hf import HFModel, openai_to_hf-from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory-import subprocessimport reimport shutil+import subprocess# from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter-import backoff+import requests++from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory+from dsp.modules.hf import HFModel, openai_to_hfERRORS = (Exception)@@ -173,7 +173,7 @@ class HFServerTGI:container_id = match.group(1)port_mapping = subprocess.check_output(['docker', 'port', container_id]).decode().strip()if f'0.0.0.0:{port}' in port_mapping:- subprocess.run(['docker', 'stop', container_id])+ subprocess.run(['docker', 'stop', container_id], check=False)def run_server(self, port, model_name=None, model_path=None, env_variable=None, gpus="all", num_shard=1, max_input_length=4000, max_total_tokens=4096, max_best_of=100):self.close_server(port)@@ -355,8 +355,7 @@ class ChatModuleClient(HFModel):def __init__(self, model, model_path):super().__init__(model=model, is_client=True)- from mlc_chat import ChatModule- from mlc_chat import ChatConfig+ from mlc_chat import ChatConfig, ChatModuleself.cm = ChatModule(model=model, lib_path=model_path, chat_config=ChatConfig(conv_template="LM"),diff --git a/dsp/modules/ollama.py b/dsp/modules/ollama.pyindex a2e3e0139..27304d271 100644--- a/dsp/modules/ollama.py+++ b/dsp/modules/ollama.py@@ -1,10 +1,10 @@-from dsp.modules.lm import LM-from typing import Any, Literal-import datetimeimport hashlib+from typing import Any, Literal+import requests+from dsp.modules.lm import LMdef post_request_metadata(model_name, prompt):diff --git a/dsp/modules/pyserini.py b/dsp/modules/pyserini.pyindex 94ce970ea..5523a76e7 100644--- a/dsp/modules/pyserini.py+++ b/dsp/modules/pyserini.py@@ -1,5 +1,6 @@-from typing import Unionimport json+from typing import Union+from datasets import Datasetfrom dsp.utils import dotdict@@ -30,8 +31,8 @@ class PyseriniRetriever:"""# Keep pyserini as an optional dependency+ from pyserini.prebuilt_index_info import FAISS_INDEX_INFO, IMPACT_INDEX_INFO, TF_INDEX_INFOfrom pyserini.search import FaissSearcher- from pyserini.prebuilt_index_info import TF_INDEX_INFO, FAISS_INDEX_INFO, IMPACT_INDEX_INFOself.encoder = FaissSearcher._init_encoder_from_str(query_encoder)self.dataset = datasetdiff --git a/dsp/primitives/__init__.py b/dsp/primitives/__init__.pyindex 3798444a1..ef278efa6 100644--- a/dsp/primitives/__init__.py+++ b/dsp/primitives/__init__.py@@ -1,6 +1,6 @@-from .primitives import *+from .compiler import *from .demonstrate import *-from .search import *-from .predict import *from .inspect import *-from .compiler import *+from .predict import *+from .primitives import *+from .search import *diff --git a/dsp/primitives/compiler.py b/dsp/primitives/compiler.pyindex 996919e85..625ae23ac 100644--- a/dsp/primitives/compiler.py+++ b/dsp/primitives/compiler.py@@ -1,12 +1,13 @@import os+import random+import subprocessimport time+import tqdmimport ujson-import random-import subprocess+from datasets.fingerprint import Hasherimport dsp-from datasets.fingerprint import Hasherif os.environ.get('DSP_NOTEBOOK_CACHEDIR'):training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler')@@ -24,7 +25,7 @@ def openai_check_finetune(jobname):command = f"""openai api fine_tunes.get -i {jobname}"""print(command)- result = subprocess.run(command.split(), stdout=subprocess.PIPE)+ result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False)output = result.stdout.decode("utf-8").strip()try:diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pyindex ae08e28ba..7dfe126b5 100644--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@ -1,5 +1,5 @@import random-from typing import Callable, Any+from typing import Any, Callableimport numpy as npdiff --git a/dsp/primitives/inspect.py b/dsp/primitives/inspect.pyindex 3fbb05c55..ebcd3b787 100644--- a/dsp/primitives/inspect.py+++ b/dsp/primitives/inspect.py@@ -1,8 +1,9 @@import inspect-import string+import jsonimport random+import string+import requests-import jsonclass FuncInspector:diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.pyindex c89256b91..41edf6c51 100644--- a/dsp/primitives/predict.py+++ b/dsp/primitives/predict.py@@ -1,11 +1,11 @@from collections import Counter-from typing import Callable, Any, Optional+from typing import Any, Callable, Optionalimport dsp-from dsp.utils import zipstar, normalize_text-from dsp.utils.utils import dotdict-from dsp.templates.template_v3 import Templatefrom dsp.primitives.demonstrate import Example+from dsp.templates.template_v3 import Template+from dsp.utils import normalize_text, zipstar+from dsp.utils.utils import dotdictclass Completions:diff --git a/dsp/primitives/primitives.py b/dsp/primitives/primitives.pyindex a7851ea33..2e4c4bfc0 100644--- a/dsp/primitives/primitives.py+++ b/dsp/primitives/primitives.py@@ -1,6 +1,8 @@-import dspfrom functools import wraps+import dsp++# applied right to left (innermost first, like function calls)def compose_decorators(*decorators):def decorator(func):diff --git a/dsp/templates/__init__.py b/dsp/templates/__init__.pyindex b4aed7643..5eaae95f7 100644--- a/dsp/templates/__init__.py+++ b/dsp/templates/__init__.py@@ -1,4 +1,4 @@-from .utils import *from .template_v2 import *from .template_v3 import *+from .utils import *diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.pyindex d6e61642e..236fd1d64 100644--- a/dsp/templates/template_v2.py+++ b/dsp/templates/template_v2.py@@ -1,9 +1,11 @@-from collections import namedtupleimport re-from typing import Union, Any+from collections import namedtuple+from typing import Any, Union+import dspfrom dsp.primitives.demonstrate import Example-from .utils import passages2text, format_answers++from .utils import format_answers, passages2textField = namedtuple("Field", "name separator input_variable output_variable description")diff --git a/dsp/templates/template_v3.py b/dsp/templates/template_v3.pyindex fb097f877..605705c15 100644--- a/dsp/templates/template_v3.py+++ b/dsp/templates/template_v3.py@@ -1,5 +1,6 @@from typing import Callable-from dsp.templates import TemplateV2, passages2text, format_answers, Field++from dsp.templates import Field, TemplateV2, format_answers, passages2textclass Type:diff --git a/dsp/templates/utils.py b/dsp/templates/utils.pyindex 0b0d4891f..53624ad2d 100644--- a/dsp/templates/utils.py+++ b/dsp/templates/utils.py@@ -1,4 +1,4 @@-from typing import Union, Optional+from typing import Optional, Uniondef passages2text(passages: Union[str, list, tuple]) -> str:diff --git a/dsp/utils/__init__.py b/dsp/utils/__init__.pyindex c41f74832..63624350a 100644--- a/dsp/utils/__init__.py+++ b/dsp/utils/__init__.py@@ -1,4 +1,4 @@+from dsp.utils.dpr import *+from dsp.utils.metrics import *from dsp.utils.settings import *from dsp.utils.utils import *-from dsp.utils.metrics import *-from dsp.utils.dpr import *diff --git a/dsp/utils/dpr.py b/dsp/utils/dpr.pyindex 2174ea4da..d4d18f84f 100644--- a/dsp/utils/dpr.py+++ b/dsp/utils/dpr.py@@ -4,9 +4,10 @@Original license: https://github.com/facebookresearch/DPR/blob/main/LICENSE"""-import regeximport unicodedata+import regex+class Tokens:"""A class to represent a list of tokenized text."""diff --git a/dsp/utils/metrics.py b/dsp/utils/metrics.pyindex f0d69f00f..ddbd51ae9 100644--- a/dsp/utils/metrics.py+++ b/dsp/utils/metrics.py@@ -1,8 +1,8 @@import reimport stringimport unicodedata-from collections import Counter+from dsp.utils.utils import print_messagediff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex 462f99d42..1de0ada7e 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -1,6 +1,7 @@+import threadingfrom contextlib import contextmanager+from dsp.utils.utils import dotdict-import threadingclass Settings:diff --git a/dsp/utils/settings_v2.py b/dsp/utils/settings_v2.pyindex 2998ca7cd..6652474d3 100644--- a/dsp/utils/settings_v2.py+++ b/dsp/utils/settings_v2.py@@ -1,7 +1,8 @@+import copyimport threading-from contextlib import contextmanagerfrom concurrent.futures import ThreadPoolExecutor, as_completed-import copy+from contextlib import contextmanager+class Settings:def __init__(self):diff --git a/dsp/utils/utils.py b/dsp/utils/utils.pyindex aeba5c970..d5a30b0e3 100644--- a/dsp/utils/utils.py+++ b/dsp/utils/utils.py@@ -1,10 +1,10 @@-import os-import tqdmimport datetimeimport itertools-+import osfrom collections import defaultdict+import tqdm+def print_message(*s, condition=True, pad=False, sep=None):s = " ".join([str(x) for x in s])@@ -77,6 +77,7 @@ def batch(group, bsize, provide_offset=False):import copy+class dotdict(dict):def __getattr__(self, key):if key.startswith('__') and key.endswith('__'):diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 98fe6fba8..b61ea8a01 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -1,18 +1,12 @@-from dsp.modules.hf_client import ChatModuleClient-from dsp.modules.hf_client import HFServerTGI, HFClientVLLM, HFClientSGLang-from .signatures import *--from .retrieve import *-from .predict import *-from .primitives import *-# from .evaluation import *--# FIXME:--import dsp+from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI++from .predict import *+from .primitives import *+from .retrieve import *+from .signatures import *settings = dsp.settingsdiff --git a/dspy/datasets/__init__.py b/dspy/datasets/__init__.pyindex 62409d6ed..bf6e3dd01 100644--- a/dspy/datasets/__init__.py+++ b/dspy/datasets/__init__.py@@ -1,4 +1,4 @@+from .colors import Colors+from .dataloader import DataLoaderfrom .dataset import Datasetfrom .hotpotqa import HotPotQA-from .colors import Colors-from .dataloader import DataLoader\ No newline at end of filediff --git a/dspy/datasets/colors.py b/dspy/datasets/colors.pyindex 265badf27..edf2e4169 100644--- a/dspy/datasets/colors.py+++ b/dspy/datasets/colors.py@@ -1,4 +1,5 @@import random+from dspy.datasets.dataset import Dataset### A bunch of colors, originally from matplotlibdiff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.pyindex 8900c107f..c9da0270d 100644--- a/dspy/datasets/dataloader.py+++ b/dspy/datasets/dataloader.py@@ -1,10 +1,12 @@-import dspyimport random-from dspy.datasets import Dataset+from collections.abc import Mapping+from typing import List, Tuple, Unionfrom datasets import load_dataset-from typing import Union, List, Tuple-from collections.abc import Mapping++import dspy+from dspy.datasets.dataset import Dataset+class DataLoader(Dataset):def __init__(self,):diff --git a/dspy/datasets/dataset.py b/dspy/datasets/dataset.pyindex 9cd279199..c66c5edc2 100644--- a/dspy/datasets/dataset.py+++ b/dspy/datasets/dataset.py@@ -1,8 +1,9 @@-import uuidimport random+import uuid-from dspy import Examplefrom dsp.utils import dotdict+from dspy import Example+class Dataset:def __init__(self, train_seed=0, train_size=None, eval_seed=0, dev_size=None, test_size=None):diff --git a/dspy/datasets/gsm8k.py b/dspy/datasets/gsm8k.pyindex 3b3514862..328d1aee8 100644--- a/dspy/datasets/gsm8k.py+++ b/dspy/datasets/gsm8k.py@@ -1,8 +1,9 @@-import tqdmimport random+import tqdmfrom datasets import load_dataset+class GSM8K:def __init__(self) -> None:super().__init__()diff --git a/dspy/datasets/hotpotqa.py b/dspy/datasets/hotpotqa.pyindex f1cc73482..9fbf08617 100644--- a/dspy/datasets/hotpotqa.py+++ b/dspy/datasets/hotpotqa.py@@ -1,6 +1,7 @@import randomfrom datasets import load_dataset+from dspy.datasets.dataset import Datasetdiff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.pyindex eea386c59..2526c2be0 100644--- a/dspy/evaluate/__init__.py+++ b/dspy/evaluate/__init__.py@@ -1,4 +1,5 @@+from dsp.utils import EM, normalize_text++from .auto_evaluation import *from .evaluate import Evaluatefrom .metrics import *-from .auto_evaluation import *-from dsp.utils import EM, normalize_textdiff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.pyindex 1ec659540..569a5b16c 100644--- a/dspy/evaluate/auto_evaluation.py+++ b/dspy/evaluate/auto_evaluation.py@@ -1,4 +1,5 @@-import dspy+import dspy+class AnswerCorrectnessSignature(dspy.Signature):"""Verify that the predicted answer matches the gold answer."""diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 0be330375..f38bcff38 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -1,11 +1,14 @@-import dsp-import tqdm-import typesimport threading+import types+import pandas as pd+import tqdm++import dsptry:- from IPython.display import display as ipython_display, HTML+ from IPython.display import HTML+ from IPython.display import display as ipython_displayexcept ImportError:ipython_display = printHTML = lambda x: xdiff --git a/dspy/evaluate/metrics.py b/dspy/evaluate/metrics.pyindex 79c6208af..0e25b5a11 100644--- a/dspy/evaluate/metrics.py+++ b/dspy/evaluate/metrics.py@@ -2,6 +2,7 @@import dsp+def answer_exact_match(example, pred, trace=None, frac=1.0):assert(type(example.answer) is str or type(example.answer) is list)diff --git a/dspy/experimental/__init__.py b/dspy/experimental/__init__.pyindex b4385639d..36477304c 100644--- a/dspy/experimental/__init__.py+++ b/dspy/experimental/__init__.py@@ -1,2 +1,2 @@-from .synthetic_data import *from .synthesizer import *+from .synthetic_data import *diff --git a/dspy/experimental/synthesizer.py b/dspy/experimental/synthesizer.pyindex e247d35f3..a66bf4d2b 100644--- a/dspy/experimental/synthesizer.py+++ b/dspy/experimental/synthesizer.py@@ -1,8 +1,11 @@-import dspyimport randomfrom typing import List-from tqdm import tqdm, trange+from datasets import Dataset+from tqdm import tqdm, trange++import dspy+def format_examples(examples: List[dspy.Example]):if isinstance(examples, str):@@ -14,11 +17,11 @@ def format_examples(examples: List[dspy.Example]):input_keys = example.inputs().keys()label_keys = example.labels().keys()- formatted_example += f"Inputs:\n"+ formatted_example += "Inputs:\n"for key in input_keys:formatted_example += f"{key}: {example[key]}\n"- formatted_example += f"Outputs:\n"+ formatted_example += "Outputs:\n"for key in label_keys:formatted_example += f"{key}: {example[key]}\n"@@ -94,7 +97,7 @@ class Synthesizer:self.generate_input_data = self.generate_input_data.insert(-1,field_name,- output_field+ output_field,)input_field = dspy.InputField(@@ -104,7 +107,7 @@ class Synthesizer:self.generate_output_data = self.generate_output_data.insert(-1,field_name,- input_field+ input_field,)for key in tqdm(output_keys, desc="Preparing Output Fields"):@@ -123,7 +126,7 @@ class Synthesizer:self.generate_output_data = self.generate_output_data.insert(-1,field_name,- output_field+ output_field,)return dspy.ChainOfThought(self.generate_input_data), dspy.Predict(self.generate_output_data)@@ -167,7 +170,7 @@ class Synthesizer:extention = mode or path.split(".")[-1]dataset = Dataset.from_list(- [example.toDict() for example in data]+ [example.toDict() for example in data],)if extention == "csv":diff --git a/dspy/experimental/synthetic_data.py b/dspy/experimental/synthetic_data.pyindex d5177fcac..37c140352 100644--- a/dspy/experimental/synthetic_data.py+++ b/dspy/experimental/synthetic_data.py@@ -1,8 +1,11 @@-from pydantic import BaseModel-import dspyimport randomfrom typing import List, Optional+from pydantic import BaseModel++import dspy++class descriptionSignature(dspy.Signature):field_name = dspy.InputField(desc="name of a field")example = dspy.InputField(desc="an example value for the field")@@ -55,7 +58,7 @@ class SyntheticDataGenerator:'__doc__': f"Generates the following outputs: {{{', '.join(properties.keys())}}}.",'sindex': dspy.InputField(desc="a random string"),**{field_name: dspy.OutputField(desc=properties[field_name].get('description', 'No description'))- for field_name in properties.keys()}+ for field_name in properties.keys()},}# # Usage examplediff --git a/dspy/functional/__init__.py b/dspy/functional/__init__.pyindex 3ccced47b..746ac9318 100644--- a/dspy/functional/__init__.py+++ b/dspy/functional/__init__.py@@ -1 +1 @@-from .functional import cot, predictor, FunctionalModule, TypedPredictor, TypedChainOfThought+from .functional import FunctionalModule, TypedChainOfThought, TypedPredictor, cot, predictordiff --git a/dspy/functional/functional.py b/dspy/functional/functional.pyindex cb7b3669a..97bbdb8d8 100644--- a/dspy/functional/functional.py+++ b/dspy/functional/functional.py@@ -1,18 +1,15 @@-from collections import defaultdictimport inspect-import os-import openai-import dspy+import jsonimport typing+from typing import Annotated, List, Tuple+import pydantic-from typing import Annotated, List, Tuple # noqa: UP035-from dsp.templates import passages2text-import json-from dspy.primitives.prediction import Prediction+import dspy+from dsp.templates.utils import passages2text+from dspy.primitives.prediction import Predictionfrom dspy.signatures.signature import ensure_signature, make_signature-MAX_RETRIES = 3@@ -199,7 +196,7 @@ class TypedPredictor(dspy.Module):else:# If there are no errors, we return the parsed resultsreturn Prediction.from_completions(- {key: [r[key] for r in parsed_results] for key in signature.output_fields}+ {key: [r[key] for r in parsed_results] for key in signature.output_fields},)raise ValueError("Too many retries trying to get the correct output format. " + "Try simplifying the requirements.",@@ -326,8 +323,8 @@ def gold_passages_retrieved(example, pred, _trace=None) -> bool:def hotpot() -> None:- from dsp.utils import deduplicateimport dspy.evaluate+ from dsp.utils import deduplicatefrom dspy.datasets import HotPotQAfrom dspy.evaluate.evaluate import Evaluatefrom dspy.teleprompt.bootstrap import BootstrapFewShotdiff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.pyindex 8b0770150..b176d2a0e 100644--- a/dspy/predict/__init__.py+++ b/dspy/predict/__init__.py@@ -1,9 +1,9 @@-from .predict import Predict+from .aggregation import majorityfrom .chain_of_thought import ChainOfThought-from .multi_chain_comparison import MultiChainComparisonfrom .chain_of_thought_with_hint import ChainOfThoughtWithHint-from .react import ReAct-from .aggregation import majority+from .knn import KNN+from .multi_chain_comparison import MultiChainComparison+from .predict import Predictfrom .program_of_thought import ProgramOfThought+from .react import ReActfrom .retry import Retry-from .knn import KNN\ No newline at end of filediff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.pyindex ca4154aa2..4cb6df3f9 100644--- a/dspy/predict/aggregation.py+++ b/dspy/predict/aggregation.py@@ -1,6 +1,5 @@-from dspy.primitives.prediction import Prediction, Completionsfrom dsp.utils import normalize_text-+from dspy.primitives.prediction import Completions, Predictiondefault_normalize = lambda s: normalize_text(s) or Nonediff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.pyindex f374aaabe..70bab0a7a 100644--- a/dspy/predict/chain_of_thought.py+++ b/dspy/predict/chain_of_thought.py@@ -4,7 +4,6 @@ from dspy.signatures.signature import ensure_signaturefrom .predict import Predict-# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.# TODO: This shouldn't inherit from Predict. It should be a module that has one or two predictors.diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.pyindex 6f7856632..484a2a38a 100644--- a/dspy/predict/chain_of_thought_with_hint.py+++ b/dspy/predict/chain_of_thought_with_hint.py@@ -3,8 +3,6 @@ import dspyfrom .predict import Predict--# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.class ChainOfThoughtWithHint(Predict):diff --git a/dspy/predict/knn.py b/dspy/predict/knn.pyindex e0e49fe2b..e275ccb6a 100644--- a/dspy/predict/knn.py+++ b/dspy/predict/knn.py@@ -1,7 +1,10 @@from typing import List+import numpy as np+import dsp+class KNN:def __init__(self, k: int, trainset: List[dsp.Example]):self.k = kdiff --git a/dspy/predict/langchain.py b/dspy/predict/langchain.pyindex 4be855e8d..86d439f50 100644--- a/dspy/predict/langchain.py+++ b/dspy/predict/langchain.py@@ -1,18 +1,17 @@import copyimport random+from langchain_core.pydantic_v1 import Extra+from langchain_core.runnables import Runnable+import dspimport dspy-from dspy.predict.parameter import Parameterfrom dspy.predict.predict import Predictfrom dspy.primitives.prediction import Predictionfrom dspy.signatures.field import InputField, OutputFieldfrom dspy.signatures.signature import infer_prefix-from langchain_core.pydantic_v1 import Extra-from langchain_core.runnables import Runnable-# TODO: This class is currently hard to test, because it hardcodes gpt-4 usage:# gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat')diff --git a/dspy/predict/multi_chain_comparison.py b/dspy/predict/multi_chain_comparison.pyindex ad69b646e..37b66afad 100644--- a/dspy/predict/multi_chain_comparison.py+++ b/dspy/predict/multi_chain_comparison.py@@ -1,8 +1,8 @@import dspyfrom dspy.signatures.signature import ensure_signature-from .predict import Predict-from ..primitives.program import Module+from ..primitives.program import Module+from .predict import Predictclass MultiChainComparison(Module):diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 60e668a83..fe6beb448 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -1,11 +1,11 @@-import dspimport random+import dspfrom dspy.predict.parameter import Parameterfrom dspy.primitives.prediction import Prediction-from dspy.signatures.signature import ensure_signature, signature_to_template+class Predict(Parameter):def __init__(self, signature, **config):self.stage = random.randbytes(8).hex()diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.pyindex f7e80eed7..1cfb54bd9 100644--- a/dspy/predict/program_of_thought.py+++ b/dspy/predict/program_of_thought.py@@ -1,8 +1,10 @@+import re+import dspyfrom dspy.signatures.signature import ensure_signature+from ..primitives.program import Modulefrom ..primitives.python_interpreter import CodePrompt, PythonInterpreter-import reclass ProgramOfThought(Module):diff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex cf3be8b73..a465ee710 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -1,6 +1,7 @@import dspimport dspyfrom dspy.signatures.signature import ensure_signature+from ..primitives.program import Modulefrom .predict import Predictdiff --git a/dspy/predict/retry.py b/dspy/predict/retry.pyindex af1d37f98..dcef1488f 100644--- a/dspy/predict/retry.py+++ b/dspy/predict/retry.py@@ -1,6 +1,7 @@import copy-import dspy+import dsp+import dspyfrom .predict import Predictdiff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.pyindex b7baf229b..e66783716 100644--- a/dspy/primitives/__init__.py+++ b/dspy/primitives/__init__.py@@ -1,5 +1,5 @@+from .assertions import *from .example import *-from .program import *from .prediction import *-from .assertions import *+from .program import *from .python_interpreter import *diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 140f8bf6a..19f1aea3a 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -1,9 +1,10 @@import inspect+import logging+import uuidfrom typing import Any+import dspimport dspy-import logging-import uuid#################### Assertion Helpers ####################diff --git a/dspy/primitives/module.py b/dspy/primitives/module.pyindex 1b4a342f7..bee80338f 100644--- a/dspy/primitives/module.py+++ b/dspy/primitives/module.py@@ -1,4 +1,5 @@import copy+import ujsondiff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex 5e063bfb0..aa499d908 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -1,8 +1,9 @@-from dspy.primitives.module import BaseModule-from dspy.primitives.assertions import *import re+from dspy.primitives.assertions import *+from dspy.primitives.module import BaseModule+class ProgramMeta(type):passdiff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex fd6166db5..1b7456c7b 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -12,10 +12,12 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========import ast+import builtinsimport difflibimport importlibimport reimport typing+from collections.abc import Mappingfrom typing import (Any,Dict,@@ -24,8 +26,6 @@ from typing import (Set,Tuple,)-from collections.abc import Mapping-import builtinsclass InterpreterError(ValueError):diff --git a/dspy/retrieve/chromadb_rm.py b/dspy/retrieve/chromadb_rm.pyindex b4fc0bcf9..07ef407d1 100644--- a/dspy/retrieve/chromadb_rm.py+++ b/dspy/retrieve/chromadb_rm.py@@ -2,10 +2,12 @@Retriever model for chromadb"""-from typing import Optional, List, Union+from typing import List, Optional, Union++import backoffimport openai+import dspy-import backofffrom dsp.utils import dotdicttry:@@ -16,13 +18,13 @@ except Exception:try:import chromadb- from chromadb.config import Settings- from chromadb.utils import embedding_functions+ import chromadb.utils.embedding_functions as effrom chromadb.api.types import (Embeddable,EmbeddingFunction,)- import chromadb.utils.embedding_functions as ef+ from chromadb.config import Settings+ from chromadb.utils import embedding_functionsexcept ImportError:chromadb = Nonediff --git a/dspy/retrieve/databricks_rm.py b/dspy/retrieve/databricks_rm.pyindex 74066ec5b..c275bdddc 100644--- a/dspy/retrieve/databricks_rm.py+++ b/dspy/retrieve/databricks_rm.py@@ -1,10 +1,13 @@-import dspyimport os-import requests-from typing import Union, Listfrom collections import defaultdict+from typing import List, Union++import requests++import dspyfrom dspy.primitives.prediction import Prediction+class DatabricksRM(dspy.Retrieve):"""A retrieval module that uses Databricks Vector Search Endpoint to return the top-k embeddings for a given query.diff --git a/dspy/retrieve/deeplake_rm.py b/dspy/retrieve/deeplake_rm.pyindex 87beb4856..235108a91 100644--- a/dspy/retrieve/deeplake_rm.py+++ b/dspy/retrieve/deeplake_rm.py@@ -2,10 +2,12 @@Retriever model for deeplake"""-from typing import Optional, List, Union+from collections import defaultdict+from typing import List, Optional, Union+import openai+import dspy-from collections import defaultdictfrom dsp.utils import dotdicttry:diff --git a/dspy/retrieve/marqo_rm.py b/dspy/retrieve/marqo_rm.pyindex a090e47f5..29c52fdb4 100644--- a/dspy/retrieve/marqo_rm.py+++ b/dspy/retrieve/marqo_rm.py@@ -1,5 +1,6 @@from collections import defaultdictfrom typing import List, Union+import dspyfrom dspy import dotdictdiff --git a/dspy/retrieve/mongodb_atlas_rm.py b/dspy/retrieve/mongodb_atlas_rm.pyindex 70e784730..3cafb05f6 100644--- a/dspy/retrieve/mongodb_atlas_rm.py+++ b/dspy/retrieve/mongodb_atlas_rm.py@@ -1,23 +1,25 @@-from typing import List, Any-import dspyimport os+from typing import Any, List++import backofffrom openai import (- OpenAI,APITimeoutError,InternalServerError,+ OpenAI,RateLimitError,UnprocessableEntityError,)-import backoff++import dspytry:from pymongo import MongoClientfrom pymongo.errors import (- ConnectionFailure,ConfigurationError,- ServerSelectionTimeoutError,+ ConnectionFailure,InvalidURI,OperationFailure,+ ServerSelectionTimeoutError,)except ImportError:raise ImportError(diff --git a/dspy/retrieve/pgvector_rm.py b/dspy/retrieve/pgvector_rm.pyindex 8403efb2e..cf1773f17 100644--- a/dspy/retrieve/pgvector_rm.py+++ b/dspy/retrieve/pgvector_rm.py@@ -1,10 +1,12 @@-import dspy-import openaifrom typing import List, Optional+import openai++import dspy+try:- from pgvector.psycopg2 import register_vectorimport psycopg2+ from pgvector.psycopg2 import register_vectorfrom psycopg2 import sqlexcept ImportError:raise ImportError(diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.pyindex b520ed5fa..0328bc6ba 100644--- a/dspy/retrieve/pinecone_rm.py+++ b/dspy/retrieve/pinecone_rm.py@@ -3,11 +3,13 @@ Retriever model for PineconeAuthor: Dhar Rawal (@drawal1)"""-from dsp.utils import dotdict-from typing import Optional, List, Union-import dspy+from typing import List, Optional, Union+import backoff+import dspy+from dsp.utils import dotdict+try:import pineconeexcept ImportError:@@ -19,6 +21,7 @@ if pinecone is None:)import openai+try:OPENAI_LEGACY = int(openai.version.__version__[0]) == 0except Exception:diff --git a/dspy/retrieve/qdrant_rm.py b/dspy/retrieve/qdrant_rm.pyindex 46977a252..5c2af050b 100644--- a/dspy/retrieve/qdrant_rm.py+++ b/dspy/retrieve/qdrant_rm.py@@ -1,11 +1,12 @@from collections import defaultdict-from typing import List, Union, Optional+from typing import List, Optional, Union+import dspyfrom dsp.utils import dotdicttry:- from qdrant_client import QdrantClientimport fastembed+ from qdrant_client import QdrantClientexcept ImportError:raise ImportError("The 'qdrant' extra is required to use QdrantRM. Install it with `pip install dspy-ai[qdrant]`",diff --git a/dspy/retrieve/retrieve.py b/dspy/retrieve/retrieve.pyindex 910858241..71b4f5734 100644--- a/dspy/retrieve/retrieve.py+++ b/dspy/retrieve/retrieve.py@@ -1,8 +1,7 @@-from typing import List, Union, Optional--import dspimport random+from typing import List, Optional, Union+import dspfrom dspy.predict.parameter import Parameterfrom dspy.primitives.prediction import Predictiondiff --git a/dspy/retrieve/vectara_rm.py b/dspy/retrieve/vectara_rm.pyindex 4d72ed7e8..047c70d6c 100644--- a/dspy/retrieve/vectara_rm.py+++ b/dspy/retrieve/vectara_rm.py@@ -1,11 +1,11 @@-from collections import defaultdict-from typing import List, Union-import dspy-from typing import Optionalimport jsonimport os+from collections import defaultdict+from typing import List, Optional, Union+import requests+import dspyfrom dsp.utils import dotdictSTART_SNIPPET = "<%START%>"diff --git a/dspy/retrieve/weaviate_rm.py b/dspy/retrieve/weaviate_rm.pyindex 1ef7950c1..61e8d1ef0 100644--- a/dspy/retrieve/weaviate_rm.py+++ b/dspy/retrieve/weaviate_rm.py@@ -1,7 +1,7 @@-from typing import List, Union+from typing import List, Optional, Union+import dspyfrom dsp.utils import dotdict-from typing import Optionaltry:import weaviatediff --git a/dspy/retrieve/you_rm.py b/dspy/retrieve/you_rm.pyindex 7321e67bd..25a62ccea 100644--- a/dspy/retrieve/you_rm.py+++ b/dspy/retrieve/you_rm.py@@ -1,9 +1,10 @@-import dspyimport os+from typing import List, Optional, Union+import requests-from dsp.utils import dotdict-from typing import Union, List, Optional+import dspy+from dsp.utils import dotdictclass YouRM(dspy.Retrieve):diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.pyindex 8c1f16164..22bc6475d 100644--- a/dspy/signatures/signature.py+++ b/dspy/signatures/signature.py@@ -1,12 +1,13 @@import ast-from copy import deepcopy+import reimport typing-import dsp+from copy import deepcopy+from typing import Any, Dict, Tuple, Type, Union # noqa: UP035+from pydantic import BaseModel, Field, create_modelfrom pydantic.fields import FieldInfo-from typing import Any, Type, Union, Dict, Tuple # noqa: UP035-import re+import dspfrom dspy.signatures.field import InputField, OutputField, new_to_old_fielddiff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.pyindex c4454725b..61425e5b9 100644--- a/dspy/teleprompt/__init__.py+++ b/dspy/teleprompt/__init__.py@@ -1,9 +1,9 @@-from .teleprompt import *from .bootstrap import *-from .vanilla import *-from .random_search import *from .finetune import *-from .teleprompt_optuna import *from .knn_fewshot import *+from .random_search import *from .signature_opt import SignatureOptimizer-from .signature_opt_bayesian import BayesianSignatureOptimizer\ No newline at end of file+from .signature_opt_bayesian import BayesianSignatureOptimizer+from .teleprompt import *+from .teleprompt_optuna import *+from .vanilla import *diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 5aaefa271..6fc9dd2ef 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -1,16 +1,15 @@-import dsp-import tqdmimport randomimport threading-import dspy+import tqdm+import dsp+import dspyfrom dspy.primitives import Examplefrom .teleprompt import Teleprompterfrom .vanilla import LabeledFewShot-# TODO: metrics should return an object with __bool__ basically, but fine if they're more complex.# They can also be sortable.diff --git a/dspy/teleprompt/finetune.py b/dspy/teleprompt/finetune.pyindex 29b7887ab..8400fe774 100644--- a/dspy/teleprompt/finetune.py+++ b/dspy/teleprompt/finetune.py@@ -1,17 +1,18 @@import os-import time-import dspimport random+import timeimport ujsonfrom datasets.fingerprint import Hasher+import dspfrom dspy.signatures.signature import signature_to_template-# from dspy.primitives import Example+from .bootstrap import BootstrapFewShot+# from dspy.primitives import Examplefrom .teleprompt import Teleprompter-from .bootstrap import BootstrapFewShot+# from .vanilla import LabeledFewShot# from dspy.evaluate.evaluate import Evaluatediff --git a/dspy/teleprompt/knn_fewshot.py b/dspy/teleprompt/knn_fewshot.pyindex 4bc447abd..b99944707 100644--- a/dspy/teleprompt/knn_fewshot.py+++ b/dspy/teleprompt/knn_fewshot.py@@ -1,9 +1,11 @@-from typing import Listimport types+from typing import List+import dsp+from dspy.teleprompt import BootstrapFewShotfrom .teleprompt import Teleprompter-from dspy.teleprompt import BootstrapFewShot+class KNNFewShot(Teleprompter):def __init__(self, KNN, k: int, trainset: List[dsp.Example]):diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex 3f7f2cd63..bc0e5ef61 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -1,13 +1,11 @@import random+from dspy.evaluate.evaluate import Evaluatefrom dspy.teleprompt.teleprompt import Teleprompterfrom .bootstrap import BootstrapFewShotfrom .vanilla import LabeledFewShot-from dspy.evaluate.evaluate import Evaluate--# TODO: Don't forget dealing with the raw demos.# TODO: Deal with the (pretty common) case of having a metric for filtering and a separate metric for eval.# The metric itself may tell though by the presence of trace.diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.pyindex 3163acf3f..d71aa69a7 100644--- a/dspy/teleprompt/signature_opt.py+++ b/dspy/teleprompt/signature_opt.py@@ -1,9 +1,10 @@+from collections import defaultdict+import dspimport dspy-from dspy.teleprompt.teleprompt import Teleprompter-from dspy.signatures import Signaturefrom dspy.evaluate.evaluate import Evaluate-from collections import defaultdict+from dspy.signatures import Signature+from dspy.teleprompt.teleprompt import Teleprompter"""USAGE SUGGESTIONS:diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.pyindex 85298276a..e316462c9 100644--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@ -1,14 +1,16 @@+import math+import random+from collections import defaultdict++import optuna+import dspimport dspy-from dspy.signatures.signature import signature_to_template-from dspy.teleprompt.teleprompt import Teleprompter-from dspy.signatures import Signaturefrom dspy.evaluate.evaluate import Evaluate-from collections import defaultdict-import random+from dspy.signatures import Signature+from dspy.signatures.signature import signature_to_templatefrom dspy.teleprompt import BootstrapFewShot-import optuna-import math+from dspy.teleprompt.teleprompt import Teleprompter"""USAGE SUGGESTIONS:diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.pyindex 3686e66b7..501bd71fd 100644--- a/dspy/teleprompt/teleprompt_optuna.py+++ b/dspy/teleprompt/teleprompt_optuna.py@@ -1,10 +1,10 @@import optuna+from dspy.evaluate.evaluate import Evaluatefrom dspy.teleprompt.teleprompt import Teleprompterfrom .bootstrap import BootstrapFewShot-from dspy.evaluate.evaluate import Evaluateclass BootstrapFewShotWithOptuna(Teleprompter):def __init__(self, metric, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, num_candidate_programs=16, num_threads=6):diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.pyindex 1f6390536..d97d59b5a 100644--- a/dspy/utils/dummies.py+++ b/dspy/utils/dummies.py@@ -1,9 +1,11 @@import random+import refrom typing import Union-from dsp.modules import LM+import numpy as np++from dsp.modules import LMfrom dsp.utils.utils import dotdict-import reclass DummyLM(LM):diff --git a/examples/longformqa/utils.py b/examples/longformqa/utils.pyindex e4ebadc82..bdae81a24 100644--- a/examples/longformqa/utils.py+++ b/examples/longformqa/utils.py@@ -1,8 +1,10 @@-import regex as reimport nltk+import regex as re+nltk.download('punkt')from nltk.tokenize import sent_tokenize+def extract_text_by_citation(paragraph):citation_regex = re.compile(r'(.*?)(\[\d+\]\.)', re.DOTALL)parts_with_citation = citation_regex.findall(paragraph)diff --git a/inspect-app/app.py b/inspect-app/app.pyindex 5ea50dd92..14ea40551 100644--- a/inspect-app/app.py+++ b/inspect-app/app.py@@ -1,8 +1,9 @@-from flask import Flask, request, jsonify-from flask_cors import CORS-import boto3import time+import boto3+from flask import Flask, jsonify, request+from flask_cors import CORS+app = Flask(__name__)CORS(app)diff --git a/pyproject.toml b/pyproject.tomlindex 1c0f75968..e2c719eaf 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -218,6 +218,8 @@ select = ["ERA",# pandas-vet"PD",+ # Import sort+ "I",# avoid shadowing"PLW",]diff --git a/setup.py b/setup.pyindex d31e9c791..3f3bb1858 100644--- a/setup.py+++ b/setup.py@@ -1,4 +1,4 @@-from setuptools import setup, find_packages+from setuptools import find_packages, setup# Read the content of the README filewith open('README.md', encoding='utf-8') as f:diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.pyindex e55f4dd4c..68d394601 100644--- a/testing/optimizer_tester.py+++ b/testing/optimizer_tester.py@@ -1,18 +1,21 @@-from .tasks.scone import ScoNeTask-from .tasks.hotpotqa import HotPotQATask-from .tasks.gsm8k import GSM8KTask-from .tasks.biodex import BioDexTask-from .tasks.tweet import TweetTask-from .tasks.tweet_metric import TweetMetricTask-from dspy.evaluate import Evaluate-from dotenv import load_dotenv-import openai-import os-import dspyimport csvimport datetime+import osfrom timeit import default_timer as timer+import openai+from dotenv import load_dotenv++import dspy+from dspy.evaluate import Evaluate++from .tasks.biodex import BioDexTask+from .tasks.gsm8k import GSM8KTask+from .tasks.hotpotqa import HotPotQATask+from .tasks.scone import ScoNeTask+from .tasks.tweet import TweetTask+from .tasks.tweet_metric import TweetMetricTask+datasets = ["ScoNe", "HotPotQA", "GSM8K", "BioDex", "Tweet"]class OptimizerTester:diff --git a/testing/tasks/__init__.py b/testing/tasks/__init__.pyindex f5034e36f..0b1ee32ec 100644--- a/testing/tasks/__init__.py+++ b/testing/tasks/__init__.py@@ -1,7 +1,7 @@from .base_task import BaseTask+from .biodex import BioDexTaskfrom .gsm8k import GSM8KTaskfrom .hotpotqa import HotPotQATaskfrom .scone import ScoNeTask-from .biodex import BioDexTaskfrom .tweet import TweetTask-from .tweet_metric import TweetMetricTask\ No newline at end of file+from .tweet_metric import TweetMetricTaskdiff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.pyindex c7f8f42b6..089e6d7be 100644--- a/testing/tasks/base_task.py+++ b/testing/tasks/base_task.py@@ -1,5 +1,6 @@from abc import ABC, abstractmethod+class BaseTask(ABC):def __init__(self):passdiff --git a/testing/tasks/biodex.py b/testing/tasks/biodex.pyindex f98da1141..6e51745b3 100644--- a/testing/tasks/biodex.py+++ b/testing/tasks/biodex.py@@ -1,20 +1,24 @@from __future__ import annotations-from .base_task import BaseTask-import dspy-from dspy.evaluate import Evaluate-from dsp.utils import deduplicate-import tqdm-import datasets+import math-from functools import lru_cacheimport os-import reimport pickle+import re+from collections import defaultdict+from enum import Enum+from functools import lru_cache+from typing import DefaultDict, Dict, List, Optional, Union++import datasets+import tqdmfrom pydantic import BaseModelfrom rapidfuzz import process-from typing import Union, Optional, Dict, DefaultDict, List-from enum import Enum-from collections import defaultdict++import dspy+from dsp.utils import deduplicate+from dspy.evaluate import Evaluate++from .base_task import BaseTask# Must point to a MedDra download with mdhier.ascdata_dir = '/future/u/okhattab/data/2023/MedDraV2/meddra_23_0_english/MedAscii' # NOTE: EDIT THIS LINE@@ -529,6 +533,7 @@ class CompilationMetric:metricR = CompilationMetric(field='reactions', metric='recall')from collections import Counter+def reduce_grounded_reactions(grounded_reactions):scores = {}for score, reaction in grounded_reactions:diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.pyindex 6724ebdcd..04f20d177 100644--- a/testing/tasks/gsm8k.py+++ b/testing/tasks/gsm8k.py@@ -1,11 +1,14 @@-import dspy-from .base_task import BaseTask-from dspy.datasets.gsm8k import gsm8k_metricimport random-import tqdm+import tqdmfrom datasets import load_dataset+import dspy+from dspy.datasets.gsm8k import gsm8k_metric++from .base_task import BaseTask++class CoT(dspy.Module):def __init__(self):super().__init__()diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.pyindex 19a5075fc..f0bf17078 100644--- a/testing/tasks/hotpotqa.py+++ b/testing/tasks/hotpotqa.py@@ -1,7 +1,9 @@import dspy+from dspy.datasets import HotPotQAfrom dspy.evaluate import Evaluate+from .base_task import BaseTask-from dspy.datasets import HotPotQA+class MultiHop(dspy.Module):def __init__(self,passages_per_hop):diff --git a/testing/tasks/scone.py b/testing/tasks/scone.pyindex 0ee5f0391..27e8691ac 100644--- a/testing/tasks/scone.py+++ b/testing/tasks/scone.py@@ -1,9 +1,13 @@import globimport os+import random+import pandas as pd+import dspy+from .base_task import BaseTask-import random+def load_scone(dirname):dfs = []diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.pyindex 30bdef68a..73870a059 100644--- a/testing/tasks/tweet.py+++ b/testing/tasks/tweet.py@@ -1,10 +1,14 @@-import dspy-from dspy.datasets import HotPotQA-from .base_task import BaseTask+import osfrom functools import lru_cache+import openaifrom dotenv import load_dotenv-import os++import dspy+from dspy.datasets import HotPotQA++from .base_task import BaseTask+class TweetSignature(dspy.Signature):("""Given context and a question, answer with a tweet""")diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.pyindex baa7024ab..b91458c48 100644--- a/testing/tasks/tweet_metric.py+++ b/testing/tasks/tweet_metric.py@@ -1,14 +1,18 @@-import dspy-from dspy.datasets import HotPotQA-from .base_task import BaseTask-from dspy import Example+import os+import uuidfrom functools import lru_cache+import openaifrom dotenv import load_dotenv-import os-import uuidfrom tqdm import tqdm+import dspy+from dspy import Example+from dspy.datasets import HotPotQA++from .base_task import BaseTask++class TweetSignature(dspy.Signature):("""Given context and a question, answer with a tweet""")