DSPy: The framework for programming—not prompting—language models
Python
feat(dspy): remove files where DSPDummyLM not used
Michael Jones53b24c84c736
diff --git a/dsp/modules/dummy_lm.py b/dsp/modules/dummy_lm.pyindex 0509004c7..49f35fa72 100644--- a/dsp/modules/dummy_lm.py+++ b/dsp/modules/dummy_lm.py@@ -5,7 +5,7 @@ from dsp.modules import LM# This testing module was moved in PR #735 to patch Arize Phoenix logging-class DspDummyLM(LM):+class DSPDummyLM(LM):"""Dummy language model for unit testing purposes."""def __init__(self, answers: Union[list[str], dict[str, str]], follow_examples: bool = False):diff --git a/tests/dsp_LM/evaluate/test_metrics.py b/tests/dsp_LM/evaluate/test_metrics.pydeleted file mode 100644index 04e32e68c..000000000--- a/tests/dsp_LM/evaluate/test_metrics.py+++ /dev/null@@ -1,36 +0,0 @@-# FILEPATH: /Users/ahle/repos/dspy/tests/evaluate/test_metrics.py--import dsp-import dspy-from dspy.evaluate.metrics import answer_exact_match-from dspy.predict import Predict---def test_answer_exact_match_string():- example = dspy.Example(- question="What is 1+1?",- answer="2",- ).with_inputs("question")- pred = Predict("question -> answer")- pred.answer = "2"- assert answer_exact_match(example, pred)---def test_answer_exact_match_list():- example = dspy.Example(- question="What is 1+1?",- answer=["2", "two"],- ).with_inputs("question")- pred = Predict("question -> answer")- pred.answer = "2"- assert answer_exact_match(example, pred)---def test_answer_exact_match_no_match():- example = dspy.Example(- question="What is 1+1?",- answer="2",- ).with_inputs("question")- pred = Predict("question -> answer")- pred.answer = "3"- assert not answer_exact_match(example, pred)diff --git a/tests/dsp_LM/examples/test_baleen.py b/tests/dsp_LM/examples/test_baleen.pydeleted file mode 100644index f0b804269..000000000--- a/tests/dsp_LM/examples/test_baleen.py+++ /dev/null@@ -1,122 +0,0 @@-import pytest--import dspy-import dspy.evaluate-from dsp.utils import deduplicate-from dspy.datasets import HotPotQA-from dspy.evaluate.evaluate import Evaluate-from dspy.teleprompt.bootstrap import BootstrapFewShot---class GenerateAnswer(dspy.Signature):- """Answer questions with short factoid answers."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- answer = dspy.OutputField(desc="often between 1 and 5 words")---class GenerateSearchQuery(dspy.Signature):- """Write a simple search query that will help answer a complex question."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- query = dspy.OutputField()---class SimplifiedBaleen(dspy.Module):- def __init__(self, passages_per_hop=3, max_hops=2):- super().__init__()-- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]- self.retrieve = dspy.Retrieve(k=passages_per_hop)- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)- self.max_hops = max_hops-- def forward(self, question):- context = []-- for hop in range(self.max_hops):- query = self.generate_query[hop](context=context, question=question).query- passages = self.retrieve(query).passages- context = deduplicate(context + passages)-- pred = self.generate_answer(context=context, question=question)- return dspy.Prediction(context=context, answer=pred.answer)---def load_hotpotqa():- # Load the dataset.- dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)- # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.- trainset = [x.with_inputs("question") for x in dataset.train]- devset = [x.with_inputs("question") for x in dataset.dev]- return trainset, devset---# @pytest.mark.slow_test-# TODO: Find a way to make this test run without openai-def _test_baleen():- lm = dspy.OpenAI(model="gpt-3.5-turbo")- rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")- dspy.settings.configure(lm=lm, rm=rm)-- # Ask any question you like to this simple RAG program.- my_question = "How many storeys are in the castle that David Gregory inherited?"-- # Get the prediction. This contains `pred.context` and `pred.answer`.- uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program- pred = uncompiled_baleen(my_question)-- assert pred.answer == "five"---def validate_context_and_answer_and_hops(example, pred, trace=None):- if not dspy.evaluate.answer_exact_match(example, pred):- return False- if not dspy.evaluate.answer_passage_match(example, pred):- return False-- hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]-- if max([len(h) for h in hops]) > 100:- return False- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))):- return False-- return True---def gold_passages_retrieved(example, pred, trace=None):- gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))-- return gold_titles.issubset(found_titles)---# @pytest.mark.slow_test-# TODO: Find a way to make this test run without the slow hotpotqa dataset-def _test_compiled_baleen():- trainset, devset = load_hotpotqa()- lm = dspy.OpenAI(model="gpt-3.5-turbo")- rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")- dspy.settings.configure(lm=lm, rm=rm)-- uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program-- teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)- compiled_baleen = teleprompter.compile(- SimplifiedBaleen(),- teacher=SimplifiedBaleen(passages_per_hop=2),- trainset=trainset,- )-- evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)- uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(- uncompiled_baleen, metric=gold_passages_retrieved, display=False- )- # assert uncompiled_baleen_retrieval_score / 100 == 18 / 50-- compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)- # assert compiled_baleen_retrieval_score / 100 == 27 / 50- assert uncompiled_baleen_retrieval_score < compiled_baleen_retrieval_scorediff --git a/tests/dsp_LM/functional/test_functional.py b/tests/dsp_LM/functional/test_functional.pyindex bc844a82c..31d0416b7 100644--- a/tests/dsp_LM/functional/test_functional.py+++ b/tests/dsp_LM/functional/test_functional.py@@ -202,7 +202,7 @@ def test_bootstrap_effectiveness():assert demos[0].output == trainset[0].output# Test the compiled student's prediction.- # We are using a DspDummyLM with follow_examples=True, which means that+ # We are using a DSPDummyLM with follow_examples=True, which means that# even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"# on the second output, if it seems an example that perfectly matches the# prompt, it will use that instead. That is why we expect "blue" here.diff --git a/tests/dsp_LM/functional/test_signature_typed.py b/tests/dsp_LM/functional/test_signature_typed.pydeleted file mode 100644index 6e875d851..000000000--- a/tests/dsp_LM/functional/test_signature_typed.py+++ /dev/null@@ -1,155 +0,0 @@-from typing import Any, Optional, Union--import pydantic-import pytest--import dspy-from dspy.functional import TypedPredictor-from dspy.signatures.signature import signature_to_template---def get_field_and_parser(signature: dspy.Signature) -> tuple[Any, Any]:- module = TypedPredictor(signature)- signature = module._prepare_signature()- assert "answer" in signature.fields, "'answer' not in signature.fields"- field = signature.fields.get("answer")- parser = field.json_schema_extra.get("parser")- return field, parser---class Mysubmodel(pydantic.BaseModel):- sub_floating: float---class MyModel(pydantic.BaseModel):- floating: float- string: str- boolean: bool- integer: int- optional: Optional[str]- sequence_of_strings: list[str]- union: Union[str, float]- submodel: Mysubmodel- optional_submodel: Optional[Mysubmodel]- optional_existing_submodule: Optional[Mysubmodel]---def build_model_instance() -> MyModel:- return MyModel(- floating=3.14,- string="foobar",- boolean=True,- integer=42,- optional=None,- sequence_of_strings=["foo", "bar"],- union=3.14,- submodel=Mysubmodel(sub_floating=42.42),- optional_submodel=None,- optional_existing_submodule=Mysubmodel(sub_floating=42.42),- )--- "test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)]-)-def test_basic_types(test_type: type, serialized: str, expected: Any):- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: test_type = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)- assert parser is test_type, "Parser is not correct for 'answer'"- assert parser(serialized) == expected, f"{test_type}({serialized})!= {expected}"---def test_boolean():- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: bool = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)- assert parser("true"), f"Parsing 'true' failed"- assert not parser("false"), f"Parsing 'false' failed"--- "test_type,serialized, expected",- [(list[str], '["foo", "bar"]', ["foo", "bar"]), (tuple[int, float], "[42, 3.14]", (42, 3.14))],-)-def test_sequences(test_type: type, serialized: str, expected: Any):- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: test_type = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)-- assert parser(serialized) == expected, f"Parsing {expected} failed"--- "test_type,serialized, expected",- [- (Optional[str], '"foobar"', "foobar"),- (Optional[str], "null", None),- (Union[str, float], "3.14", 3.14),- (Union[str, bool], "true", True),- ],-)-def test_unions(test_type: type, serialized: str, expected: Any):- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: test_type = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)-- assert parser(serialized) == expected, f"Parsing {expected} failed"---def test_pydantic():- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: MyModel = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)-- instance = build_model_instance()- parsed_instance = parser(instance.model_dump_json())-- assert parsed_instance == instance, f"{instance} != {parsed_instance}"---def test_optional_pydantic():- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: Optional[MyModel] = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)-- instance = build_model_instance()- parsed_instance = parser(instance.model_dump_json())- assert parsed_instance == instance, f"{instance} != {parsed_instance}"-- # Check null case- parsed_instance = parser("null")- assert parsed_instance == None, "Optional[MyModel] should be None"---def test_dataclass():- from dataclasses import dataclass-- @dataclass(frozen=True)- class MyDataclass:- string: str- number: int- floating: float- boolean: bool-- class MySignature(dspy.Signature):- question: str = dspy.InputField()- answer: MyDataclass = dspy.OutputField()-- _, parser = get_field_and_parser(MySignature)-- instance = MyDataclass("foobar", 42, 3.14, True)- parsed_instance = parser('{"string": "foobar", "number": 42, "floating": 3.14, "boolean": true}')- assert parsed_instance == instance, f"{instance} != {parsed_instance}"diff --git a/tests/dsp_LM/modules/test_aws_models.py b/tests/dsp_LM/modules/test_aws_models.pydeleted file mode 100644index b6e018b33..000000000--- a/tests/dsp_LM/modules/test_aws_models.py+++ /dev/null@@ -1,70 +0,0 @@-"""Tests for AWS models.-Note: Requires configuration of your AWS credentials with the AWS CLI and creating sagemaker endpoints.-TODO: Create mock fixtures for pytest to remove the need for AWS credentials and endpoints.-"""--import dsp-import dspy---def get_lm(lm_provider: str, model_path: str, **kwargs) -> dsp.modules.lm.LM:- """get the language model"""- # extract model vendor and name from model name- # Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>- model_vendor = model_path.split("/")[0]- model_name = model_path.split("/")[1]-- if lm_provider == "Bedrock":- bedrock = dspy.Bedrock(region_name="us-west-2")- if model_vendor == "mistral":- return dspy.AWSMistral(bedrock, model_name, **kwargs)- elif model_vendor == "anthropic":- return dspy.AWSAnthropic(bedrock, model_name, **kwargs)- elif model_vendor == "meta":- return dspy.AWSMeta(bedrock, model_name, **kwargs)- else:- raise ValueError(- "Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>"- )- elif lm_provider == "Sagemaker":- sagemaker = dspy.Sagemaker(region_name="us-west-2")- if model_vendor == "mistral":- return dspy.AWSMistral(sagemaker, model_name, **kwargs)- elif model_vendor == "meta":- return dspy.AWSMeta(sagemaker, model_name, **kwargs)- else:- raise ValueError(- "Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>"- )- else:- raise ValueError(f"Unsupported model: {model_name}")---def run_tests():- """Test the providers and models"""- # Configure your AWS credentials with the AWS CLI before running this script- provider_model_tuples = [- ("Bedrock", "mistral/mistral.mixtral-8x7b-instruct-v0:1"),- ("Bedrock", "anthropic/anthropic.claude-3-haiku-20240307-v1:0"),- ("Bedrock", "anthropic/anthropic.claude-3-sonnet-20240229-v1:0"),- ("Bedrock", "meta/meta.llama2-70b-chat-v1"),- ("Bedrock", "meta/meta.llama3-8b-instruct-v1:0"),- ("Bedrock", "meta/meta.llama3-70b-instruct-v1:0"),- # ('Sagemaker', 'mistral/<YOUR_ENDPOINT_NAME>'), # REPLACE YOUR_ENDPOINT_NAME with your sagemaker endpoint- ]-- predict_func = dspy.Predict("question -> answer")- for provider, model_path in provider_model_tuples:- print(f"Provider: {provider}, Model: {model_path}")- lm = get_lm(provider, model_path)- with dspy.context(lm=lm):- question = "What is the capital of France?"- answer = predict_func(question=question).answer- print(f"Question: {question}\nAnswer: {answer}")- print("---------------------------------")- lm.inspect_history()- print("---------------------------------\n")---if __name__ == "__main__":- run_tests()diff --git a/tests/dsp_LM/modules/test_cloudflare_models.py b/tests/dsp_LM/modules/test_cloudflare_models.pydeleted file mode 100644index 188d2424e..000000000--- a/tests/dsp_LM/modules/test_cloudflare_models.py+++ /dev/null@@ -1,61 +0,0 @@-"""Tests for Cloudflare models.-Note: Requires configuration of your Cloudflare account_id and api_key.-"""--import dspy--models = {- "@cf/qwen/qwen1.5-0.5b-chat": "https://huggingface.co/qwen/qwen1.5-0.5b-chat",- "@hf/meta-llama/meta-llama-3-8b-instruct": "https://llama.meta.com",- "@hf/nexusflow/starling-lm-7b-beta": "https://huggingface.co/Nexusflow/Starling-LM-7B-beta",- "@cf/meta/llama-3-8b-instruct": "https://llama.meta.com",- "@hf/thebloke/neural-chat-7b-v3-1-awq": "",- "@cf/meta/llama-2-7b-chat-fp16": "https://ai.meta.com/llama/",- "@cf/mistral/mistral-7b-instruct-v0.1": "https://mistral.ai/news/announcing-mistral-7b/",- "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",- "@hf/mistral/mistral-7b-instruct-v0.2": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",- "@cf/fblgit/una-cybertron-7b-v2-bf16": "",- "@hf/thebloke/codellama-7b-instruct-awq": "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-AWQ",- "@cf/thebloke/discolm-german-7b-v1-awq": "https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-AWQ",- "@cf/meta/llama-2-7b-chat-int8": "https://ai.meta.com/llama/",- "@hf/thebloke/mistral-7b-instruct-v0.1-awq": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-AWQ",- "@hf/thebloke/openchat_3.5-awq": "",- "@cf/qwen/qwen1.5-7b-chat-awq": "https://huggingface.co/qwen/qwen1.5-7b-chat-awq",- "@hf/thebloke/llama-2-13b-chat-awq": "https://huggingface.co/TheBloke/Llama-2-13B-chat-AWQ",- "@hf/thebloke/deepseek-coder-6.7b-base-awq": "",- "@hf/thebloke/openhermes-2.5-mistral-7b-awq": "",- "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": "",- "@cf/deepseek-ai/deepseek-math-7b-instruct": "https://huggingface.co/deepseek-ai/deepseek-math-7b-instruct",- "@cf/tiiuae/falcon-7b-instruct": "https://huggingface.co/tiiuae/falcon-7b-instruct",- "@hf/nousresearch/hermes-2-pro-mistral-7b": "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B",- "@hf/thebloke/zephyr-7b-beta-awq": "https://huggingface.co/TheBloke/zephyr-7B-beta-AWQ",- "@cf/qwen/qwen1.5-1.8b-chat": "https://huggingface.co/qwen/qwen1.5-1.8b-chat",- "@cf/defog/sqlcoder-7b-2": "https://huggingface.co/defog/sqlcoder-7b-2",- "@cf/microsoft/phi-2": "https://huggingface.co/microsoft/phi-2",- "@hf/google/gemma-7b-it": "https://ai.google.dev/gemma/docs",-}---def get_lm(name: str): # -> dspy.LM:- return dspy.CloudflareAI(model=name)---def run_tests():- """Test the providers and models"""- # Configure your AWS credentials with the AWS CLI before running this script- models-- predict_func = dspy.Predict("question -> answer")- for model_name in models.keys():- lm = get_lm(model_name)- with dspy.context(lm=lm):- question = "What is the capital of France?"- answer = predict_func(question=question).answer- print(f"Question: {question}\nAnswer: {answer}")- print("---------------------------------")- lm.inspect_history()- print("---------------------------------\n")---if __name__ == "__main__":- run_tests()diff --git a/tests/dsp_LM/modules/test_hf_model.py b/tests/dsp_LM/modules/test_hf_model.pydeleted file mode 100644index 0b06c8042..000000000--- a/tests/dsp_LM/modules/test_hf_model.py+++ /dev/null@@ -1,31 +0,0 @@-from pytest_mock.plugin import MockerFixture-from transformers import AutoModelForSeq2SeqLM--import dspy---class MockConfig:- def __init__(self, architectures: list[str]):- self.architectures = architectures---def test_load_gated_model(mocker: MockerFixture):- conf = MockConfig(architectures=["ConditionalGeneration"])- mocker.patch("transformers.AutoModelForSeq2SeqLM.from_pretrained")- mocker.patch("transformers.AutoConfig.from_pretrained", return_value=conf)- mocker.patch("transformers.AutoTokenizer.from_pretrained")-- some_token = "asdfasdfasdf"- model = "google/gemma-7b"- _ = dspy.HFModel(model, token=some_token)- AutoModelForSeq2SeqLM.from_pretrained.assert_called_with(model, device_map="auto", token=some_token)---def test_load_ungated_model(mocker: MockerFixture):- conf = MockConfig(architectures=["ConditionalGeneration"])- mocker.patch("transformers.AutoModelForSeq2SeqLM.from_pretrained")- mocker.patch("transformers.AutoConfig.from_pretrained", return_value=conf)- mocker.patch("transformers.AutoTokenizer.from_pretrained")- _ = dspy.HFModel("openai-community/gpt2")- # no token used in automodel- AutoModelForSeq2SeqLM.from_pretrained.assert_called_with("openai-community/gpt2", device_map="auto", token=None)diff --git a/tests/dsp_LM/modules/vectorizer/test_fastembed.py b/tests/dsp_LM/modules/vectorizer/test_fastembed.pydeleted file mode 100644index e9a7335a4..000000000--- a/tests/dsp_LM/modules/vectorizer/test_fastembed.py+++ /dev/null@@ -1,43 +0,0 @@-import pytest--from dsp.modules.sentence_vectorizer import FastEmbedVectorizer-from dspy.primitives.example import Example--# Skip the test if the 'fastembed' package is not installed-pytest.importorskip("fastembed", reason="'fastembed' is not installed. Use `pip install fastembed` to install it.")--- "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]-)-def test_fastembed_with_examples(n_dims, model_name):- vectorizer = FastEmbedVectorizer(model_name)-- examples = [- Example(query="What's the price today?", response="The price is $10.00").with_inputs("query", "response"),- Example(query="What's the weather today?", response="The weather is sunny").with_inputs("query", "response"),- Example(query="Who was leading the team?", response="It was Jim. Rather enthusiastic guy.").with_inputs(- "query", "response"- ),- ]-- embeddings = vectorizer(examples)-- assert embeddings.shape == (len(examples), n_dims)--- "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]-)-def test_fastembed_with_strings(n_dims, model_name):- vectorizer = FastEmbedVectorizer(model_name)-- inputs = [- "Jonathan Kent is a fictional character appearing in American comic books published by DC Comics.",- "Clark Kent is a fictional character appearing in American comic books published by DC Comics.",- "Martha Kent is a fictional character appearing in American comic books published by DC Comics.",- ]-- embeddings = vectorizer(inputs)-- assert embeddings.shape == (len(inputs), n_dims)diff --git a/tests/dsp_LM/predict/test_aggregation.py b/tests/dsp_LM/predict/test_aggregation.pydeleted file mode 100644index eb1d97536..000000000--- a/tests/dsp_LM/predict/test_aggregation.py+++ /dev/null@@ -1,43 +0,0 @@-from dsp.utils import normalize_text-from dspy.predict.aggregation import majority-from dspy.primitives.prediction import Completions, Prediction---def test_majority_with_prediction():- prediction = Prediction.from_completions([{"answer": "2"}, {"answer": "2"}, {"answer": "3"}])- result = majority(prediction)- assert result.completions[0]["answer"] == "2"---def test_majority_with_completions():- completions = Completions([{"answer": "2"}, {"answer": "2"}, {"answer": "3"}])- result = majority(completions)- assert result.completions[0]["answer"] == "2"---def test_majority_with_list():- completions = [{"answer": "2"}, {"answer": "2"}, {"answer": "3"}]- result = majority(completions)- assert result.completions[0]["answer"] == "2"---def test_majority_with_normalize():- completions = [{"answer": "2"}, {"answer": " 2"}, {"answer": "3"}]- result = majority(completions, normalize=normalize_text)- assert result.completions[0]["answer"] == "2"---def test_majority_with_field():- completions = [- {"answer": "2", "other": "1"},- {"answer": "2", "other": "1"},- {"answer": "3", "other": "2"},- ]- result = majority(completions, field="other")- assert result.completions[0]["other"] == "1"---def test_majority_with_no_majority():- completions = [{"answer": "2"}, {"answer": "3"}, {"answer": "4"}]- result = majority(completions)- assert result.completions[0]["answer"] == "2" # The first completion is returned in case of a tiediff --git a/tests/dsp_LM/predict/test_knn.py b/tests/dsp_LM/predict/test_knn.pydeleted file mode 100644index 7f35b42e4..000000000--- a/tests/dsp_LM/predict/test_knn.py+++ /dev/null@@ -1,51 +0,0 @@-import numpy as np-import pytest--import dsp-import dspy-from dspy.predict import KNN-from dspy.utils import DummyVectorizer---def mock_example(question: str, answer: str) -> dsp.Example:- """Creates a mock DSP example with specified question and answer."""- return dspy.Example(question=question, answer=answer).with_inputs("question")---def setup_knn():- """Sets up a KNN instance with a mocked vectorizer for testing."""- dsp.SentenceTransformersVectorizer = DummyVectorizer- trainset = [- mock_example("What is the capital of France?", "Paris"),- mock_example("What is the largest ocean?", "Pacific"),- mock_example("What is 2+2?", "4"),- ]- knn = KNN(k=2, trainset=trainset)- return knn---def test_knn_initialization(setup_knn):- """Tests the KNN initialization and checks if the trainset vectors are correctly created."""- knn = setup_knn- assert knn.k == 2, "Incorrect k value"- assert len(knn.trainset_vectors) == 3, "Incorrect size of trainset vectors"- assert isinstance(knn.trainset_vectors, np.ndarray), "Trainset vectors should be a NumPy array"---def test_knn_query(setup_knn):- """Tests the KNN query functionality for retrieving the nearest neighbors."""- knn = setup_knn- query = {"question": "What is 3+3?"} # A query close to "What is 2+2?"- nearest_samples = knn(**query)- assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"- assert nearest_samples[0].answer == "4", "Incorrect nearest sample returned"---def test_knn_query_specificity(setup_knn):- """Tests the KNN query functionality for specificity of returned examples."""- knn = setup_knn- query = {"question": "What is the capital of Germany?"} # A query close to "What is the capital of France?"- nearest_samples = knn(**query)- assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"- assert "Paris" in [sample.answer for sample in nearest_samples], "Expected Paris to be a nearest sample answer"diff --git a/tests/dsp_LM/predict/test_langchain.py b/tests/dsp_LM/predict/test_langchain.pydeleted file mode 100644index 89fa1d75d..000000000--- a/tests/dsp_LM/predict/test_langchain.py+++ /dev/null@@ -1,55 +0,0 @@-import pytest--pytest.importorskip("langchain")--import os--from langchain import hub-from langchain_chroma import Chroma-from langchain_community.embeddings import FakeEmbeddings-from langchain_core.documents import Document-from langchain_core.output_parsers import StrOutputParser-from langchain_core.runnables import RunnablePassthrough-from langchain_openai import ChatOpenAI-from langchain_text_splitters import RecursiveCharacterTextSplitter--from dspy.predict.langchain import LangChainModule, LangChainPredict---def test_copying_module():- os.environ["OPENAI_API_KEY"] = "fake-key"- llm = ChatOpenAI(model="gpt-4o-mini")- docs = [Document(page_content="Hello, world!", metadata={"source": "https://example.com"})]-- text_splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=10)- splits = text_splitter.split_documents(docs)- vectorstore = Chroma.from_documents(documents=splits, embedding=FakeEmbeddings(size=5))-- # Retrieve and generate using the relevant snippets of the blog.- retriever = vectorstore.as_retriever()-- prompt = hub.pull("rlm/rag-prompt")-- def format_docs(docs):- return "\n\n".join(doc.page_content for doc in docs)-- rag_chain = (- {"context": retriever | format_docs, "question": RunnablePassthrough()}- | LangChainPredict(prompt, llm)- | StrOutputParser()- )- # Now we wrap it in LangChainModule.- rag_dspy_module = LangChainModule(rag_chain)-- copied_module = rag_dspy_module.reset_copy()- assert len(copied_module.chain.steps) == len(rag_dspy_module.chain.steps)- for module, copied_module in zip(rag_dspy_module.chain.steps, copied_module.chain.steps):- if isinstance(module, LangChainPredict):- # The LangChainPredict modules are deep copied.- assert module != copied_module- assert module.langchain_llm.model_name == copied_module.langchain_llm.model_name- else:- # The rest of the modules are just copied by reference.- assert module == copied_module- # Clean up.- os.environ["OPENAI_API_KEY"] = Nonediff --git a/tests/dsp_LM/primitives/test_example.py b/tests/dsp_LM/primitives/test_example.pydeleted file mode 100644index 39ad0b749..000000000--- a/tests/dsp_LM/primitives/test_example.py+++ /dev/null@@ -1,110 +0,0 @@-import pytest--from dspy import Example---def test_example_initialization():- example = Example(a=1, b=2)- assert example.a == 1- assert example.b == 2---def test_example_initialization_from_base():- base = Example(a=1, b=2)- example = Example(base=base, c=3)- assert example.a == 1- assert example.b == 2- assert example.c == 3---def test_example_initialization_from_dict():- base_dict = {"a": 1, "b": 2}- example = Example(base=base_dict, c=3)- assert example.a == 1- assert example.b == 2- assert example.c == 3---def test_example_set_get_item():- example = Example()- example["a"] = 1- assert example["a"] == 1---def test_example_attribute_access():- example = Example(a=1)- assert example.a == 1- example.a = 2- assert example.a == 2---def test_example_deletion():- example = Example(a=1, b=2)- del example["a"]- with pytest.raises(AttributeError):- _ = example.a---def test_example_len():- example = Example(a=1, b=2, dspy_hidden=3)- assert len(example) == 2---def test_example_repr_str():- example = Example(a=1)- assert repr(example) == "Example({'a': 1}) (input_keys=None)"- assert str(example) == "Example({'a': 1}) (input_keys=None)"---def test_example_eq():- example1 = Example(a=1, b=2)- example2 = Example(a=1, b=2)- assert example1 == example2- assert example1 != ""---def test_example_hash():- example1 = Example(a=1, b=2)- example2 = Example(a=1, b=2)- assert hash(example1) == hash(example2)---def test_example_keys_values_items():- example = Example(a=1, b=2, dspy_hidden=3)- assert set(example.keys()) == {"a", "b"}- assert 1 in example.values()- assert ("b", 2) in example.items()---def test_example_get():- example = Example(a=1, b=2)- assert example.get("a") == 1- assert example.get("c", "default") == "default"---def test_example_with_inputs():- example = Example(a=1, b=2).with_inputs("a")- assert example._input_keys == {"a"}---def test_example_inputs_labels():- example = Example(a=1, b=2).with_inputs("a")- inputs = example.inputs()- assert inputs.toDict() == {"a": 1}- labels = example.labels()- assert labels.toDict() == {"b": 2}---def test_example_copy_without():- example = Example(a=1, b=2)- copied = example.copy(c=3)- assert copied.a == 1- assert copied.c == 3- without_a = copied.without("a")- with pytest.raises(AttributeError):- _ = without_a.a---def test_example_to_dict():- example = Example(a=1, b=2)- assert example.toDict() == {"a": 1, "b": 2}diff --git a/tests/dsp_LM/primitives/test_module.py b/tests/dsp_LM/primitives/test_module.pydeleted file mode 100644index 4ba6df220..000000000--- a/tests/dsp_LM/primitives/test_module.py+++ /dev/null@@ -1,49 +0,0 @@-import threading--import dspy---def test_deepcopy_basic():- signature = dspy.Signature("q -> a")- cot = dspy.ChainOfThought(signature)- cot_copy = cot.deepcopy()- assert len(cot.parameters()) == len(cot_copy.parameters())- # Parameters should be different objects with the same values.- assert id(cot.parameters()[0]) != id(cot_copy.parameters()[0])- assert cot.parameters()[0].__dict__ == cot_copy.parameters()[0].__dict__---def test_deepcopy_with_uncopyable_modules():- class CustomClass(dspy.Module):- def __init__(self):- self.lock = threading.Lock() # Non-copyable object.- self.cot = dspy.ChainOfThought(dspy.Signature("q -> a"))-- model = CustomClass()- model_copy = model.deepcopy()- assert len(model.parameters()) == len(model_copy.parameters())- # The lock should be refer to the same object (shallow copy).- assert id(model.lock) == id(model_copy.lock)- # Parameters should be different objects with the same values.- assert id(model.parameters()[0]) != id(model_copy.parameters()[0])- assert model.parameters()[0].__dict__ == model_copy.parameters()[0].__dict__---def test_deepcopy_with_nested_modules():- class CustomClass1(dspy.Module):- def __init__(self):- self.lock = threading.Lock() # Non-copyable object.- self.cot = dspy.ChainOfThought(dspy.Signature("q -> a"))-- class CustomClass2(dspy.Module):- def __init__(self):- self.submodel = CustomClass1()-- model = CustomClass2()- model_copy = model.deepcopy()- assert len(model.parameters()) == len(model_copy.parameters())- # The lock should be refer to the same object (shallow copy).- assert id(model.submodel.lock) == id(model_copy.submodel.lock)- # Parameters should be different objects with the same values.- assert id(model.parameters()[0]) != id(model_copy.parameters()[0])- assert model.parameters()[0].__dict__ == model_copy.parameters()[0].__dict__diff --git a/tests/dsp_LM/primitives/test_python_interpreter.py b/tests/dsp_LM/primitives/test_python_interpreter.pydeleted file mode 100644index a1c071388..000000000--- a/tests/dsp_LM/primitives/test_python_interpreter.py+++ /dev/null@@ -1,53 +0,0 @@-import pytest--from dspy.primitives.python_interpreter import CodePrompt, PythonInterpreter, TextPrompt---def test_execute_simple_code():- interpreter = PythonInterpreter(action_space={"print": print})- code = "print('Hello, World!')"- result = interpreter.execute(code)- assert result is None, "Simple print statement should return None"---def test_action_space_limitation():- def func(string):- pass-- interpreter = PythonInterpreter(action_space={})- code = "func('This should not execute')"- with pytest.raises(Exception):- interpreter.execute(code)---def test_import_whitelist():- interpreter = PythonInterpreter(action_space={}, import_white_list=["math"])- code = "import math\nresult = math.sqrt(4)"- result = interpreter.execute(code)- assert result == 2, "Should be able to import and use math.sqrt"---def test_fuzzy_variable_matching():- interpreter = PythonInterpreter(action_space={})- code = "result = number + 1"- result = interpreter.execute(code, fuzz_state={"number": 4})- assert result == 5, "Fuzzy variable matching should work"---def test_text_prompt_keyword_extraction():- prompt = TextPrompt("Hello {name}, how are you?")- assert "name" in prompt.key_words, "Keyword 'name' should be extracted"---def test_text_prompt_formatting():- prompt = TextPrompt("Hello {name}, how are you?")- formatted = prompt.format(name="Alice")- assert formatted == "Hello Alice, how are you?", "Should format with provided value"---def test_code_prompt_execution():- action_space = {"len": len}- interpreter = PythonInterpreter(action_space=action_space)- code_prompt = CodePrompt("result = len('hello')")- result, _ = code_prompt.execute(interpreter)- assert result == 5, "Code execution should return the length of 'hello'"diff --git a/tests/dsp_LM/retrieve/integration_test_pgvectorrm.py b/tests/dsp_LM/retrieve/integration_test_pgvectorrm.pydeleted file mode 100644index 524209bea..000000000--- a/tests/dsp_LM/retrieve/integration_test_pgvectorrm.py+++ /dev/null@@ -1,94 +0,0 @@-"""Instructions:-Add to dev container features:- "ghcr.io/itsmechlark/features/postgresql:1": {},- "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {}-Add to .personalization.sh:- sudo apt install -y postgresql-16-pgvector-- sudo /etc/init.d/postgresql restart-- psql -v ON_ERROR_STOP=1 --user ${PGUSER} <<EOF- create extension if not exists vector;- EOF-poetry install -E postgres-"""-import psycopg2-import pytest--from dspy.primitives.example import Example-from dspy.retrieve.pgvector_rm import PgVectorRM--DB_URL = "postgresql://postgres:password@localhost/postgres"-PG_TABLE_NAME = "test_table"---def get_pgvectorrm():- openai_client = None # Mock or use a real OpenAI client- pgvectorrm = PgVectorRM(- DB_URL, PG_TABLE_NAME, openai_client=openai_client, embedding_func=lambda x: "[2,3,4]", include_similarity=True- )- return pgvectorrm---def setup_pgvectorrm():- pgvectorrm = get_pgvectorrm()- conn = psycopg2.connect(DB_URL)- cursor = conn.cursor()- cursor.execute(f"DROP TABLE IF EXISTS {PG_TABLE_NAME}")- conn.commit()-- cursor.execute(f"CREATE TABLE IF NOT EXISTS {PG_TABLE_NAME} (id SERIAL PRIMARY KEY, text TEXT, embedding VECTOR)")- cursor.execute(- f"INSERT INTO {PG_TABLE_NAME} (text, embedding) VALUES ('Dummy text1', '[1,2,3]'), ('Dummy text2', '[4,5,6]')"- )- conn.commit()-- yield pgvectorrm-- cursor.execute(f"TRUNCATE TABLE {PG_TABLE_NAME}")- conn.commit()-- cursor.close()- conn.close()---def test_pgvectorrm_retrieve(setup_pgvectorrm):- pgvectorrm = setup_pgvectorrm- query = "test query"- results = pgvectorrm(query)- assert len(results) == 2- assert results == [- Example(text="Dummy text2", similarity=0.9946115458726394),- Example(text="Dummy text1", similarity=0.9925833339709302),- ]--- "k, expected",- [- (1, 1),- (2, 2),- (3, 2), # Assuming only 2 entries exist- ],-)-def test_pgvectorrm_retrieve_diff_k(setup_pgvectorrm, k, expected):- setup_pgvectorrm.k = k- query = "test query"- results = setup_pgvectorrm(query)- assert len(results) == expected---def test_empty_table():- # Assuming setup_pgvectorrm cleans up after yielding- query = "test query"- results = get_pgvectorrm()(query)- assert len(results) == 0---def test_retrieval_without_similarity(setup_pgvectorrm):- setup_pgvectorrm.include_similarity = False- query = "test query"- results = setup_pgvectorrm(query)- # Ensure 'similarity' key is not in results- assert all("similarity" not in result for result in results)diff --git a/tests/dsp_LM/retrieve/test_llama_index_rm.py b/tests/dsp_LM/retrieve/test_llama_index_rm.pyindex 497ed87fc..735c1a940 100644--- a/tests/dsp_LM/retrieve/test_llama_index_rm.py+++ b/tests/dsp_LM/retrieve/test_llama_index_rm.py@@ -3,7 +3,7 @@ import loggingimport pytestimport dspy-from dsp.modules.dummy_lm import DspDummyLM+from dsp.modules.dummy_lm import DSPDummyLMfrom dspy.datasets import HotPotQAtry:@@ -38,7 +38,7 @@ def rag_setup() -> dict:"index": index,"retriever": retriever,"rm": rm,- "lm": DspDummyLM(answers=dummyset),+ "lm": DSPDummyLM(answers=dummyset),"trainset": trainset,"devset": devset,}diff --git a/tests/dsp_LM/teleprompt/test_bootstrap.py b/tests/dsp_LM/teleprompt/test_bootstrap.pyindex 6bc41ec61..936daf8e4 100644--- a/tests/dsp_LM/teleprompt/test_bootstrap.py+++ b/tests/dsp_LM/teleprompt/test_bootstrap.py@@ -65,7 +65,7 @@ def test_bootstrap_effectiveness():assert compiled_student.predictor.demos[0].output == trainset[0].output# Test the compiled student's prediction.- # We are using a DspDummyLM with follow_examples=True, which means that+ # We are using a DSPDummyLM with follow_examples=True, which means that# even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"# on the second output, if it seems an example that perfectly matches the# prompt, it will use that instead. That is why we expect "blue" here.@@ -115,7 +115,7 @@ def test_error_handling_during_bootstrap():student = SimpleModule("input -> output")teacher = BuggyModule("input -> output")- # Setup DspDummyLM to simulate an error scenario+ # Setup DSPDummyLM to simulate an error scenariolm = DSPDummyLM(["Initial thoughts", # Simulate initial teacher's predictiondiff --git a/tests/dsp_LM/teleprompt/test_ensemble.py b/tests/dsp_LM/teleprompt/test_ensemble.pydeleted file mode 100644index c779e8782..000000000--- a/tests/dsp_LM/teleprompt/test_ensemble.py+++ /dev/null@@ -1,59 +0,0 @@-import pytest--import dspy-from dspy.teleprompt import Ensemble---class MockProgram(dspy.Module):- def __init__(self, output):- super().__init__()- self.output = output-- def forward(self, *args, **kwargs):- return self.output---# Simple reduction function to test with-def mock_reduce_fn(outputs):- return sum(outputs) / len(outputs)---def test_ensemble_without_reduction():- """Test that Ensemble correctly combines outputs without applying a reduce_fn."""- programs = [MockProgram(i) for i in range(5)]- ensemble = Ensemble()- ensembled_program = ensemble.compile(programs)-- outputs = ensembled_program()- assert len(outputs) == 5, "Ensemble did not combine the correct number of outputs"---def test_ensemble_with_reduction():- """Test that Ensemble correctly applies a reduce_fn to combine outputs."""- programs = [MockProgram(i) for i in range(5)]- ensemble = Ensemble(reduce_fn=mock_reduce_fn)- ensembled_program = ensemble.compile(programs)-- output = ensembled_program()- expected_output = sum(range(5)) / 5- assert output == expected_output, "Ensemble did not correctly apply the reduce_fn"---def test_ensemble_with_size_limitation():- """Test that specifying a size limits the number of programs used in the ensemble."""- programs = [MockProgram(i) for i in range(10)]- ensemble_size = 3- ensemble = Ensemble(size=ensemble_size)- ensembled_program = ensemble.compile(programs)-- outputs = ensembled_program()- assert len(outputs) == ensemble_size, "Ensemble did not respect the specified size limitation"---def test_ensemble_deterministic_behavior():- """Verify that the Ensemble class raises an assertion for deterministic behavior."""- with pytest.raises(- AssertionError,- match="TODO: Implement example hashing for deterministic ensemble.",- ):- Ensemble(deterministic=True)diff --git a/tests/dsp_LM/teleprompt/test_knn_fewshot.py b/tests/dsp_LM/teleprompt/test_knn_fewshot.pyindex 771e01d55..97c2dbbe3 100644--- a/tests/dsp_LM/teleprompt/test_knn_fewshot.py+++ b/tests/dsp_LM/teleprompt/test_knn_fewshot.py@@ -43,7 +43,7 @@ def _test_knn_few_shot_compile(setup_knn_few_shot):student = SimpleModule("input -> output")teacher = SimpleModule("input -> output") # Assuming teacher uses the same module type- # Setup DspDummyLM with a response for a query similar to one of the training examples+ # Setup DSPDummyLM with a response for a query similar to one of the training exampleslm = DSPDummyLM(["Madrid", "10"])dspy.settings.configure(lm=lm) # Responses for the capital of Spain and the result of 5+5)@@ -60,6 +60,6 @@ def _test_knn_few_shot_compile(setup_knn_few_shot):print("CONVO")print(lm.get_convo(-1))- # Validate that the output corresponds to one of the expected DspDummyLM responses+ # Validate that the output corresponds to one of the expected DSPDummyLM responses# This assumes the compiled_student's forward method will execute the predictor with the given queryassert output in ["Madrid", "10"], "The compiled student did not return the correct output based on the query"diff --git a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py b/tests/dsp_LM/teleprompt/test_mipro_optimizer.pyindex 9ed1b594f..86d8c00d0 100644--- a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py+++ b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py@@ -81,7 +81,7 @@ class ConditionalLM(LM):answer = "think deeply.\nOutput: " + answerRED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m"- print("=== DspDummyLM ===")+ print("=== DSPDummyLM ===")print(prompt, end="")print(f"{RED}{answer}{RESET}")print("===")@@ -173,7 +173,7 @@ def test_signature_optimizer_bad_lm():track_stats=False,)- # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DspDummyLM,+ # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DSPDummyLM,# which only outputs "Optimized instruction i" this means that none of the bootstrapped examples are successful,# and therefore the set of examples that we're using to generate new prompts is emptywith pytest.raises(ValueError):