Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): remove files where DSPDummyLM not used

Michael Jones53b24c84c736
diff --git a/dsp/modules/dummy_lm.py b/dsp/modules/dummy_lm.py
index 0509004c7..49f35fa72 100644
--- a/dsp/modules/dummy_lm.py
+++ b/dsp/modules/dummy_lm.py
@@ -5,7 +5,7 @@ from dsp.modules import LM
# This testing module was moved in PR #735 to patch Arize Phoenix logging
-class DspDummyLM(LM):
+class DSPDummyLM(LM):
"""Dummy language model for unit testing purposes."""
def __init__(self, answers: Union[list[str], dict[str, str]], follow_examples: bool = False):
diff --git a/tests/dsp_LM/evaluate/test_metrics.py b/tests/dsp_LM/evaluate/test_metrics.py
deleted file mode 100644
index 04e32e68c..000000000
--- a/tests/dsp_LM/evaluate/test_metrics.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# FILEPATH: /Users/ahle/repos/dspy/tests/evaluate/test_metrics.py
-
-import dsp
-import dspy
-from dspy.evaluate.metrics import answer_exact_match
-from dspy.predict import Predict
-
-
-def test_answer_exact_match_string():
- example = dspy.Example(
- question="What is 1+1?",
- answer="2",
- ).with_inputs("question")
- pred = Predict("question -> answer")
- pred.answer = "2"
- assert answer_exact_match(example, pred)
-
-
-def test_answer_exact_match_list():
- example = dspy.Example(
- question="What is 1+1?",
- answer=["2", "two"],
- ).with_inputs("question")
- pred = Predict("question -> answer")
- pred.answer = "2"
- assert answer_exact_match(example, pred)
-
-
-def test_answer_exact_match_no_match():
- example = dspy.Example(
- question="What is 1+1?",
- answer="2",
- ).with_inputs("question")
- pred = Predict("question -> answer")
- pred.answer = "3"
- assert not answer_exact_match(example, pred)
diff --git a/tests/dsp_LM/examples/test_baleen.py b/tests/dsp_LM/examples/test_baleen.py
deleted file mode 100644
index f0b804269..000000000
--- a/tests/dsp_LM/examples/test_baleen.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import pytest
-
-import dspy
-import dspy.evaluate
-from dsp.utils import deduplicate
-from dspy.datasets import HotPotQA
-from dspy.evaluate.evaluate import Evaluate
-from dspy.teleprompt.bootstrap import BootstrapFewShot
-
-
-class GenerateAnswer(dspy.Signature):
- """Answer questions with short factoid answers."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- answer = dspy.OutputField(desc="often between 1 and 5 words")
-
-
-class GenerateSearchQuery(dspy.Signature):
- """Write a simple search query that will help answer a complex question."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- query = dspy.OutputField()
-
-
-class SimplifiedBaleen(dspy.Module):
- def __init__(self, passages_per_hop=3, max_hops=2):
- super().__init__()
-
- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
- self.retrieve = dspy.Retrieve(k=passages_per_hop)
- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
- self.max_hops = max_hops
-
- def forward(self, question):
- context = []
-
- for hop in range(self.max_hops):
- query = self.generate_query[hop](context=context, question=question).query
- passages = self.retrieve(query).passages
- context = deduplicate(context + passages)
-
- pred = self.generate_answer(context=context, question=question)
- return dspy.Prediction(context=context, answer=pred.answer)
-
-
-def load_hotpotqa():
- # Load the dataset.
- dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
- # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
- trainset = [x.with_inputs("question") for x in dataset.train]
- devset = [x.with_inputs("question") for x in dataset.dev]
- return trainset, devset
-
-
-# @pytest.mark.slow_test
-# TODO: Find a way to make this test run without openai
-def _test_baleen():
- lm = dspy.OpenAI(model="gpt-3.5-turbo")
- rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
- dspy.settings.configure(lm=lm, rm=rm)
-
- # Ask any question you like to this simple RAG program.
- my_question = "How many storeys are in the castle that David Gregory inherited?"
-
- # Get the prediction. This contains `pred.context` and `pred.answer`.
- uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program
- pred = uncompiled_baleen(my_question)
-
- assert pred.answer == "five"
-
-
-def validate_context_and_answer_and_hops(example, pred, trace=None):
- if not dspy.evaluate.answer_exact_match(example, pred):
- return False
- if not dspy.evaluate.answer_passage_match(example, pred):
- return False
-
- hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]
-
- if max([len(h) for h in hops]) > 100:
- return False
- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))):
- return False
-
- return True
-
-
-def gold_passages_retrieved(example, pred, trace=None):
- gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))
- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))
-
- return gold_titles.issubset(found_titles)
-
-
-# @pytest.mark.slow_test
-# TODO: Find a way to make this test run without the slow hotpotqa dataset
-def _test_compiled_baleen():
- trainset, devset = load_hotpotqa()
- lm = dspy.OpenAI(model="gpt-3.5-turbo")
- rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
- dspy.settings.configure(lm=lm, rm=rm)
-
- uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program
-
- teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
- compiled_baleen = teleprompter.compile(
- SimplifiedBaleen(),
- teacher=SimplifiedBaleen(passages_per_hop=2),
- trainset=trainset,
- )
-
- evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)
- uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(
- uncompiled_baleen, metric=gold_passages_retrieved, display=False
- )
- # assert uncompiled_baleen_retrieval_score / 100 == 18 / 50
-
- compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)
- # assert compiled_baleen_retrieval_score / 100 == 27 / 50
- assert uncompiled_baleen_retrieval_score < compiled_baleen_retrieval_score
diff --git a/tests/dsp_LM/functional/test_functional.py b/tests/dsp_LM/functional/test_functional.py
index bc844a82c..31d0416b7 100644
--- a/tests/dsp_LM/functional/test_functional.py
+++ b/tests/dsp_LM/functional/test_functional.py
@@ -202,7 +202,7 @@ def test_bootstrap_effectiveness():
assert demos[0].output == trainset[0].output
# Test the compiled student's prediction.
- # We are using a DspDummyLM with follow_examples=True, which means that
+ # We are using a DSPDummyLM with follow_examples=True, which means that
# even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"
# on the second output, if it seems an example that perfectly matches the
# prompt, it will use that instead. That is why we expect "blue" here.
diff --git a/tests/dsp_LM/functional/test_signature_typed.py b/tests/dsp_LM/functional/test_signature_typed.py
deleted file mode 100644
index 6e875d851..000000000
--- a/tests/dsp_LM/functional/test_signature_typed.py
+++ /dev/null
@@ -1,155 +0,0 @@
-from typing import Any, Optional, Union
-
-import pydantic
-import pytest
-
-import dspy
-from dspy.functional import TypedPredictor
-from dspy.signatures.signature import signature_to_template
-
-
-def get_field_and_parser(signature: dspy.Signature) -> tuple[Any, Any]:
- module = TypedPredictor(signature)
- signature = module._prepare_signature()
- assert "answer" in signature.fields, "'answer' not in signature.fields"
- field = signature.fields.get("answer")
- parser = field.json_schema_extra.get("parser")
- return field, parser
-
-
-class Mysubmodel(pydantic.BaseModel):
- sub_floating: float
-
-
-class MyModel(pydantic.BaseModel):
- floating: float
- string: str
- boolean: bool
- integer: int
- optional: Optional[str]
- sequence_of_strings: list[str]
- union: Union[str, float]
- submodel: Mysubmodel
- optional_submodel: Optional[Mysubmodel]
- optional_existing_submodule: Optional[Mysubmodel]
-
-
-def build_model_instance() -> MyModel:
- return MyModel(
- floating=3.14,
- string="foobar",
- boolean=True,
- integer=42,
- optional=None,
- sequence_of_strings=["foo", "bar"],
- union=3.14,
- submodel=Mysubmodel(sub_floating=42.42),
- optional_submodel=None,
- optional_existing_submodule=Mysubmodel(sub_floating=42.42),
- )
-
-
- "test_type,serialized, expected", [(str, "foo", "foo"), (int, "42", 42), (float, "42.42", 42.42)]
-)
-def test_basic_types(test_type: type, serialized: str, expected: Any):
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: test_type = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
- assert parser is test_type, "Parser is not correct for 'answer'"
- assert parser(serialized) == expected, f"{test_type}({serialized})!= {expected}"
-
-
-def test_boolean():
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: bool = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
- assert parser("true"), f"Parsing 'true' failed"
- assert not parser("false"), f"Parsing 'false' failed"
-
-
- "test_type,serialized, expected",
- [(list[str], '["foo", "bar"]', ["foo", "bar"]), (tuple[int, float], "[42, 3.14]", (42, 3.14))],
-)
-def test_sequences(test_type: type, serialized: str, expected: Any):
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: test_type = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
-
- assert parser(serialized) == expected, f"Parsing {expected} failed"
-
-
- "test_type,serialized, expected",
- [
- (Optional[str], '"foobar"', "foobar"),
- (Optional[str], "null", None),
- (Union[str, float], "3.14", 3.14),
- (Union[str, bool], "true", True),
- ],
-)
-def test_unions(test_type: type, serialized: str, expected: Any):
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: test_type = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
-
- assert parser(serialized) == expected, f"Parsing {expected} failed"
-
-
-def test_pydantic():
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: MyModel = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
-
- instance = build_model_instance()
- parsed_instance = parser(instance.model_dump_json())
-
- assert parsed_instance == instance, f"{instance} != {parsed_instance}"
-
-
-def test_optional_pydantic():
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: Optional[MyModel] = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
-
- instance = build_model_instance()
- parsed_instance = parser(instance.model_dump_json())
- assert parsed_instance == instance, f"{instance} != {parsed_instance}"
-
- # Check null case
- parsed_instance = parser("null")
- assert parsed_instance == None, "Optional[MyModel] should be None"
-
-
-def test_dataclass():
- from dataclasses import dataclass
-
- @dataclass(frozen=True)
- class MyDataclass:
- string: str
- number: int
- floating: float
- boolean: bool
-
- class MySignature(dspy.Signature):
- question: str = dspy.InputField()
- answer: MyDataclass = dspy.OutputField()
-
- _, parser = get_field_and_parser(MySignature)
-
- instance = MyDataclass("foobar", 42, 3.14, True)
- parsed_instance = parser('{"string": "foobar", "number": 42, "floating": 3.14, "boolean": true}')
- assert parsed_instance == instance, f"{instance} != {parsed_instance}"
diff --git a/tests/dsp_LM/modules/test_aws_models.py b/tests/dsp_LM/modules/test_aws_models.py
deleted file mode 100644
index b6e018b33..000000000
--- a/tests/dsp_LM/modules/test_aws_models.py
+++ /dev/null
@@ -1,70 +0,0 @@
-"""Tests for AWS models.
-Note: Requires configuration of your AWS credentials with the AWS CLI and creating sagemaker endpoints.
-TODO: Create mock fixtures for pytest to remove the need for AWS credentials and endpoints.
-"""
-
-import dsp
-import dspy
-
-
-def get_lm(lm_provider: str, model_path: str, **kwargs) -> dsp.modules.lm.LM:
- """get the language model"""
- # extract model vendor and name from model name
- # Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>
- model_vendor = model_path.split("/")[0]
- model_name = model_path.split("/")[1]
-
- if lm_provider == "Bedrock":
- bedrock = dspy.Bedrock(region_name="us-west-2")
- if model_vendor == "mistral":
- return dspy.AWSMistral(bedrock, model_name, **kwargs)
- elif model_vendor == "anthropic":
- return dspy.AWSAnthropic(bedrock, model_name, **kwargs)
- elif model_vendor == "meta":
- return dspy.AWSMeta(bedrock, model_name, **kwargs)
- else:
- raise ValueError(
- "Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>"
- )
- elif lm_provider == "Sagemaker":
- sagemaker = dspy.Sagemaker(region_name="us-west-2")
- if model_vendor == "mistral":
- return dspy.AWSMistral(sagemaker, model_name, **kwargs)
- elif model_vendor == "meta":
- return dspy.AWSMeta(sagemaker, model_name, **kwargs)
- else:
- raise ValueError(
- "Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>"
- )
- else:
- raise ValueError(f"Unsupported model: {model_name}")
-
-
-def run_tests():
- """Test the providers and models"""
- # Configure your AWS credentials with the AWS CLI before running this script
- provider_model_tuples = [
- ("Bedrock", "mistral/mistral.mixtral-8x7b-instruct-v0:1"),
- ("Bedrock", "anthropic/anthropic.claude-3-haiku-20240307-v1:0"),
- ("Bedrock", "anthropic/anthropic.claude-3-sonnet-20240229-v1:0"),
- ("Bedrock", "meta/meta.llama2-70b-chat-v1"),
- ("Bedrock", "meta/meta.llama3-8b-instruct-v1:0"),
- ("Bedrock", "meta/meta.llama3-70b-instruct-v1:0"),
- # ('Sagemaker', 'mistral/<YOUR_ENDPOINT_NAME>'), # REPLACE YOUR_ENDPOINT_NAME with your sagemaker endpoint
- ]
-
- predict_func = dspy.Predict("question -> answer")
- for provider, model_path in provider_model_tuples:
- print(f"Provider: {provider}, Model: {model_path}")
- lm = get_lm(provider, model_path)
- with dspy.context(lm=lm):
- question = "What is the capital of France?"
- answer = predict_func(question=question).answer
- print(f"Question: {question}\nAnswer: {answer}")
- print("---------------------------------")
- lm.inspect_history()
- print("---------------------------------\n")
-
-
-if __name__ == "__main__":
- run_tests()
diff --git a/tests/dsp_LM/modules/test_cloudflare_models.py b/tests/dsp_LM/modules/test_cloudflare_models.py
deleted file mode 100644
index 188d2424e..000000000
--- a/tests/dsp_LM/modules/test_cloudflare_models.py
+++ /dev/null
@@ -1,61 +0,0 @@
-"""Tests for Cloudflare models.
-Note: Requires configuration of your Cloudflare account_id and api_key.
-"""
-
-import dspy
-
-models = {
- "@cf/qwen/qwen1.5-0.5b-chat": "https://huggingface.co/qwen/qwen1.5-0.5b-chat",
- "@hf/meta-llama/meta-llama-3-8b-instruct": "https://llama.meta.com",
- "@hf/nexusflow/starling-lm-7b-beta": "https://huggingface.co/Nexusflow/Starling-LM-7B-beta",
- "@cf/meta/llama-3-8b-instruct": "https://llama.meta.com",
- "@hf/thebloke/neural-chat-7b-v3-1-awq": "",
- "@cf/meta/llama-2-7b-chat-fp16": "https://ai.meta.com/llama/",
- "@cf/mistral/mistral-7b-instruct-v0.1": "https://mistral.ai/news/announcing-mistral-7b/",
- "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
- "@hf/mistral/mistral-7b-instruct-v0.2": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
- "@cf/fblgit/una-cybertron-7b-v2-bf16": "",
- "@hf/thebloke/codellama-7b-instruct-awq": "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-AWQ",
- "@cf/thebloke/discolm-german-7b-v1-awq": "https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-AWQ",
- "@cf/meta/llama-2-7b-chat-int8": "https://ai.meta.com/llama/",
- "@hf/thebloke/mistral-7b-instruct-v0.1-awq": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-AWQ",
- "@hf/thebloke/openchat_3.5-awq": "",
- "@cf/qwen/qwen1.5-7b-chat-awq": "https://huggingface.co/qwen/qwen1.5-7b-chat-awq",
- "@hf/thebloke/llama-2-13b-chat-awq": "https://huggingface.co/TheBloke/Llama-2-13B-chat-AWQ",
- "@hf/thebloke/deepseek-coder-6.7b-base-awq": "",
- "@hf/thebloke/openhermes-2.5-mistral-7b-awq": "",
- "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": "",
- "@cf/deepseek-ai/deepseek-math-7b-instruct": "https://huggingface.co/deepseek-ai/deepseek-math-7b-instruct",
- "@cf/tiiuae/falcon-7b-instruct": "https://huggingface.co/tiiuae/falcon-7b-instruct",
- "@hf/nousresearch/hermes-2-pro-mistral-7b": "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B",
- "@hf/thebloke/zephyr-7b-beta-awq": "https://huggingface.co/TheBloke/zephyr-7B-beta-AWQ",
- "@cf/qwen/qwen1.5-1.8b-chat": "https://huggingface.co/qwen/qwen1.5-1.8b-chat",
- "@cf/defog/sqlcoder-7b-2": "https://huggingface.co/defog/sqlcoder-7b-2",
- "@cf/microsoft/phi-2": "https://huggingface.co/microsoft/phi-2",
- "@hf/google/gemma-7b-it": "https://ai.google.dev/gemma/docs",
-}
-
-
-def get_lm(name: str): # -> dspy.LM:
- return dspy.CloudflareAI(model=name)
-
-
-def run_tests():
- """Test the providers and models"""
- # Configure your AWS credentials with the AWS CLI before running this script
- models
-
- predict_func = dspy.Predict("question -> answer")
- for model_name in models.keys():
- lm = get_lm(model_name)
- with dspy.context(lm=lm):
- question = "What is the capital of France?"
- answer = predict_func(question=question).answer
- print(f"Question: {question}\nAnswer: {answer}")
- print("---------------------------------")
- lm.inspect_history()
- print("---------------------------------\n")
-
-
-if __name__ == "__main__":
- run_tests()
diff --git a/tests/dsp_LM/modules/test_hf_model.py b/tests/dsp_LM/modules/test_hf_model.py
deleted file mode 100644
index 0b06c8042..000000000
--- a/tests/dsp_LM/modules/test_hf_model.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from pytest_mock.plugin import MockerFixture
-from transformers import AutoModelForSeq2SeqLM
-
-import dspy
-
-
-class MockConfig:
- def __init__(self, architectures: list[str]):
- self.architectures = architectures
-
-
-def test_load_gated_model(mocker: MockerFixture):
- conf = MockConfig(architectures=["ConditionalGeneration"])
- mocker.patch("transformers.AutoModelForSeq2SeqLM.from_pretrained")
- mocker.patch("transformers.AutoConfig.from_pretrained", return_value=conf)
- mocker.patch("transformers.AutoTokenizer.from_pretrained")
-
- some_token = "asdfasdfasdf"
- model = "google/gemma-7b"
- _ = dspy.HFModel(model, token=some_token)
- AutoModelForSeq2SeqLM.from_pretrained.assert_called_with(model, device_map="auto", token=some_token)
-
-
-def test_load_ungated_model(mocker: MockerFixture):
- conf = MockConfig(architectures=["ConditionalGeneration"])
- mocker.patch("transformers.AutoModelForSeq2SeqLM.from_pretrained")
- mocker.patch("transformers.AutoConfig.from_pretrained", return_value=conf)
- mocker.patch("transformers.AutoTokenizer.from_pretrained")
- _ = dspy.HFModel("openai-community/gpt2")
- # no token used in automodel
- AutoModelForSeq2SeqLM.from_pretrained.assert_called_with("openai-community/gpt2", device_map="auto", token=None)
diff --git a/tests/dsp_LM/modules/vectorizer/test_fastembed.py b/tests/dsp_LM/modules/vectorizer/test_fastembed.py
deleted file mode 100644
index e9a7335a4..000000000
--- a/tests/dsp_LM/modules/vectorizer/test_fastembed.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import pytest
-
-from dsp.modules.sentence_vectorizer import FastEmbedVectorizer
-from dspy.primitives.example import Example
-
-# Skip the test if the 'fastembed' package is not installed
-pytest.importorskip("fastembed", reason="'fastembed' is not installed. Use `pip install fastembed` to install it.")
-
-
- "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]
-)
-def test_fastembed_with_examples(n_dims, model_name):
- vectorizer = FastEmbedVectorizer(model_name)
-
- examples = [
- Example(query="What's the price today?", response="The price is $10.00").with_inputs("query", "response"),
- Example(query="What's the weather today?", response="The weather is sunny").with_inputs("query", "response"),
- Example(query="Who was leading the team?", response="It was Jim. Rather enthusiastic guy.").with_inputs(
- "query", "response"
- ),
- ]
-
- embeddings = vectorizer(examples)
-
- assert embeddings.shape == (len(examples), n_dims)
-
-
- "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]
-)
-def test_fastembed_with_strings(n_dims, model_name):
- vectorizer = FastEmbedVectorizer(model_name)
-
- inputs = [
- "Jonathan Kent is a fictional character appearing in American comic books published by DC Comics.",
- "Clark Kent is a fictional character appearing in American comic books published by DC Comics.",
- "Martha Kent is a fictional character appearing in American comic books published by DC Comics.",
- ]
-
- embeddings = vectorizer(inputs)
-
- assert embeddings.shape == (len(inputs), n_dims)
diff --git a/tests/dsp_LM/predict/test_aggregation.py b/tests/dsp_LM/predict/test_aggregation.py
deleted file mode 100644
index eb1d97536..000000000
--- a/tests/dsp_LM/predict/test_aggregation.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from dsp.utils import normalize_text
-from dspy.predict.aggregation import majority
-from dspy.primitives.prediction import Completions, Prediction
-
-
-def test_majority_with_prediction():
- prediction = Prediction.from_completions([{"answer": "2"}, {"answer": "2"}, {"answer": "3"}])
- result = majority(prediction)
- assert result.completions[0]["answer"] == "2"
-
-
-def test_majority_with_completions():
- completions = Completions([{"answer": "2"}, {"answer": "2"}, {"answer": "3"}])
- result = majority(completions)
- assert result.completions[0]["answer"] == "2"
-
-
-def test_majority_with_list():
- completions = [{"answer": "2"}, {"answer": "2"}, {"answer": "3"}]
- result = majority(completions)
- assert result.completions[0]["answer"] == "2"
-
-
-def test_majority_with_normalize():
- completions = [{"answer": "2"}, {"answer": " 2"}, {"answer": "3"}]
- result = majority(completions, normalize=normalize_text)
- assert result.completions[0]["answer"] == "2"
-
-
-def test_majority_with_field():
- completions = [
- {"answer": "2", "other": "1"},
- {"answer": "2", "other": "1"},
- {"answer": "3", "other": "2"},
- ]
- result = majority(completions, field="other")
- assert result.completions[0]["other"] == "1"
-
-
-def test_majority_with_no_majority():
- completions = [{"answer": "2"}, {"answer": "3"}, {"answer": "4"}]
- result = majority(completions)
- assert result.completions[0]["answer"] == "2" # The first completion is returned in case of a tie
diff --git a/tests/dsp_LM/predict/test_knn.py b/tests/dsp_LM/predict/test_knn.py
deleted file mode 100644
index 7f35b42e4..000000000
--- a/tests/dsp_LM/predict/test_knn.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import numpy as np
-import pytest
-
-import dsp
-import dspy
-from dspy.predict import KNN
-from dspy.utils import DummyVectorizer
-
-
-def mock_example(question: str, answer: str) -> dsp.Example:
- """Creates a mock DSP example with specified question and answer."""
- return dspy.Example(question=question, answer=answer).with_inputs("question")
-
-
-def setup_knn():
- """Sets up a KNN instance with a mocked vectorizer for testing."""
- dsp.SentenceTransformersVectorizer = DummyVectorizer
- trainset = [
- mock_example("What is the capital of France?", "Paris"),
- mock_example("What is the largest ocean?", "Pacific"),
- mock_example("What is 2+2?", "4"),
- ]
- knn = KNN(k=2, trainset=trainset)
- return knn
-
-
-def test_knn_initialization(setup_knn):
- """Tests the KNN initialization and checks if the trainset vectors are correctly created."""
- knn = setup_knn
- assert knn.k == 2, "Incorrect k value"
- assert len(knn.trainset_vectors) == 3, "Incorrect size of trainset vectors"
- assert isinstance(knn.trainset_vectors, np.ndarray), "Trainset vectors should be a NumPy array"
-
-
-def test_knn_query(setup_knn):
- """Tests the KNN query functionality for retrieving the nearest neighbors."""
- knn = setup_knn
- query = {"question": "What is 3+3?"} # A query close to "What is 2+2?"
- nearest_samples = knn(**query)
- assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
- assert nearest_samples[0].answer == "4", "Incorrect nearest sample returned"
-
-
-def test_knn_query_specificity(setup_knn):
- """Tests the KNN query functionality for specificity of returned examples."""
- knn = setup_knn
- query = {"question": "What is the capital of Germany?"} # A query close to "What is the capital of France?"
- nearest_samples = knn(**query)
- assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
- assert "Paris" in [sample.answer for sample in nearest_samples], "Expected Paris to be a nearest sample answer"
diff --git a/tests/dsp_LM/predict/test_langchain.py b/tests/dsp_LM/predict/test_langchain.py
deleted file mode 100644
index 89fa1d75d..000000000
--- a/tests/dsp_LM/predict/test_langchain.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import pytest
-
-pytest.importorskip("langchain")
-
-import os
-
-from langchain import hub
-from langchain_chroma import Chroma
-from langchain_community.embeddings import FakeEmbeddings
-from langchain_core.documents import Document
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.runnables import RunnablePassthrough
-from langchain_openai import ChatOpenAI
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-
-from dspy.predict.langchain import LangChainModule, LangChainPredict
-
-
-def test_copying_module():
- os.environ["OPENAI_API_KEY"] = "fake-key"
- llm = ChatOpenAI(model="gpt-4o-mini")
- docs = [Document(page_content="Hello, world!", metadata={"source": "https://example.com"})]
-
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=10)
- splits = text_splitter.split_documents(docs)
- vectorstore = Chroma.from_documents(documents=splits, embedding=FakeEmbeddings(size=5))
-
- # Retrieve and generate using the relevant snippets of the blog.
- retriever = vectorstore.as_retriever()
-
- prompt = hub.pull("rlm/rag-prompt")
-
- def format_docs(docs):
- return "\n\n".join(doc.page_content for doc in docs)
-
- rag_chain = (
- {"context": retriever | format_docs, "question": RunnablePassthrough()}
- | LangChainPredict(prompt, llm)
- | StrOutputParser()
- )
- # Now we wrap it in LangChainModule.
- rag_dspy_module = LangChainModule(rag_chain)
-
- copied_module = rag_dspy_module.reset_copy()
- assert len(copied_module.chain.steps) == len(rag_dspy_module.chain.steps)
- for module, copied_module in zip(rag_dspy_module.chain.steps, copied_module.chain.steps):
- if isinstance(module, LangChainPredict):
- # The LangChainPredict modules are deep copied.
- assert module != copied_module
- assert module.langchain_llm.model_name == copied_module.langchain_llm.model_name
- else:
- # The rest of the modules are just copied by reference.
- assert module == copied_module
- # Clean up.
- os.environ["OPENAI_API_KEY"] = None
diff --git a/tests/dsp_LM/primitives/test_example.py b/tests/dsp_LM/primitives/test_example.py
deleted file mode 100644
index 39ad0b749..000000000
--- a/tests/dsp_LM/primitives/test_example.py
+++ /dev/null
@@ -1,110 +0,0 @@
-import pytest
-
-from dspy import Example
-
-
-def test_example_initialization():
- example = Example(a=1, b=2)
- assert example.a == 1
- assert example.b == 2
-
-
-def test_example_initialization_from_base():
- base = Example(a=1, b=2)
- example = Example(base=base, c=3)
- assert example.a == 1
- assert example.b == 2
- assert example.c == 3
-
-
-def test_example_initialization_from_dict():
- base_dict = {"a": 1, "b": 2}
- example = Example(base=base_dict, c=3)
- assert example.a == 1
- assert example.b == 2
- assert example.c == 3
-
-
-def test_example_set_get_item():
- example = Example()
- example["a"] = 1
- assert example["a"] == 1
-
-
-def test_example_attribute_access():
- example = Example(a=1)
- assert example.a == 1
- example.a = 2
- assert example.a == 2
-
-
-def test_example_deletion():
- example = Example(a=1, b=2)
- del example["a"]
- with pytest.raises(AttributeError):
- _ = example.a
-
-
-def test_example_len():
- example = Example(a=1, b=2, dspy_hidden=3)
- assert len(example) == 2
-
-
-def test_example_repr_str():
- example = Example(a=1)
- assert repr(example) == "Example({'a': 1}) (input_keys=None)"
- assert str(example) == "Example({'a': 1}) (input_keys=None)"
-
-
-def test_example_eq():
- example1 = Example(a=1, b=2)
- example2 = Example(a=1, b=2)
- assert example1 == example2
- assert example1 != ""
-
-
-def test_example_hash():
- example1 = Example(a=1, b=2)
- example2 = Example(a=1, b=2)
- assert hash(example1) == hash(example2)
-
-
-def test_example_keys_values_items():
- example = Example(a=1, b=2, dspy_hidden=3)
- assert set(example.keys()) == {"a", "b"}
- assert 1 in example.values()
- assert ("b", 2) in example.items()
-
-
-def test_example_get():
- example = Example(a=1, b=2)
- assert example.get("a") == 1
- assert example.get("c", "default") == "default"
-
-
-def test_example_with_inputs():
- example = Example(a=1, b=2).with_inputs("a")
- assert example._input_keys == {"a"}
-
-
-def test_example_inputs_labels():
- example = Example(a=1, b=2).with_inputs("a")
- inputs = example.inputs()
- assert inputs.toDict() == {"a": 1}
- labels = example.labels()
- assert labels.toDict() == {"b": 2}
-
-
-def test_example_copy_without():
- example = Example(a=1, b=2)
- copied = example.copy(c=3)
- assert copied.a == 1
- assert copied.c == 3
- without_a = copied.without("a")
- with pytest.raises(AttributeError):
- _ = without_a.a
-
-
-def test_example_to_dict():
- example = Example(a=1, b=2)
- assert example.toDict() == {"a": 1, "b": 2}
diff --git a/tests/dsp_LM/primitives/test_module.py b/tests/dsp_LM/primitives/test_module.py
deleted file mode 100644
index 4ba6df220..000000000
--- a/tests/dsp_LM/primitives/test_module.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import threading
-
-import dspy
-
-
-def test_deepcopy_basic():
- signature = dspy.Signature("q -> a")
- cot = dspy.ChainOfThought(signature)
- cot_copy = cot.deepcopy()
- assert len(cot.parameters()) == len(cot_copy.parameters())
- # Parameters should be different objects with the same values.
- assert id(cot.parameters()[0]) != id(cot_copy.parameters()[0])
- assert cot.parameters()[0].__dict__ == cot_copy.parameters()[0].__dict__
-
-
-def test_deepcopy_with_uncopyable_modules():
- class CustomClass(dspy.Module):
- def __init__(self):
- self.lock = threading.Lock() # Non-copyable object.
- self.cot = dspy.ChainOfThought(dspy.Signature("q -> a"))
-
- model = CustomClass()
- model_copy = model.deepcopy()
- assert len(model.parameters()) == len(model_copy.parameters())
- # The lock should be refer to the same object (shallow copy).
- assert id(model.lock) == id(model_copy.lock)
- # Parameters should be different objects with the same values.
- assert id(model.parameters()[0]) != id(model_copy.parameters()[0])
- assert model.parameters()[0].__dict__ == model_copy.parameters()[0].__dict__
-
-
-def test_deepcopy_with_nested_modules():
- class CustomClass1(dspy.Module):
- def __init__(self):
- self.lock = threading.Lock() # Non-copyable object.
- self.cot = dspy.ChainOfThought(dspy.Signature("q -> a"))
-
- class CustomClass2(dspy.Module):
- def __init__(self):
- self.submodel = CustomClass1()
-
- model = CustomClass2()
- model_copy = model.deepcopy()
- assert len(model.parameters()) == len(model_copy.parameters())
- # The lock should be refer to the same object (shallow copy).
- assert id(model.submodel.lock) == id(model_copy.submodel.lock)
- # Parameters should be different objects with the same values.
- assert id(model.parameters()[0]) != id(model_copy.parameters()[0])
- assert model.parameters()[0].__dict__ == model_copy.parameters()[0].__dict__
diff --git a/tests/dsp_LM/primitives/test_python_interpreter.py b/tests/dsp_LM/primitives/test_python_interpreter.py
deleted file mode 100644
index a1c071388..000000000
--- a/tests/dsp_LM/primitives/test_python_interpreter.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import pytest
-
-from dspy.primitives.python_interpreter import CodePrompt, PythonInterpreter, TextPrompt
-
-
-def test_execute_simple_code():
- interpreter = PythonInterpreter(action_space={"print": print})
- code = "print('Hello, World!')"
- result = interpreter.execute(code)
- assert result is None, "Simple print statement should return None"
-
-
-def test_action_space_limitation():
- def func(string):
- pass
-
- interpreter = PythonInterpreter(action_space={})
- code = "func('This should not execute')"
- with pytest.raises(Exception):
- interpreter.execute(code)
-
-
-def test_import_whitelist():
- interpreter = PythonInterpreter(action_space={}, import_white_list=["math"])
- code = "import math\nresult = math.sqrt(4)"
- result = interpreter.execute(code)
- assert result == 2, "Should be able to import and use math.sqrt"
-
-
-def test_fuzzy_variable_matching():
- interpreter = PythonInterpreter(action_space={})
- code = "result = number + 1"
- result = interpreter.execute(code, fuzz_state={"number": 4})
- assert result == 5, "Fuzzy variable matching should work"
-
-
-def test_text_prompt_keyword_extraction():
- prompt = TextPrompt("Hello {name}, how are you?")
- assert "name" in prompt.key_words, "Keyword 'name' should be extracted"
-
-
-def test_text_prompt_formatting():
- prompt = TextPrompt("Hello {name}, how are you?")
- formatted = prompt.format(name="Alice")
- assert formatted == "Hello Alice, how are you?", "Should format with provided value"
-
-
-def test_code_prompt_execution():
- action_space = {"len": len}
- interpreter = PythonInterpreter(action_space=action_space)
- code_prompt = CodePrompt("result = len('hello')")
- result, _ = code_prompt.execute(interpreter)
- assert result == 5, "Code execution should return the length of 'hello'"
diff --git a/tests/dsp_LM/retrieve/integration_test_pgvectorrm.py b/tests/dsp_LM/retrieve/integration_test_pgvectorrm.py
deleted file mode 100644
index 524209bea..000000000
--- a/tests/dsp_LM/retrieve/integration_test_pgvectorrm.py
+++ /dev/null
@@ -1,94 +0,0 @@
-"""Instructions:
-Add to dev container features:
- "ghcr.io/itsmechlark/features/postgresql:1": {},
- "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {}
-Add to .personalization.sh:
- sudo apt install -y postgresql-16-pgvector
-
- sudo /etc/init.d/postgresql restart
-
- psql -v ON_ERROR_STOP=1 --user ${PGUSER} <<EOF
- create extension if not exists vector;
- EOF
-poetry install -E postgres
-"""
-import psycopg2
-import pytest
-
-from dspy.primitives.example import Example
-from dspy.retrieve.pgvector_rm import PgVectorRM
-
-DB_URL = "postgresql://postgres:password@localhost/postgres"
-PG_TABLE_NAME = "test_table"
-
-
-def get_pgvectorrm():
- openai_client = None # Mock or use a real OpenAI client
- pgvectorrm = PgVectorRM(
- DB_URL, PG_TABLE_NAME, openai_client=openai_client, embedding_func=lambda x: "[2,3,4]", include_similarity=True
- )
- return pgvectorrm
-
-
-def setup_pgvectorrm():
- pgvectorrm = get_pgvectorrm()
- conn = psycopg2.connect(DB_URL)
- cursor = conn.cursor()
- cursor.execute(f"DROP TABLE IF EXISTS {PG_TABLE_NAME}")
- conn.commit()
-
- cursor.execute(f"CREATE TABLE IF NOT EXISTS {PG_TABLE_NAME} (id SERIAL PRIMARY KEY, text TEXT, embedding VECTOR)")
- cursor.execute(
- f"INSERT INTO {PG_TABLE_NAME} (text, embedding) VALUES ('Dummy text1', '[1,2,3]'), ('Dummy text2', '[4,5,6]')"
- )
- conn.commit()
-
- yield pgvectorrm
-
- cursor.execute(f"TRUNCATE TABLE {PG_TABLE_NAME}")
- conn.commit()
-
- cursor.close()
- conn.close()
-
-
-def test_pgvectorrm_retrieve(setup_pgvectorrm):
- pgvectorrm = setup_pgvectorrm
- query = "test query"
- results = pgvectorrm(query)
- assert len(results) == 2
- assert results == [
- Example(text="Dummy text2", similarity=0.9946115458726394),
- Example(text="Dummy text1", similarity=0.9925833339709302),
- ]
-
-
- "k, expected",
- [
- (1, 1),
- (2, 2),
- (3, 2), # Assuming only 2 entries exist
- ],
-)
-def test_pgvectorrm_retrieve_diff_k(setup_pgvectorrm, k, expected):
- setup_pgvectorrm.k = k
- query = "test query"
- results = setup_pgvectorrm(query)
- assert len(results) == expected
-
-
-def test_empty_table():
- # Assuming setup_pgvectorrm cleans up after yielding
- query = "test query"
- results = get_pgvectorrm()(query)
- assert len(results) == 0
-
-
-def test_retrieval_without_similarity(setup_pgvectorrm):
- setup_pgvectorrm.include_similarity = False
- query = "test query"
- results = setup_pgvectorrm(query)
- # Ensure 'similarity' key is not in results
- assert all("similarity" not in result for result in results)
diff --git a/tests/dsp_LM/retrieve/test_llama_index_rm.py b/tests/dsp_LM/retrieve/test_llama_index_rm.py
index 497ed87fc..735c1a940 100644
--- a/tests/dsp_LM/retrieve/test_llama_index_rm.py
+++ b/tests/dsp_LM/retrieve/test_llama_index_rm.py
@@ -3,7 +3,7 @@ import logging
import pytest
import dspy
-from dsp.modules.dummy_lm import DspDummyLM
+from dsp.modules.dummy_lm import DSPDummyLM
from dspy.datasets import HotPotQA
try:
@@ -38,7 +38,7 @@ def rag_setup() -> dict:
"index": index,
"retriever": retriever,
"rm": rm,
- "lm": DspDummyLM(answers=dummyset),
+ "lm": DSPDummyLM(answers=dummyset),
"trainset": trainset,
"devset": devset,
}
diff --git a/tests/dsp_LM/teleprompt/test_bootstrap.py b/tests/dsp_LM/teleprompt/test_bootstrap.py
index 6bc41ec61..936daf8e4 100644
--- a/tests/dsp_LM/teleprompt/test_bootstrap.py
+++ b/tests/dsp_LM/teleprompt/test_bootstrap.py
@@ -65,7 +65,7 @@ def test_bootstrap_effectiveness():
assert compiled_student.predictor.demos[0].output == trainset[0].output
# Test the compiled student's prediction.
- # We are using a DspDummyLM with follow_examples=True, which means that
+ # We are using a DSPDummyLM with follow_examples=True, which means that
# even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"
# on the second output, if it seems an example that perfectly matches the
# prompt, it will use that instead. That is why we expect "blue" here.
@@ -115,7 +115,7 @@ def test_error_handling_during_bootstrap():
student = SimpleModule("input -> output")
teacher = BuggyModule("input -> output")
- # Setup DspDummyLM to simulate an error scenario
+ # Setup DSPDummyLM to simulate an error scenario
lm = DSPDummyLM(
[
"Initial thoughts", # Simulate initial teacher's prediction
diff --git a/tests/dsp_LM/teleprompt/test_ensemble.py b/tests/dsp_LM/teleprompt/test_ensemble.py
deleted file mode 100644
index c779e8782..000000000
--- a/tests/dsp_LM/teleprompt/test_ensemble.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import pytest
-
-import dspy
-from dspy.teleprompt import Ensemble
-
-
-class MockProgram(dspy.Module):
- def __init__(self, output):
- super().__init__()
- self.output = output
-
- def forward(self, *args, **kwargs):
- return self.output
-
-
-# Simple reduction function to test with
-def mock_reduce_fn(outputs):
- return sum(outputs) / len(outputs)
-
-
-def test_ensemble_without_reduction():
- """Test that Ensemble correctly combines outputs without applying a reduce_fn."""
- programs = [MockProgram(i) for i in range(5)]
- ensemble = Ensemble()
- ensembled_program = ensemble.compile(programs)
-
- outputs = ensembled_program()
- assert len(outputs) == 5, "Ensemble did not combine the correct number of outputs"
-
-
-def test_ensemble_with_reduction():
- """Test that Ensemble correctly applies a reduce_fn to combine outputs."""
- programs = [MockProgram(i) for i in range(5)]
- ensemble = Ensemble(reduce_fn=mock_reduce_fn)
- ensembled_program = ensemble.compile(programs)
-
- output = ensembled_program()
- expected_output = sum(range(5)) / 5
- assert output == expected_output, "Ensemble did not correctly apply the reduce_fn"
-
-
-def test_ensemble_with_size_limitation():
- """Test that specifying a size limits the number of programs used in the ensemble."""
- programs = [MockProgram(i) for i in range(10)]
- ensemble_size = 3
- ensemble = Ensemble(size=ensemble_size)
- ensembled_program = ensemble.compile(programs)
-
- outputs = ensembled_program()
- assert len(outputs) == ensemble_size, "Ensemble did not respect the specified size limitation"
-
-
-def test_ensemble_deterministic_behavior():
- """Verify that the Ensemble class raises an assertion for deterministic behavior."""
- with pytest.raises(
- AssertionError,
- match="TODO: Implement example hashing for deterministic ensemble.",
- ):
- Ensemble(deterministic=True)
diff --git a/tests/dsp_LM/teleprompt/test_knn_fewshot.py b/tests/dsp_LM/teleprompt/test_knn_fewshot.py
index 771e01d55..97c2dbbe3 100644
--- a/tests/dsp_LM/teleprompt/test_knn_fewshot.py
+++ b/tests/dsp_LM/teleprompt/test_knn_fewshot.py
@@ -43,7 +43,7 @@ def _test_knn_few_shot_compile(setup_knn_few_shot):
student = SimpleModule("input -> output")
teacher = SimpleModule("input -> output") # Assuming teacher uses the same module type
- # Setup DspDummyLM with a response for a query similar to one of the training examples
+ # Setup DSPDummyLM with a response for a query similar to one of the training examples
lm = DSPDummyLM(["Madrid", "10"])
dspy.settings.configure(lm=lm) # Responses for the capital of Spain and the result of 5+5)
@@ -60,6 +60,6 @@ def _test_knn_few_shot_compile(setup_knn_few_shot):
print("CONVO")
print(lm.get_convo(-1))
- # Validate that the output corresponds to one of the expected DspDummyLM responses
+ # Validate that the output corresponds to one of the expected DSPDummyLM responses
# This assumes the compiled_student's forward method will execute the predictor with the given query
assert output in ["Madrid", "10"], "The compiled student did not return the correct output based on the query"
diff --git a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
index 9ed1b594f..86d8c00d0 100644
--- a/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
+++ b/tests/dsp_LM/teleprompt/test_mipro_optimizer.py
@@ -81,7 +81,7 @@ class ConditionalLM(LM):
answer = "think deeply.\nOutput: " + answer
RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m"
- print("=== DspDummyLM ===")
+ print("=== DSPDummyLM ===")
print(prompt, end="")
print(f"{RED}{answer}{RESET}")
print("===")
@@ -173,7 +173,7 @@ def test_signature_optimizer_bad_lm():
track_stats=False,
)
- # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DspDummyLM,
+ # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DSPDummyLM,
# which only outputs "Optimized instruction i" this means that none of the bootstrapped examples are successful,
# and therefore the set of examples that we're using to generate new prompts is empty
with pytest.raises(ValueError):