Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fixes and removes non-existent dependency on dsp.SentenceTransformersVectorizer from KNN and KNNFewShot (#7884)

Cezar Cocu21b21e877365
diff --git a/docs/docs/cheatsheet.md b/docs/docs/cheatsheet.md
index c6d3a8f0c..491e49026 100644
--- a/docs/docs/cheatsheet.md
+++ b/docs/docs/cheatsheet.md
@@ -2,7 +2,6 @@
sidebar_position: 999
---
-
# DSPy Cheatsheet
This page will contain snippets for frequent usage patterns.
@@ -216,7 +215,7 @@ def parse_integer_answer(answer, only_first_line=True):
except (ValueError, IndexError):
# print(answer)
answer = 0
-
+
return answer
# Metric Function
@@ -254,7 +253,8 @@ evaluate_program(your_dspy_program)
## DSPy Optimizers
-### LabeledFewShot
+### LabeledFewShot
+
```python
from dspy.teleprompt import LabeledFewShot
@@ -262,7 +262,8 @@ labeled_fewshot_optimizer = LabeledFewShot(k=8)
your_dspy_program_compiled = labeled_fewshot_optimizer.compile(student = your_dspy_program, trainset=trainset)
```
-### BootstrapFewShot
+### BootstrapFewShot
+
```python
from dspy.teleprompt import BootstrapFewShot
@@ -272,6 +273,7 @@ your_dspy_program_compiled = fewshot_optimizer.compile(student = your_dspy_progr
```
#### Using another LM for compilation, specifying in teacher_settings
+
```python
from dspy.teleprompt import BootstrapFewShot
@@ -306,7 +308,6 @@ loaded_program.load(path=save_path)
Detailed documentation on BootstrapFewShotWithRandomSearch can be found [here](deep-dive/optimizers/bootstrap-fewshot.md).
-
```python
from dspy.teleprompt import BootstrapFewShotWithRandomSearch
@@ -315,8 +316,8 @@ fewshot_optimizer = BootstrapFewShotWithRandomSearch(metric=your_defined_metric,
your_dspy_program_compiled = fewshot_optimizer.compile(student = your_dspy_program, trainset=trainset, valset=devset)
```
-Other custom configurations are similar to customizing the `BootstrapFewShot` optimizer.
+Other custom configurations are similar to customizing the `BootstrapFewShot` optimizer.
### Ensemble
@@ -363,7 +364,6 @@ for p in finetune_program.predictors():
Detailed documentation on COPRO can be found [here](deep-dive/optimizers/copro.md).
-
```python
from dspy.teleprompt import COPRO
@@ -376,7 +376,6 @@ compiled_program_optimized_signature = copro_teleprompter.compile(your_dspy_prog
### MIPRO
-
```python
from dspy.teleprompt import MIPRO
@@ -392,7 +391,9 @@ compiled_program_optimized_bayesian_signature = teleprompter.compile(your_dspy_p
Note: detailed documentation can be found [here](deep-dive/optimizers/miprov2.md). `MIPROv2` is the latest extension of `MIPRO` which includes updates such as (1) improvements to instruction proposal and (2) more efficient search with minibatching.
#### Optimizing with MIPROv2
+
This shows how to perform an easy out-of-the box run with `auto=light`, which configures many hyperparameters for you and performs a light optimization run. You can alternatively set `auto=medium` or `auto=heavy` to perform longer optimization runs. The more detailed `MIPROv2` documentation [here](deep-dive/optimizers/miprov2.md) also provides more information about how to set hyperparameters by hand.
+
```python
# Import the optimizer
from dspy.teleprompt import MIPROv2
@@ -422,6 +423,7 @@ evaluate(optimized_program, devset=devset[:])
```
#### Optimizing instructions only with MIPROv2 (0-Shot)
+
```python
# Import the optimizer
from dspy.teleprompt import MIPROv2
@@ -449,6 +451,7 @@ optimized_program.save(f"mipro_optimized")
print(f"Evaluate optimized program...")
evaluate(optimized_program, devset=devset[:])
```
+
### Signature Optimizer with Types
```python
@@ -466,12 +469,14 @@ compiled_program = optimize_signature(
### KNNFewShot
```python
-from dspy.predict import KNN
+from sentence_transformers import SentenceTransformer
+from dspy import Embedder
from dspy.teleprompt import KNNFewShot
+from dspy import ChainOfThought
-knn_optimizer = KNNFewShot(KNN, k=3, trainset=trainset)
+knn_optimizer = KNNFewShot(k=3, trainset=trainset, vectorizer=Embedder(SentenceTransformer("all-MiniLM-L6-v2").encode))
-your_dspy_program_compiled = knn_optimizer.compile(student=your_dspy_program, trainset=trainset, valset=devset)
+qa_compiled = knn_optimizer.compile(student=ChainOfThought("question -> answer"))
```
### BootstrapFewShotWithOptuna
@@ -483,21 +488,22 @@ fewshot_optuna_optimizer = BootstrapFewShotWithOptuna(metric=your_defined_metric
your_dspy_program_compiled = fewshot_optuna_optimizer.compile(student=your_dspy_program, trainset=trainset, valset=devset)
```
-Other custom configurations are similar to customizing the `dspy.BootstrapFewShot` optimizer.
+Other custom configurations are similar to customizing the `dspy.BootstrapFewShot` optimizer.
## DSPy Assertions
### Including `dspy.Assert` and `dspy.Suggest` statements
+
```python
dspy.Assert(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModule")
dspy.Suggest(your_validation_fn(model_outputs), "your feedback message", target_module="YourDSPyModule")
```
-### Activating DSPy Program with Assertions
+### Activating DSPy Program with Assertions
-**Note**: To use Assertions properly, you must **activate** a DSPy program that includes `dspy.Assert` or `dspy.Suggest` statements from either of the methods above.
+**Note**: To use Assertions properly, you must **activate** a DSPy program that includes `dspy.Assert` or `dspy.Suggest` statements from either of the methods above.
```python
#1. Using `assert_transform_module:
diff --git a/dspy/predict/knn.py b/dspy/predict/knn.py
index c0cd2df12..85b80bca4 100644
--- a/dspy/predict/knn.py
+++ b/dspy/predict/knn.py
@@ -1,28 +1,30 @@
import numpy as np
+from dspy.clients import Embedder
+from dspy.primitives import Example
+
class KNN:
- def __init__(self, k: int, trainset: list, vectorizer=None):
+ def __init__(self, k: int, trainset: list[Example], vectorizer: Embedder):
"""
A k-nearest neighbors retriever that finds similar examples from a training set.
Args:
k: Number of nearest neighbors to retrieve
trainset: List of training examples to search through
- vectorizer: Optional dspy.Embedder for computing embeddings. If None, uses sentence-transformers.
+ vectorizer: The `Embedder` to use for vectorization
Example:
- >>> trainset = [dsp.Example(input="hello", output="world"), ...]
- >>> knn = KNN(k=3, trainset=trainset)
+ >>> import dspy
+ >>> from sentence_transformers import SentenceTransformer
+ >>>
+ >>> trainset = [dspy.Example(input="hello", output="world"), ...]
+ >>> knn = KNN(k=3, trainset=trainset, vectorizer=dspy.Embedder(SentenceTransformer("all-MiniLM-L6-v2").encode))
>>> similar_examples = knn(input="hello")
"""
-
- import dspy.dsp as dsp
- import dspy
-
self.k = k
self.trainset = trainset
- self.embedding = vectorizer or dspy.Embedder(dsp.SentenceTransformersVectorizer())
+ self.embedding = vectorizer
trainset_casted_to_vectorize = [
" | ".join([f"{key}: {value}" for key, value in example.items() if key in example._input_keys])
for example in self.trainset
@@ -33,5 +35,4 @@ class KNN:
input_example_vector = self.embedding([" | ".join([f"{key}: {val}" for key, val in kwargs.items()])])
scores = np.dot(self.trainset_vectors, input_example_vector.T).squeeze()
nearest_samples_idxs = scores.argsort()[-self.k :][::-1]
- train_sampled = [self.trainset[cur_idx] for cur_idx in nearest_samples_idxs]
- return train_sampled
+ return [self.trainset[cur_idx] for cur_idx in nearest_samples_idxs]
diff --git a/dspy/teleprompt/knn_fewshot.py b/dspy/teleprompt/knn_fewshot.py
index 5fe079147..56e70440a 100644
--- a/dspy/teleprompt/knn_fewshot.py
+++ b/dspy/teleprompt/knn_fewshot.py
@@ -1,16 +1,40 @@
import types
+
+from dspy.clients import Embedder
from dspy.predict.knn import KNN
+from dspy.primitives import Example
from dspy.teleprompt import BootstrapFewShot
from .teleprompt import Teleprompter
class KNNFewShot(Teleprompter):
- def __init__(self, k: int, trainset: list, vectorizer=None, **few_shot_bootstrap_args):
+ def __init__(self, k: int, trainset: list[Example], vectorizer: Embedder, **few_shot_bootstrap_args):
+ """
+ KNNFewShot is an optimizer that uses an in-memory KNN retriever to find the k nearest neighbors
+ in a trainset at test time. For each input example in a forward call, it identifies the k most
+ similar examples from the trainset and attaches them as demonstrations to the student module.
+
+ Args:
+ k: The number of nearest neighbors to attach to the student model.
+ trainset: The training set to use for few-shot prompting.
+ vectorizer: The `Embedder` to use for vectorization
+ **few_shot_bootstrap_args: Additional arguments for the `BootstrapFewShot` optimizer.
+
+ Example:
+ >>> import dspy
+ >>> from sentence_transformers import SentenceTransformer
+ >>>
+ >>> qa = dspy.ChainOfThought("question -> answer")
+ >>> trainset = [dspy.Example(question="What is the capital of France?", answer="Paris").with_inputs("question"), ...]
+ >>> knn_few_shot = KNNFewShot(k=3, trainset=trainset, vectorizer=dspy.Embedder(SentenceTransformer("all-MiniLM-L6-v2").encode))
+ >>> compiled_qa = knn_few_shot.compile(qa)
+ >>> compiled_qa("What is the capital of Belgium?")
+ """
self.KNN = KNN(k, trainset, vectorizer=vectorizer)
self.few_shot_bootstrap_args = few_shot_bootstrap_args
- def compile(self, student, *, teacher=None, trainset=None, valset=None):
+ def compile(self, student, *, teacher=None):
student_copy = student.reset_copy()
def forward_pass(_, **kwargs):
diff --git a/tests/predict/test_knn.py b/tests/predict/test_knn.py
index 0cbe6a205..be22fdd12 100644
--- a/tests/predict/test_knn.py
+++ b/tests/predict/test_knn.py
@@ -1,8 +1,9 @@
-import pytest
import numpy as np
+import pytest
+
import dspy
-from dspy.utils import DummyVectorizer
from dspy.predict import KNN
+from dspy.utils import DummyVectorizer
def mock_example(question: str, answer: str) -> dspy.Example:
@@ -10,46 +11,38 @@ def mock_example(question: str, answer: str) -> dspy.Example:
return dspy.Example(question=question, answer=answer).with_inputs("question")
-# @pytest.fixture
-# def setup_knn():
-# """Sets up a KNN instance with a mocked vectorizer for testing."""
-# dsp.SentenceTransformersVectorizer = DummyVectorizer
-# trainset = [
-# mock_example("What is the capital of France?", "Paris"),
-# mock_example("What is the largest ocean?", "Pacific"),
-# mock_example("What is 2+2?", "4"),
-# ]
-# knn = KNN(k=2, trainset=trainset)
-# return knn
-
-
-# def test_knn_initialization(setup_knn):
-# """Tests the KNN initialization and checks if the trainset vectors are correctly created."""
-# knn = setup_knn
-# assert knn.k == 2, "Incorrect k value"
-# assert len(knn.trainset_vectors) == 3, "Incorrect size of trainset vectors"
-# assert isinstance(
-# knn.trainset_vectors, np.ndarray
-# ), "Trainset vectors should be a NumPy array"
-
-
-# def test_knn_query(setup_knn):
-# """Tests the KNN query functionality for retrieving the nearest neighbors."""
-# knn = setup_knn
-# query = {"question": "What is 3+3?"} # A query close to "What is 2+2?"
-# nearest_samples = knn(**query)
-# assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
-# assert nearest_samples[0].answer == "4", "Incorrect nearest sample returned"
-
-
-# def test_knn_query_specificity(setup_knn):
-# """Tests the KNN query functionality for specificity of returned examples."""
-# knn = setup_knn
-# query = {
-# "question": "What is the capital of Germany?"
-# } # A query close to "What is the capital of France?"
-# nearest_samples = knn(**query)
-# assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
-# assert "Paris" in [
-# sample.answer for sample in nearest_samples
-# ], "Expected Paris to be a nearest sample answer"
+def setup_knn() -> KNN:
+ """Sets up a KNN instance with a mocked vectorizer for testing."""
+ trainset = [
+ mock_example("What is the capital of France?", "Paris"),
+ mock_example("What is the largest ocean?", "Pacific"),
+ mock_example("What is 2+2?", "4"),
+ ]
+ return KNN(k=2, trainset=trainset, vectorizer=dspy.Embedder(DummyVectorizer()))
+
+
+def test_knn_initialization(setup_knn):
+ """Tests the KNN initialization and checks if the trainset vectors are correctly created."""
+ knn = setup_knn
+ assert knn.k == 2, "Incorrect k value"
+ assert len(knn.trainset_vectors) == 3, "Incorrect size of trainset vectors"
+ assert isinstance(knn.trainset_vectors, np.ndarray), "Trainset vectors should be a NumPy array"
+
+
+def test_knn_query(setup_knn):
+ """Tests the KNN query functionality for retrieving the nearest neighbors."""
+ knn = setup_knn
+ query = {"question": "What is 3+3?"} # A query close to "What is 2+2?"
+ nearest_samples = knn(**query)
+ assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
+ assert nearest_samples[0].answer == "4", "Incorrect nearest sample returned"
+
+
+def test_knn_query_specificity(setup_knn):
+ """Tests the KNN query functionality for specificity of returned examples."""
+ knn = setup_knn
+ query = {"question": "What is the capital of Germany?"} # A query close to "What is the capital of France?"
+ nearest_samples = knn(**query)
+ assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"
+ assert "Paris" in [sample.answer for sample in nearest_samples], "Expected Paris to be a nearest sample answer"
diff --git a/tests/teleprompt/test_knn_fewshot.py b/tests/teleprompt/test_knn_fewshot.py
index 83600493d..db72c58e5 100644
--- a/tests/teleprompt/test_knn_fewshot.py
+++ b/tests/teleprompt/test_knn_fewshot.py
@@ -1,4 +1,5 @@
import pytest
+
import dspy
from dspy.teleprompt.knn_fewshot import KNNFewShot
from dspy.utils.dummies import DummyLM, DummyVectorizer
@@ -9,62 +10,57 @@ def mock_example(question: str, answer: str) -> dspy.Example:
return dspy.Example(question=question, answer=answer).with_inputs("question")
-# @pytest.fixture
-# def setup_knn_few_shot():
-# """Sets up a KNNFewShot instance for testing."""
-# trainset = [
-# mock_example("What is the capital of France?", "Paris"),
-# mock_example("What is the largest ocean?", "Pacific"),
-# mock_example("What is 2+2?", "4"),
-# ]
-# dsp.SentenceTransformersVectorizer = DummyVectorizer
-# knn_few_shot = KNNFewShot(k=2, trainset=trainset)
-# return knn_few_shot
-
+def setup_knn_few_shot() -> KNNFewShot:
+ """Sets up a KNNFewShot instance for testing."""
+ trainset = [
+ mock_example("What is the capital of France?", "Paris"),
+ mock_example("What is the largest ocean?", "Pacific"),
+ mock_example("What is 2+2?", "4"),
+ ]
+ return KNNFewShot(k=2, trainset=trainset, vectorizer=dspy.Embedder(DummyVectorizer()))
-# def test_knn_few_shot_initialization(setup_knn_few_shot):
-# """Tests the KNNFewShot initialization."""
-# knn_few_shot = setup_knn_few_shot
-# assert knn_few_shot.KNN.k == 2, "Incorrect k value for KNN"
-# assert len(knn_few_shot.KNN.trainset) == 3, "Incorrect trainset size for KNN"
+def test_knn_few_shot_initialization(setup_knn_few_shot):
+ """Tests the KNNFewShot initialization."""
+ knn_few_shot = setup_knn_few_shot
+ assert knn_few_shot.KNN.k == 2, "Incorrect k value for KNN"
+ assert len(knn_few_shot.KNN.trainset) == 3, "Incorrect trainset size for KNN"
-# class SimpleModule(dspy.Module):
-# def __init__(self, signature):
-# super().__init__()
-# self.predictor = dspy.Predict(signature)
-# def forward(self, *args, **kwargs):
-# return self.predictor(**kwargs)
+class SimpleModule(dspy.Module):
+ def __init__(self, signature):
+ super().__init__()
+ self.predictor = dspy.Predict(signature)
-# def reset_copy(self):
-# # Creates a new instance of SimpleModule with the same predictor
-# return SimpleModule(self.predictor.signature)
+ def forward(self, *args, **kwargs):
+ return self.predictor(**kwargs)
+ def reset_copy(self):
+ # Creates a new instance of SimpleModule with the same predictor
+ return SimpleModule(self.predictor.signature)
-# # TODO: Test not working yet
-# def _test_knn_few_shot_compile(setup_knn_few_shot):
-# """Tests the compile method of KNNFewShot with SimpleModule as student."""
-# student = SimpleModule("input -> output")
-# teacher = SimpleModule("input -> output") # Assuming teacher uses the same module type
-# # Setup DummyLM with a response for a query similar to one of the training examples
-# lm = DummyLM(["Madrid", "10"])
-# dspy.settings.configure(lm=lm) # Responses for the capital of Spain and the result of 5+5)
+# TODO: Test not working yet
+def _test_knn_few_shot_compile(setup_knn_few_shot):
+ """Tests the compile method of KNNFewShot with SimpleModule as student."""
+ student = SimpleModule("input -> output")
+ teacher = SimpleModule("input -> output") # Assuming teacher uses the same module type
-# knn_few_shot = setup_knn_few_shot
-# trainset = knn_few_shot.KNN.trainset
-# compiled_student = knn_few_shot.compile(student, teacher=teacher, trainset=trainset, valset=None)
+ # Setup DummyLM with a response for a query similar to one of the training examples
+ lm = DummyLM(["Madrid", "10"])
+ dspy.settings.configure(lm=lm) # Responses for the capital of Spain and the result of 5+5)
-# assert len(compiled_student.predictor.demos) == 1
-# assert compiled_student.predictor.demos[0].input == trainset[0].input
-# assert compiled_student.predictor.demos[0].output == trainset[0].output
-# # Simulate a query that is similar to one of the training examples
-# output = compiled_student.forward(input="What is the capital of Spain?").output
+ knn_few_shot = setup_knn_few_shot
+ trainset = knn_few_shot.KNN.trainset
+ compiled_student = knn_few_shot.compile(student, teacher=teacher, trainset=trainset, valset=None)
-# print("CONVO")
-# print(lm.get_convo(-1))
+ assert len(compiled_student.predictor.demos) == 1
+ assert compiled_student.predictor.demos[0].input == trainset[0].input
+ assert compiled_student.predictor.demos[0].output == trainset[0].output
+ # Simulate a query that is similar to one of the training examples
+ output = compiled_student.forward(input="What is the capital of Spain?").output
-# # Validate that the output corresponds to one of the expected DummyLM responses
-# # This assumes the compiled_student's forward method will execute the predictor with the given query
-# assert output in ["Madrid", "10"], "The compiled student did not return the correct output based on the query"
+ # Validate that the output corresponds to one of the expected DummyLM responses
+ # This assumes the compiled_student's forward method will execute the predictor with the given query
+ assert output in ["Madrid", "10"], "The compiled student did not return the correct output based on the query"