Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(retrievers): add EmbeddingsWithScores for similarity score access (#9478)

Serj Smorb0ea262ca914
diff --git a/dspy/retrievers/__init__.py b/dspy/retrievers/__init__.py
index ffaf603bb..89339ca03 100644
--- a/dspy/retrievers/__init__.py
+++ b/dspy/retrievers/__init__.py
@@ -1,4 +1,4 @@
-from dspy.retrievers.embeddings import Embeddings
+from dspy.retrievers.embeddings import Embeddings, EmbeddingsWithScores
from dspy.retrievers.retrieve import Retrieve
-__all__ = ["Embeddings", "Retrieve"]
+__all__ = ["Embeddings", "EmbeddingsWithScores", "Retrieve"]
diff --git a/dspy/retrievers/embeddings.py b/dspy/retrievers/embeddings.py
index 94bbc589f..82ba02e8b 100644
--- a/dspy/retrievers/embeddings.py
+++ b/dspy/retrievers/embeddings.py
@@ -8,6 +8,13 @@ from dspy.utils.unbatchify import Unbatchify
class Embeddings:
+ """DSPy Embeddings retriever.
+
+ This class retrieves the top-k most similar passages from a corpus using embedding-based similarity search.
+ For large corpora, a FAISS index is built for fast approximate candidate retrieval, followed by exact
+ re-ranking. For small corpora, brute-force search is used.
+ """
+
def __init__(
self,
corpus: list[str],
@@ -35,9 +42,17 @@ class Embeddings:
return self.forward(query)
def forward(self, query: str):
+ """Search for the top-k passages most similar to the query.
+
+ Args:
+ query (str): The search query string
+
+ Returns:
+ dspy.Prediction: A prediction containing passages and their corpus indices.
+ """
import dspy
- passages, indices = self.search_fn(query)
+ passages, indices, _scores = self.search_fn(query)
return dspy.Prediction(passages=passages, indices=indices)
def _batch_forward(self, queries: list[str]):
@@ -81,8 +96,13 @@ class Embeddings:
top_k_indices = np.argsort(-scores, axis=1)[:, : self.k]
top_indices = candidate_indices[np.arange(len(q_embeds))[:, None], top_k_indices]
+ top_scores = scores[np.arange(len(q_embeds))[:, None], top_k_indices]
- return [([self.corpus[idx] for idx in indices], [idx for idx in indices]) for indices in top_indices] # noqa: C416
+ results = []
+ for indices, query_scores in zip(top_indices, top_scores, strict=True):
+ passages = [self.corpus[idx] for idx in indices]
+ results.append((passages, indices.tolist(), query_scores.tolist()))
+ return results
def _normalize(self, embeddings: np.ndarray):
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
@@ -214,3 +234,25 @@ class Embeddings:
instance.search_fn = Unbatchify(instance._batch_forward)
instance.load(path, embedder)
return instance
+
+
+class EmbeddingsWithScores(Embeddings):
+ """DSPy EmbeddingsWithScores retriever.
+
+ This class extends the Embeddings retriever to also return similarity scores alongside passages and indices.
+ Similarity scores enable downstream such as thresholding and re-ranking.
+ """
+
+ def forward(self, query: str):
+ """Search for the top-k passages most similar to the query.
+
+ Args:
+ query (str): The search query string.
+
+ Returns:
+ dspy.Prediction: A prediction containing passages, indices, and similarity scores.
+ """
+ import dspy
+
+ passages, indices, scores = self.search_fn(query)
+ return dspy.Prediction(passages=passages, indices=indices, scores=scores)
diff --git a/tests/retrievers/test_embeddings.py b/tests/retrievers/test_embeddings.py
index fe1996562..c06413895 100644
--- a/tests/retrievers/test_embeddings.py
+++ b/tests/retrievers/test_embeddings.py
@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
import numpy as np
import pytest
-from dspy.retrievers.embeddings import Embeddings
+from dspy.retrievers.embeddings import Embeddings, EmbeddingsWithScores
def dummy_corpus():
@@ -132,3 +132,38 @@ def test_embeddings_from_saved():
def test_embeddings_load_nonexistent_path():
with pytest.raises((FileNotFoundError, OSError)):
Embeddings.from_saved("/nonexistent/path", dummy_embedder)
+
+
+def test_embeddings_with_scores_basic_search():
+ corpus = dummy_corpus()
+ retriever = EmbeddingsWithScores(corpus=corpus, embedder=dummy_embedder, k=2)
+
+ result = retriever("A dog is barking.")
+
+ assert result.passages == ["The dog barked at the mailman.", "The cat sat on the mat."]
+ assert result.indices == [1, 0]
+ assert result.scores == pytest.approx([1.0, 0.0])
+
+
+def test_embeddings_with_scores_save_load():
+ corpus = dummy_corpus()
+ original_retriever = EmbeddingsWithScores(
+ corpus=corpus,
+ embedder=dummy_embedder,
+ k=2,
+ normalize=False,
+ brute_force_threshold=1000,
+ )
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ save_path = os.path.join(temp_dir, "test_embeddings_with_scores")
+
+ original_retriever.save(save_path)
+ loaded_retriever = EmbeddingsWithScores.from_saved(save_path, dummy_embedder)
+
+ original_result = original_retriever("cat sitting")
+ loaded_result = loaded_retriever("cat sitting")
+
+ assert loaded_result.passages == original_result.passages
+ assert loaded_result.indices == original_result.indices
+ assert loaded_result.scores == pytest.approx(original_result.scores)