DSPy: The framework for programming—not prompting—language models
Python
feat(retrievers): add EmbeddingsWithScores for similarity score access (#9478)
Serj Smorb0ea262ca914
diff --git a/dspy/retrievers/__init__.py b/dspy/retrievers/__init__.pyindex ffaf603bb..89339ca03 100644--- a/dspy/retrievers/__init__.py+++ b/dspy/retrievers/__init__.py@@ -1,4 +1,4 @@-from dspy.retrievers.embeddings import Embeddings+from dspy.retrievers.embeddings import Embeddings, EmbeddingsWithScoresfrom dspy.retrievers.retrieve import Retrieve-__all__ = ["Embeddings", "Retrieve"]+__all__ = ["Embeddings", "EmbeddingsWithScores", "Retrieve"]diff --git a/dspy/retrievers/embeddings.py b/dspy/retrievers/embeddings.pyindex 94bbc589f..82ba02e8b 100644--- a/dspy/retrievers/embeddings.py+++ b/dspy/retrievers/embeddings.py@@ -8,6 +8,13 @@ from dspy.utils.unbatchify import Unbatchifyclass Embeddings:+ """DSPy Embeddings retriever.++ This class retrieves the top-k most similar passages from a corpus using embedding-based similarity search.+ For large corpora, a FAISS index is built for fast approximate candidate retrieval, followed by exact+ re-ranking. For small corpora, brute-force search is used.+ """+def __init__(self,corpus: list[str],@@ -35,9 +42,17 @@ class Embeddings:return self.forward(query)def forward(self, query: str):+ """Search for the top-k passages most similar to the query.++ Args:+ query (str): The search query string++ Returns:+ dspy.Prediction: A prediction containing passages and their corpus indices.+ """import dspy- passages, indices = self.search_fn(query)+ passages, indices, _scores = self.search_fn(query)return dspy.Prediction(passages=passages, indices=indices)def _batch_forward(self, queries: list[str]):@@ -81,8 +96,13 @@ class Embeddings:top_k_indices = np.argsort(-scores, axis=1)[:, : self.k]top_indices = candidate_indices[np.arange(len(q_embeds))[:, None], top_k_indices]+ top_scores = scores[np.arange(len(q_embeds))[:, None], top_k_indices]- return [([self.corpus[idx] for idx in indices], [idx for idx in indices]) for indices in top_indices] # noqa: C416+ results = []+ for indices, query_scores in zip(top_indices, top_scores, strict=True):+ passages = [self.corpus[idx] for idx in indices]+ results.append((passages, indices.tolist(), query_scores.tolist()))+ return resultsdef _normalize(self, embeddings: np.ndarray):norms = np.linalg.norm(embeddings, axis=1, keepdims=True)@@ -214,3 +234,25 @@ class Embeddings:instance.search_fn = Unbatchify(instance._batch_forward)instance.load(path, embedder)return instance+++class EmbeddingsWithScores(Embeddings):+ """DSPy EmbeddingsWithScores retriever.++ This class extends the Embeddings retriever to also return similarity scores alongside passages and indices.+ Similarity scores enable downstream such as thresholding and re-ranking.+ """++ def forward(self, query: str):+ """Search for the top-k passages most similar to the query.++ Args:+ query (str): The search query string.++ Returns:+ dspy.Prediction: A prediction containing passages, indices, and similarity scores.+ """+ import dspy++ passages, indices, scores = self.search_fn(query)+ return dspy.Prediction(passages=passages, indices=indices, scores=scores)diff --git a/tests/retrievers/test_embeddings.py b/tests/retrievers/test_embeddings.pyindex fe1996562..c06413895 100644--- a/tests/retrievers/test_embeddings.py+++ b/tests/retrievers/test_embeddings.py@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutorimport numpy as npimport pytest-from dspy.retrievers.embeddings import Embeddings+from dspy.retrievers.embeddings import Embeddings, EmbeddingsWithScoresdef dummy_corpus():@@ -132,3 +132,38 @@ def test_embeddings_from_saved():def test_embeddings_load_nonexistent_path():with pytest.raises((FileNotFoundError, OSError)):Embeddings.from_saved("/nonexistent/path", dummy_embedder)+++def test_embeddings_with_scores_basic_search():+ corpus = dummy_corpus()+ retriever = EmbeddingsWithScores(corpus=corpus, embedder=dummy_embedder, k=2)++ result = retriever("A dog is barking.")++ assert result.passages == ["The dog barked at the mailman.", "The cat sat on the mat."]+ assert result.indices == [1, 0]+ assert result.scores == pytest.approx([1.0, 0.0])+++def test_embeddings_with_scores_save_load():+ corpus = dummy_corpus()+ original_retriever = EmbeddingsWithScores(+ corpus=corpus,+ embedder=dummy_embedder,+ k=2,+ normalize=False,+ brute_force_threshold=1000,+ )++ with tempfile.TemporaryDirectory() as temp_dir:+ save_path = os.path.join(temp_dir, "test_embeddings_with_scores")++ original_retriever.save(save_path)+ loaded_retriever = EmbeddingsWithScores.from_saved(save_path, dummy_embedder)++ original_result = original_retriever("cat sitting")+ loaded_result = loaded_retriever("cat sitting")++ assert loaded_result.passages == original_result.passages+ assert loaded_result.indices == original_result.indices+ assert loaded_result.scores == pytest.approx(original_result.scores)