DSPy: The framework for programming—not prompting—language models
Python
feature(search): reranker
Sri Vardhamanana2e5cb19c5f3
diff --git a/.gitignore b/.gitignoreindex 1a8f23201..08d3e2f50 100644--- a/.gitignore+++ b/.gitignore@@ -32,4 +32,5 @@ build/*.jsonl*.json!/data/*.json-/dist/\ No newline at end of file+/dist/+**/*.pkl\ No newline at end of filediff --git a/dsp/modules/sbert.py b/dsp/modules/sbert.pynew file mode 100644index 000000000..f8ac3f80c--- /dev/null+++ b/dsp/modules/sbert.py@@ -0,0 +1,16 @@+class SentenceTransformersCrossEncoder:+ """Wrapper for sentence-transformers cross-encoder model.+ """+ def __init__(+ self, model_name_or_path: str = "cross-encoder/ms-marco-MiniLM-L-12-v2"+ ):+ try:+ from sentence_transformers.cross_encoder import CrossEncoder+ except ImportError:+ raise ModuleNotFoundError(+ "You need to install sentence-transformers library to use SentenceTransformersCrossEncoder."+ )+ self.model = CrossEncoder(model_name_or_path)++ def __call__(self, query: str, passage: list[str]) -> list[float]:+ return self.model.predict([[query, p] for p in passage]).tolist()diff --git a/dsp/primitives/search.py b/dsp/primitives/search.pyindex 4a77d7d55..b472ca9d8 100644--- a/dsp/primitives/search.py+++ b/dsp/primitives/search.py@@ -1,3 +1,4 @@+import numpy as npimport dsp@@ -7,17 +8,42 @@ def retrieve(query: str, k: int) -> list[str]:raise AssertionError("No RM is loaded.")passages = dsp.settings.rm(query, k=k)passages = [psg.long_text for psg in passages]++ if dsp.settings.reranker:+ passages_cs_scores = dsp.settings.reranker(query, passages)+ passages_cs_scores_sorted = np.argsort(passages_cs_scores)[::-1]+ passages = [passages[idx] for idx in passages_cs_scores_sorted]return passages+def retrieveRerankEnsemble(queries: list[str], k: int) -> list[str]:+ if not (dsp.settings.rm and dsp.settings.reranker):+ raise AssertionError("Both RM and Reranker are needed to retrieve & re-rank.")+ print("reranking")+ queries = [q for q in queries if q]+ passages = {}+ for query in queries:+ retrieved_passages = dsp.settings.rm(query, k=k*3)+ passages_cs_scores = dsp.settings.reranker(query, [psg.long_text for psg in retrieved_passages])+ for idx in np.argsort(passages_cs_scores)[::-1]:+ psg = retrieved_passages[idx]+ passages[psg.long_text] = passages.get(psg.long_text, []) + [+ passages_cs_scores[idx]+ ]++ passages = [(np.average(score), text) for text, score in passages.items()]+ return [text for _, text in sorted(passages, reverse=True)[:k]]++def retrieveEnsemble(queries: list[str], k: int, by_prob: bool = True) -> list[str]:"""Retrieves passages from the RM for each query in queries and returns the top k passagesbased on the probability or score."""if not dsp.settings.rm:raise AssertionError("No RM is loaded.")-+ if dsp.settings.reranker:+ return retrieveRerankEnsemble(queries, k)queries = [q for q in queries if q]passages = {}diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex 1a69ca639..65c628bd1 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -1,4 +1,3 @@-from typing import Callable, Optionalfrom contextlib import contextmanagerfrom dsp.utils.utils import dotdict@@ -21,6 +20,7 @@ class Settings(object):config = dotdict(lm=None,rm=None,+ reranker=None,compiled_lm=None,force_reuse_cached_compilation=False,compiling=False,