Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feature(search): reranker

Sri Vardhamanana2e5cb19c5f3
diff --git a/.gitignore b/.gitignore
index 1a8f23201..08d3e2f50 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,4 +32,5 @@ build/
*.jsonl
*.json
!/data/*.json
-/dist/
\ No newline at end of file
+/dist/
+**/*.pkl
\ No newline at end of file
diff --git a/dsp/modules/sbert.py b/dsp/modules/sbert.py
new file mode 100644
index 000000000..f8ac3f80c
--- /dev/null
+++ b/dsp/modules/sbert.py
@@ -0,0 +1,16 @@
+class SentenceTransformersCrossEncoder:
+ """Wrapper for sentence-transformers cross-encoder model.
+ """
+ def __init__(
+ self, model_name_or_path: str = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+ ):
+ try:
+ from sentence_transformers.cross_encoder import CrossEncoder
+ except ImportError:
+ raise ModuleNotFoundError(
+ "You need to install sentence-transformers library to use SentenceTransformersCrossEncoder."
+ )
+ self.model = CrossEncoder(model_name_or_path)
+
+ def __call__(self, query: str, passage: list[str]) -> list[float]:
+ return self.model.predict([[query, p] for p in passage]).tolist()
diff --git a/dsp/primitives/search.py b/dsp/primitives/search.py
index 4a77d7d55..b472ca9d8 100644
--- a/dsp/primitives/search.py
+++ b/dsp/primitives/search.py
@@ -1,3 +1,4 @@
+import numpy as np
import dsp
@@ -7,17 +8,42 @@ def retrieve(query: str, k: int) -> list[str]:
raise AssertionError("No RM is loaded.")
passages = dsp.settings.rm(query, k=k)
passages = [psg.long_text for psg in passages]
+
+ if dsp.settings.reranker:
+ passages_cs_scores = dsp.settings.reranker(query, passages)
+ passages_cs_scores_sorted = np.argsort(passages_cs_scores)[::-1]
+ passages = [passages[idx] for idx in passages_cs_scores_sorted]
return passages
+def retrieveRerankEnsemble(queries: list[str], k: int) -> list[str]:
+ if not (dsp.settings.rm and dsp.settings.reranker):
+ raise AssertionError("Both RM and Reranker are needed to retrieve & re-rank.")
+ print("reranking")
+ queries = [q for q in queries if q]
+ passages = {}
+ for query in queries:
+ retrieved_passages = dsp.settings.rm(query, k=k*3)
+ passages_cs_scores = dsp.settings.reranker(query, [psg.long_text for psg in retrieved_passages])
+ for idx in np.argsort(passages_cs_scores)[::-1]:
+ psg = retrieved_passages[idx]
+ passages[psg.long_text] = passages.get(psg.long_text, []) + [
+ passages_cs_scores[idx]
+ ]
+
+ passages = [(np.average(score), text) for text, score in passages.items()]
+ return [text for _, text in sorted(passages, reverse=True)[:k]]
+
+
def retrieveEnsemble(queries: list[str], k: int, by_prob: bool = True) -> list[str]:
"""Retrieves passages from the RM for each query in queries and returns the top k passages
based on the probability or score.
"""
if not dsp.settings.rm:
raise AssertionError("No RM is loaded.")
-
+ if dsp.settings.reranker:
+ return retrieveRerankEnsemble(queries, k)
queries = [q for q in queries if q]
passages = {}
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py
index 1a69ca639..65c628bd1 100644
--- a/dsp/utils/settings.py
+++ b/dsp/utils/settings.py
@@ -1,4 +1,3 @@
-from typing import Callable, Optional
from contextlib import contextmanager
from dsp.utils.utils import dotdict
@@ -21,6 +20,7 @@ class Settings(object):
config = dotdict(
lm=None,
rm=None,
+ reranker=None,
compiled_lm=None,
force_reuse_cached_compilation=False,
compiling=False,