Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

change linear searching to index lookup

Jasper Xiandaea29811351
diff --git a/dsp/modules/pyserini.py b/dsp/modules/pyserini.py
index f0ad28b2e..94ce970ea 100644
--- a/dsp/modules/pyserini.py
+++ b/dsp/modules/pyserini.py
@@ -42,6 +42,11 @@ class PyseriniRetriever:
self.searcher = FaissSearcher.from_prebuilt_index(index, self.encoder)
else:
self.searcher = FaissSearcher(index_dir=index, query_encoder=self.encoder)
+ assert self.dataset is not None
+ self.dataset_id_to_index = {}
+ for i, docid in enumerate(self.dataset[self.id_field]):
+ self.dataset_id_to_index[docid] = i
+
def __call__(
self, query: str, k: int = 10, threads: int = 16,
@@ -51,8 +56,7 @@ class PyseriniRetriever:
topk = []
for rank, hit in enumerate(hits, start=1):
if self.dataset is not None:
- # Linearly search through user's dataset for the document, maybe some room for future improvement here
- row = self.dataset[self.id_field].index(hit.docid)
+ row = self.dataset_id_to_index[hit.docid]
text = ' '.join(self.dataset[field][row] for field in self.text_fields)
pid = self.dataset[self.id_field][row]
else: