DSPy: The framework for programming—not prompting—language models
Python
deeplake retriever implementation
Aniruddha Mandal08db447a77de
diff --git a/dspy/retrieve/deeplake_rm.py b/dspy/retrieve/deeplake_rm.pynew file mode 100644index 000000000..80c47d6d8--- /dev/null+++ b/dspy/retrieve/deeplake_rm.py@@ -0,0 +1,96 @@+"""+Retriever model for deeplake+"""++from typing import Optional, List, Union+import openai+import dspy+from collections import defaultdict++try:+ import openai.error++ ERRORS = (+ openai.RateLimitError,+ openai.error.ServiceUnavailableError,+ openai.error.APIError,+ )+except Exception:+ ERRORS = (openai.RateLimitError, openai.APIError)++try:+ from deeplake import VectorStore+except ImportError:+ deeplake = None++if deeplake is None:+ raise ImportError(+ "The deeplake library is required to use DeeplakeRM. Install it with `pip install dspy-ai[deeplake]`"+ )+++class DeeplakeRM(dspy.RetrieverModel):+ """+ A retriever module that uses deeplake to return the top passages for a given query.++ """++ def __init__(+ self,+ deeplake_vectorstore_name: str,+ deeplake_client: VectorStore,+ k: int = 3,+ ):+ self._deeplake_vectorstore_name = deeplake_vectorstore_name+ self._deeplake_client = deeplake_client++ super().__init__(k=k)++ def embedding_function(texts, model="text-embedding-ada-002"):+ if isinstance(texts, str):+ texts = [texts]++ texts = [t.replace("\n", " ") for t in texts]+ return [+ data["embedding"]+ for data in openai.Embedding.create(input=texts, model=model)["data"]+ ]++ def forward(+ self, query_or_queries: Union[str, List[str]], k: Optional[int]+ ) -> dspy.Prediction:+ """Search with DeepLake for self.k top passages for query++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.++ Returns:+ dspy.Prediction: An object containing the retrieved passages.+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ queries = [q for q in queries if q] # Filter empty queries++ k = k if k is not None else self.k+ # batch_results = self._deeplake_client(+ # path=self._deeplake_vectorstore_name+ # ).search(queries, embedding_function=self.embedding_function, k=k)++ passages = defaultdict(float)+ for query in queries:+ results = self._deeplake_client(+ path=self._deeplake_vectorstore_name,+ embedding_function=self.embedding_function+ ).search(query, k=k)++ for score,text in zip(results.get('score',0.0),results.get('text',"")):+ passages[text] += score++ sorted_passages = sorted(+ passages.items(), key=lambda x: x[1], reverse=True)[:k]++ return dspy.Prediction(passages=[passage for passage, _ in sorted_passages])\ No newline at end of file