Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Change Vector Search to Hybrid Search, remove comments

Connor Shorten89cda4fe9606
diff --git a/dspy/retrieve/weaviate_rm.py b/dspy/retrieve/weaviate_rm.py
index 763ef5a5b..adb825503 100644
--- a/dspy/retrieve/weaviate_rm.py
+++ b/dspy/retrieve/weaviate_rm.py
@@ -3,8 +3,6 @@ from typing import List, Union
import dspy
from typing import Optional
-# Note, using Python v3 Weaviate Client -- need to make sure this is tied to `pip install dspy-ai[weaviate]`
-
try:
import weaviate
except ImportError:
@@ -52,7 +50,6 @@ class WeaviateRM(dspy.Retrieve):
super().__init__(k=k)
- # ToDo -- interface query batching
def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:
"""Search with Weaviate for self.k top passages for query
@@ -63,31 +60,23 @@ class WeaviateRM(dspy.Retrieve):
dspy.Prediction: An object containing the retrieved passages.
"""
- # Please note, as stated above the WeaviateRM assumes you have a text key payload named: `content`
k = k if k is not None else self.k
queries = (
[query_or_queries]
if isinstance(query_or_queries, str)
else query_or_queries
)
- queries = [q for q in queries if q] # Filter empty queries
+ queries = [q for q in queries if q]
passages = []
- # Note this assumes you are using a text2vec embedding service orchestrated in Weaviate
for query in queries:
- # We are currently hard-coding the payload / text key `content`
results = self._weaviate_client.query\
.get(self._weaviate_collection_name, ["content"])\
- .with_near_text({"concepts": [query]})\
+ .with_hybrid(query=query)\
.with_limit(k)\
.do()
results = results["data"]["Get"][self._weaviate_collection_name]
parsed_results = [result["content"] for result in results]
- # This assumes you will call `dspy.utils.deduplicate` in the forward pass
- # I am skeptical how often you will have duplicate documents,
- # but an idea could be to extend this with the distance scores.
- # However, it probably makes more sense for us (Weaviate) to cook this into our
- # batch query API (which will be in the v4 client, expected to go GA in 1.24).
passages.extend(parsed_results)
return dspy.Prediction(passages=passages)