Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Remove community retrievers (#8073)

TomuHiratac3e79ce87b42
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 9cc6c90c9..a9118851e 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -1,11 +1,9 @@
from dspy.predict import *
from dspy.primitives import *
-from dspy.retrieve import *
+from dspy.retrievers import *
from dspy.signatures import *
from dspy.teleprompt import *
-import dspy.retrievers
-
from dspy.evaluate import Evaluate # isort: skip
from dspy.clients import * # isort: skip
from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, History, BaseType, Tool, ToolCalls # isort: skip
diff --git a/dspy/retrieve/__init__.py b/dspy/retrieve/__init__.py
deleted file mode 100644
index 072c7d527..000000000
--- a/dspy/retrieve/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from dspy.retrieve.retrieve import Retrieve
-
-__all__ = [
- "Retrieve",
-]
diff --git a/dspy/retrieve/azureaisearch_rm.py b/dspy/retrieve/azureaisearch_rm.py
deleted file mode 100644
index ffb1f91b2..000000000
--- a/dspy/retrieve/azureaisearch_rm.py
+++ /dev/null
@@ -1,397 +0,0 @@
-"""
-Retriever module for Azure AI Search
-Author: Prajapati Harishkumar Kishorkumar (@HARISHKUMAR1112001)
-"""
-
-import warnings
-from typing import Any, Callable, List, Optional, Union
-
-import dspy
-from dspy.dsp.utils.utils import dotdict
-
-try:
- from azure.core.credentials import AzureKeyCredential
- from azure.search.documents import SearchClient
- from azure.search.documents._paging import SearchItemPaged
- from azure.search.documents.models import QueryType, VectorFilterMode, VectorizedQuery
-except ImportError:
- raise ImportError(
- "You need to install azure-search-documents library"
- "Please use the command: pip install azure-search-documents==11.6.0b1",
- )
-
-try:
- import openai
-except ImportError:
- warnings.warn(
- "`openai` is not installed. Install it with `pip install openai` to use AzureOpenAI embedding models.",
- category=ImportWarning,
- )
-
-
-class AzureAISearchRM(dspy.Retrieve):
-
- """
- A retrieval module that utilizes Azure AI Search to retrieve top passages for a given query.
-
- Args:
- search_service_name (str): The name of the Azure AI Search service.
- search_api_key (str): The API key for accessing the Azure AI Search service.
- search_index_name (str): The name of the search index in the Azure AI Search service.
- field_text (str): The name of the field containing text content in the search index. This field will be mapped to the "content" field in the dsp framework.
- field_vector (Optional[str]): The name of the field containing vector content in the search index. Defaults to None.
- k (int, optional): The default number of top passages to retrieve. Defaults to 3.
- azure_openai_client (Optional[openai.AzureOpenAI]): An instance of the AzureOpenAI client. Either openai_client or embedding_func must be provided. Defaults to None.
- openai_embed_model (Optional[str]): The name of the OpenAI embedding model. Defaults to "text-embedding-ada-002".
- embedding_func (Optional[Callable]): A function for generating embeddings. Either openai_client or embedding_func must be provided. Defaults to None.
- semantic_ranker (bool, optional): Whether to use semantic ranking. Defaults to False.
- filter (str, optional): Additional filter query. Defaults to None.
- query_language (str, optional): The language of the query. Defaults to "en-Us".
- query_speller (str, optional): The speller mode. Defaults to "lexicon".
- use_semantic_captions (bool, optional): Whether to use semantic captions. Defaults to False.
- query_type (Optional[QueryType], optional): The type of query. Defaults to QueryType.FULL.
- semantic_configuration_name (str, optional): The name of the semantic configuration. Defaults to None.
- is_vector_search (Optional[bool]): Whether to enable vector search. Defaults to False.
- is_hybrid_search (Optional[bool]): Whether to enable hybrid search. Defaults to False.
- is_fulltext_search (Optional[bool]): Whether to enable fulltext search. Defaults to True.
- vector_filter_mode (Optional[VectorFilterMode]): The vector filter mode. Defaults to None.
-
- Examples:
- Below is a code snippet that demonstrates how to instantiate and use the AzureAISearchRM class:
- ```python
- search_service_name = "your_search_service_name"
- search_api_key = "your_search_api_key"
- search_index_name = "your_search_index_name"
- field_text = "text_content_field"
-
- azure_search_retriever = AzureAISearchRM(search_service_name, search_api_key, search_index_name, field_text)
- ```
-
- Attributes:
- search_service_name (str): The name of the Azure AI Search service.
- search_api_key (str): The API key for accessing the Azure AI Search service.
- search_index_name (str): The name of the search index in the Azure AI Search service.
- endpoint (str): The endpoint URL for the Azure AI Search service.
- field_text (str): The name of the field containing text content in the search index.
- field_vector (Optional[str]): The name of the field containing vector content in the search index.
- azure_openai_client (Optional[openai.AzureOpenAI]): An instance of the AzureOpenAI client.
- openai_embed_model (Optional[str]): The name of the OpenAI embedding model.
- embedding_func (Optional[Callable]): A function for generating embeddings.
- credential (AzureKeyCredential): The Azure key credential for accessing the service.
- client (SearchClient): The Azure AI Search client instance.
- semantic_ranker (bool): Whether to use semantic ranking.
- filter (str): Additional filter query.
- query_language (str): The language of the query.
- query_speller (str): The speller mode.
- use_semantic_captions (bool): Whether to use semantic captions.
- query_type (Optional[QueryType]): The type of query.
- semantic_configuration_name (str): The name of the semantic configuration.
- is_vector_search (Optional[bool]): Whether to enable vector search.
- is_hybrid_search (Optional[bool]): Whether to enable hybrid search.
- is_fulltext_search (Optional[bool]): Whether to enable fulltext search.
- vector_filter_mode (Optional[VectorFilterMode]): The vector filter mode.
-
- Methods:
- forward(query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:
- Search for the top passages corresponding to the given query or queries.
-
- azure_search_request(
- self,
- key_content: str,
- client: SearchClient,
- query: str,
- top: int,
- semantic_ranker: bool,
- filter: str,
- query_language: str,
- query_speller: str,
- use_semantic_captions: bool,
- query_type: QueryType,
- semantic_configuration_name: str,
- is_vector_search: bool,
- is_hybrid_search: bool,
- is_fulltext_search: bool,
- field_vector: str,
- vector_filter_mode: VectorFilterMode
- ) -> List[dict]:
- Perform a search request to the Azure AI Search service.
-
- process_azure_result(
- self,
- results:SearchItemPaged,
- content_key:str,
- content_score: str
- ) -> List[dict]:
- Process the results received from the Azure AI Search service and map them to the correct format.
-
- get_embeddings(
- self,
- query: str,
- k_nearest_neighbors: int,
- field_vector: str
- ) -> List | Any:
- Returns embeddings for the given query.
-
- check_semantic_configuration(
- self,
- semantic_configuration_name,
- query_type
- ):
- Checks semantic configuration.
-
- Raises:
- ImportError: If the required Azure AI Search libraries are not installed.
-
- Note:
- This class relies on the 'azure-search-documents' library for interacting with the Azure AI Search service.
- Ensure that you have the necessary permissions and correct configurations set up in Azure before using this class.
- """
-
- def __init__(
- self,
- search_service_name: str,
- search_api_key: str,
- search_index_name: str,
- field_text: str,
- field_vector: Optional[str] = None,
- k: int = 3,
- azure_openai_client: Optional[openai.AzureOpenAI] = None,
- openai_embed_model: Optional[str] = "text-embedding-ada-002",
- embedding_func: Optional[Callable] = None,
- semantic_ranker: bool = False,
- filter: str = None,
- query_language: str = "en-Us",
- query_speller: str = "lexicon",
- use_semantic_captions: bool = False,
- query_type: Optional[QueryType] = QueryType.FULL,
- semantic_configuration_name: str = None,
- is_vector_search: Optional[bool] = False,
- is_hybrid_search: Optional[bool] = False,
- is_fulltext_search: Optional[bool] = True,
- vector_filter_mode: Optional[VectorFilterMode.PRE_FILTER] = None,
- ):
- self.search_service_name = search_service_name
- self.search_api_key = search_api_key
- self.search_index_name = search_index_name
- self.endpoint = f"https://{self.search_service_name}.search.windows.net"
- self.field_text = field_text # field name of the text content
- self.field_vector = field_vector # field name of the vector content
- self.azure_openai_client = azure_openai_client
- self.openai_embed_model = openai_embed_model
- self.embedding_func = embedding_func
- # Create a client
- self.credential = AzureKeyCredential(self.search_api_key)
- self.client = SearchClient(
- endpoint=self.endpoint, index_name=self.search_index_name, credential=self.credential,
- )
- self.semantic_ranker = semantic_ranker
- self.filter = filter
- self.query_language = query_language
- self.query_speller = query_speller
- self.use_semantic_captions = use_semantic_captions
- self.query_type = query_type
- self.semantic_configuration_name = semantic_configuration_name
- self.is_vector_search = is_vector_search
- self.is_hybrid_search = is_hybrid_search
- self.is_fulltext_search = is_fulltext_search
- self.vector_filter_mode = vector_filter_mode
-
- super().__init__(k=k)
-
- def azure_search_request(
- self,
- key_content: str,
- client: SearchClient,
- query: str,
- top: int,
- semantic_ranker: bool,
- filter: str,
- query_language: str,
- query_speller: str,
- use_semantic_captions: bool,
- query_type: QueryType,
- semantic_configuration_name: str,
- is_vector_search: bool,
- is_hybrid_search: bool,
- is_fulltext_search: bool,
- field_vector: str,
- vector_filter_mode: VectorFilterMode,
- ):
- """
- Search in Azure AI Search Index
- """
-
- if is_vector_search:
- vector_query = self.get_embeddings(query, top, field_vector)
- if semantic_ranker:
- self.check_semantic_configuration(semantic_configuration_name, query_type)
- results = client.search(
- search_text=None,
- filter=filter,
- query_type=query_type,
- vector_queries=vector_query,
- vector_filter_mode=vector_filter_mode,
- semantic_configuration_name=semantic_configuration_name,
- top=top,
- query_caption=("extractive|highlight-false" if use_semantic_captions else None),
- )
- else:
- results = client.search(
- search_text=None,
- filter=filter,
- vector_queries=vector_query,
- vector_filter_mode=vector_filter_mode,
- top=top,
- )
- if is_hybrid_search:
- vector_query = self.get_embeddings(query, top, field_vector)
- if semantic_ranker:
- self.check_semantic_configuration(semantic_configuration_name, query_type)
- results = client.search(
- search_text=query,
- filter=filter,
- query_type=query_type,
- query_language=query_language,
- query_speller=query_speller,
- semantic_configuration_name=semantic_configuration_name,
- top=top,
- vector_queries=vector_query,
- vector_filter_mode=vector_filter_mode,
- query_caption=("extractive|highlight-false" if use_semantic_captions else None),
- )
- else:
- results = client.search(
- search_text=query,
- filter=filter,
- query_language=query_language,
- query_speller=query_speller,
- top=top,
- vector_queries=vector_query,
- vector_filter_mode=vector_filter_mode,
- )
- if is_fulltext_search:
- if semantic_ranker:
- self.check_semantic_configuration(semantic_configuration_name, query_type)
- results = client.search(
- search_text=query,
- filter=filter,
- query_type=query_type,
- query_language=query_language,
- query_speller=query_speller,
- semantic_configuration_name=semantic_configuration_name,
- top=top,
- query_caption=("extractive|highlight-false" if use_semantic_captions else None),
- )
- else:
- results = client.search(search_text=query, top=top, filter=filter)
-
- sorted_results = sorted(results, key=lambda x: x["@search.score"], reverse=True)
-
- sorted_results = self.process_azure_result(sorted_results, key_content, key_content)
-
- return sorted_results
-
- def process_azure_result(self, results: SearchItemPaged, content_key: str, content_score: str):
- """
- process received result from Azure AI Search as dictionary array and map content and score to correct format
- """
- res = []
- for result in results:
- tmp = {}
- for key, value in result.items():
- if key == content_key:
- tmp["text"] = value # assign content
- elif key == content_score:
- tmp["score"] = value
- else:
- tmp[key] = value
- res.append(tmp)
- return res
-
- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:
- """
- Search with pinecone for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
-
- k = k if k is not None else self.k
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
- queries = [q for q in queries if q] # Filter empty queries
-
- passages = []
- for query in queries:
- results = self.azure_search_request(
- self.field_text,
- self.client,
- query,
- k,
- self.semantic_ranker,
- self.filter,
- self.query_language,
- self.query_speller,
- self.use_semantic_captions,
- self.query_type,
- self.semantic_configuration_name,
- self.is_vector_search,
- self.is_hybrid_search,
- self.is_fulltext_search,
- self.field_vector,
- self.vector_filter_mode,
- )
- passages.extend(dotdict({"long_text": d["text"]}) for d in results)
-
- return passages
-
- def get_embeddings(self, query: str, k_nearest_neighbors: int, field_vector: str) -> List | Any:
- """
- Returns embeddings for the given query.
-
- Args:
- query (str): The query for which embeddings are to be retrieved.
- k_nearest_neighbors (int): The number of nearest neighbors to consider.
- field_vector (str): The field vector to use for embeddings.
-
- Returns:
- list: A list containing the vectorized query.
- Any: The result of embedding_func if azure_openai_client is not provided.
-
- Raises:
- AssertionError: If neither azure_openai_client nor embedding_func is provided,
- or if field_vector is not provided.
- """
- assert (
- self.azure_openai_client or self.embedding_func
- ), "Either azure_openai_client or embedding_func must be provided."
-
- if self.azure_openai_client is not None:
- assert field_vector, "field_vector must be provided."
-
- embedding = (
- self.azure_openai_client.embeddings.create(input=query, model=self.openai_embed_model).data[0].embedding
- )
- vector_query = VectorizedQuery(
- vector=embedding, k_nearest_neighbors=k_nearest_neighbors, fields=field_vector,
- )
- return [vector_query]
- else:
- return self.embedding_func(query)
-
- def check_semantic_configuration(self, semantic_configuration_name, query_type):
- """
- Checks semantic configuration.
-
- Args:
- semantic_configuration_name: The name of the semantic configuration.
- query_type: The type of the query.
-
- Raises:
- AssertionError: If semantic_configuration_name is not provided
- or if query_type is not QueryType.SEMANTIC.
- """
- assert semantic_configuration_name, "Semantic configuration name must be provided."
- assert query_type == QueryType.SEMANTIC, "Query type must be QueryType.SEMANTIC."
diff --git a/dspy/retrieve/chromadb_rm.py b/dspy/retrieve/chromadb_rm.py
deleted file mode 100644
index c102f8565..000000000
--- a/dspy/retrieve/chromadb_rm.py
+++ /dev/null
@@ -1,170 +0,0 @@
-"""
-Retriever model for chromadb
-"""
-
-from typing import List, Optional, Union
-
-import backoff
-import openai
-
-from dspy import Retrieve, Prediction
-from dspy.dsp.utils.settings import settings
-from dspy.dsp.utils import dotdict
-
-try:
- import openai.error
- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)
-except Exception:
- ERRORS = (openai.RateLimitError, openai.APIError)
-
-try:
- import chromadb
- import chromadb.utils.embedding_functions as ef
- from chromadb.api.types import (
- Embeddable,
- EmbeddingFunction,
- )
- from chromadb.config import Settings
-except ImportError:
- raise ImportError(
- "The chromadb library is required to use ChromadbRM. Install it with `pip install dspy-ai[chromadb]`",
- )
-
-
-class ChromadbRM(Retrieve):
- """
- A retrieval module that uses chromadb to return the top passages for a given query.
-
- Assumes that the chromadb index has been created and populated with the following metadata:
- - documents: The text of the passage
-
- Args:
- collection_name (str): chromadb collection name
- persist_directory (str): chromadb persist directory
- embedding_function (Optional[EmbeddingFunction[Embeddable]]): Optional function to use to embed documents. Defaults to DefaultEmbeddingFunction.
- k (int, optional): The number of top passages to retrieve. Defaults to 7.
- client(Optional[chromadb.Client]): Optional chromadb client provided by user, default to None
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
-
- Examples:
- Below is a code snippet that shows how to use this as the default retriever:
- ```python
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- # using default chromadb client
- retriever_model = ChromadbRM('collection_name', 'db_path')
- dspy.settings.configure(lm=llm, rm=retriever_model)
- # to test the retriever with "my query"
- retriever_model("my query")
- ```
-
- Use provided chromadb client
- ```python
- import chromadb
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- # say you have a chromadb running on a different port
- client = chromadb.HttpClient(host='localhost', port=8889)
- retriever_model = ChromadbRM('collection_name', 'db_path', client=client)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- # to test the retriever with "my query"
- retriever_model("my query")
- ```
-
- Below is a code snippet that shows how to use this in the forward() function of a module
- ```python
- self.retrieve = ChromadbRM('collection_name', 'db_path', k=num_passages)
- ```
- """
-
- def __init__(
- self,
- collection_name: str,
- persist_directory: str,
- embedding_function: Optional[
- EmbeddingFunction[Embeddable]
- ] = ef.DefaultEmbeddingFunction(),
- client: Optional[chromadb.Client] = None,
- k: int = 7,
- ):
- self._init_chromadb(collection_name, persist_directory, client=client)
- self.ef = embedding_function
-
- super().__init__(k=k)
-
- def _init_chromadb(
- self,
- collection_name: str,
- persist_directory: str,
- client: Optional[chromadb.Client] = None,
- ) -> chromadb.Collection:
- """Initialize chromadb and return the loaded index.
-
- Args:
- collection_name (str): chromadb collection name
- persist_directory (str): chromadb persist directory
- client (chromadb.Client): chromadb client provided by user
-
- Returns: collection per collection_name
- """
-
- if client:
- self._chromadb_client = client
- else:
- self._chromadb_client = chromadb.Client(
- Settings(
- persist_directory=persist_directory,
- is_persistent=True,
- ),
- )
- self._chromadb_collection = self._chromadb_client.get_or_create_collection(
- name=collection_name,
- )
-
- @backoff.on_exception(
- backoff.expo,
- ERRORS,
- max_time=settings.backoff_time,
- )
- def _get_embeddings(self, queries: List[str]) -> List[List[float]]:
- """Return query vector after creating embedding using OpenAI
-
- Args:
- queries (list): List of query strings to embed.
-
- Returns:
- List[List[float]]: List of embeddings corresponding to each query.
- """
- return self.ef(queries)
-
- def forward(
- self, query_or_queries: Union[str, List[str]], k: Optional[int] = None, **kwargs,
- ) -> Prediction:
- """Search with db for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q] # Filter empty queries
- embeddings = self._get_embeddings(queries)
-
- k = self.k if k is None else k
- results = self._chromadb_collection.query(
- query_embeddings=embeddings, n_results=k,**kwargs,
- )
-
- zipped_results = zip(
- results["ids"][0],
- results["distances"][0],
- results["documents"][0],
- results["metadatas"][0])
- results = [dotdict({"id": id, "score": dist, "long_text": doc, "metadatas": meta }) for id, dist, doc, meta in zipped_results]
- return results
diff --git a/dspy/retrieve/clarifai_rm.py b/dspy/retrieve/clarifai_rm.py
deleted file mode 100644
index 9ba13c7e3..000000000
--- a/dspy/retrieve/clarifai_rm.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""Clarifai as retriver to retrieve hits"""
-import os
-from concurrent.futures import ThreadPoolExecutor
-from typing import List, Optional, Union
-
-import requests
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- from clarifai.client.search import Search
-except ImportError as err:
- raise ImportError(
- "Clarifai is not installed. Install it using `pip install clarifai`",
- ) from err
-
-
-class ClarifaiRM(dspy.Retrieve):
- """
- Retrieval module uses clarifai to return the Top K relevant pasages for the given query.
- Assuming that you have ingested the source documents into clarifai App, where it is indexed and stored.
-
- Args:
- clarifai_user_id (str): Clarifai unique user_id.
- clarfiai_app_id (str): Clarifai App ID, where the documents are stored.
- clarifai_pat (str): Clarifai PAT key.
- k (int): Top K documents to retrieve.
-
- Examples:
- TODO
- """
-
- def __init__(
- self,
- clarifai_user_id: str,
- clarfiai_app_id: str,
- clarifai_pat: Optional[str] = None,
- k: int = 3,
- ):
- self.app_id = clarfiai_app_id
- self.user_id = clarifai_user_id
- self.pat = (
- clarifai_pat if clarifai_pat is not None else os.environ["CLARIFAI_PAT"]
- )
- self.k = k
- self.clarifai_search = Search(
- user_id=self.user_id, app_id=self.app_id, top_k=k, pat=self.pat,
- )
- super().__init__(k=k)
-
- def retrieve_hits(self, hits):
- header = {"Authorization": f"Key {self.pat}"}
- request = requests.get(hits.input.data.text.url, headers=header)
- request.encoding = request.apparent_encoding
- requested_text = request.text
- return requested_text
-
- def forward(
- self, query_or_queries: Union[str, List[str]], k: Optional[int] = None,**kwargs,
- ) -> dspy.Prediction:
- """Uses clarifai-python SDK search function and retrieves top_k similar passages for given query,
- Args:
- query_or_queries : single query or list of queries
- k : Top K relevant documents to return
-
- Returns:
- passages in format of dotdict
-
- Examples:
- Below is a code snippet that shows how to use Marqo as the default retriver:
- ```python
- import clarifai
- llm = dspy.Clarifai(model=MODEL_URL, api_key="YOUR CLARIFAI_PAT")
- retriever_model = ClarifaiRM(clarifai_user_id="USER_ID", clarfiai_app_id="APP_ID", clarifai_pat="YOUR CLARIFAI_PAT")
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- passages = []
- queries = [q for q in queries if q]
-
- for query in queries:
- search_response = self.clarifai_search.query(ranks=[{"text_raw": query}],**kwargs)
-
- # Retrieve hits
- hits = [hit for data in search_response for hit in data.hits]
- with ThreadPoolExecutor(max_workers=10) as executor:
- results = list(executor.map(self.retrieve_hits, hits))
- passages.extend(dotdict({"long_text": d}) for d in results)
-
- return passages
diff --git a/dspy/retrieve/deeplake_rm.py b/dspy/retrieve/deeplake_rm.py
deleted file mode 100644
index a126941dd..000000000
--- a/dspy/retrieve/deeplake_rm.py
+++ /dev/null
@@ -1,113 +0,0 @@
-"""
-Retriever model for deeplake
-"""
-
-from collections import defaultdict
-from typing import List, Optional, Union
-
-import openai
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- import openai
-
- ERRORS = (
- openai.RateLimitError,
- openai.APIError,
- )
-except Exception:
- ERRORS = (openai.RateLimitError, openai.APIError)
-
-
-class DeeplakeRM(dspy.Retrieve):
-
- """
- A retriever module that uses deeplake to return the top passages for a given query.
-
- Assumes that a Deep Lake Vector Store has been created and populated with the following payload:
- - text: The text of the passage
-
- Args:
- deeplake_vectorstore_name (str): The name or path of the Deep Lake Vector Store.
- deeplake_client (VectorStore): An instance of the Deep Lake client.
- k (int, optional): The default number of top passages to retrieve. Defaults to 3.
-
- Examples:
- Below is a code snippet that shows how to use Deep Lake as the default retriver:
- ```python
- from deeplake import VectorStore
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- deeplake_client = VectorStore
- retriever_model = DeeplakeRM("my_vectorstore_path", deeplake_client=deeplake_client)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use Deep Lake in the forward() function of a module
- ```python
- self.retrieve = DeeplakeRM("my_vectorstore_path", deeplake_client=deeplake_client, k=num_passages)
- ```
- """
-
- def __init__(
- self,
- deeplake_vectorstore_name: str,
- deeplake_client,
- k: int = 3,
- ):
- try:
- from deeplake import VectorStore # noqa: F401
- except ImportError:
- raise ImportError("The 'deeplake' extra is required to use DeepLakeRM. Install it with `pip install dspy-ai[deeplake]`",)
-
- self._deeplake_vectorstore_name = deeplake_vectorstore_name
- self._deeplake_client = deeplake_client(
- path=self._deeplake_vectorstore_name,
- embedding_function=self.embedding_function,
- )
-
- super().__init__(k=k)
-
- def embedding_function(self, texts, model="text-embedding-ada-002"):
- if isinstance(texts, str):
- texts = [texts]
-
- texts = [t.replace("\n", " ") for t in texts]
-
- return [data.embedding for data in openai.embeddings.create(input = texts, model=model).data]
-
- def forward(
- self, query_or_queries: Union[str, List[str]], k: Optional[int],**kwargs,
- ) -> dspy.Prediction:
-
- """Search with DeepLake for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q] # Filter empty queries
-
- k = k if k is not None else self.k
-
- passages = defaultdict(float)
- #deeplake doesn't support batch querying, manually querying each query and storing them
- for query in queries:
- results = self._deeplake_client.search(query, k=k, **kwargs)
-
- for score,text in zip(results.get('score',0.0),results.get('text',"")):
- passages[text] += score
-
- sorted_passages = sorted(
- passages.items(), key=lambda x: x[1], reverse=True)[:k]
-
- return [dotdict({"long_text": p}) for p, _ in sorted_passages]
\ No newline at end of file
diff --git a/dspy/retrieve/epsilla_rm.py b/dspy/retrieve/epsilla_rm.py
deleted file mode 100644
index 6257b745b..000000000
--- a/dspy/retrieve/epsilla_rm.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from collections import defaultdict # noqa: F401
-from typing import Dict, List, Union # noqa: UP035
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- from pyepsilla import vectordb
-except ImportError:
- raise ImportError( # noqa: B904
- "The 'pyepsilla' extra is required to use EpsillaRM. Install it with `pip install dspy-ai[epsilla]`",
- )
-
-
-class EpsillaRM(dspy.Retrieve):
- def __init__(
- self,
- epsilla_client: vectordb.Client,
- db_name: str,
- db_path: str,
- table_name: str,
- k: int = 3,
- page_content: str = "document",
- ):
- self._epsilla_client = epsilla_client
- self._epsilla_client.load_db(db_name=db_name, db_path=db_path)
- self._epsilla_client.use_db(db_name=db_name)
- self.page_content = page_content
- self.table_name = table_name
-
- super().__init__(k=k)
-
- def forward(self, query_or_queries: Union[str, List[str]], k: Union[int, None] = None, **kwargs) -> dspy.Prediction: # noqa: ARG002
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
- queries = [q for q in queries if q]
- limit = k if k else self.k
- all_query_results: list = []
-
- passages: Dict = defaultdict(float)
-
- for result_dict in all_query_results:
- for result in result_dict:
- passages[result[self.page_content]] += result["@distance"]
- sorted_passages = sorted(passages.items(), key=lambda x: x[1], reverse=False)[:limit]
- return dspy.Prediction(passages=[dotdict({"long_text": passage}) for passage, _ in sorted_passages])
diff --git a/dspy/retrieve/faiss_rm.py b/dspy/retrieve/faiss_rm.py
deleted file mode 100755
index 5ac4de971..000000000
--- a/dspy/retrieve/faiss_rm.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# """Retriever model for faiss: https://github.com/facebookresearch/faiss.
-# Author: Jagane Sundar: https://github.com/jagane.
-# """
-
-# import logging
-# from typing import Optional, Union
-
-# import numpy as np
-
-# import dspy
-# from dspy.dsp.modules.sentence_vectorizer import SentenceTransformersVectorizer
-# from dspy.dsp.utils import dotdict
-
-# try:
-# import faiss
-# except ImportError:
-# faiss = None
-
-# if faiss is None:
-# raise ImportError(
-# """
-# The faiss package is required. Install it using `pip install dspy-ai[faiss-cpu]`
-# """,
-# )
-
-
-# logger = logging.getLogger(__name__)
-# class FaissRM(dspy.Retrieve):
-# """A retrieval module that uses an in-memory Faiss to return the top passages for a given query.
-
-# Args:
-# document_chunks: the input text chunks
-# vectorizer: an object that is a subclass of BaseSentenceVectorizer
-# k (int, optional): The number of top passages to retrieve. Defaults to 3.
-
-# Returns:
-# dspy.Prediction: An object containing the retrieved passages.
-
-# Examples:
-# Below is a code snippet that shows how to use this as the default retriver:
-# ```python
-# import dspy
-# from dspy.retrieve import faiss_rm
-
-# document_chunks = [
-# "The superbowl this year was played between the San Francisco 49ers and the Kanasas City Chiefs",
-# "Pop corn is often served in a bowl",
-# "The Rice Bowl is a Chinese Restaurant located in the city of Tucson, Arizona",
-# "Mars is the fourth planet in the Solar System",
-# "An aquarium is a place where children can learn about marine life",
-# "The capital of the United States is Washington, D.C",
-# "Rock and Roll musicians are honored by being inducted in the Rock and Roll Hall of Fame",
-# "Music albums were published on Long Play Records in the 70s and 80s",
-# "Sichuan cuisine is a spicy cuisine from central China",
-# "The interest rates for mortgages is considered to be very high in 2024",
-# ]
-
-# frm = faiss_rm.FaissRM(document_chunks)
-# turbo = dspy.OpenAI(model="gpt-3.5-turbo")
-# dspy.settings.configure(lm=turbo, rm=frm)
-# print(frm(["I am in the mood for Chinese food"]))
-# ```
-
-# Below is a code snippet that shows how to use this in the forward() function of a module
-# ```python
-# self.retrieve = FaissRM(k=num_passages)
-# ```
-# """
-
-# def __init__(self, document_chunks, vectorizer=None, k: int = 3):
-# """Inits the faiss retriever.
-
-# Args:
-# document_chunks: a list of input strings.
-# vectorizer: an object that is a subclass of BaseTransformersVectorizer.
-# k: number of matches to return.
-# """
-# if vectorizer:
-# self._vectorizer = vectorizer
-# else:
-# self._vectorizer = SentenceTransformersVectorizer()
-# embeddings = self._vectorizer(document_chunks)
-# xb = np.array(embeddings)
-# d = len(xb[0])
-# logger.info(f"FaissRM: embedding size={d}")
-# if len(xb) < 100:
-# self._faiss_index = faiss.IndexFlatL2(d)
-# self._faiss_index.add(xb)
-# else:
-# # if we have at least 100 vectors, we use Voronoi cells
-# nlist = 100
-# quantizer = faiss.IndexFlatL2(d)
-# self._faiss_index = faiss.IndexIVFFlat(quantizer, d, nlist)
-# self._faiss_index.train(xb)
-# self._faiss_index.add(xb)
-
-# logger.info(f"{self._faiss_index.ntotal} vectors in faiss index")
-# self._document_chunks = document_chunks # save the input document chunks
-
-# super().__init__(k=k)
-
-# def _dump_raw_results(self, queries, index_list, distance_list) -> None:
-# for i in range(len(queries)):
-# indices = index_list[i]
-# distances = distance_list[i]
-# logger.debug(f"Query: {queries[i]}")
-# for j in range(len(indices)):
-# logger.debug(f" Hit {j} = {indices[j]}/{distances[j]}: {self._document_chunks[indices[j]]}")
-# return
-
-# def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None, **kwargs) -> dspy.Prediction:
-# """Search the faiss index for k or self.k top passages for query.
-
-# Args:
-# query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
-# Returns:
-# dspy.Prediction: An object containing the retrieved passages.
-# """
-# queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
-# queries = [q for q in queries if q] # Filter empty queries
-# embeddings = self._vectorizer(queries)
-# emb_npa = np.array(embeddings)
-# # For single query, just look up the top k passages
-# if len(queries) == 1:
-# distance_list, index_list = self._faiss_index.search(emb_npa, k or self.k)
-# # self._dump_raw_results(queries, index_list, distance_list)
-# passages = [(self._document_chunks[ind], ind) for ind in index_list[0]]
-# return [dotdict({"long_text": passage[0], "index": passage[1]}) for passage in passages]
-
-# distance_list, index_list = self._faiss_index.search(emb_npa, (k or self.k) * 3, **kwargs)
-# # self._dump_raw_results(queries, index_list, distance_list)
-# passage_scores = {}
-# for emb in range(len(embeddings)):
-# indices = index_list[emb] # indices of neighbors for embeddings[emb] - this is an array of k*3 integers
-# distances = distance_list[
-# emb
-# ] # distances of neighbors for embeddings[emb] - this is an array of k*3 floating point numbers
-# for res in range((k or self.k) * 3):
-# neighbor = indices[res]
-# distance = distances[res]
-# if neighbor in passage_scores:
-# passage_scores[neighbor].append(distance)
-# else:
-# passage_scores[neighbor] = [distance]
-# # Note re. sorting:
-# # first degree sort: number of queries that got a hit with any particular document chunk. More
-# # is a better match. This is len(queries)-len(x[1])
-# # second degree sort: sum of the distances of each hit returned by faiss. Smaller distance is a better match
-# sorted_passages = sorted(passage_scores.items(), key=lambda x: (len(queries) - len(x[1]), sum(x[1])))[
-# : k or self.k
-# ]
-# return [
-# dotdict({"long_text": self._document_chunks[passage_index], "index": passage_index})
-# for passage_index, _ in sorted_passages
-# ]
diff --git a/dspy/retrieve/falkordb_rm.py b/dspy/retrieve/falkordb_rm.py
deleted file mode 100644
index 5897db620..000000000
--- a/dspy/retrieve/falkordb_rm.py
+++ /dev/null
@@ -1,213 +0,0 @@
-import os
-from typing import List, Optional, Union
-import string
-import random
-
-import backoff
-from openai import (
- APITimeoutError,
- InternalServerError,
- OpenAI,
- RateLimitError,
- UnprocessableEntityError,
-)
-
-from dspy import Retrieve, Prediction
-from dspy.dsp.utils.settings import settings
-from dspy.dsp.utils import dotdict
-
-try:
- import falkordb
-except ImportError:
- raise ImportError(
- "Please install the falkordb package by running `pip install dspy-ai[falkordb]`"
- )
-import redis.exceptions
-
-
-def generate_random_string(length: int) -> str:
- characters = string.ascii_letters
- random_string = "".join(random.choice(characters) for _ in range(length))
- return random_string
-
-
-class Embedder:
- def __init__(self, provider: str, model: str):
- self.provider = provider
- if self.provider == "openai":
- api_key = os.getenv("OPENAI_API_KEY")
- if not api_key:
- raise ValueError(
- "Environment variable OPENAI_API_KEY must be set to"
- "use openai as embedding provider"
- )
- self.client = OpenAI()
- self.model = model
- else:
- raise ValueError(f"Unsupported provider: {provider}")
-
- @backoff.on_exception(
- backoff.expo,
- (
- APITimeoutError,
- InternalServerError,
- RateLimitError,
- UnprocessableEntityError,
- ),
- max_time=settings.backoff_time,
- )
- def __call__(self, queries: Union[str, List[str]]) -> List[List[float]]:
- if isinstance(queries, str):
- queries = [queries]
-
- if self.provider == "openai":
- embedding = self.client.embeddings.create(input=queries, model=self.model)
- return [result.embedding for result in embedding.data]
-
-
-DEFAULT_INDEX_QUERY = "CALL db.idx.vector.queryNodes($node_label, $embedding_node_property, $k, vecf32($embedding)) YIELD node, score "
-
-
-class FalkordbRM(Retrieve):
- """
- Implements a retriever that utilizes FalkorDB for retrieving passages.
- This class manages a connection to a FalkorDB database using official FalkorDB Python drivers and requires
- the database credentials. That is, if using a local FalkorDB session, host and port else if using a FalkorDB cloud session,
- host, port, username, and password to be set as environment variables and optionally the database name.
- Additionally, it utilizes an embedding provider (defaulting to OpenAI's services) to compute query embeddings,
- which are then used to find the most relevant nodes in the FalkorDB graph based on the specified node property or custom retrieval query.
-
- Returns a list of passages in the form of `dspy.Prediction` objects
-
- Args:
- Args:
- node_label (str): The label of the node in the FalkorDB database to query against
- text_node_property (str): The property of the node containing the text.
- embedding_node_property (List[float]): The property of the node containing the embeddings.
- k (Optional[int]): The default number of top passages to retrieve. Defaults to 5.
- retrieval_query (Optional[str]): Custom Cypher query for retrieving passages.
- embedding_provider (str): The provider of the embedding service. Defaults to "openai".
- embedding_model (str): The model identifier for generating embeddings. Defaults to "text-embedding-ada-002".
-
- Examples:
- Below is a code snippet showcasing how to initialize FalkordbRM with environment variables for the database connection and OpenAI as the embedding provider:
-
- ```python
- import os
-
- import dspy
- import openai
-
- os.environ["FALKORDB_HOST"] = "localhost"
- os.environ["FALORDB_PORT"] = "6379"
- os.environ["OPENAI_API_KEY"] = "sk-" (Only if using openai as embedding's provider)
-
- # Uncomment and set the following if you are using FalkorDB cloud
- # os.environ["FALKORDB_USERNAME"] = "falkordb"
- # os.environ["FALKORDB_PASSWORD"] = "password"
-
-
- falkordb_retriever = FalkordbRM(
- node_label="myIndex",
- text_node_property="text",
- k=10,
- embedding_provider="openai",
- embedding_model="text-embedding-ada-002",
- )
-
- dspy.settings.configure(rm=falkordb_retriever)
- ```
-
- In this example, `FalkordbRM` is configured to retrieve nodes based on the "text" property from an index on a node labeled "myIndex",
- using embeddings computed by OpenAI's "text-embedding-ada-002" model.
- """
-
- def __init__(
- self,
- node_label: str,
- text_node_property: str = None,
- embedding_node_property: str = None,
- k: int = 5,
- retrieval_query: Optional[str] = None,
- embedding_provider: str = "openai",
- embedding_model: str = "text-embedding-ada-002",
- ):
- super().__init__(k=k)
- self.node_label = node_label
- self.username = os.getenv("FALKORDB_USERNAME", None)
- self.password = os.getenv("FALKORDB_PASSWORD", None)
- self.host = os.getenv("FALKORDB_HOST", "localhost")
- self.port = int(os.getenv("FALKORDB_PORT", 6379))
-
- self.database = os.getenv("FALKORDB_DATABASE", generate_random_string(4))
- self.k = k
- self.retrieval_query = retrieval_query
- self.text_node_property = text_node_property
- self.embedding_node_property = embedding_node_property
- if not self.text_node_property and not self.retrieval_query:
- raise ValueError(
- "Either `text_node_property` or `retrieval_query` must be set"
- )
- if not embedding_node_property:
- raise ValueError("`embedding_node_property` must be set")
- try:
- self.driver = falkordb.FalkorDB(
- host=self.host,
- port=self.port,
- username=self.username,
- password=self.password,
- ).select_graph(self.database)
-
- except (
- redis.exceptions.ConnectionError,
- redis.exceptions.AuthenticationError,
- ) as e:
- raise ConnectionError("Failed to connect to FalkorDB database") from e
-
- self.embedder = Embedder(provider=embedding_provider, model=embedding_model)
-
- def forward(
- self, query_or_queries: Union[str, List[str]], k: Optional[int]
- ) -> Prediction:
- if not isinstance(query_or_queries, list):
- query_or_queries = [query_or_queries]
- query_vectors = self.embedder(query_or_queries)
- contents = []
- retrieval_query = (
- self.retrieval_query
- or f"RETURN node.{self.text_node_property} AS text, score"
- )
- if not k:
- k = self.k
-
- for vector in query_vectors:
- params = {
- "embedding": vector,
- "node_label": self.node_label,
- "text_node_property": self.text_node_property,
- "embedding_node_property": self.embedding_node_property,
- "k": k,
- }
- try:
- records = self.driver.query(
- DEFAULT_INDEX_QUERY + retrieval_query,
- params=params,
- ).result_set
- except Exception as e:
- if "Invalid arguments" in str(e):
- raise ValueError(
- f"There is no vector index on node label, {self.node_label}"
- f" and node property, {self.embedding_node_property}"
- )
- contents.extend(
- [
- {"passage": dotdict({"long_text": r[1]}), "score": r[0]}
- for r in records
- ]
- )
- sorted_passages = sorted(
- contents,
- key=lambda x: x["score"],
- reverse=True,
- )[:k]
- return [el["passage"] for el in sorted_passages]
diff --git a/dspy/retrieve/lancedb_rm.py b/dspy/retrieve/lancedb_rm.py
deleted file mode 100644
index d2769b24e..000000000
--- a/dspy/retrieve/lancedb_rm.py
+++ /dev/null
@@ -1,139 +0,0 @@
-"""
-Retriever model for LanceDB
-Author: Prashant Dixit (@PrashantDixit0)
-"""
-
-from typing import List, Union
-
-import backoff
-import lancedb.table
-
-from dspy import Retrieve, Prediction
-from dspy.dsp.utils.settings import settings
-from dspy.dsp.utils import dotdict
-
-try:
- import lancedb
-except ImportError:
- lancedb = None
-
-if lancedb is None:
- raise ImportError(
- "The lancedb library is required to use LancedbRM. Install it with `pip install dspy-ai[lancedb]`",
- )
-
-
-import openai
-
-try:
- OPENAI_LEGACY = int(openai.version.__version__[0]) == 0
-except Exception:
- OPENAI_LEGACY = True
-
-try:
- import openai.error
- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)
-except Exception:
- ERRORS = (openai.RateLimitError, openai.APIError)
-
-
-class LancedbRM(Retrieve):
- """
- A retrieval module that uses LanceDB to return the top passages for a given query.
-
- Assumes that the LanceDB table has been created and populated with the following metadata:
- - text: The text of the passage
-
- Args:
- table_name (str): The name of the table to query against.
- persist_directory (str): directory where database is stored.
- k (int, optional): The number of top passages to retrieve. Defaults to 3.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
-
- Examples:
- Below is a code snippet that shows how to use this as the default retriever:
- ```python
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- retriever_model = LancedbRM()
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use this in the forward() function of a module
- ```python
- self.retrieve = LancedbRM(k=num_passages)
- ```
- """
-
- def __init__(
- self,
- table_name: str,
- persist_directory: str,
- k: int = 3,
- ):
-
- self._table = self._init_lancedb(
- table_name, persist_directory,
- )
-
- super().__init__(k=k)
-
- def _init_lancedb(
- self,
- table_name: str,
- persist_directory: str,
- ):
- """
- Initialize LanceDB database and return the table instance.
-
- Args:
- table_name (str): The name of the table to load. If the table does not exist, it will be Raise Value Error.
-
- Raises:
- ValueError: If table is already not created
-
- Returns:
- Lancedb.table: The loaded table.
- """
-
- self.db = lancedb.connect(persist_directory)
- table_names = self.db.table_names()
-
- if table_name not in table_names:
- raise ValueError(
- f"Table does not exist in lancedb database persist_directory. Create table with name {table_name}, then reintialize retriever instance",
- )
- else:
- table = self.db.open_table(table_name)
- return table
-
-
- @backoff.on_exception(
- backoff.expo,
- ERRORS,
- max_time=settings.backoff_time,
- )
-
- def forward(self, query_or_queries: Union[str, List[str]]) -> Prediction:
- """Search with Lancedb for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q] # Filter empty queries
-
- passages = []
- for q in queries:
- results = self._table.search(q).limit(self.k).to_list()
- for r in results:
- passages.append(r['text'])
- return Prediction(passages=[dotdict({"long_text": passage}) for passage in passages])
diff --git a/dspy/retrieve/llama_index_rm.py b/dspy/retrieve/llama_index_rm.py
deleted file mode 100644
index 639d849ba..000000000
--- a/dspy/retrieve/llama_index_rm.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import logging
-from typing import Optional
-
-import dspy
-
-try:
- from llama_index.core.base.base_retriever import BaseRetriever
-except ImportError:
- err = "The 'llama_index' package is required to use LlamaIndexRM. Install it with 'pip install llama_index'."
- raise ImportError(err) from None
-
-NO_TOP_K_WARNING = "The underlying LlamaIndex retriever does not support top k retrieval. Ignoring k value."
-
-
-class LlamaIndexRM(dspy.Retrieve):
- """Implements a retriever which wraps over a LlamaIndex retriever.
-
- This is done to bridge LlamaIndex and DSPy and allow the various retrieval
- abstractions in LlamaIndex to be used in DSPy.
-
- To-do (maybe):
- - Async support (DSPy lacks this entirely it seems, so not a priority until the rest of the repo catches on)
- - Text/video retrieval (Available in LI, not sure if this will be a priority in DSPy)
-
- Args:
- retriever (BaseRetriever): A LlamaIndex retriever object - text based only
- k (int): Optional; the number of examples to retrieve (similarity_top_k)
-
- If the underlying LI retriever does not have the property similarity_top_k, k will be ignored.
-
- Returns:
- DSPy RM Object - this is a retriever object that can be used in DSPy
- """
-
- retriever: BaseRetriever
-
- def __init__(
- self,
- retriever: BaseRetriever,
- k: Optional[int] = None,
- ):
- self.retriever = retriever
-
- if k:
- self.k = k
-
- @property
- def k(self) -> Optional[int]:
- """Get similarity top k of retriever."""
- if not hasattr(self.retriever, "similarity_top_k"):
- logging.warning(NO_TOP_K_WARNING)
- return None
-
- return self.retriever.similarity_top_k
-
- @k.setter
- def k(self, k: int) -> None:
- """Set similarity top k of retriever."""
- if hasattr(self.retriever, "similarity_top_k"):
- self.retriever.similarity_top_k = k
- else:
- logging.warning(NO_TOP_K_WARNING)
-
- def forward(self, query: str, k: Optional[int] = None) -> list[dspy.Example]:
- """Forward function for the LI retriever.
-
- This is the function that is called to retrieve the top k examples for a given query.
- Top k is set via the setter similarity_top_k or at LI instantiation.
-
- Args:
- query (str): The query to retrieve examples for
- k (int): Optional; the number of examples to retrieve (similarity_top_k)
-
- If the underlying LI retriever does not have the property similarity_top_k, k will be ignored.
-
- Returns:
- List[dspy.Example]: A list of examples retrieved by the retriever
- """
- if k:
- self.k = k
-
- raw = self.retriever.retrieve(query)
-
- return [
- dspy.Example(
- text=result.text,
- score=result.score,
- )
- for result in raw
- ]
diff --git a/dspy/retrieve/marqo_rm.py b/dspy/retrieve/marqo_rm.py
deleted file mode 100644
index 25f1a6ee3..000000000
--- a/dspy/retrieve/marqo_rm.py
+++ /dev/null
@@ -1,96 +0,0 @@
-from collections import defaultdict
-from typing import List, Union
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- import marqo
-except ImportError:
- raise ImportError(
- "The 'marqo' extra is required to use MarqoRM. Install it with `pip install dspy-ai[marqo]`",
- )
-
-class MarqoRM(dspy.Retrieve):
- """
- A retrieval module that uses Marqo to return the top passages for a given query.
-
- Assumes that a Marqo index has been created and populated with the following payload:
- - document: The text of the passage
-
- Args:
- marqo_index_name (str): The name of the marqo index.
- marqo_client (marqo.client.Client): A marqo client instance.
- k (int, optional): The number of top passages to retrieve. Defaults to 3.
- page_content (str, optional): The name of the field in the marqo index that contains the text of the passage. Defaults to 'document'.
- filter_string (str, optional): A filter string to use when searching. Defaults to None.
- **kwargs: Additional keyword arguments to pass to the marqo search function.
-
- Examples:
- Below is a code snippet that shows how to use Marqo as the default retriver:
- ```python
- import marqo
- marqo_client = marqo.Client(url="http://0.0.0.0:8882")
-
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- retriever_model = MarqoRM("my_index_name", marqo_client=marqo_client)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use Marqo in the forward() function of a module
- ```python
- self.retrieve = MarqoRM("my_index_name", marqo_client=marqo_client, k=num_passages)
- ```
- """
-
- def __init__(
- self,
- marqo_index_name: str,
- marqo_client: marqo.client.Client,
- k: int = 3,
- page_content: str = 'document',
- filter_string: str = None,
- ):
- self._marqo_index_name = marqo_index_name
- self._marqo_client = marqo_client
- self.page_content = page_content
- self.filter_string = filter_string
-
- super().__init__(k=k)
-
- def forward(self, query_or_queries: Union[str, List[str]], k=None, **kwargs) -> dspy.Prediction:
- """Search with Marqo for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q]
- limit = k if k else self.k
-
- all_query_results = []
- for query in queries:
- _result = self._marqo_client.index(self._marqo_index_name).search(
- q=query,
- limit=limit,
- filter_string=self.filter_string,
- **kwargs,
- )
- all_query_results.append(_result)
-
- passages = defaultdict(float)
-
- for result_dict in all_query_results:
- for result in result_dict['hits']:
- passages[result[self.page_content]] += result['_score']
-
- sorted_passages = sorted(
- passages.items(), key=lambda x: x[1], reverse=True)[:limit]
- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]
diff --git a/dspy/retrieve/milvus_rm.py b/dspy/retrieve/milvus_rm.py
deleted file mode 100644
index a49570391..000000000
--- a/dspy/retrieve/milvus_rm.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""
-Retriever model for Milvus or Zilliz Cloud
-"""
-
-from typing import Callable, List, Optional, Union
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- from pymilvus import MilvusClient
-except ImportError:
- raise ImportError(
- "The pymilvus library is required to use MilvusRM. Install it with `pip install dspy-ai[milvus]`",
- )
-
-
-def openai_embedding_function(texts: List[str]):
- from openai import OpenAI
-
- client = OpenAI()
- response = client.embeddings.create(
- input=texts,
- model="text-embedding-3-small",
- )
- return [x.embedding for x in response.data]
-
-
-class MilvusRM(dspy.Retrieve):
- """
- A retrieval module that uses Milvus to return passages for a given query.
-
- Assumes that a Milvus collection has been created and populated with the following field:
- - text: The text of the passage
-
- Args:
- collection_name (str): The name of the Milvus collection to query against.
- uri (str, optional): The Milvus connection uri. Defaults to "http://localhost:19530".
- token (str, optional): The Milvus connection token. Defaults to None.
- db_name (str, optional): The Milvus database name. Defaults to "default".
- embedding_function (callable, optional): The function to convert a list of text to embeddings.
- The embedding function should take a list of text strings as input and output a list of embeddings.
- Defaults to None. By default, it will get OpenAI client by the environment variable OPENAI_API_KEY
- and use OpenAI's embedding model "text-embedding-3-small" with the default dimension.
- k (int, optional): The number of top passages to retrieve. Defaults to 3.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
-
- Examples:
- Below is a code snippet that shows how to use this as the default retriever:
- ```python
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- retriever_model = MilvusRM(
- collection_name="<YOUR_COLLECTION_NAME>",
- uri="<YOUR_MILVUS_URI>",
- token="<YOUR_MILVUS_TOKEN>"
- )
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use this in the forward() function of a module
- ```python
- self.retrieve = MilvusRM(k=num_passages)
- ```
- """
-
- def __init__(
- self,
- collection_name: str,
- uri: Optional[str] = "http://localhost:19530",
- token: Optional[str] = None,
- db_name: Optional[str] = "default",
- embedding_function: Optional[Callable] = None,
- k: int = 3,
- ):
- self.milvus_client = MilvusClient(uri=uri, token=token, db_name=db_name)
-
- # check collection existence
- if collection_name not in self.milvus_client.list_collections():
- raise AttributeError(f"Milvus collection not found: {collection_name}")
- self.collection_name = collection_name
-
- self.embedding_function = embedding_function or openai_embedding_function
- self.k = k
-
- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int] = None) -> dspy.Prediction:
- if isinstance(query_or_queries, str):
- query_or_queries = [query_or_queries]
- query_embeddings = self.embedding_function(query_or_queries)
-
- k = k or self.k
-
- milvus_res = self.milvus_client.search(
- collection_name=self.collection_name,
- data=query_embeddings,
- output_fields=["text"],
- limit=k,
- )
- passage_scores = {}
- for res in milvus_res:
- for r in res:
- passage_scores[r["entity"]["text"]] = r["distance"]
- sorted_passages = sorted(
- passage_scores.items(), key=lambda x: x[1], reverse=True,
- )[:k]
- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]
\ No newline at end of file
diff --git a/dspy/retrieve/mongodb_atlas_rm.py b/dspy/retrieve/mongodb_atlas_rm.py
deleted file mode 100644
index 0a83017c6..000000000
--- a/dspy/retrieve/mongodb_atlas_rm.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import os
-from typing import Any, List
-
-import backoff
-from openai import (
- APITimeoutError,
- InternalServerError,
- OpenAI,
- RateLimitError,
- UnprocessableEntityError,
-)
-
-from dspy.dsp.utils.settings import settings
-from dspy import Retrieve, Prediction
-
-try:
- from pymongo import MongoClient
- from pymongo.errors import (
- ConfigurationError,
- ConnectionFailure,
- InvalidURI,
- OperationFailure,
- ServerSelectionTimeoutError,
- )
-except ImportError:
- raise ImportError(
- "Please install the pymongo package by running `pip install dspy-ai[mongodb]`",
- )
-
-
-def build_vector_search_pipeline(
- index_name: str, query_vector: List[float], num_candidates: int, limit: int,
-) -> List[dict[str, Any]]:
- return [
- {
- "$vectorSearch": {
- "index": index_name,
- "path": "embedding",
- "queryVector": query_vector,
- "numCandidates": num_candidates,
- "limit": limit,
- },
- },
- {"$project": {"_id": 0, "text": 1, "score": {"$meta": "vectorSearchScore"}}},
- ]
-
-
-class Embedder:
- def __init__(self, provider: str, model: str):
- if provider == "openai":
- api_key = os.getenv("OPENAI_API_KEY")
- if not api_key:
- raise ValueError("Environment variable OPENAI_API_KEY must be set")
- self.client = OpenAI()
- self.model = model
-
- @backoff.on_exception(
- backoff.expo,
- (
- APITimeoutError,
- InternalServerError,
- RateLimitError,
- UnprocessableEntityError,
- ),
- max_time=settings.backoff_time,
- )
- def __call__(self, queries) -> Any:
- embedding = self.client.embeddings.create(input=queries, model=self.model)
- return [result.embedding for result in embedding.data]
-
-
-class MongoDBAtlasRM(Retrieve):
- def __init__(
- self,
- db_name: str,
- collection_name: str,
- index_name: str,
- k: int = 5,
- embedding_provider: str = "openai",
- embedding_model: str = "text-embedding-ada-002",
- ):
- super().__init__(k=k)
- self.db_name = db_name
- self.collection_name = collection_name
- self.index_name = index_name
- self.username = os.getenv("ATLAS_USERNAME")
- self.password = os.getenv("ATLAS_PASSWORD")
- self.cluster_url = os.getenv("ATLAS_CLUSTER_URL")
- if not self.username:
- raise ValueError("Environment variable ATLAS_USERNAME must be set")
- if not self.password:
- raise ValueError("Environment variable ATLAS_PASSWORD must be set")
- if not self.cluster_url:
- raise ValueError("Environment variable ATLAS_CLUSTER_URL must be set")
- try:
- self.client = MongoClient(
- f"mongodb+srv://{self.username}:{self.password}@{self.cluster_url}/{self.db_name}"
- "?retryWrites=true&w=majority",
- )
- except (
- InvalidURI,
- ConfigurationError,
- ConnectionFailure,
- ServerSelectionTimeoutError,
- OperationFailure,
- ) as e:
- raise ConnectionError("Failed to connect to MongoDB Atlas") from e
-
- self.embedder = Embedder(provider=embedding_provider, model=embedding_model)
-
- def forward(self, query_or_queries: str) -> Prediction:
- query_vector = self.embedder([query_or_queries])
- pipeline = build_vector_search_pipeline(
- index_name=self.index_name,
- query_vector=query_vector[0],
- num_candidates=self.k * 10,
- limit=self.k,
- )
- contents = self.client[self.db_name][self.collection_name].aggregate(pipeline)
- return Prediction(passages=list(contents))
diff --git a/dspy/retrieve/my_scale_rm.py b/dspy/retrieve/my_scale_rm.py
deleted file mode 100644
index 8d61d0028..000000000
--- a/dspy/retrieve/my_scale_rm.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# import functools
-# import os
-# from typing import List, Optional
-
-# import openai
-
-# import dspy
-# from dspy.dsp.modules.cache_utils import NotebookCacheMemory, cache_turn_on
-# from dspy.dsp.utils import dotdict
-
-# # Check for necessary libraries and suggest installation if not found.
-# try:
-# import clickhouse_connect
-# except ImportError:
-# raise ImportError(
-# "The 'myscale' extra is required to use MyScaleRM. Install it with `pip install dspy-ai[myscale]`",
-# )
-
-# # Verify the compatibility of the OpenAI library version installed.
-# try:
-# major, minor, _ = map(int, openai.__version__.split("."))
-# OPENAI_VERSION_COMPATIBLE = major >= 1 and minor >= 16
-# except Exception:
-# OPENAI_VERSION_COMPATIBLE = False
-
-# if not OPENAI_VERSION_COMPATIBLE:
-# raise ImportError(
-# "An incompatible OpenAI library version is installed. Ensure you have version 1.16.1 or later.",
-# )
-
-# # Attempt to handle specific OpenAI errors; fallback to general ones if necessary.
-# try:
-# import openai.error
-
-# ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)
-# except Exception:
-# ERRORS = (openai.RateLimitError, openai.APIError)
-
-
-# class MyScaleRM(dspy.Retrieve):
-# """
-# A retrieval module that uses MyScaleDB to return the top passages for a given query.
-
-# MyScaleDB is a fork of ClickHouse that focuses on vector similarity search and full
-# text search. MyScaleRM is designed to facilitate easy retrieval of information from
-# MyScaleDB using embeddings. It supports embedding generation through either a local
-# model or the OpenAI API. This class abstracts away the complexities of connecting to
-# MyScaleDB, managing API keys, and processing queries to return semantically
-# relevant results.
-
-# Assumes that a table named `database.table` exists in MyScaleDB, and that the
-# table has column named `vector_column` that stores vector data and a vector index has
-# been created on this column. Other metadata are stored in `metadata_columns`.
-
-# Args:
-# client (clickhouse_connect.driver.client.Client): A client connection to the MyScaleDB.
-# table (str): Name of the table within the database to perform queries against.
-# database (str, optional): Name of the database to query within MyScaleDB.
-# metadata_columns(List[str], optional): A list of columns to include in the results.
-# vector_column (str, optional): The name of the column in the table that stores vector data.
-# k (int, optional): The number of closest matches to retrieve for a given query.
-# openai_api_key (str, optional): The API key for accessing OpenAI's services.
-# model (str, optional): Specifies the particular OpenAI model to use for embedding generation.
-# use_local_model (bool): Flag indicating whether a local model is used for embeddings.
-
-# """
-
-# def __init__(
-# self,
-# client: clickhouse_connect.driver.client.Client,
-# table: str,
-# database: str = "default",
-# metadata_columns: List[str] = ["text"],
-# vector_column: str = "vector",
-# k: int = 3,
-# openai_api_key: Optional[str] = None,
-# openai_model: Optional[str] = None,
-# local_embed_model: Optional[str] = None,
-# ):
-# self.client = client
-# self.database = database
-# self.table = table
-# if not metadata_columns:
-# raise ValueError("metadata_columns is required")
-# self.metadata_columns = metadata_columns
-# self.vector_column = vector_column
-# self.k = k
-# self.openai_api_key = openai_api_key
-# self.model = openai_model
-# self.use_local_model = False
-
-# if local_embed_model:
-# self.setup_local_model(local_embed_model)
-# elif openai_api_key:
-# os.environ["OPENAI_API_KEY"] = self.openai_api_key
-
-# def setup_local_model(self, model_name: str):
-# """
-# Configures a local model for embedding generation, including model and tokenizer loading.
-
-# Args:
-# model_name: The name or path to the pre-trained model to load.
-
-# Raises:
-# ModuleNotFoundError: If necessary libraries (torch or transformers) are not installed.
-# """
-# try:
-# import torch
-# from transformers import AutoModel, AutoTokenizer
-# except ImportError as exc:
-# raise ModuleNotFoundError(
-# """You need to install PyTorch and Hugging Face's transformers library to use a local embedding model.
-# Install the pytorch using `pip install torch` and transformers using `pip install transformers` """,
-# ) from exc
-
-# try:
-# self._local_embed_model = AutoModel.from_pretrained(model_name)
-# self._local_tokenizer = AutoTokenizer.from_pretrained(model_name)
-# self.use_local_model = True
-# except Exception as e:
-# raise ValueError(f"Failed to load model or tokenizer. Error: {str(e)}")
-
-# if torch.cuda.is_available():
-# self.device = torch.device("cuda:0")
-# elif torch.backends.mps.is_available():
-# self.device = torch.device("mps")
-# else:
-# self.device = torch.device("cpu")
-
-# self._local_embed_model.to(self.device)
-
-# @functools.lru_cache(maxsize=None if cache_turn_on else 0)
-# @NotebookCacheMemory.cache
-# def get_embeddings(self, query: str) -> List[float]:
-# """
-# Determines the appropriate source (OpenAI or local model) for embedding generation based on class configuration,
-# and retrieves embeddings for the provided queries.
-
-# Args:
-# query: A query to generate embeddings for.
-
-# Returns:
-# A list of embeddings corresponding to the query in the input list.
-
-# Raises:
-# ValueError: If neither an OpenAI API key nor a local model has been configured.
-# """
-# if self.openai_api_key and self.model:
-# return self._get_embeddings_from_openai(query)
-# elif self.use_local_model:
-# return self._get_embedding_from_local_model(query)
-# else:
-# raise ValueError("No valid method for obtaining embeddings is configured.")
-
-# # TO DO Add this method as Util method outside MyScaleRM
-# def _get_embeddings_from_openai(self, query: str) -> List[float]:
-# """
-# Uses the OpenAI API to generate embeddings for the given query.
-
-# Args:
-# query: A string for which to generate embeddings.
-
-# Returns:
-# A list containing the embedding of a query.
-# """
-# response = openai.embeddings.create(model=self.model, input=query)
-# return response.data[0].embedding
-
-# # TO DO Add this method as Util method outside MyScaleRM
-# def _get_embedding_from_local_model(self, query: str) -> List[float]:
-# """
-# Generates embeddings for a single query using the configured local model.
-
-# Args:
-# query: The text query to generate an embedding for.
-
-# Returns:
-# A list representing the query's embedding.
-# """
-# import torch
-
-# self._local_embed_model.eval() # Ensure the model is in evaluation mode
-
-# inputs = self._local_tokenizer(query, return_tensors="pt", padding=True, truncation=True).to(self.device)
-# with torch.no_grad():
-# output = self._local_embed_model(**inputs)
-
-# return output.last_hidden_state.mean(dim=1).squeeze().numpy().tolist()
-
-# def forward(self, user_query: str, k: Optional[int] = None) -> List[dotdict]:
-# """
-# Executes a retrieval operation based on a user's query and returns the top k relevant results.
-
-# Args:
-# user_query: The query text to search for.
-# k: Optional; The number of top matches to return. Defaults to the class's configured k value.
-
-# Returns:
-# A list of dotdict objects containing the formatted retrieval results.
-
-# Raises:
-# ValueError: If the user_query is None.
-# """
-# if user_query is None:
-# raise ValueError("Query is required")
-# k = k or self.k
-# embeddings = self.get_embeddings(user_query)
-# columns_string = ", ".join(self.metadata_columns)
-# result = self.client.query(f"""
-# SELECT {columns_string},
-# distance({self.vector_column}, {embeddings}) as dist FROM {self.database}.{self.table} ORDER BY dist LIMIT {k}
-# """)
-# result = self.client.query(f"""
-# SELECT {columns_string}, distance({self.vector_column}, {embeddings}) AS dist
-# FROM {self.database}.{self.table}
-# ORDER BY dist
-# LIMIT {k}
-# """)
-
-# # Convert the metadata into strings to pass to dspy.Prediction
-# results = []
-# for row in result.named_results():
-# if len(self.metadata_columns) == 1:
-# results.append(row[self.metadata_columns[0]])
-# else:
-# row_strings = [f"{column}: {row[column]}" for column in self.metadata_columns] # Format row data
-# row_string = "\n".join(row_strings) # Combine formatted data
-# results.append(row_string) # Append to results
-
-# # Return list of dotdict
-# return [dotdict({"long_text": passage}) for passage in results]
diff --git a/dspy/retrieve/neo4j_rm.py b/dspy/retrieve/neo4j_rm.py
deleted file mode 100644
index 3d4f65a12..000000000
--- a/dspy/retrieve/neo4j_rm.py
+++ /dev/null
@@ -1,160 +0,0 @@
-import os
-from typing import Any, List, Optional, Union, Callable
-
-import backoff
-from openai import (
- APITimeoutError,
- InternalServerError,
- OpenAI,
- RateLimitError,
- UnprocessableEntityError,
-)
-
-from dspy import Retrieve, Prediction
-from dspy.dsp.utils.settings import settings
-from dspy.dsp.utils import dotdict
-
-try:
- from neo4j import GraphDatabase
- from neo4j.exceptions import (
- AuthError,
- ServiceUnavailable,
- )
-except ImportError:
- raise ImportError(
- "Please install the neo4j package by running `pip install dspy-ai[neo4j]`",
- )
-
-
-class Embedder:
- def __init__(self, provider: str, model: str):
- if provider == "openai":
- api_key = os.getenv("OPENAI_API_KEY")
- if not api_key:
- raise ValueError("Environment variable OPENAI_API_KEY must be set")
- self.client = OpenAI()
- self.model = model
-
- @backoff.on_exception(
- backoff.expo,
- (
- APITimeoutError,
- InternalServerError,
- RateLimitError,
- UnprocessableEntityError,
- ),
- max_time=settings.backoff_time,
- )
- def __call__(self, queries) -> Any:
- embedding = self.client.embeddings.create(input=queries, model=self.model)
- return [result.embedding for result in embedding.data]
-
-
-DEFAULT_INDEX_QUERY = "CALL db.index.vector.queryNodes($index, $k, $embedding) YIELD node, score "
-
-
-class Neo4jRM(Retrieve):
- """
- Implements a retriever that utilizes Neo4j for retrieving passages.
- This class manages a connection to a Neo4j database using official Neo4j Python drivers and requires
- the database credentials (username, password, URI, and optionally the database name) to be set as environment variables.
- Additionally, it utilizes an embedding provider (defaulting to OpenAI's services) to compute query embeddings,
- which are then used to find the most relevant nodes in the Neo4j graph based on the specified node property or custom retrieval query.
-
- Returns a list of passages in the form of `dspy.Prediction` objects.
-
- Args:
- index_name (str): The name of the vector index in the Neo4j database to query against.
- text_node_property (Optional[str]): The property of the node containing the text. Required if `retrieval_query` is not set.
- k (Optional[int]): The default number of top passages to retrieve. Defaults to 5.
- retrieval_query (Optional[str]): Custom Cypher query for retrieving passages. Required if `text_node_property` is not set.
- embedding_provider (str): The provider of the embedding service. Defaults to "openai".
- embedding_model (str): The model identifier for generating embeddings. Defaults to "text-embedding-ada-002".
-
- Examples:
- Below is a code snippet showcasing how to initialize Neo4jRM with environment variables for the database connection and OpenAI as the embedding provider:
-
- ```python
- import os
-
- import dspy
- import openai
-
- os.environ["NEO4J_URI"] = "bolt://localhost:7687"
- os.environ["NEO4J_USERNAME"] = "neo4j"
- os.environ["NEO4J_PASSWORD"] = "password"
- os.environ["OPENAI_API_KEY"] = "sk-"
-
- neo4j_retriever = Neo4jRM(
- index_name="myIndex",
- text_node_property="text",
- k=10,
- embedding_provider="openai",
- embedding_model="text-embedding-ada-002",
- )
-
- dspy.settings.configure(rm=neo4j_retriever)
- ```
-
- In this example, `Neo4jRM` is configured to retrieve nodes based on the "text" property from an index named "myIndex",
- using embeddings computed by OpenAI's "text-embedding-ada-002" model.
- """
-
- def __init__(
- self,
- index_name: str,
- text_node_property: str = None,
- k: int = 5,
- retrieval_query: str = None,
- embedding_provider: str = "openai",
- embedding_model: str = "text-embedding-ada-002",
- embedding_function: Optional[Callable] = None,
- ):
- super().__init__(k=k)
- self.index_name = index_name
- self.username = os.getenv("NEO4J_USERNAME")
- self.password = os.getenv("NEO4J_PASSWORD")
- self.uri = os.getenv("NEO4J_URI")
- self.database = os.getenv("NEO4J_DATABASE", "neo4j")
- self.k = k
- self.retrieval_query = retrieval_query
- self.text_node_property = text_node_property
- if not self.username:
- raise ValueError("Environment variable NEO4J_USERNAME must be set")
- if not self.password:
- raise ValueError("Environment variable NEO4J_PASSWORD must be set")
- if not self.uri:
- raise ValueError("Environment variable NEO4J_URI must be set")
- if not self.text_node_property and not self.retrieval_query:
- raise ValueError("Either `text_node_property` or `retrieval_query` parameters must be defined")
- try:
- self.driver = GraphDatabase.driver(self.uri, auth=(self.username, self.password))
- self.driver.verify_connectivity()
-
- except (
- ServiceUnavailable,
- AuthError,
- ) as e:
- raise ConnectionError("Failed to connect to Neo4j database") from e
-
- self.embedder = embedding_function or Embedder(provider=embedding_provider, model=embedding_model)
-
- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> Prediction:
- if not isinstance(query_or_queries, list):
- query_or_queries = [query_or_queries]
- query_vectors = self.embedder(query_or_queries)
- contents = []
- retrieval_query = self.retrieval_query or f"RETURN node.{self.text_node_property} AS text, score"
- for vector in query_vectors:
- records, _, _ = self.driver.execute_query(
- DEFAULT_INDEX_QUERY + retrieval_query,
- {"embedding": vector, "index": self.index_name, "k": k or self.k},
- database_=self.database,
- )
- contents.extend([{"passage": dotdict({"long_text": r["text"]}), "score": r["score"]} for r in records])
- sorted_passages = sorted(
- contents,
- key=lambda x: x["score"],
- reverse=True,
- )[: k or self.k]
- return [el["passage"] for el in sorted_passages]
diff --git a/dspy/retrieve/pgvector_rm.py b/dspy/retrieve/pgvector_rm.py
deleted file mode 100644
index b05767c4a..000000000
--- a/dspy/retrieve/pgvector_rm.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import warnings
-from typing import Callable, Optional
-
-import dspy
-
-try:
- import psycopg2
- from pgvector.psycopg2 import register_vector
- from psycopg2 import sql
-except ImportError:
- raise ImportError(
- "The 'pgvector' extra is required to use PgVectorRM. Install it with `pip install dspy-ai[pgvector]`. Also, try `pip install pgvector psycopg2`.",
- )
-try:
- import openai
-except ImportError:
- warnings.warn(
- "`openai` is not installed. Install it with `pip install openai` to use OpenAI embedding models.",
- category=ImportWarning,
- )
-
-
-class PgVectorRM(dspy.Retrieve):
- """
- Implements a retriever that (as the name suggests) uses pgvector to retrieve passages,
- using a raw SQL query and a postgresql connection managed by psycopg2.
-
- It needs to register the pgvector extension with the psycopg2 connection
-
- Returns a list of dspy.Example objects
-
- Args:
- db_url (str): A PostgreSQL database URL in psycopg2's DSN format
- pg_table_name (Optional[str]): name of the table containing passages
- openai_client (openai.OpenAI): OpenAI client to use for computing query embeddings. Either openai_client or embedding_func must be provided.
- embedding_func (Callable): A function to use for computing query embeddings. Either openai_client or embedding_func must be provided.
- content_field (str = "text"): Field containing the passage text. Defaults to "text"
- k (Optional[int]): Default number of top passages to retrieve. Defaults to 20
- embedding_field (str = "embedding"): Field containing passage embeddings. Defaults to "embedding"
- fields (List[str] = ['text']): Fields to retrieve from the table. Defaults to "text"
- embedding_model (str = "text-embedding-ada-002"): Field containing the OpenAI embedding model to use. Defaults to "text-embedding-ada-002"
-
- Examples:
- Below is a code snippet that shows how to use PgVector as the default retriever
-
- ```python
- import dspy
- import openai
- import psycopg2
-
- openai.api_key = os.environ.get("OPENAI_API_KEY", None)
- openai_client = openai.OpenAI()
-
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
-
- DATABASE_URL should be in the format postgresql://user:password@host/database
- db_url=os.getenv("DATABASE_URL")
-
- retriever_model = PgVectorRM(conn, openai_client=openai_client, "paragraphs", fields=["text", "document_id"], k=20)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use PgVector in the forward() function of a module
- ```python
- self.retrieve = PgVectorRM(db_url, openai_client=openai_client, "paragraphs", fields=["text", "document_id"], k=20)
- ```
- """
-
- def __init__(
- self,
- db_url: str,
- pg_table_name: str,
- openai_client: Optional[openai.OpenAI] = None,
- embedding_func: Optional[Callable] = None,
- k: int = 20,
- embedding_field: str = "embedding",
- fields: Optional[list[str]] = None,
- content_field: str = "text",
- embedding_model: str = "text-embedding-ada-002",
- include_similarity: bool = False,
- ):
- """
- k = 20 is the number of paragraphs to retrieve
- """
- assert (
- openai_client or embedding_func
- ), "Either openai_client or embedding_func must be provided."
- self.openai_client = openai_client
- self.embedding_func = embedding_func
-
- self.conn = psycopg2.connect(db_url)
- register_vector(self.conn)
- self.pg_table_name = pg_table_name
- self.fields = fields or ["text"]
- self.content_field = content_field
- self.embedding_field = embedding_field
- self.embedding_model = embedding_model
- self.include_similarity = include_similarity
-
- super().__init__(k=k)
-
- def forward(self, query: str, k: int = None):
- """Search with PgVector for k top passages for query using cosine similarity
-
- Args:
- query (str): The query to search for
- k (int): The number of top passages to retrieve. Defaults to the value set in the constructor.
- Returns:
- dspy.Prediction: an object containing the retrieved passages.
- """
- # Embed query
- query_embedding = self._get_embeddings(query)
-
- retrieved_docs = []
-
- fields = sql.SQL(",").join([sql.Identifier(f) for f in self.fields])
- if self.include_similarity:
- similarity_field = sql.SQL(",") + sql.SQL(
- "1 - ({embedding_field} <=> %s::vector) AS similarity",
- ).format(embedding_field=sql.Identifier(self.embedding_field))
- fields += similarity_field
- args = (query_embedding, query_embedding, k if k else self.k)
- else:
- args = (query_embedding, k if k else self.k)
-
- sql_query = sql.SQL(
- "select {fields} from {table} order by {embedding_field} <=> %s::vector limit %s",
- ).format(
- fields=fields,
- table=sql.Identifier(self.pg_table_name),
- embedding_field=sql.Identifier(self.embedding_field),
- )
-
- with self.conn as conn:
- with conn.cursor() as cur:
- cur.execute(sql_query, args)
- rows = cur.fetchall()
- columns = [descrip[0] for descrip in cur.description]
- for row in rows:
- data = dict(zip(columns, row))
- data["long_text"] = data[self.content_field]
- retrieved_docs.append(dspy.Example(**data))
- # Return Prediction
- return retrieved_docs
-
- def _get_embeddings(self, query: str) -> list[float]:
- if self.openai_client is not None:
- return (
- self.openai_client.embeddings.create(
- model=self.embedding_model,
- input=query,
- encoding_format="float",
- )
- .data[0]
- .embedding
- )
- else:
- return self.embedding_func(query)
diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.py
deleted file mode 100644
index f5fe245af..000000000
--- a/dspy/retrieve/pinecone_rm.py
+++ /dev/null
@@ -1,273 +0,0 @@
-"""
-Retriever model for Pinecone
-Author: Dhar Rawal (@drawal1)
-"""
-
-from typing import List, Optional, Union
-
-import backoff
-
-from dspy import Retrieve, Prediction
-from dspy.dsp.utils.settings import settings
-from dspy.dsp.utils import dotdict
-
-try:
- import pinecone
-except ImportError:
- pinecone = None
-
-if pinecone is None:
- raise ImportError(
- "The pinecone library is required to use PineconeRM. Install it with `pip install dspy-ai[pinecone]`",
- )
-
-import openai
-
-try:
- OPENAI_LEGACY = int(openai.version.__version__[0]) == 0
-except Exception:
- OPENAI_LEGACY = True
-
-try:
- import openai.error
- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)
-except Exception:
- ERRORS = (openai.RateLimitError, openai.APIError)
-
-class PineconeRM(Retrieve):
- """
- A retrieval module that uses Pinecone to return the top passages for a given query.
-
- Assumes that the Pinecone index has been created and populated with the following metadata:
- - text: The text of the passage
-
- Args:
- pinecone_index_name (str): The name of the Pinecone index to query against.
- pinecone_api_key (str, optional): The Pinecone API key. Defaults to None.
- pinecone_env (str, optional): The Pinecone environment. Defaults to None.
- local_embed_model (str, optional): The local embedding model to use. A popular default is "sentence-transformers/all-mpnet-base-v2".
- openai_embed_model (str, optional): The OpenAI embedding model to use. Defaults to "text-embedding-ada-002".
- openai_api_key (str, optional): The API key for OpenAI. Defaults to None.
- openai_org (str, optional): The organization for OpenAI. Defaults to None.
- k (int, optional): The number of top passages to retrieve. Defaults to 3.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
-
- Examples:
- Below is a code snippet that shows how to use this as the default retriever:
- ```python
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- retriever_model = PineconeRM(openai.api_key)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use this in the forward() function of a module
- ```python
- self.retrieve = PineconeRM(k=num_passages)
- ```
- """
-
- def __init__(
- self,
- pinecone_index_name: str,
- pinecone_api_key: Optional[str] = None,
- pinecone_env: Optional[str] = None,
- local_embed_model: Optional[str] = None,
- openai_embed_model: Optional[str] = "text-embedding-ada-002",
- openai_api_key: Optional[str] = None,
- openai_org: Optional[str] = None,
- k: int = 3,
- ):
- if local_embed_model is not None:
- try:
- import torch
- from transformers import AutoModel, AutoTokenizer
- except ImportError as exc:
- raise ModuleNotFoundError(
- "You need to install Hugging Face transformers (with torch dependencies - pip install transformers[torch]) library to use a local embedding model with PineconeRM.",
- ) from exc
-
- self._local_embed_model = AutoModel.from_pretrained(local_embed_model)
- self._local_tokenizer = AutoTokenizer.from_pretrained(local_embed_model)
- self.use_local_model = True
- self.device = torch.device(
- 'cuda:0' if torch.cuda.is_available() else
- 'mps' if torch.backends.mps.is_available()
- else 'cpu',
- )
- elif openai_embed_model is not None:
- self._openai_embed_model = openai_embed_model
- self.use_local_model = False
- # If not provided, defaults to env vars OPENAI_API_KEY and OPENAI_ORGANIZATION
- if openai_api_key:
- openai.api_key = openai_api_key
- if openai_org:
- openai.organization = openai_org
- else:
- raise ValueError(
- "Either local_embed_model or openai_embed_model must be provided.",
- )
-
- self._pinecone_index = self._init_pinecone(
- pinecone_index_name, pinecone_api_key, pinecone_env,
- )
-
- super().__init__(k=k)
-
- def _init_pinecone(
- self,
- index_name: str,
- api_key: Optional[str] = None,
- environment: Optional[str] = None,
- dimension: Optional[int] = None,
- distance_metric: Optional[str] = None,
- ) -> pinecone.Index:
- """Initialize pinecone and return the loaded index.
-
- Args:
- index_name (str): The name of the index to load. If the index is not does not exist, it will be created.
- api_key (str, optional): The Pinecone API key, defaults to env var PINECONE_API_KEY if not provided.
- environment (str, optional): The environment (ie. `us-west1-gcp` or `gcp-starter`. Defaults to env PINECONE_ENVIRONMENT.
-
- Raises:
- ValueError: If api_key or environment is not provided and not set as an environment variable.
-
- Returns:
- pinecone.Index: The loaded index.
- """
-
- # Pinecone init overrides default if kwargs are present, so we need to exclude if None
- kwargs = {}
- if api_key:
- kwargs["api_key"] = api_key
- if environment:
- kwargs["environment"] = environment
- pinecone.init(**kwargs)
-
- active_indexes = pinecone.list_indexes()
- if index_name not in active_indexes:
- if dimension is None and distance_metric is None:
- raise ValueError(
- "dimension and distance_metric must be provided since the index provided does not exist.",
- )
-
- pinecone.create_index(
- name=index_name,
- dimension=dimension,
- metric=distance_metric,
- )
-
- return pinecone.Index(index_name)
-
- def _mean_pooling(
- self,
- model_output,
- attention_mask,
- ):
- try:
- import torch
- except ImportError as exc:
- raise ModuleNotFoundError(
- "You need to install torch to use a local embedding model with PineconeRM.",
- ) from exc
-
- token_embeddings = model_output[0] # First element of model_output contains all token embeddings
- input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
- return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
- @backoff.on_exception(
- backoff.expo,
- ERRORS,
- max_time=settings.backoff_time,
- )
- def _get_embeddings(
- self,
- queries: List[str],
- ) -> List[List[float]]:
- """Return query vector after creating embedding using OpenAI
-
- Args:
- queries (list): List of query strings to embed.
-
- Returns:
- List[List[float]]: List of embeddings corresponding to each query.
- """
- try:
- import torch
- except ImportError as exc:
- raise ModuleNotFoundError(
- "You need to install torch to use a local embedding model with PineconeRM.",
- ) from exc
-
- if not self.use_local_model:
- if OPENAI_LEGACY:
- embedding = openai.Embedding.create(
- input=queries, model=self._openai_embed_model,
- )
- else:
- embedding = openai.embeddings.create(
- input=queries, model=self._openai_embed_model,
- ).model_dump()
- return [embedding["embedding"] for embedding in embedding["data"]]
-
- # Use local model
- encoded_input = self._local_tokenizer(queries, padding=True, truncation=True, return_tensors="pt").to(self.device)
- with torch.no_grad():
- model_output = self._local_embed_model(**encoded_input.to(self.device))
-
- embeddings = self._mean_pooling(model_output, encoded_input['attention_mask'])
- normalized_embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
- return normalized_embeddings.cpu().numpy().tolist()
-
- # we need a pooling strategy to get a single vector representation of the input
- # so the default is to take the mean of the hidden states
-
- def forward(self, query_or_queries: Union[str, List[str]]) -> Prediction:
- """Search with pinecone for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q] # Filter empty queries
- embeddings = self._get_embeddings(queries)
-
- # For single query, just look up the top k passages
- if len(queries) == 1:
- results_dict = self._pinecone_index.query(
- embeddings[0], top_k=self.k, include_metadata=True,
- )
-
- # Sort results by score
- sorted_results = sorted(
- results_dict["matches"], key=lambda x: x.get("scores", 0.0), reverse=True,
- )
- passages = [result["metadata"]["text"] for result in sorted_results]
- passages = [dotdict({"long_text": passage for passage in passages})]
- return Prediction(passages=passages)
-
- # For multiple queries, query each and return the highest scoring passages
- # If a passage is returned multiple times, the score is accumulated. For this reason we increase top_k by 3x
- passage_scores = {}
- for embedding in embeddings:
- results_dict = self._pinecone_index.query(
- embedding, top_k=self.k * 3, include_metadata=True,
- )
- for result in results_dict["matches"]:
- passage_scores[result["metadata"]["text"]] = (
- passage_scores.get(result["metadata"]["text"], 0.0)
- + result["score"]
- )
-
- sorted_passages = sorted(
- passage_scores.items(), key=lambda x: x[1], reverse=True,
- )[: self.k]
- return Prediction(passages=[dotdict({"long_text": passage}) for passage, _ in sorted_passages])
diff --git a/dspy/retrieve/qdrant_rm.py b/dspy/retrieve/qdrant_rm.py
deleted file mode 100644
index 956adeab4..000000000
--- a/dspy/retrieve/qdrant_rm.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# from collections import defaultdict
-# from typing import Optional, Union
-
-# import dspy
-# from dspy.dsp.modules.sentence_vectorizer import BaseSentenceVectorizer, FastEmbedVectorizer
-# from dspy.dsp.utils import dotdict
-
-# try:
-# from qdrant_client import QdrantClient, models
-# except ImportError as e:
-# raise ImportError(
-# "The 'qdrant' extra is required to use QdrantRM. Install it with `pip install dspy-ai[qdrant]`",
-# ) from e
-
-
-# class QdrantRM(dspy.Retrieve):
-# """A retrieval module that uses Qdrant to return the top passages for a given query.
-
-# Args:
-# qdrant_collection_name (str): The name of the Qdrant collection.
-# qdrant_client (QdrantClient): An instance of `qdrant_client.QdrantClient`.
-# k (int, optional): The default number of top passages to retrieve. Default: 3.
-# document_field (str, optional): The key in the Qdrant payload with the content. Default: `"document"`.
-# vectorizer (BaseSentenceVectorizer, optional): An implementation `BaseSentenceVectorizer`.
-# Default: `FastEmbedVectorizer`.
-# vector_name (str, optional): Name of the vector in the collection. Default: The first available vector name.
-
-# Examples:
-# Below is a code snippet that shows how to use Qdrant as the default retriver:
-# ```python
-# from qdrant_client import QdrantClient
-
-# llm = dspy.OpenAI(model="gpt-3.5-turbo")
-# qdrant_client = QdrantClient()
-# retriever_model = QdrantRM("my_collection_name", qdrant_client=qdrant_client)
-# dspy.settings.configure(lm=llm, rm=retriever_model)
-# ```
-
-# Below is a code snippet that shows how to use Qdrant in the forward() function of a module
-# ```python
-# self.retrieve = QdrantRM(question, k=num_passages, filter=filter)
-# ```
-# """
-
-# def __init__(
-# self,
-# qdrant_collection_name: str,
-# qdrant_client: QdrantClient,
-# k: int = 3,
-# document_field: str = "document",
-# vectorizer: Optional[BaseSentenceVectorizer] = None,
-# vector_name: Optional[str] = None,
-# ):
-# self._collection_name = qdrant_collection_name
-# self._client = qdrant_client
-
-# self._vectorizer = vectorizer or FastEmbedVectorizer(self._client.embedding_model_name)
-
-# self._document_field = document_field
-
-# self._vector_name = vector_name or self._get_first_vector_name()
-
-# super().__init__(k=k)
-
-# def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None, filter: Optional[models.Filter]=None) -> dspy.Prediction:
-# """Search with Qdrant for self.k top passages for query.
-
-# Args:
-# query_or_queries (Union[str, List[str]]): The query or queries to search for.
-# k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
-# filter (Optional["Filter"]): "Look only for points which satisfies this conditions". Default: None.
-
-# Returns:
-# dspy.Prediction: An object containing the retrieved passages.
-# """
-# queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
-# queries = [q for q in queries if q] # Filter empty queries
-
-# vectors = self._vectorizer(queries)
-
-# search_requests = [
-# models.QueryRequest(
-# query=vector,
-# using=self._vector_name,
-# limit=k or self.k,
-# with_payload=[self._document_field],
-# filter=filter,
-# )
-# for vector in vectors
-# ]
-# batch_results = self._client.query_batch_points(self._collection_name, requests=search_requests)
-
-# passages_scores = defaultdict(float)
-# for batch in batch_results:
-# for result in batch.points:
-# # If a passage is returned multiple times, the score is accumulated.
-# document = result.payload.get(self._document_field)
-# passages_scores[document] += result.score
-
-# # Sort passages by their accumulated scores in descending order
-# sorted_passages = sorted(passages_scores.items(), key=lambda x: x[1], reverse=True)[:k]
-
-# # Wrap each sorted passage in a dotdict with 'long_text'
-# return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]
-
-# def _get_first_vector_name(self) -> Optional[str]:
-# vectors = self._client.get_collection(self._collection_name).config.params.vectors
-
-# if not isinstance(vectors, dict):
-# # The collection only has the default, unnamed vector
-# return None
-
-# first_vector_name = list(vectors.keys())[0]
-
-# # The collection has multiple vectors. Could also include the falsy unnamed vector - Empty string("")
-# return first_vector_name or None
diff --git a/dspy/retrieve/ragatouille_rm.py b/dspy/retrieve/ragatouille_rm.py
deleted file mode 100644
index 54f31baa0..000000000
--- a/dspy/retrieve/ragatouille_rm.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from typing import Optional, Union
-
-import dspy
-from dspy.dsp.utils.utils import dotdict
-
-try:
- from ragatouille import RAGPretrainedModel
-except ImportError:
- raise Exception(
- "You need to install RAGAtouille library, Please use the command: pip install ragatouille",
- )
-
-
-class RAGatouilleRM(dspy.Retrieve):
- """A retrieval model that uses RAGatouille library to return the top passages for a given query.
-
- Assumes that you already have an index created with RAGatouille.
- Reference: https://github.com/bclavie/RAGatouille
-
- Args:
- index_root (str): Folder path where you index is stored.
- index_name (str): Name of the index you want to retrieve from.
- k (int, optional): The default number of passages to retrieve. Defaults to 3.
-
- Examples:
- Below is a code snippet that shows how to use RAGatouille index as the default retriver:
- ```python
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- rm = RAGatouilleRM(index_root="ragatouille/colbert/indexes", index_name="my_index")
- dspy.settings.configure(lm=llm, rm=rm)
- ```
- """
-
- def __init__(self, index_root: str, index_name: str, k: int = 3):
- super().__init__(k=k)
- self.index_path = f"{index_root}/{index_name}"
- try:
- self.model = RAGPretrainedModel.from_index(index_path=self.index_path)
- except FileNotFoundError as e:
- raise FileNotFoundError(f"Index not found: {e}")
-
- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]) -> dspy.Prediction:
- """Search with RAGAtouille based index for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- k = k if k is not None else self.k
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q]
- passages = []
- for query in queries:
- results = self.model.search(query=query, k=k)
- passages.extend(dotdict({"long_text": d["content"]}) for d in results)
- return dspy.Prediction(passages=passages)
\ No newline at end of file
diff --git a/dspy/retrieve/snowflake_rm.py b/dspy/retrieve/snowflake_rm.py
deleted file mode 100644
index 14048197b..000000000
--- a/dspy/retrieve/snowflake_rm.py
+++ /dev/null
@@ -1,291 +0,0 @@
-import json
-import re
-from typing import Any, Optional, Type, Union
-
-from pydantic import BaseModel, Field, ValidationError
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-try:
- from snowflake.core import Root
- from snowflake.snowpark.functions import col
-except ImportError:
- raise ImportError(
- "The snowflake-snowpark-python library is required to use SnowflakeRM. Install it with dspy-ai[snowflake]",
- )
-
-
-class SnowflakeRM(dspy.Retrieve):
- """A retrieval module that uses Snowflake's Cortex Search service to return the top relevant passages for a given query.
-
- Assumes that a Snowflake Cortex Search endpoint has been configured by the use.
-
- For more information on configuring the Cortex Search service, visit: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-search/cortex-search-overview
-
- Args:
- snowflake_session (object): Snowflake Snowpark session for accessing the service.
- cortex_search_service(str): Name of the Cortex Search service to be used.
- snowflake_database (str): The name of the Snowflake table containing document embeddings.
- snowflake_schema (str): The name of the Snowflake table containing document embeddings.
- auto_filter (bool): Auto generate metadata filter based on user query and push it down prior to retrieving Cortex Search results.
- k (int, optional): The default number of top passages to retrieve. Defaults to 3.
- """
-
- def __init__(
- self,
- snowflake_session: object,
- cortex_search_service: str,
- snowflake_database: str,
- snowflake_schema: str,
- auto_filter=False,
- k: int = 3,
- max_retries=3,
- ):
- super().__init__(k=k)
- self.k = k
- self.auto_filter = auto_filter
- self.max_retries = max_retries
- self.cortex_search_service_name = cortex_search_service
- self.client = self._fetch_cortex_service(
- snowflake_session, snowflake_database, snowflake_schema, cortex_search_service
- )
-
- if self.auto_filter:
- self.sample_values = self._get_sample_values(
- snowpark_session=snowflake_session, cortex_search_service=cortex_search_service
- )
- self.optimized_filter_gen = SmartSearch()
- self.filter_lm = dspy.Snowflake(session=snowflake_session, model="mixtral-8x7b")
-
- def forward(
- self,
- query_or_queries: Union[str, list[str]],
- retrieval_columns: list[str],
- filter: Optional[dict] = None,
- k: Optional[int] = None,
- ) -> dspy.Prediction:
- """Query Cortex Search endpoint for top k relevant passages.
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
- retrieval_columns (List[str]): Columns to include in response.
- filter (Optional[json]):Filter query.
- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- k = self.k if k is None else k
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
- queries = [q for q in queries if q]
- passages = []
-
- for cortex_query in queries:
- if self.auto_filter:
- response_chunks = self._cortex_search_with_auto_filter(
- cortex_search_service=self.client, query=cortex_query, columns=retrieval_columns, k=k
- )
-
- else:
- response_chunks = self._query_cortex_search(
- cortex_search_service=self.client,
- query=cortex_query,
- columns=retrieval_columns,
- filter=filter,
- k=k,
- )
-
- if len(retrieval_columns) == 1:
- passages.extend(
- dotdict({"long_text": passage[retrieval_columns[0]]}) for passage in response_chunks["results"]
- )
- else:
- passages.extend(dotdict({"long_text": str(passage)}) for passage in response_chunks["results"])
-
- return passages
-
- def _fetch_cortex_service(self, snowpark_session, snowflake_database, snowflake_schema, cortex_search_service_name):
- """Fetch the Cortex Search service to be used"""
- snowpark_session.query_tag = {"origin": "sf_sit", "name": "dspy", "version": {"major": 1, "minor": 0}}
- root = Root(snowpark_session)
-
- # fetch service
- search_service = (
- root.databases[snowflake_database]
- .schemas[snowflake_schema]
- .cortex_search_services[cortex_search_service_name]
- )
-
- return search_service
-
- def _query_cortex_search(self, cortex_search_service, query, columns, filter, k):
- """Query Cortex Search endpoint for top relevant passages .
-
- Args:
- cortex_search_service (object): cortex search service for querying
- query (str): The query or queries to search for.
- columns (Optional[list]): A comma-separated list of columns to return for each relevant result in the response. These columns must be included in the source query for the service.
- filter (Optional[json]): A filter object for filtering results based on data in the ATTRIBUTES columns. See Filter syntax.
- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- # query service
- resp = cortex_search_service.search(query=query, columns=columns, filter=filter, limit=k)
-
- return json.loads(resp.to_json())
-
- def _cortex_search_with_auto_filter(self, cortex_search_service, query, columns, k):
- """Cortex Search Query with automatic metadata filter generation."""
-
- search_response = None
- with dspy.settings.context(lm=self.filter_lm):
- if self.auto_filter:
- for _ in range(self.max_retries):
- raw_filter_query = self.optimized_filter_gen(
- query=query, attributes=str([*self.sample_values]), sample_values=str(self.sample_values)
- )["answer"]
-
- try:
- filter_query = json.loads(raw_filter_query)
- search_response = self._query_cortex_search(
- cortex_search_service=cortex_search_service,
- query=query,
- columns=columns,
- filter=filter_query,
- k=k,
- )
- break
- except json.JSONDecodeError:
- continue
- except Exception:
- continue
-
- else:
- search_response = self._query_cortex_search(
- cortex_search_service=cortex_search_service, query=query, columns=columns, filter=None, k=k
- )
-
- return search_response
-
- def _get_search_attributes(self, snowpark_session, search_service_name):
- df = snowpark_session.sql("SHOW CORTEX SEARCH SERVICES")
- raw_atts = (
- df.where(col('"name"') == search_service_name).select('"attribute_columns"').to_pandas().loc[0].values[0]
- )
- attribute_list = raw_atts.split(",")
-
- return attribute_list
-
- def _get_search_table(self, snowpark_session, search_service_name):
- df = snowpark_session.sql("SHOW CORTEX SEARCH SERVICES")
- table_def = df.where(col('"name"') == search_service_name).select('"definition"').to_pandas().loc[0].values[0]
-
- pattern = r"FROM\s+([\w\.]+)"
- match = re.search(pattern, table_def)
-
- if match:
- from_value = match.group(1)
- return from_value
- else:
- print("No match found.")
-
- return table_def
-
- def _get_sample_values(self, snowpark_session, cortex_search_service, max_samples=10):
- sample_values = {}
- attributes = self._get_search_attributes(
- snowpark_session=snowpark_session, search_service_name=cortex_search_service
- )
- table_name = self._get_search_table(
- snowpark_session=snowpark_session, search_service_name=cortex_search_service
- )
-
- for attribute in attributes:
- query = f"""SELECT DISTINCT({attribute}) FROM {table_name} LIMIT {max_samples}"""
- sample_values[attribute] = list(snowpark_session.sql(query).to_pandas()[attribute].values)
-
- return sample_values
-
-
-class JSONFilter(BaseModel):
- answer: str = Field(description="The filter_query in valid JSON format")
-
- @classmethod
- def model_validate_json(cls, json_data: str, *, strict: bool | None = None, context: dict[str, Any] | None = None):
- __tracebackhide__ = True
- try:
- return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)
- except ValidationError:
- min_length = get_min_length(cls)
- for substring_length in range(len(json_data), min_length - 1, -1):
- for start in range(len(json_data) - substring_length + 1):
- substring = json_data[start : start + substring_length]
- try:
- res = cls.__pydantic_validator__.validate_json(substring, strict=strict, context=context)
- return res
- except ValidationError:
- pass
- raise ValueError("Could not find valid json")
-
-
-class GenerateFilter(dspy.Signature):
- """
- Given a query, attributes in the data, and example values of each attribute, generate a filter in valid JSON format.
- Ensure the filter only uses valid operators: @eq, @contains,@and,@or,@not
- Ensure only the valid JSON is output with no other reasoning.
-
- ---
- Query: What was the sentiment of CEOs between 2021 and 2024?
- Attributes: industry,hq,date
- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}
- Answer: {"@or":[{"@eq":{"year":"2021"}},{"@eq":{"year":"2022"}},{"@eq":{"year":"2023"}},{"@eq":{"year":"2024"}}]}
-
- Query: What is the sentiment of Biotech CEO's of companies based in New York?
- Attributes: industry,hq,date
- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}
- Answer: {"@and": [ { "@eq": { "industry"": "biotechnology" } }, { "@eq": { "HQ": "NY,US" } }]}
-
- Query: What is the sentiment of Biotech CEOs outside of California?
- Attributes: industry,hq,date
- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}
- Answer: {"@and":[{ "@eq": { "industry": "biotechnology" } },{"@not":{"@eq":{"HQ":"CA,US"}}}]}
-
- Query: What is the sentiment of Biotech CEOs outside of California?
- Attributes: industry,hq,date
- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}
- Answer: {"@and":[{ "@eq": { "industry": "biotechnology" } },{"@not":{"@eq":{"HQ":"CA,US"}}}]}
-
- Query: What is sentiment towards ag and biotech companies based outside of the US?
- Attributes: industry,hq,date
- Sample Values: {"industry"":["biotechnology","healthcare","agriculture"],"COUNTRY":["United States","Ireland","Russia","Georgia","Spain"],"month":["01","02","03","06","11","12""],""year"":["2022","2023","2024"]}
- Answer:{"@and": [{ "@or": [{"@eq":{ "industry": "biotechnology" } },{"@eq":{"industry":"agriculture"}}]},{ "@not": {"@eq": { "COUNTRY": "United States" } }}]}
-
- """
-
- query = dspy.InputField(desc="user query")
- attributes = dspy.InputField(desc="attributes to filter on")
- sample_values = dspy.InputField(desc="examples of values per attribute")
- answer: JSONFilter = dspy.OutputField(
- desc="filter query in valid JSON format. ONLY output the filter query in JSON, no reasoning"
- )
-
-
-class SmartSearch(dspy.Module):
- def __init__(self):
- super().__init__()
- self.filter_gen = dspy.ChainOfThought(GenerateFilter)
-
- def forward(self, query, attributes, sample_values):
- filter_query = self.filter_gen(query=query, attributes=attributes, sample_values=sample_values)
-
- return filter_query
-
-
-def get_min_length(model: Type[BaseModel]):
- min_length = 0
- for key, field in model.model_fields.items():
- if issubclass(field.annotation, BaseModel):
- min_length += get_min_length(field.annotation)
- min_length += len(key)
- return min_length
diff --git a/dspy/retrieve/vectara_rm.py b/dspy/retrieve/vectara_rm.py
deleted file mode 100644
index fc1bc9bc0..000000000
--- a/dspy/retrieve/vectara_rm.py
+++ /dev/null
@@ -1,167 +0,0 @@
-import json
-import os
-from collections import defaultdict
-from typing import List, Optional, Union
-
-import requests
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-START_SNIPPET = "<%START%>"
-END_SNIPPET = "<%END%>"
-
-def remove_snippet(s: str) -> str:
- return s.replace(START_SNIPPET, "").replace(END_SNIPPET, "")
-
-class VectaraRM(dspy.Retrieve):
- """
- A retrieval module that uses Vectara to return the top passages for a given query.
-
- Assumes that a Vectara corpora have been created and populated with the following payload:
- - document: The text of the passage
-
- Args:
- vectara_customer_id (str): Vectara Customer ID. defaults to VECTARA_CUSTOMER_ID environment variable
- vectara_corpus_id (str): Vectara Corpus ID. defaults to VECTARA_CORPUS_ID environment variable
- vectara_api_key (str): Vectara API Key. defaults to VECTARA_API_KEY environment variable
- k (int, optional): The default number of top passages to retrieve. Defaults to 3.
-
- Examples:
- Below is a code snippet that shows how to use Vectara as the default retriver:
- ```python
- from vectara_client import vectaraClient
-
- llm = dspy.OpenAI(model="gpt-3.5-turbo")
- retriever_model = vectaraRM("<VECTARA_CUSTOMER_ID>", "<VECTARA_CORPUS_ID>", "<VECTARA_API_KEY>")
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use Vectara in the forward() function of a module
- ```python
- self.retrieve = vectaraRM("<VECTARA_CUSTOMER_ID>", "<VECTARA_CORPUS_ID>", "<VECTARA_API_KEY>", k=num_passages)
- ```
- """
-
- def __init__(
- self,
- vectara_customer_id: Optional[str] = None,
- vectara_corpus_id: Optional[str] = None,
- vectara_api_key: Optional[str] = None,
- k: int = 5,
- ):
- if vectara_customer_id is None:
- vectara_customer_id = os.environ.get("VECTARA_CUSTOMER_ID", "")
- if vectara_corpus_id is None:
- vectara_corpus_id = os.environ.get("VECTARA_CORPUS_ID", "")
- if vectara_api_key is None:
- vectara_api_key = os.environ.get("VECTARA_API_KEY", "")
-
- self._vectara_customer_id = vectara_customer_id
- self._vectara_corpus_id = vectara_corpus_id
- self._vectara_api_key = vectara_api_key
- self._n_sentences_before = self._n_sentences_after = 2
- self._vectara_timeout = 120
- super().__init__(k=k)
-
- def _vectara_query(
- self,
- query: str,
- limit: int = 5,
- ) -> List[str]:
- """Query Vectara index to get for top k matching passages.
- Args:
- query: query string
- """
- # If multiple corpus ids are provided (comma-separated), create a list of corpus keys
- # otherwise by default, the `split(',')` is a no-op so retains the single corpus id
- corpus_key = [
- {
- "customerId": self._vectara_customer_id,
- "corpusId": corpus_id,
- "lexicalInterpolationConfig": {"lambda": 0.025 },
- } for corpus_id in self._vectara_corpus_id.split(',')
- ]
-
- data = {
- "query": [
- {
- "query": query,
- "start": 0,
- "numResults": limit,
- "contextConfig": {
- "sentencesBefore": self._n_sentences_before,
- "sentencesAfter": self._n_sentences_after,
- "startTag": START_SNIPPET,
- "endTag": END_SNIPPET,
- },
- "corpusKey": corpus_key,
- },
- ],
- }
-
- headers = {
- "x-api-key": self._vectara_api_key,
- "customer-id": self._vectara_customer_id,
- "Content-Type": "application/json",
- "X-Source": "dspy",
- }
-
- response = requests.post(
- headers=headers,
- url="https://api.vectara.io/v1/query",
- data=json.dumps(data),
- timeout=self._vectara_timeout,
- )
-
- if response.status_code != 200:
- print(
- "Query failed %s",
- f"(code {response.status_code}, reason {response.reason}, details "
- f"{response.text})",
- )
- return []
-
- result = response.json()
- responses = result["responseSet"][0]["response"]
-
- res = [
- {
- "text": remove_snippet(x["text"]),
- "score": x["score"],
- } for x in responses
- ]
- return res
-
- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:
- """Search with Vectara for self.k top passages for query
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = (
- [query_or_queries]
- if isinstance(query_or_queries, str)
- else query_or_queries
- )
- queries = [q for q in queries if q] # Filter empty queries
- k = k if k is not None else self.k
-
- all_res = []
- limit = 3*k if len(queries) > 1 else k
- for query in queries:
- _res = self._vectara_query(query, limit=limit)
- all_res.append(_res)
-
- passages = defaultdict(float)
- for res_list in all_res:
- for res in res_list:
- passages[res["text"]] += res["score"]
- sorted_passages = sorted(
- passages.items(), key=lambda x: x[1], reverse=True)[:k]
-
- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]
-
\ No newline at end of file
diff --git a/dspy/retrieve/watson_discovery_rm.py b/dspy/retrieve/watson_discovery_rm.py
deleted file mode 100644
index 47b03f804..000000000
--- a/dspy/retrieve/watson_discovery_rm.py
+++ /dev/null
@@ -1,107 +0,0 @@
-import json
-from typing import Optional, Union
-
-import requests
-from requests.auth import HTTPBasicAuth
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-
-class WatsonDiscoveryRM(dspy.Retrieve):
- """A retrieval module that uses Watson Discovery to return the top passages for a given query.
-
- Args:
- apikey (str): apikey for authentication purposes,
- url (str): endpoint URL that includes the service instance ID
- version (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.
- project_id (str): The Universally Unique Identifier (UUID) of the project.
- collection_ids (list): An array containing the collections on which the search will be executed.
- k (int, optional): The number of top passages to retrieve. Defaults to 5.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
-
- def __init__(
- self,
- apikey: str,
- url:str,
- version:str,
- project_id:str,
- collection_ids:list=None,
- k: int = 5,
- ):
- if collection_ids is None:
- collection_ids = []
- self.apikey=apikey
- self.url=url,
- self.version:version
- self.project_id:project_id
- self.collection_ids=collection_ids
- self.k: k
- self.query_url=url + "/v2/projects/" + project_id + "/query?version=" + version
-
- super().__init__(k=k)
-
- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:
- """Search with Watson Discovery for self.k top passages for query.
-
- Args:
- query_or_queries (Union[str, List[str]]): The query or queries to search for.
- k (int, optional): The number of top passages to retrieve.
-
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
- """
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
- queries = [q for q in queries if q] # Filter empty queries
- k = self.k if k is None else k
-
- response=[]
- try:
- for query in queries:
- payload = json.dumps({
- "collection_ids": self.collection_ids,
- "natural_language_query": "text:"+query,
- "passages": {
- "enabled": True,
- "count": k,
- "fields":["text"],
- "characters": 500,
- "find_answers": False,
- "per_document": False,
- },
- "highlight": False,
- })
- headers = {'Content-Type': 'application/json'}
-
- discovery_results = requests.request(
- "POST",
- url=self.query_url,
- headers=headers,
- auth=HTTPBasicAuth("apikey", self.apikey),
- data=payload,
- )
-
- discovery_results.raise_for_status()
-
- doc_dict={}
- for d in discovery_results.json()["results"]:
- doc_dict[d["document_id"]]=d["title"]
-
- for d in discovery_results.json()["passages"]:
- response.append(dotdict({
- "title":doc_dict[d["document_id"]],
- "long_text": doc_dict[d["document_id"]]+" | " + d["passage_text"],
- "passage_score": d["passage_score"],
- "document_id": d["document_id"],
- "collection_id": d["collection_id"],
- "start_offset": d["start_offset"],
- "end_offset": d["end_offset"],
- "field": d["field"],
- }))
-
- except requests.exceptions.RequestException as err:
- raise SystemExit(err) from err
- return response
\ No newline at end of file
diff --git a/dspy/retrieve/you_rm.py b/dspy/retrieve/you_rm.py
deleted file mode 100644
index cf7498b3c..000000000
--- a/dspy/retrieve/you_rm.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import os
-import warnings
-from typing import Any, Literal, Optional, Union
-
-import requests
-
-import dspy
-from dspy.dsp.utils import dotdict
-
-
-class YouRM(dspy.Retrieve):
- """Retriever for You.com's Search and News API.
-
- [API reference](https://documentation.you.com/api-reference/)
-
- Args:
- ydc_api_key: you.com API key, if `YDC_API_KEY` is not set in the environment
- k: If ``endpoint="search"``, the max snippets to return per search hit.
- If ``endpoint="news"``, the max articles to return.
- endpoint: you.com endpoints
- num_web_results: The max number of web results to return, must be under 20
- safesearch: Safesearch settings, one of "off", "moderate", "strict", defaults to moderate
- country: Country code, ex: 'US' for United States, see API reference for more info
- search_lang: (News API) Language codes, ex: 'en' for English, see API reference for more info
- ui_lang: (News API) User interface language for the response, ex: 'en' for English.
- See API reference for more info
- spellcheck: (News API) Whether to spell check query or not, defaults to True
- """
-
- def __init__(
- self,
- ydc_api_key: Optional[str] = None,
- k: int = 3,
- endpoint: Literal["search", "news"] = "search",
- num_web_results: Optional[int] = None,
- safesearch: Optional[Literal["off", "moderate", "strict"]] = None,
- country: Optional[str] = None,
- search_lang: Optional[str] = None,
- ui_lang: Optional[str] = None,
- spellcheck: Optional[bool] = None,
- ):
- super().__init__(k=k)
-
- # Data validation
- if not ydc_api_key and not os.environ.get("YDC_API_KEY"):
- raise RuntimeError('You must supply `ydc_api_key` or set environment variable "YDC_API_KEY"')
-
- if endpoint not in ("search", "news"):
- raise ValueError('`endpoint` must be either "search" or "news"')
-
- # Raise warning if News API-specific fields are set but endpoint is not "news"
- if endpoint != "news":
- news_api_fields = (search_lang, ui_lang, spellcheck)
- for field in news_api_fields:
- if field:
- warnings.warn(
- (
- f"News API-specific field '{field}' is set but `{endpoint=}`. "
- "This will have no effect."
- ),
- UserWarning,
- )
-
- self.ydc_api_key = ydc_api_key or os.environ.get("YDC_API_KEY")
- self.endpoint = endpoint
- self.num_web_results = num_web_results
- self.safesearch = safesearch
- self.country = country
- self.search_lang = search_lang
- self.ui_lang = ui_lang
- self.spellcheck = spellcheck
-
- def _generate_params(self, query: str) -> dict[str, Any]:
- params = {"safesearch": self.safesearch, "country": self.country}
-
- if self.endpoint == "search":
- params.update(
- query=query,
- num_web_results=self.num_web_results,
- )
- elif self.endpoint == "news":
- params.update(
- q=query,
- count=self.num_web_results,
- search_lang=self.search_lang,
- ui_lang=self.ui_lang,
- spellcheck=self.spellcheck,
- )
-
- # Remove `None` values
- params = {k: v for k, v in params.items() if v is not None}
- return params
-
- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None) -> dspy.Prediction:
- k = k if k is not None else self.k
-
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
- docs: list[str]
- for query in queries:
- headers = {"X-API-Key": self.ydc_api_key}
- params = self._generate_params(query)
- response = requests.get(
- f"https://api.ydc-index.io/{self.endpoint}",
- params=params,
- headers=headers,
- )
- response.raise_for_status()
- results = response.json()
-
- if self.endpoint == "search":
- docs = [snippet for hits in results["hits"][:k] for snippet in hits["snippets"]]
- elif self.endpoint == "news":
- docs = [article["description"] for article in results["news"]["results"][:k]]
- return [dotdict({"long_text": document}) for document in docs]
diff --git a/dspy/retrievers/__init__.py b/dspy/retrievers/__init__.py
index a8eb9e231..ffaf603bb 100644
--- a/dspy/retrievers/__init__.py
+++ b/dspy/retrievers/__init__.py
@@ -1,3 +1,4 @@
from dspy.retrievers.embeddings import Embeddings
+from dspy.retrievers.retrieve import Retrieve
-__all__ = ["Embeddings"]
+__all__ = ["Embeddings", "Retrieve"]
diff --git a/dspy/retrieve/databricks_rm.py b/dspy/retrievers/databricks_rm.py
similarity index 100%
rename from dspy/retrieve/databricks_rm.py
rename to dspy/retrievers/databricks_rm.py
diff --git a/dspy/retrieve/retrieve.py b/dspy/retrievers/retrieve.py
similarity index 100%
rename from dspy/retrieve/retrieve.py
rename to dspy/retrievers/retrieve.py
diff --git a/dspy/retrieve/weaviate_rm.py b/dspy/retrievers/weaviate_rm.py
similarity index 99%
rename from dspy/retrieve/weaviate_rm.py
rename to dspy/retrievers/weaviate_rm.py
index 7d4cf436a..434afc6ec 100644
--- a/dspy/retrieve/weaviate_rm.py
+++ b/dspy/retrievers/weaviate_rm.py
@@ -5,9 +5,10 @@ from dspy.dsp.utils import dotdict
from dspy.primitives.prediction import Prediction
try:
+ from uuid import uuid4
+
import weaviate
from weaviate.util import get_valid_uuid
- from uuid import uuid4
except ImportError as err:
raise ImportError(
"The 'weaviate' extra is required to use WeaviateRM. Install it with `pip install dspy-ai[weaviate]`",
@@ -109,7 +110,7 @@ class WeaviateRM(dspy.Retrieve):
passages.extend(dotdict({"long_text": d}) for d in parsed_results)
return passages
-
+
def get_objects(self, num_samples: int, fields: List[str]) -> List[dict]:
"""Get objects from Weaviate using the cursor API."""
if self._client_type == "WeaviateClient":
@@ -127,7 +128,7 @@ class WeaviateRM(dspy.Retrieve):
return objects
else:
raise ValueError("`get_objects` is not supported for the v3 Weaviate Python client, please upgrade to v4.")
-
+
def insert(self, new_object_properties: dict):
if self._client_type == "WeaviateClient":
self._weaviate_collection.data.insert(
diff --git a/tests/retrieve/integration_test_pgvectorrm.py b/tests/retrieve/integration_test_pgvectorrm.py
deleted file mode 100644
index be6c18f7a..000000000
--- a/tests/retrieve/integration_test_pgvectorrm.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""Instructions:
-Add to dev container features:
- "ghcr.io/itsmechlark/features/postgresql:1": {},
- "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {}
-Add to .personalization.sh:
- sudo apt install -y postgresql-16-pgvector
-
- sudo /etc/init.d/postgresql restart
-
- psql -v ON_ERROR_STOP=1 --user ${PGUSER} <<EOF
- create extension if not exists vector;
- EOF
-poetry install -E postgres
-"""
-from dspy.primitives.example import Example
-import pytest
-import psycopg2
-from dspy.retrieve.pgvector_rm import PgVectorRM
-
-DB_URL = "postgresql://postgres:password@localhost/postgres"
-PG_TABLE_NAME = "test_table"
-
-
-def get_pgvectorrm():
- openai_client = None # Mock or use a real OpenAI client
- pgvectorrm = PgVectorRM(DB_URL, PG_TABLE_NAME, openai_client=openai_client, embedding_func=lambda x: '[2,3,4]', include_similarity=True)
- return pgvectorrm
-
-
-def setup_pgvectorrm():
- pgvectorrm = get_pgvectorrm()
- conn = psycopg2.connect(DB_URL)
- cursor = conn.cursor()
- cursor.execute(f'DROP TABLE IF EXISTS {PG_TABLE_NAME}')
- conn.commit()
-
- cursor.execute(f"CREATE TABLE IF NOT EXISTS {PG_TABLE_NAME} (id SERIAL PRIMARY KEY, text TEXT, embedding VECTOR)")
- cursor.execute(f"INSERT INTO {PG_TABLE_NAME} (text, embedding) VALUES ('Dummy text1', '[1,2,3]'), ('Dummy text2', '[4,5,6]')")
- conn.commit()
-
- yield pgvectorrm
-
- cursor.execute(f'TRUNCATE TABLE {PG_TABLE_NAME}')
- conn.commit()
-
- cursor.close()
- conn.close()
-
-
-def test_pgvectorrm_retrieve(setup_pgvectorrm):
- pgvectorrm = setup_pgvectorrm
- query = "test query"
- results = pgvectorrm(query)
- assert len(results) == 2
- assert results == [
- Example(text='Dummy text2', similarity=0.9946115458726394),
- Example(text='Dummy text1', similarity=0.9925833339709302)
- ]
-
-
[email protected]("k, expected", [
- (1, 1),
- (2, 2),
- (3, 2), # Assuming only 2 entries exist
-])
-def test_pgvectorrm_retrieve_diff_k(setup_pgvectorrm, k, expected):
- setup_pgvectorrm.k = k
- query = "test query"
- results = setup_pgvectorrm(query)
- assert len(results) == expected
-
-
-def test_empty_table():
- # Assuming setup_pgvectorrm cleans up after yielding
- query = "test query"
- results = get_pgvectorrm()(query)
- assert len(results) == 0
-
-
-def test_retrieval_without_similarity(setup_pgvectorrm):
- setup_pgvectorrm.include_similarity = False
- query = "test query"
- results = setup_pgvectorrm(query)
- # Ensure 'similarity' key is not in results
- assert all('similarity' not in result for result in results)
diff --git a/tests/retrieve/test_llama_index_rm.py b/tests/retrieve/test_llama_index_rm.py
deleted file mode 100644
index 9a4246fa9..000000000
--- a/tests/retrieve/test_llama_index_rm.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import logging
-
-import pytest
-
-import dspy
-from dspy.datasets import HotPotQA
-from dspy.utils.dummies import DummyLM
-
-try:
- from llama_index.core import Settings, VectorStoreIndex
- from llama_index.core.base.base_retriever import BaseRetriever
- from llama_index.core.embeddings.mock_embed_model import MockEmbedding
- from llama_index.core.readers.string_iterable import StringIterableReader
-
- from dspy.retrieve.llama_index_rm import LlamaIndexRM
-
-except ImportError:
- logging.info("Optional dependency llama-index is not installed - skipping LlamaIndexRM tests.")
-
-
-def rag_setup() -> dict:
- """Builds the necessary fixtures to test LI"""
- pytest.importorskip("llama_index")
- dataset = HotPotQA(train_seed=1, train_size=8, eval_seed=2023, dev_size=4, test_size=0)
- trainset = [x.with_inputs("question") for x in dataset.train]
- devset = [x.with_inputs("question") for x in dataset.dev]
- ragset = [f"Question: {x.question} Answer: {x.answer}" for x in dataset.train]
- dummyset = {x.question: x.answer for x in dataset.train}
-
- Settings.embed_model = MockEmbedding(8)
- docs = StringIterableReader().load_data(texts=ragset)
- index = VectorStoreIndex.from_documents(documents=docs)
- retriever = index.as_retriever()
- rm = LlamaIndexRM(retriever)
-
- return {
- "index": index,
- "retriever": retriever,
- "rm": rm,
- "lm": DummyLM(answers=dummyset),
- "trainset": trainset,
- "devset": devset,
- }
-
-
-def test_lirm_as_rm(rag_setup):
- """Test the retriever as retriever method"""
- pytest.importorskip("llama_index")
- retriever = rag_setup.get("retriever")
- test_res_li = retriever.retrieve("At My Window was released by which American singer-songwriter?")
- rm = rag_setup.get("rm")
- test_res_dspy = rm.forward("At My Window was released by which American singer-songwriter?")
-
- assert isinstance(retriever, BaseRetriever), "Ensuring that the retriever is a LI Retriever object"
- assert isinstance(test_res_li, list), "Ensuring results are a list from LI Retriever"
-
- assert isinstance(rm, dspy.Retrieve), "Ensuring the RM is a retriever object from dspy"
- assert isinstance(test_res_dspy, list), "Ensuring the results are a list from the DSPy retriever"
-
- assert len(test_res_li) == len(test_res_dspy), "Rough equality check of the results"
-
-
-def test_save_load_llama_index_rag(rag_setup, tmp_path):
- pytest.importorskip("llama_index")
-
- class RAG(dspy.Module):
- def __init__(self):
- super().__init__()
- self.retriever = dspy.Retrieve(k=3)
- self.cot = dspy.ChainOfThought("question, context -> answer")
-
- rag = RAG()
- rag.retriever.k = 4
-
- file_path = tmp_path / "rag.json"
- rag.save(file_path)
- loaded_rag = RAG()
- # Before loading, the retriever k should be 3.
- assert loaded_rag.retriever.k == 3
- # After loading, the retriever k should be 4.
- loaded_rag.load(file_path)
- assert loaded_rag.retriever.k == 4