DSPy: The framework for programming—not prompting—language models
Python
Remove community retrievers (#8073)
TomuHiratac3e79ce87b42
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 9cc6c90c9..a9118851e 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -1,11 +1,9 @@from dspy.predict import *from dspy.primitives import *-from dspy.retrieve import *+from dspy.retrievers import *from dspy.signatures import *from dspy.teleprompt import *-import dspy.retrievers-from dspy.evaluate import Evaluate # isort: skipfrom dspy.clients import * # isort: skipfrom dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, History, BaseType, Tool, ToolCalls # isort: skipdiff --git a/dspy/retrieve/__init__.py b/dspy/retrieve/__init__.pydeleted file mode 100644index 072c7d527..000000000--- a/dspy/retrieve/__init__.py+++ /dev/null@@ -1,5 +0,0 @@-from dspy.retrieve.retrieve import Retrieve--__all__ = [- "Retrieve",-]diff --git a/dspy/retrieve/azureaisearch_rm.py b/dspy/retrieve/azureaisearch_rm.pydeleted file mode 100644index ffb1f91b2..000000000--- a/dspy/retrieve/azureaisearch_rm.py+++ /dev/null@@ -1,397 +0,0 @@-"""-Retriever module for Azure AI Search-Author: Prajapati Harishkumar Kishorkumar (@HARISHKUMAR1112001)-"""--import warnings-from typing import Any, Callable, List, Optional, Union--import dspy-from dspy.dsp.utils.utils import dotdict--try:- from azure.core.credentials import AzureKeyCredential- from azure.search.documents import SearchClient- from azure.search.documents._paging import SearchItemPaged- from azure.search.documents.models import QueryType, VectorFilterMode, VectorizedQuery-except ImportError:- raise ImportError(- "You need to install azure-search-documents library"- "Please use the command: pip install azure-search-documents==11.6.0b1",- )--try:- import openai-except ImportError:- warnings.warn(- "`openai` is not installed. Install it with `pip install openai` to use AzureOpenAI embedding models.",- category=ImportWarning,- )---class AzureAISearchRM(dspy.Retrieve):-- """- A retrieval module that utilizes Azure AI Search to retrieve top passages for a given query.-- Args:- search_service_name (str): The name of the Azure AI Search service.- search_api_key (str): The API key for accessing the Azure AI Search service.- search_index_name (str): The name of the search index in the Azure AI Search service.- field_text (str): The name of the field containing text content in the search index. This field will be mapped to the "content" field in the dsp framework.- field_vector (Optional[str]): The name of the field containing vector content in the search index. Defaults to None.- k (int, optional): The default number of top passages to retrieve. Defaults to 3.- azure_openai_client (Optional[openai.AzureOpenAI]): An instance of the AzureOpenAI client. Either openai_client or embedding_func must be provided. Defaults to None.- openai_embed_model (Optional[str]): The name of the OpenAI embedding model. Defaults to "text-embedding-ada-002".- embedding_func (Optional[Callable]): A function for generating embeddings. Either openai_client or embedding_func must be provided. Defaults to None.- semantic_ranker (bool, optional): Whether to use semantic ranking. Defaults to False.- filter (str, optional): Additional filter query. Defaults to None.- query_language (str, optional): The language of the query. Defaults to "en-Us".- query_speller (str, optional): The speller mode. Defaults to "lexicon".- use_semantic_captions (bool, optional): Whether to use semantic captions. Defaults to False.- query_type (Optional[QueryType], optional): The type of query. Defaults to QueryType.FULL.- semantic_configuration_name (str, optional): The name of the semantic configuration. Defaults to None.- is_vector_search (Optional[bool]): Whether to enable vector search. Defaults to False.- is_hybrid_search (Optional[bool]): Whether to enable hybrid search. Defaults to False.- is_fulltext_search (Optional[bool]): Whether to enable fulltext search. Defaults to True.- vector_filter_mode (Optional[VectorFilterMode]): The vector filter mode. Defaults to None.-- Examples:- Below is a code snippet that demonstrates how to instantiate and use the AzureAISearchRM class:- ```python- search_service_name = "your_search_service_name"- search_api_key = "your_search_api_key"- search_index_name = "your_search_index_name"- field_text = "text_content_field"-- azure_search_retriever = AzureAISearchRM(search_service_name, search_api_key, search_index_name, field_text)- ```-- Attributes:- search_service_name (str): The name of the Azure AI Search service.- search_api_key (str): The API key for accessing the Azure AI Search service.- search_index_name (str): The name of the search index in the Azure AI Search service.- endpoint (str): The endpoint URL for the Azure AI Search service.- field_text (str): The name of the field containing text content in the search index.- field_vector (Optional[str]): The name of the field containing vector content in the search index.- azure_openai_client (Optional[openai.AzureOpenAI]): An instance of the AzureOpenAI client.- openai_embed_model (Optional[str]): The name of the OpenAI embedding model.- embedding_func (Optional[Callable]): A function for generating embeddings.- credential (AzureKeyCredential): The Azure key credential for accessing the service.- client (SearchClient): The Azure AI Search client instance.- semantic_ranker (bool): Whether to use semantic ranking.- filter (str): Additional filter query.- query_language (str): The language of the query.- query_speller (str): The speller mode.- use_semantic_captions (bool): Whether to use semantic captions.- query_type (Optional[QueryType]): The type of query.- semantic_configuration_name (str): The name of the semantic configuration.- is_vector_search (Optional[bool]): Whether to enable vector search.- is_hybrid_search (Optional[bool]): Whether to enable hybrid search.- is_fulltext_search (Optional[bool]): Whether to enable fulltext search.- vector_filter_mode (Optional[VectorFilterMode]): The vector filter mode.-- Methods:- forward(query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:- Search for the top passages corresponding to the given query or queries.-- azure_search_request(- self,- key_content: str,- client: SearchClient,- query: str,- top: int,- semantic_ranker: bool,- filter: str,- query_language: str,- query_speller: str,- use_semantic_captions: bool,- query_type: QueryType,- semantic_configuration_name: str,- is_vector_search: bool,- is_hybrid_search: bool,- is_fulltext_search: bool,- field_vector: str,- vector_filter_mode: VectorFilterMode- ) -> List[dict]:- Perform a search request to the Azure AI Search service.-- process_azure_result(- self,- results:SearchItemPaged,- content_key:str,- content_score: str- ) -> List[dict]:- Process the results received from the Azure AI Search service and map them to the correct format.-- get_embeddings(- self,- query: str,- k_nearest_neighbors: int,- field_vector: str- ) -> List | Any:- Returns embeddings for the given query.-- check_semantic_configuration(- self,- semantic_configuration_name,- query_type- ):- Checks semantic configuration.-- Raises:- ImportError: If the required Azure AI Search libraries are not installed.-- Note:- This class relies on the 'azure-search-documents' library for interacting with the Azure AI Search service.- Ensure that you have the necessary permissions and correct configurations set up in Azure before using this class.- """-- def __init__(- self,- search_service_name: str,- search_api_key: str,- search_index_name: str,- field_text: str,- field_vector: Optional[str] = None,- k: int = 3,- azure_openai_client: Optional[openai.AzureOpenAI] = None,- openai_embed_model: Optional[str] = "text-embedding-ada-002",- embedding_func: Optional[Callable] = None,- semantic_ranker: bool = False,- filter: str = None,- query_language: str = "en-Us",- query_speller: str = "lexicon",- use_semantic_captions: bool = False,- query_type: Optional[QueryType] = QueryType.FULL,- semantic_configuration_name: str = None,- is_vector_search: Optional[bool] = False,- is_hybrid_search: Optional[bool] = False,- is_fulltext_search: Optional[bool] = True,- vector_filter_mode: Optional[VectorFilterMode.PRE_FILTER] = None,- ):- self.search_service_name = search_service_name- self.search_api_key = search_api_key- self.search_index_name = search_index_name- self.endpoint = f"https://{self.search_service_name}.search.windows.net"- self.field_text = field_text # field name of the text content- self.field_vector = field_vector # field name of the vector content- self.azure_openai_client = azure_openai_client- self.openai_embed_model = openai_embed_model- self.embedding_func = embedding_func- # Create a client- self.credential = AzureKeyCredential(self.search_api_key)- self.client = SearchClient(- endpoint=self.endpoint, index_name=self.search_index_name, credential=self.credential,- )- self.semantic_ranker = semantic_ranker- self.filter = filter- self.query_language = query_language- self.query_speller = query_speller- self.use_semantic_captions = use_semantic_captions- self.query_type = query_type- self.semantic_configuration_name = semantic_configuration_name- self.is_vector_search = is_vector_search- self.is_hybrid_search = is_hybrid_search- self.is_fulltext_search = is_fulltext_search- self.vector_filter_mode = vector_filter_mode-- super().__init__(k=k)-- def azure_search_request(- self,- key_content: str,- client: SearchClient,- query: str,- top: int,- semantic_ranker: bool,- filter: str,- query_language: str,- query_speller: str,- use_semantic_captions: bool,- query_type: QueryType,- semantic_configuration_name: str,- is_vector_search: bool,- is_hybrid_search: bool,- is_fulltext_search: bool,- field_vector: str,- vector_filter_mode: VectorFilterMode,- ):- """- Search in Azure AI Search Index- """-- if is_vector_search:- vector_query = self.get_embeddings(query, top, field_vector)- if semantic_ranker:- self.check_semantic_configuration(semantic_configuration_name, query_type)- results = client.search(- search_text=None,- filter=filter,- query_type=query_type,- vector_queries=vector_query,- vector_filter_mode=vector_filter_mode,- semantic_configuration_name=semantic_configuration_name,- top=top,- query_caption=("extractive|highlight-false" if use_semantic_captions else None),- )- else:- results = client.search(- search_text=None,- filter=filter,- vector_queries=vector_query,- vector_filter_mode=vector_filter_mode,- top=top,- )- if is_hybrid_search:- vector_query = self.get_embeddings(query, top, field_vector)- if semantic_ranker:- self.check_semantic_configuration(semantic_configuration_name, query_type)- results = client.search(- search_text=query,- filter=filter,- query_type=query_type,- query_language=query_language,- query_speller=query_speller,- semantic_configuration_name=semantic_configuration_name,- top=top,- vector_queries=vector_query,- vector_filter_mode=vector_filter_mode,- query_caption=("extractive|highlight-false" if use_semantic_captions else None),- )- else:- results = client.search(- search_text=query,- filter=filter,- query_language=query_language,- query_speller=query_speller,- top=top,- vector_queries=vector_query,- vector_filter_mode=vector_filter_mode,- )- if is_fulltext_search:- if semantic_ranker:- self.check_semantic_configuration(semantic_configuration_name, query_type)- results = client.search(- search_text=query,- filter=filter,- query_type=query_type,- query_language=query_language,- query_speller=query_speller,- semantic_configuration_name=semantic_configuration_name,- top=top,- query_caption=("extractive|highlight-false" if use_semantic_captions else None),- )- else:- results = client.search(search_text=query, top=top, filter=filter)-- sorted_results = sorted(results, key=lambda x: x["@search.score"], reverse=True)-- sorted_results = self.process_azure_result(sorted_results, key_content, key_content)-- return sorted_results-- def process_azure_result(self, results: SearchItemPaged, content_key: str, content_score: str):- """- process received result from Azure AI Search as dictionary array and map content and score to correct format- """- res = []- for result in results:- tmp = {}- for key, value in result.items():- if key == content_key:- tmp["text"] = value # assign content- elif key == content_score:- tmp["score"] = value- else:- tmp[key] = value- res.append(tmp)- return res-- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:- """- Search with pinecone for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """-- k = k if k is not None else self.k- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries- queries = [q for q in queries if q] # Filter empty queries-- passages = []- for query in queries:- results = self.azure_search_request(- self.field_text,- self.client,- query,- k,- self.semantic_ranker,- self.filter,- self.query_language,- self.query_speller,- self.use_semantic_captions,- self.query_type,- self.semantic_configuration_name,- self.is_vector_search,- self.is_hybrid_search,- self.is_fulltext_search,- self.field_vector,- self.vector_filter_mode,- )- passages.extend(dotdict({"long_text": d["text"]}) for d in results)-- return passages-- def get_embeddings(self, query: str, k_nearest_neighbors: int, field_vector: str) -> List | Any:- """- Returns embeddings for the given query.-- Args:- query (str): The query for which embeddings are to be retrieved.- k_nearest_neighbors (int): The number of nearest neighbors to consider.- field_vector (str): The field vector to use for embeddings.-- Returns:- list: A list containing the vectorized query.- Any: The result of embedding_func if azure_openai_client is not provided.-- Raises:- AssertionError: If neither azure_openai_client nor embedding_func is provided,- or if field_vector is not provided.- """- assert (- self.azure_openai_client or self.embedding_func- ), "Either azure_openai_client or embedding_func must be provided."-- if self.azure_openai_client is not None:- assert field_vector, "field_vector must be provided."-- embedding = (- self.azure_openai_client.embeddings.create(input=query, model=self.openai_embed_model).data[0].embedding- )- vector_query = VectorizedQuery(- vector=embedding, k_nearest_neighbors=k_nearest_neighbors, fields=field_vector,- )- return [vector_query]- else:- return self.embedding_func(query)-- def check_semantic_configuration(self, semantic_configuration_name, query_type):- """- Checks semantic configuration.-- Args:- semantic_configuration_name: The name of the semantic configuration.- query_type: The type of the query.-- Raises:- AssertionError: If semantic_configuration_name is not provided- or if query_type is not QueryType.SEMANTIC.- """- assert semantic_configuration_name, "Semantic configuration name must be provided."- assert query_type == QueryType.SEMANTIC, "Query type must be QueryType.SEMANTIC."diff --git a/dspy/retrieve/chromadb_rm.py b/dspy/retrieve/chromadb_rm.pydeleted file mode 100644index c102f8565..000000000--- a/dspy/retrieve/chromadb_rm.py+++ /dev/null@@ -1,170 +0,0 @@-"""-Retriever model for chromadb-"""--from typing import List, Optional, Union--import backoff-import openai--from dspy import Retrieve, Prediction-from dspy.dsp.utils.settings import settings-from dspy.dsp.utils import dotdict--try:- import openai.error- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)-except Exception:- ERRORS = (openai.RateLimitError, openai.APIError)--try:- import chromadb- import chromadb.utils.embedding_functions as ef- from chromadb.api.types import (- Embeddable,- EmbeddingFunction,- )- from chromadb.config import Settings-except ImportError:- raise ImportError(- "The chromadb library is required to use ChromadbRM. Install it with `pip install dspy-ai[chromadb]`",- )---class ChromadbRM(Retrieve):- """- A retrieval module that uses chromadb to return the top passages for a given query.-- Assumes that the chromadb index has been created and populated with the following metadata:- - documents: The text of the passage-- Args:- collection_name (str): chromadb collection name- persist_directory (str): chromadb persist directory- embedding_function (Optional[EmbeddingFunction[Embeddable]]): Optional function to use to embed documents. Defaults to DefaultEmbeddingFunction.- k (int, optional): The number of top passages to retrieve. Defaults to 7.- client(Optional[chromadb.Client]): Optional chromadb client provided by user, default to None-- Returns:- dspy.Prediction: An object containing the retrieved passages.-- Examples:- Below is a code snippet that shows how to use this as the default retriever:- ```python- llm = dspy.OpenAI(model="gpt-3.5-turbo")- # using default chromadb client- retriever_model = ChromadbRM('collection_name', 'db_path')- dspy.settings.configure(lm=llm, rm=retriever_model)- # to test the retriever with "my query"- retriever_model("my query")- ```-- Use provided chromadb client- ```python- import chromadb- llm = dspy.OpenAI(model="gpt-3.5-turbo")- # say you have a chromadb running on a different port- client = chromadb.HttpClient(host='localhost', port=8889)- retriever_model = ChromadbRM('collection_name', 'db_path', client=client)- dspy.settings.configure(lm=llm, rm=retriever_model)- # to test the retriever with "my query"- retriever_model("my query")- ```-- Below is a code snippet that shows how to use this in the forward() function of a module- ```python- self.retrieve = ChromadbRM('collection_name', 'db_path', k=num_passages)- ```- """-- def __init__(- self,- collection_name: str,- persist_directory: str,- embedding_function: Optional[- EmbeddingFunction[Embeddable]- ] = ef.DefaultEmbeddingFunction(),- client: Optional[chromadb.Client] = None,- k: int = 7,- ):- self._init_chromadb(collection_name, persist_directory, client=client)- self.ef = embedding_function-- super().__init__(k=k)-- def _init_chromadb(- self,- collection_name: str,- persist_directory: str,- client: Optional[chromadb.Client] = None,- ) -> chromadb.Collection:- """Initialize chromadb and return the loaded index.-- Args:- collection_name (str): chromadb collection name- persist_directory (str): chromadb persist directory- client (chromadb.Client): chromadb client provided by user-- Returns: collection per collection_name- """-- if client:- self._chromadb_client = client- else:- self._chromadb_client = chromadb.Client(- Settings(- persist_directory=persist_directory,- is_persistent=True,- ),- )- self._chromadb_collection = self._chromadb_client.get_or_create_collection(- name=collection_name,- )-- @backoff.on_exception(- backoff.expo,- ERRORS,- max_time=settings.backoff_time,- )- def _get_embeddings(self, queries: List[str]) -> List[List[float]]:- """Return query vector after creating embedding using OpenAI-- Args:- queries (list): List of query strings to embed.-- Returns:- List[List[float]]: List of embeddings corresponding to each query.- """- return self.ef(queries)-- def forward(- self, query_or_queries: Union[str, List[str]], k: Optional[int] = None, **kwargs,- ) -> Prediction:- """Search with db for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q] # Filter empty queries- embeddings = self._get_embeddings(queries)-- k = self.k if k is None else k- results = self._chromadb_collection.query(- query_embeddings=embeddings, n_results=k,**kwargs,- )-- zipped_results = zip(- results["ids"][0],- results["distances"][0],- results["documents"][0],- results["metadatas"][0])- results = [dotdict({"id": id, "score": dist, "long_text": doc, "metadatas": meta }) for id, dist, doc, meta in zipped_results]- return resultsdiff --git a/dspy/retrieve/clarifai_rm.py b/dspy/retrieve/clarifai_rm.pydeleted file mode 100644index 9ba13c7e3..000000000--- a/dspy/retrieve/clarifai_rm.py+++ /dev/null@@ -1,96 +0,0 @@-"""Clarifai as retriver to retrieve hits"""-import os-from concurrent.futures import ThreadPoolExecutor-from typing import List, Optional, Union--import requests--import dspy-from dspy.dsp.utils import dotdict--try:- from clarifai.client.search import Search-except ImportError as err:- raise ImportError(- "Clarifai is not installed. Install it using `pip install clarifai`",- ) from err---class ClarifaiRM(dspy.Retrieve):- """- Retrieval module uses clarifai to return the Top K relevant pasages for the given query.- Assuming that you have ingested the source documents into clarifai App, where it is indexed and stored.-- Args:- clarifai_user_id (str): Clarifai unique user_id.- clarfiai_app_id (str): Clarifai App ID, where the documents are stored.- clarifai_pat (str): Clarifai PAT key.- k (int): Top K documents to retrieve.-- Examples:- TODO- """-- def __init__(- self,- clarifai_user_id: str,- clarfiai_app_id: str,- clarifai_pat: Optional[str] = None,- k: int = 3,- ):- self.app_id = clarfiai_app_id- self.user_id = clarifai_user_id- self.pat = (- clarifai_pat if clarifai_pat is not None else os.environ["CLARIFAI_PAT"]- )- self.k = k- self.clarifai_search = Search(- user_id=self.user_id, app_id=self.app_id, top_k=k, pat=self.pat,- )- super().__init__(k=k)-- def retrieve_hits(self, hits):- header = {"Authorization": f"Key {self.pat}"}- request = requests.get(hits.input.data.text.url, headers=header)- request.encoding = request.apparent_encoding- requested_text = request.text- return requested_text-- def forward(- self, query_or_queries: Union[str, List[str]], k: Optional[int] = None,**kwargs,- ) -> dspy.Prediction:- """Uses clarifai-python SDK search function and retrieves top_k similar passages for given query,- Args:- query_or_queries : single query or list of queries- k : Top K relevant documents to return-- Returns:- passages in format of dotdict-- Examples:- Below is a code snippet that shows how to use Marqo as the default retriver:- ```python- import clarifai- llm = dspy.Clarifai(model=MODEL_URL, api_key="YOUR CLARIFAI_PAT")- retriever_model = ClarifaiRM(clarifai_user_id="USER_ID", clarfiai_app_id="APP_ID", clarifai_pat="YOUR CLARIFAI_PAT")- dspy.settings.configure(lm=llm, rm=retriever_model)- ```- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- passages = []- queries = [q for q in queries if q]-- for query in queries:- search_response = self.clarifai_search.query(ranks=[{"text_raw": query}],**kwargs)-- # Retrieve hits- hits = [hit for data in search_response for hit in data.hits]- with ThreadPoolExecutor(max_workers=10) as executor:- results = list(executor.map(self.retrieve_hits, hits))- passages.extend(dotdict({"long_text": d}) for d in results)-- return passagesdiff --git a/dspy/retrieve/deeplake_rm.py b/dspy/retrieve/deeplake_rm.pydeleted file mode 100644index a126941dd..000000000--- a/dspy/retrieve/deeplake_rm.py+++ /dev/null@@ -1,113 +0,0 @@-"""-Retriever model for deeplake-"""--from collections import defaultdict-from typing import List, Optional, Union--import openai--import dspy-from dspy.dsp.utils import dotdict--try:- import openai-- ERRORS = (- openai.RateLimitError,- openai.APIError,- )-except Exception:- ERRORS = (openai.RateLimitError, openai.APIError)---class DeeplakeRM(dspy.Retrieve):-- """- A retriever module that uses deeplake to return the top passages for a given query.-- Assumes that a Deep Lake Vector Store has been created and populated with the following payload:- - text: The text of the passage-- Args:- deeplake_vectorstore_name (str): The name or path of the Deep Lake Vector Store.- deeplake_client (VectorStore): An instance of the Deep Lake client.- k (int, optional): The default number of top passages to retrieve. Defaults to 3.-- Examples:- Below is a code snippet that shows how to use Deep Lake as the default retriver:- ```python- from deeplake import VectorStore- llm = dspy.OpenAI(model="gpt-3.5-turbo")- deeplake_client = VectorStore- retriever_model = DeeplakeRM("my_vectorstore_path", deeplake_client=deeplake_client)- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use Deep Lake in the forward() function of a module- ```python- self.retrieve = DeeplakeRM("my_vectorstore_path", deeplake_client=deeplake_client, k=num_passages)- ```- """-- def __init__(- self,- deeplake_vectorstore_name: str,- deeplake_client,- k: int = 3,- ):- try:- from deeplake import VectorStore # noqa: F401- except ImportError:- raise ImportError("The 'deeplake' extra is required to use DeepLakeRM. Install it with `pip install dspy-ai[deeplake]`",)-- self._deeplake_vectorstore_name = deeplake_vectorstore_name- self._deeplake_client = deeplake_client(- path=self._deeplake_vectorstore_name,- embedding_function=self.embedding_function,- )-- super().__init__(k=k)-- def embedding_function(self, texts, model="text-embedding-ada-002"):- if isinstance(texts, str):- texts = [texts]-- texts = [t.replace("\n", " ") for t in texts]-- return [data.embedding for data in openai.embeddings.create(input = texts, model=model).data]-- def forward(- self, query_or_queries: Union[str, List[str]], k: Optional[int],**kwargs,- ) -> dspy.Prediction:-- """Search with DeepLake for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q] # Filter empty queries-- k = k if k is not None else self.k-- passages = defaultdict(float)- #deeplake doesn't support batch querying, manually querying each query and storing them- for query in queries:- results = self._deeplake_client.search(query, k=k, **kwargs)-- for score,text in zip(results.get('score',0.0),results.get('text',"")):- passages[text] += score-- sorted_passages = sorted(- passages.items(), key=lambda x: x[1], reverse=True)[:k]-- return [dotdict({"long_text": p}) for p, _ in sorted_passages]\ No newline at end of filediff --git a/dspy/retrieve/epsilla_rm.py b/dspy/retrieve/epsilla_rm.pydeleted file mode 100644index 6257b745b..000000000--- a/dspy/retrieve/epsilla_rm.py+++ /dev/null@@ -1,45 +0,0 @@-from collections import defaultdict # noqa: F401-from typing import Dict, List, Union # noqa: UP035--import dspy-from dspy.dsp.utils import dotdict--try:- from pyepsilla import vectordb-except ImportError:- raise ImportError( # noqa: B904- "The 'pyepsilla' extra is required to use EpsillaRM. Install it with `pip install dspy-ai[epsilla]`",- )---class EpsillaRM(dspy.Retrieve):- def __init__(- self,- epsilla_client: vectordb.Client,- db_name: str,- db_path: str,- table_name: str,- k: int = 3,- page_content: str = "document",- ):- self._epsilla_client = epsilla_client- self._epsilla_client.load_db(db_name=db_name, db_path=db_path)- self._epsilla_client.use_db(db_name=db_name)- self.page_content = page_content- self.table_name = table_name-- super().__init__(k=k)-- def forward(self, query_or_queries: Union[str, List[str]], k: Union[int, None] = None, **kwargs) -> dspy.Prediction: # noqa: ARG002- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries- queries = [q for q in queries if q]- limit = k if k else self.k- all_query_results: list = []-- passages: Dict = defaultdict(float)-- for result_dict in all_query_results:- for result in result_dict:- passages[result[self.page_content]] += result["@distance"]- sorted_passages = sorted(passages.items(), key=lambda x: x[1], reverse=False)[:limit]- return dspy.Prediction(passages=[dotdict({"long_text": passage}) for passage, _ in sorted_passages])diff --git a/dspy/retrieve/faiss_rm.py b/dspy/retrieve/faiss_rm.pydeleted file mode 100755index 5ac4de971..000000000--- a/dspy/retrieve/faiss_rm.py+++ /dev/null@@ -1,156 +0,0 @@-# """Retriever model for faiss: https://github.com/facebookresearch/faiss.-# Author: Jagane Sundar: https://github.com/jagane.-# """--# import logging-# from typing import Optional, Union--# import numpy as np--# import dspy-# from dspy.dsp.modules.sentence_vectorizer import SentenceTransformersVectorizer-# from dspy.dsp.utils import dotdict--# try:-# import faiss-# except ImportError:-# faiss = None--# if faiss is None:-# raise ImportError(-# """-# The faiss package is required. Install it using `pip install dspy-ai[faiss-cpu]`-# """,-# )---# logger = logging.getLogger(__name__)-# class FaissRM(dspy.Retrieve):-# """A retrieval module that uses an in-memory Faiss to return the top passages for a given query.--# Args:-# document_chunks: the input text chunks-# vectorizer: an object that is a subclass of BaseSentenceVectorizer-# k (int, optional): The number of top passages to retrieve. Defaults to 3.--# Returns:-# dspy.Prediction: An object containing the retrieved passages.--# Examples:-# Below is a code snippet that shows how to use this as the default retriver:-# ```python-# import dspy-# from dspy.retrieve import faiss_rm--# document_chunks = [-# "The superbowl this year was played between the San Francisco 49ers and the Kanasas City Chiefs",-# "Pop corn is often served in a bowl",-# "The Rice Bowl is a Chinese Restaurant located in the city of Tucson, Arizona",-# "Mars is the fourth planet in the Solar System",-# "An aquarium is a place where children can learn about marine life",-# "The capital of the United States is Washington, D.C",-# "Rock and Roll musicians are honored by being inducted in the Rock and Roll Hall of Fame",-# "Music albums were published on Long Play Records in the 70s and 80s",-# "Sichuan cuisine is a spicy cuisine from central China",-# "The interest rates for mortgages is considered to be very high in 2024",-# ]--# frm = faiss_rm.FaissRM(document_chunks)-# turbo = dspy.OpenAI(model="gpt-3.5-turbo")-# dspy.settings.configure(lm=turbo, rm=frm)-# print(frm(["I am in the mood for Chinese food"]))-# ```--# Below is a code snippet that shows how to use this in the forward() function of a module-# ```python-# self.retrieve = FaissRM(k=num_passages)-# ```-# """--# def __init__(self, document_chunks, vectorizer=None, k: int = 3):-# """Inits the faiss retriever.--# Args:-# document_chunks: a list of input strings.-# vectorizer: an object that is a subclass of BaseTransformersVectorizer.-# k: number of matches to return.-# """-# if vectorizer:-# self._vectorizer = vectorizer-# else:-# self._vectorizer = SentenceTransformersVectorizer()-# embeddings = self._vectorizer(document_chunks)-# xb = np.array(embeddings)-# d = len(xb[0])-# logger.info(f"FaissRM: embedding size={d}")-# if len(xb) < 100:-# self._faiss_index = faiss.IndexFlatL2(d)-# self._faiss_index.add(xb)-# else:-# # if we have at least 100 vectors, we use Voronoi cells-# nlist = 100-# quantizer = faiss.IndexFlatL2(d)-# self._faiss_index = faiss.IndexIVFFlat(quantizer, d, nlist)-# self._faiss_index.train(xb)-# self._faiss_index.add(xb)--# logger.info(f"{self._faiss_index.ntotal} vectors in faiss index")-# self._document_chunks = document_chunks # save the input document chunks--# super().__init__(k=k)--# def _dump_raw_results(self, queries, index_list, distance_list) -> None:-# for i in range(len(queries)):-# indices = index_list[i]-# distances = distance_list[i]-# logger.debug(f"Query: {queries[i]}")-# for j in range(len(indices)):-# logger.debug(f" Hit {j} = {indices[j]}/{distances[j]}: {self._document_chunks[indices[j]]}")-# return--# def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None, **kwargs) -> dspy.Prediction:-# """Search the faiss index for k or self.k top passages for query.--# Args:-# query_or_queries (Union[str, List[str]]): The query or queries to search for.--# Returns:-# dspy.Prediction: An object containing the retrieved passages.-# """-# queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries-# queries = [q for q in queries if q] # Filter empty queries-# embeddings = self._vectorizer(queries)-# emb_npa = np.array(embeddings)-# # For single query, just look up the top k passages-# if len(queries) == 1:-# distance_list, index_list = self._faiss_index.search(emb_npa, k or self.k)-# # self._dump_raw_results(queries, index_list, distance_list)-# passages = [(self._document_chunks[ind], ind) for ind in index_list[0]]-# return [dotdict({"long_text": passage[0], "index": passage[1]}) for passage in passages]--# distance_list, index_list = self._faiss_index.search(emb_npa, (k or self.k) * 3, **kwargs)-# # self._dump_raw_results(queries, index_list, distance_list)-# passage_scores = {}-# for emb in range(len(embeddings)):-# indices = index_list[emb] # indices of neighbors for embeddings[emb] - this is an array of k*3 integers-# distances = distance_list[-# emb-# ] # distances of neighbors for embeddings[emb] - this is an array of k*3 floating point numbers-# for res in range((k or self.k) * 3):-# neighbor = indices[res]-# distance = distances[res]-# if neighbor in passage_scores:-# passage_scores[neighbor].append(distance)-# else:-# passage_scores[neighbor] = [distance]-# # Note re. sorting:-# # first degree sort: number of queries that got a hit with any particular document chunk. More-# # is a better match. This is len(queries)-len(x[1])-# # second degree sort: sum of the distances of each hit returned by faiss. Smaller distance is a better match-# sorted_passages = sorted(passage_scores.items(), key=lambda x: (len(queries) - len(x[1]), sum(x[1])))[-# : k or self.k-# ]-# return [-# dotdict({"long_text": self._document_chunks[passage_index], "index": passage_index})-# for passage_index, _ in sorted_passages-# ]diff --git a/dspy/retrieve/falkordb_rm.py b/dspy/retrieve/falkordb_rm.pydeleted file mode 100644index 5897db620..000000000--- a/dspy/retrieve/falkordb_rm.py+++ /dev/null@@ -1,213 +0,0 @@-import os-from typing import List, Optional, Union-import string-import random--import backoff-from openai import (- APITimeoutError,- InternalServerError,- OpenAI,- RateLimitError,- UnprocessableEntityError,-)--from dspy import Retrieve, Prediction-from dspy.dsp.utils.settings import settings-from dspy.dsp.utils import dotdict--try:- import falkordb-except ImportError:- raise ImportError(- "Please install the falkordb package by running `pip install dspy-ai[falkordb]`"- )-import redis.exceptions---def generate_random_string(length: int) -> str:- characters = string.ascii_letters- random_string = "".join(random.choice(characters) for _ in range(length))- return random_string---class Embedder:- def __init__(self, provider: str, model: str):- self.provider = provider- if self.provider == "openai":- api_key = os.getenv("OPENAI_API_KEY")- if not api_key:- raise ValueError(- "Environment variable OPENAI_API_KEY must be set to"- "use openai as embedding provider"- )- self.client = OpenAI()- self.model = model- else:- raise ValueError(f"Unsupported provider: {provider}")-- @backoff.on_exception(- backoff.expo,- (- APITimeoutError,- InternalServerError,- RateLimitError,- UnprocessableEntityError,- ),- max_time=settings.backoff_time,- )- def __call__(self, queries: Union[str, List[str]]) -> List[List[float]]:- if isinstance(queries, str):- queries = [queries]-- if self.provider == "openai":- embedding = self.client.embeddings.create(input=queries, model=self.model)- return [result.embedding for result in embedding.data]---DEFAULT_INDEX_QUERY = "CALL db.idx.vector.queryNodes($node_label, $embedding_node_property, $k, vecf32($embedding)) YIELD node, score "---class FalkordbRM(Retrieve):- """- Implements a retriever that utilizes FalkorDB for retrieving passages.- This class manages a connection to a FalkorDB database using official FalkorDB Python drivers and requires- the database credentials. That is, if using a local FalkorDB session, host and port else if using a FalkorDB cloud session,- host, port, username, and password to be set as environment variables and optionally the database name.- Additionally, it utilizes an embedding provider (defaulting to OpenAI's services) to compute query embeddings,- which are then used to find the most relevant nodes in the FalkorDB graph based on the specified node property or custom retrieval query.-- Returns a list of passages in the form of `dspy.Prediction` objects-- Args:- Args:- node_label (str): The label of the node in the FalkorDB database to query against- text_node_property (str): The property of the node containing the text.- embedding_node_property (List[float]): The property of the node containing the embeddings.- k (Optional[int]): The default number of top passages to retrieve. Defaults to 5.- retrieval_query (Optional[str]): Custom Cypher query for retrieving passages.- embedding_provider (str): The provider of the embedding service. Defaults to "openai".- embedding_model (str): The model identifier for generating embeddings. Defaults to "text-embedding-ada-002".-- Examples:- Below is a code snippet showcasing how to initialize FalkordbRM with environment variables for the database connection and OpenAI as the embedding provider:-- ```python- import os-- import dspy- import openai-- os.environ["FALKORDB_HOST"] = "localhost"- os.environ["FALORDB_PORT"] = "6379"- os.environ["OPENAI_API_KEY"] = "sk-" (Only if using openai as embedding's provider)-- # Uncomment and set the following if you are using FalkorDB cloud- # os.environ["FALKORDB_USERNAME"] = "falkordb"- # os.environ["FALKORDB_PASSWORD"] = "password"--- falkordb_retriever = FalkordbRM(- node_label="myIndex",- text_node_property="text",- k=10,- embedding_provider="openai",- embedding_model="text-embedding-ada-002",- )-- dspy.settings.configure(rm=falkordb_retriever)- ```-- In this example, `FalkordbRM` is configured to retrieve nodes based on the "text" property from an index on a node labeled "myIndex",- using embeddings computed by OpenAI's "text-embedding-ada-002" model.- """-- def __init__(- self,- node_label: str,- text_node_property: str = None,- embedding_node_property: str = None,- k: int = 5,- retrieval_query: Optional[str] = None,- embedding_provider: str = "openai",- embedding_model: str = "text-embedding-ada-002",- ):- super().__init__(k=k)- self.node_label = node_label- self.username = os.getenv("FALKORDB_USERNAME", None)- self.password = os.getenv("FALKORDB_PASSWORD", None)- self.host = os.getenv("FALKORDB_HOST", "localhost")- self.port = int(os.getenv("FALKORDB_PORT", 6379))-- self.database = os.getenv("FALKORDB_DATABASE", generate_random_string(4))- self.k = k- self.retrieval_query = retrieval_query- self.text_node_property = text_node_property- self.embedding_node_property = embedding_node_property- if not self.text_node_property and not self.retrieval_query:- raise ValueError(- "Either `text_node_property` or `retrieval_query` must be set"- )- if not embedding_node_property:- raise ValueError("`embedding_node_property` must be set")- try:- self.driver = falkordb.FalkorDB(- host=self.host,- port=self.port,- username=self.username,- password=self.password,- ).select_graph(self.database)-- except (- redis.exceptions.ConnectionError,- redis.exceptions.AuthenticationError,- ) as e:- raise ConnectionError("Failed to connect to FalkorDB database") from e-- self.embedder = Embedder(provider=embedding_provider, model=embedding_model)-- def forward(- self, query_or_queries: Union[str, List[str]], k: Optional[int]- ) -> Prediction:- if not isinstance(query_or_queries, list):- query_or_queries = [query_or_queries]- query_vectors = self.embedder(query_or_queries)- contents = []- retrieval_query = (- self.retrieval_query- or f"RETURN node.{self.text_node_property} AS text, score"- )- if not k:- k = self.k-- for vector in query_vectors:- params = {- "embedding": vector,- "node_label": self.node_label,- "text_node_property": self.text_node_property,- "embedding_node_property": self.embedding_node_property,- "k": k,- }- try:- records = self.driver.query(- DEFAULT_INDEX_QUERY + retrieval_query,- params=params,- ).result_set- except Exception as e:- if "Invalid arguments" in str(e):- raise ValueError(- f"There is no vector index on node label, {self.node_label}"- f" and node property, {self.embedding_node_property}"- )- contents.extend(- [- {"passage": dotdict({"long_text": r[1]}), "score": r[0]}- for r in records- ]- )- sorted_passages = sorted(- contents,- key=lambda x: x["score"],- reverse=True,- )[:k]- return [el["passage"] for el in sorted_passages]diff --git a/dspy/retrieve/lancedb_rm.py b/dspy/retrieve/lancedb_rm.pydeleted file mode 100644index d2769b24e..000000000--- a/dspy/retrieve/lancedb_rm.py+++ /dev/null@@ -1,139 +0,0 @@-"""-Retriever model for LanceDB-Author: Prashant Dixit (@PrashantDixit0)-"""--from typing import List, Union--import backoff-import lancedb.table--from dspy import Retrieve, Prediction-from dspy.dsp.utils.settings import settings-from dspy.dsp.utils import dotdict--try:- import lancedb-except ImportError:- lancedb = None--if lancedb is None:- raise ImportError(- "The lancedb library is required to use LancedbRM. Install it with `pip install dspy-ai[lancedb]`",- )---import openai--try:- OPENAI_LEGACY = int(openai.version.__version__[0]) == 0-except Exception:- OPENAI_LEGACY = True--try:- import openai.error- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)-except Exception:- ERRORS = (openai.RateLimitError, openai.APIError)---class LancedbRM(Retrieve):- """- A retrieval module that uses LanceDB to return the top passages for a given query.-- Assumes that the LanceDB table has been created and populated with the following metadata:- - text: The text of the passage-- Args:- table_name (str): The name of the table to query against.- persist_directory (str): directory where database is stored.- k (int, optional): The number of top passages to retrieve. Defaults to 3.-- Returns:- dspy.Prediction: An object containing the retrieved passages.-- Examples:- Below is a code snippet that shows how to use this as the default retriever:- ```python- llm = dspy.OpenAI(model="gpt-3.5-turbo")- retriever_model = LancedbRM()- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use this in the forward() function of a module- ```python- self.retrieve = LancedbRM(k=num_passages)- ```- """-- def __init__(- self,- table_name: str,- persist_directory: str,- k: int = 3,- ):-- self._table = self._init_lancedb(- table_name, persist_directory,- )-- super().__init__(k=k)-- def _init_lancedb(- self,- table_name: str,- persist_directory: str,- ):- """- Initialize LanceDB database and return the table instance.-- Args:- table_name (str): The name of the table to load. If the table does not exist, it will be Raise Value Error.-- Raises:- ValueError: If table is already not created-- Returns:- Lancedb.table: The loaded table.- """-- self.db = lancedb.connect(persist_directory)- table_names = self.db.table_names()-- if table_name not in table_names:- raise ValueError(- f"Table does not exist in lancedb database persist_directory. Create table with name {table_name}, then reintialize retriever instance",- )- else:- table = self.db.open_table(table_name)- return table--- @backoff.on_exception(- backoff.expo,- ERRORS,- max_time=settings.backoff_time,- )-- def forward(self, query_or_queries: Union[str, List[str]]) -> Prediction:- """Search with Lancedb for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q] # Filter empty queries-- passages = []- for q in queries:- results = self._table.search(q).limit(self.k).to_list()- for r in results:- passages.append(r['text'])- return Prediction(passages=[dotdict({"long_text": passage}) for passage in passages])diff --git a/dspy/retrieve/llama_index_rm.py b/dspy/retrieve/llama_index_rm.pydeleted file mode 100644index 639d849ba..000000000--- a/dspy/retrieve/llama_index_rm.py+++ /dev/null@@ -1,90 +0,0 @@-import logging-from typing import Optional--import dspy--try:- from llama_index.core.base.base_retriever import BaseRetriever-except ImportError:- err = "The 'llama_index' package is required to use LlamaIndexRM. Install it with 'pip install llama_index'."- raise ImportError(err) from None--NO_TOP_K_WARNING = "The underlying LlamaIndex retriever does not support top k retrieval. Ignoring k value."---class LlamaIndexRM(dspy.Retrieve):- """Implements a retriever which wraps over a LlamaIndex retriever.-- This is done to bridge LlamaIndex and DSPy and allow the various retrieval- abstractions in LlamaIndex to be used in DSPy.-- To-do (maybe):- - Async support (DSPy lacks this entirely it seems, so not a priority until the rest of the repo catches on)- - Text/video retrieval (Available in LI, not sure if this will be a priority in DSPy)-- Args:- retriever (BaseRetriever): A LlamaIndex retriever object - text based only- k (int): Optional; the number of examples to retrieve (similarity_top_k)-- If the underlying LI retriever does not have the property similarity_top_k, k will be ignored.-- Returns:- DSPy RM Object - this is a retriever object that can be used in DSPy- """-- retriever: BaseRetriever-- def __init__(- self,- retriever: BaseRetriever,- k: Optional[int] = None,- ):- self.retriever = retriever-- if k:- self.k = k-- @property- def k(self) -> Optional[int]:- """Get similarity top k of retriever."""- if not hasattr(self.retriever, "similarity_top_k"):- logging.warning(NO_TOP_K_WARNING)- return None-- return self.retriever.similarity_top_k-- @k.setter- def k(self, k: int) -> None:- """Set similarity top k of retriever."""- if hasattr(self.retriever, "similarity_top_k"):- self.retriever.similarity_top_k = k- else:- logging.warning(NO_TOP_K_WARNING)-- def forward(self, query: str, k: Optional[int] = None) -> list[dspy.Example]:- """Forward function for the LI retriever.-- This is the function that is called to retrieve the top k examples for a given query.- Top k is set via the setter similarity_top_k or at LI instantiation.-- Args:- query (str): The query to retrieve examples for- k (int): Optional; the number of examples to retrieve (similarity_top_k)-- If the underlying LI retriever does not have the property similarity_top_k, k will be ignored.-- Returns:- List[dspy.Example]: A list of examples retrieved by the retriever- """- if k:- self.k = k-- raw = self.retriever.retrieve(query)-- return [- dspy.Example(- text=result.text,- score=result.score,- )- for result in raw- ]diff --git a/dspy/retrieve/marqo_rm.py b/dspy/retrieve/marqo_rm.pydeleted file mode 100644index 25f1a6ee3..000000000--- a/dspy/retrieve/marqo_rm.py+++ /dev/null@@ -1,96 +0,0 @@-from collections import defaultdict-from typing import List, Union--import dspy-from dspy.dsp.utils import dotdict--try:- import marqo-except ImportError:- raise ImportError(- "The 'marqo' extra is required to use MarqoRM. Install it with `pip install dspy-ai[marqo]`",- )--class MarqoRM(dspy.Retrieve):- """- A retrieval module that uses Marqo to return the top passages for a given query.-- Assumes that a Marqo index has been created and populated with the following payload:- - document: The text of the passage-- Args:- marqo_index_name (str): The name of the marqo index.- marqo_client (marqo.client.Client): A marqo client instance.- k (int, optional): The number of top passages to retrieve. Defaults to 3.- page_content (str, optional): The name of the field in the marqo index that contains the text of the passage. Defaults to 'document'.- filter_string (str, optional): A filter string to use when searching. Defaults to None.- **kwargs: Additional keyword arguments to pass to the marqo search function.-- Examples:- Below is a code snippet that shows how to use Marqo as the default retriver:- ```python- import marqo- marqo_client = marqo.Client(url="http://0.0.0.0:8882")-- llm = dspy.OpenAI(model="gpt-3.5-turbo")- retriever_model = MarqoRM("my_index_name", marqo_client=marqo_client)- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use Marqo in the forward() function of a module- ```python- self.retrieve = MarqoRM("my_index_name", marqo_client=marqo_client, k=num_passages)- ```- """-- def __init__(- self,- marqo_index_name: str,- marqo_client: marqo.client.Client,- k: int = 3,- page_content: str = 'document',- filter_string: str = None,- ):- self._marqo_index_name = marqo_index_name- self._marqo_client = marqo_client- self.page_content = page_content- self.filter_string = filter_string-- super().__init__(k=k)-- def forward(self, query_or_queries: Union[str, List[str]], k=None, **kwargs) -> dspy.Prediction:- """Search with Marqo for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q]- limit = k if k else self.k-- all_query_results = []- for query in queries:- _result = self._marqo_client.index(self._marqo_index_name).search(- q=query,- limit=limit,- filter_string=self.filter_string,- **kwargs,- )- all_query_results.append(_result)-- passages = defaultdict(float)-- for result_dict in all_query_results:- for result in result_dict['hits']:- passages[result[self.page_content]] += result['_score']-- sorted_passages = sorted(- passages.items(), key=lambda x: x[1], reverse=True)[:limit]- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]diff --git a/dspy/retrieve/milvus_rm.py b/dspy/retrieve/milvus_rm.pydeleted file mode 100644index a49570391..000000000--- a/dspy/retrieve/milvus_rm.py+++ /dev/null@@ -1,107 +0,0 @@-"""-Retriever model for Milvus or Zilliz Cloud-"""--from typing import Callable, List, Optional, Union--import dspy-from dspy.dsp.utils import dotdict--try:- from pymilvus import MilvusClient-except ImportError:- raise ImportError(- "The pymilvus library is required to use MilvusRM. Install it with `pip install dspy-ai[milvus]`",- )---def openai_embedding_function(texts: List[str]):- from openai import OpenAI-- client = OpenAI()- response = client.embeddings.create(- input=texts,- model="text-embedding-3-small",- )- return [x.embedding for x in response.data]---class MilvusRM(dspy.Retrieve):- """- A retrieval module that uses Milvus to return passages for a given query.-- Assumes that a Milvus collection has been created and populated with the following field:- - text: The text of the passage-- Args:- collection_name (str): The name of the Milvus collection to query against.- uri (str, optional): The Milvus connection uri. Defaults to "http://localhost:19530".- token (str, optional): The Milvus connection token. Defaults to None.- db_name (str, optional): The Milvus database name. Defaults to "default".- embedding_function (callable, optional): The function to convert a list of text to embeddings.- The embedding function should take a list of text strings as input and output a list of embeddings.- Defaults to None. By default, it will get OpenAI client by the environment variable OPENAI_API_KEY- and use OpenAI's embedding model "text-embedding-3-small" with the default dimension.- k (int, optional): The number of top passages to retrieve. Defaults to 3.-- Returns:- dspy.Prediction: An object containing the retrieved passages.-- Examples:- Below is a code snippet that shows how to use this as the default retriever:- ```python- llm = dspy.OpenAI(model="gpt-3.5-turbo")- retriever_model = MilvusRM(- collection_name="<YOUR_COLLECTION_NAME>",- uri="<YOUR_MILVUS_URI>",- token="<YOUR_MILVUS_TOKEN>"- )- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use this in the forward() function of a module- ```python- self.retrieve = MilvusRM(k=num_passages)- ```- """-- def __init__(- self,- collection_name: str,- uri: Optional[str] = "http://localhost:19530",- token: Optional[str] = None,- db_name: Optional[str] = "default",- embedding_function: Optional[Callable] = None,- k: int = 3,- ):- self.milvus_client = MilvusClient(uri=uri, token=token, db_name=db_name)-- # check collection existence- if collection_name not in self.milvus_client.list_collections():- raise AttributeError(f"Milvus collection not found: {collection_name}")- self.collection_name = collection_name-- self.embedding_function = embedding_function or openai_embedding_function- self.k = k-- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int] = None) -> dspy.Prediction:- if isinstance(query_or_queries, str):- query_or_queries = [query_or_queries]- query_embeddings = self.embedding_function(query_or_queries)-- k = k or self.k-- milvus_res = self.milvus_client.search(- collection_name=self.collection_name,- data=query_embeddings,- output_fields=["text"],- limit=k,- )- passage_scores = {}- for res in milvus_res:- for r in res:- passage_scores[r["entity"]["text"]] = r["distance"]- sorted_passages = sorted(- passage_scores.items(), key=lambda x: x[1], reverse=True,- )[:k]- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]\ No newline at end of filediff --git a/dspy/retrieve/mongodb_atlas_rm.py b/dspy/retrieve/mongodb_atlas_rm.pydeleted file mode 100644index 0a83017c6..000000000--- a/dspy/retrieve/mongodb_atlas_rm.py+++ /dev/null@@ -1,120 +0,0 @@-import os-from typing import Any, List--import backoff-from openai import (- APITimeoutError,- InternalServerError,- OpenAI,- RateLimitError,- UnprocessableEntityError,-)--from dspy.dsp.utils.settings import settings-from dspy import Retrieve, Prediction--try:- from pymongo import MongoClient- from pymongo.errors import (- ConfigurationError,- ConnectionFailure,- InvalidURI,- OperationFailure,- ServerSelectionTimeoutError,- )-except ImportError:- raise ImportError(- "Please install the pymongo package by running `pip install dspy-ai[mongodb]`",- )---def build_vector_search_pipeline(- index_name: str, query_vector: List[float], num_candidates: int, limit: int,-) -> List[dict[str, Any]]:- return [- {- "$vectorSearch": {- "index": index_name,- "path": "embedding",- "queryVector": query_vector,- "numCandidates": num_candidates,- "limit": limit,- },- },- {"$project": {"_id": 0, "text": 1, "score": {"$meta": "vectorSearchScore"}}},- ]---class Embedder:- def __init__(self, provider: str, model: str):- if provider == "openai":- api_key = os.getenv("OPENAI_API_KEY")- if not api_key:- raise ValueError("Environment variable OPENAI_API_KEY must be set")- self.client = OpenAI()- self.model = model-- @backoff.on_exception(- backoff.expo,- (- APITimeoutError,- InternalServerError,- RateLimitError,- UnprocessableEntityError,- ),- max_time=settings.backoff_time,- )- def __call__(self, queries) -> Any:- embedding = self.client.embeddings.create(input=queries, model=self.model)- return [result.embedding for result in embedding.data]---class MongoDBAtlasRM(Retrieve):- def __init__(- self,- db_name: str,- collection_name: str,- index_name: str,- k: int = 5,- embedding_provider: str = "openai",- embedding_model: str = "text-embedding-ada-002",- ):- super().__init__(k=k)- self.db_name = db_name- self.collection_name = collection_name- self.index_name = index_name- self.username = os.getenv("ATLAS_USERNAME")- self.password = os.getenv("ATLAS_PASSWORD")- self.cluster_url = os.getenv("ATLAS_CLUSTER_URL")- if not self.username:- raise ValueError("Environment variable ATLAS_USERNAME must be set")- if not self.password:- raise ValueError("Environment variable ATLAS_PASSWORD must be set")- if not self.cluster_url:- raise ValueError("Environment variable ATLAS_CLUSTER_URL must be set")- try:- self.client = MongoClient(- f"mongodb+srv://{self.username}:{self.password}@{self.cluster_url}/{self.db_name}"- "?retryWrites=true&w=majority",- )- except (- InvalidURI,- ConfigurationError,- ConnectionFailure,- ServerSelectionTimeoutError,- OperationFailure,- ) as e:- raise ConnectionError("Failed to connect to MongoDB Atlas") from e-- self.embedder = Embedder(provider=embedding_provider, model=embedding_model)-- def forward(self, query_or_queries: str) -> Prediction:- query_vector = self.embedder([query_or_queries])- pipeline = build_vector_search_pipeline(- index_name=self.index_name,- query_vector=query_vector[0],- num_candidates=self.k * 10,- limit=self.k,- )- contents = self.client[self.db_name][self.collection_name].aggregate(pipeline)- return Prediction(passages=list(contents))diff --git a/dspy/retrieve/my_scale_rm.py b/dspy/retrieve/my_scale_rm.pydeleted file mode 100644index 8d61d0028..000000000--- a/dspy/retrieve/my_scale_rm.py+++ /dev/null@@ -1,231 +0,0 @@-# import functools-# import os-# from typing import List, Optional--# import openai--# import dspy-# from dspy.dsp.modules.cache_utils import NotebookCacheMemory, cache_turn_on-# from dspy.dsp.utils import dotdict--# # Check for necessary libraries and suggest installation if not found.-# try:-# import clickhouse_connect-# except ImportError:-# raise ImportError(-# "The 'myscale' extra is required to use MyScaleRM. Install it with `pip install dspy-ai[myscale]`",-# )--# # Verify the compatibility of the OpenAI library version installed.-# try:-# major, minor, _ = map(int, openai.__version__.split("."))-# OPENAI_VERSION_COMPATIBLE = major >= 1 and minor >= 16-# except Exception:-# OPENAI_VERSION_COMPATIBLE = False--# if not OPENAI_VERSION_COMPATIBLE:-# raise ImportError(-# "An incompatible OpenAI library version is installed. Ensure you have version 1.16.1 or later.",-# )--# # Attempt to handle specific OpenAI errors; fallback to general ones if necessary.-# try:-# import openai.error--# ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)-# except Exception:-# ERRORS = (openai.RateLimitError, openai.APIError)---# class MyScaleRM(dspy.Retrieve):-# """-# A retrieval module that uses MyScaleDB to return the top passages for a given query.--# MyScaleDB is a fork of ClickHouse that focuses on vector similarity search and full-# text search. MyScaleRM is designed to facilitate easy retrieval of information from-# MyScaleDB using embeddings. It supports embedding generation through either a local-# model or the OpenAI API. This class abstracts away the complexities of connecting to-# MyScaleDB, managing API keys, and processing queries to return semantically-# relevant results.--# Assumes that a table named `database.table` exists in MyScaleDB, and that the-# table has column named `vector_column` that stores vector data and a vector index has-# been created on this column. Other metadata are stored in `metadata_columns`.--# Args:-# client (clickhouse_connect.driver.client.Client): A client connection to the MyScaleDB.-# table (str): Name of the table within the database to perform queries against.-# database (str, optional): Name of the database to query within MyScaleDB.-# metadata_columns(List[str], optional): A list of columns to include in the results.-# vector_column (str, optional): The name of the column in the table that stores vector data.-# k (int, optional): The number of closest matches to retrieve for a given query.-# openai_api_key (str, optional): The API key for accessing OpenAI's services.-# model (str, optional): Specifies the particular OpenAI model to use for embedding generation.-# use_local_model (bool): Flag indicating whether a local model is used for embeddings.--# """--# def __init__(-# self,-# client: clickhouse_connect.driver.client.Client,-# table: str,-# database: str = "default",-# metadata_columns: List[str] = ["text"],-# vector_column: str = "vector",-# k: int = 3,-# openai_api_key: Optional[str] = None,-# openai_model: Optional[str] = None,-# local_embed_model: Optional[str] = None,-# ):-# self.client = client-# self.database = database-# self.table = table-# if not metadata_columns:-# raise ValueError("metadata_columns is required")-# self.metadata_columns = metadata_columns-# self.vector_column = vector_column-# self.k = k-# self.openai_api_key = openai_api_key-# self.model = openai_model-# self.use_local_model = False--# if local_embed_model:-# self.setup_local_model(local_embed_model)-# elif openai_api_key:-# os.environ["OPENAI_API_KEY"] = self.openai_api_key--# def setup_local_model(self, model_name: str):-# """-# Configures a local model for embedding generation, including model and tokenizer loading.--# Args:-# model_name: The name or path to the pre-trained model to load.--# Raises:-# ModuleNotFoundError: If necessary libraries (torch or transformers) are not installed.-# """-# try:-# import torch-# from transformers import AutoModel, AutoTokenizer-# except ImportError as exc:-# raise ModuleNotFoundError(-# """You need to install PyTorch and Hugging Face's transformers library to use a local embedding model.-# Install the pytorch using `pip install torch` and transformers using `pip install transformers` """,-# ) from exc--# try:-# self._local_embed_model = AutoModel.from_pretrained(model_name)-# self._local_tokenizer = AutoTokenizer.from_pretrained(model_name)-# self.use_local_model = True-# except Exception as e:-# raise ValueError(f"Failed to load model or tokenizer. Error: {str(e)}")--# if torch.cuda.is_available():-# self.device = torch.device("cuda:0")-# elif torch.backends.mps.is_available():-# self.device = torch.device("mps")-# else:-# self.device = torch.device("cpu")--# self._local_embed_model.to(self.device)--# @functools.lru_cache(maxsize=None if cache_turn_on else 0)-# @NotebookCacheMemory.cache-# def get_embeddings(self, query: str) -> List[float]:-# """-# Determines the appropriate source (OpenAI or local model) for embedding generation based on class configuration,-# and retrieves embeddings for the provided queries.--# Args:-# query: A query to generate embeddings for.--# Returns:-# A list of embeddings corresponding to the query in the input list.--# Raises:-# ValueError: If neither an OpenAI API key nor a local model has been configured.-# """-# if self.openai_api_key and self.model:-# return self._get_embeddings_from_openai(query)-# elif self.use_local_model:-# return self._get_embedding_from_local_model(query)-# else:-# raise ValueError("No valid method for obtaining embeddings is configured.")--# # TO DO Add this method as Util method outside MyScaleRM-# def _get_embeddings_from_openai(self, query: str) -> List[float]:-# """-# Uses the OpenAI API to generate embeddings for the given query.--# Args:-# query: A string for which to generate embeddings.--# Returns:-# A list containing the embedding of a query.-# """-# response = openai.embeddings.create(model=self.model, input=query)-# return response.data[0].embedding--# # TO DO Add this method as Util method outside MyScaleRM-# def _get_embedding_from_local_model(self, query: str) -> List[float]:-# """-# Generates embeddings for a single query using the configured local model.--# Args:-# query: The text query to generate an embedding for.--# Returns:-# A list representing the query's embedding.-# """-# import torch--# self._local_embed_model.eval() # Ensure the model is in evaluation mode--# inputs = self._local_tokenizer(query, return_tensors="pt", padding=True, truncation=True).to(self.device)-# with torch.no_grad():-# output = self._local_embed_model(**inputs)--# return output.last_hidden_state.mean(dim=1).squeeze().numpy().tolist()--# def forward(self, user_query: str, k: Optional[int] = None) -> List[dotdict]:-# """-# Executes a retrieval operation based on a user's query and returns the top k relevant results.--# Args:-# user_query: The query text to search for.-# k: Optional; The number of top matches to return. Defaults to the class's configured k value.--# Returns:-# A list of dotdict objects containing the formatted retrieval results.--# Raises:-# ValueError: If the user_query is None.-# """-# if user_query is None:-# raise ValueError("Query is required")-# k = k or self.k-# embeddings = self.get_embeddings(user_query)-# columns_string = ", ".join(self.metadata_columns)-# result = self.client.query(f"""-# SELECT {columns_string},-# distance({self.vector_column}, {embeddings}) as dist FROM {self.database}.{self.table} ORDER BY dist LIMIT {k}-# """)-# result = self.client.query(f"""-# SELECT {columns_string}, distance({self.vector_column}, {embeddings}) AS dist-# FROM {self.database}.{self.table}-# ORDER BY dist-# LIMIT {k}-# """)--# # Convert the metadata into strings to pass to dspy.Prediction-# results = []-# for row in result.named_results():-# if len(self.metadata_columns) == 1:-# results.append(row[self.metadata_columns[0]])-# else:-# row_strings = [f"{column}: {row[column]}" for column in self.metadata_columns] # Format row data-# row_string = "\n".join(row_strings) # Combine formatted data-# results.append(row_string) # Append to results--# # Return list of dotdict-# return [dotdict({"long_text": passage}) for passage in results]diff --git a/dspy/retrieve/neo4j_rm.py b/dspy/retrieve/neo4j_rm.pydeleted file mode 100644index 3d4f65a12..000000000--- a/dspy/retrieve/neo4j_rm.py+++ /dev/null@@ -1,160 +0,0 @@-import os-from typing import Any, List, Optional, Union, Callable--import backoff-from openai import (- APITimeoutError,- InternalServerError,- OpenAI,- RateLimitError,- UnprocessableEntityError,-)--from dspy import Retrieve, Prediction-from dspy.dsp.utils.settings import settings-from dspy.dsp.utils import dotdict--try:- from neo4j import GraphDatabase- from neo4j.exceptions import (- AuthError,- ServiceUnavailable,- )-except ImportError:- raise ImportError(- "Please install the neo4j package by running `pip install dspy-ai[neo4j]`",- )---class Embedder:- def __init__(self, provider: str, model: str):- if provider == "openai":- api_key = os.getenv("OPENAI_API_KEY")- if not api_key:- raise ValueError("Environment variable OPENAI_API_KEY must be set")- self.client = OpenAI()- self.model = model-- @backoff.on_exception(- backoff.expo,- (- APITimeoutError,- InternalServerError,- RateLimitError,- UnprocessableEntityError,- ),- max_time=settings.backoff_time,- )- def __call__(self, queries) -> Any:- embedding = self.client.embeddings.create(input=queries, model=self.model)- return [result.embedding for result in embedding.data]---DEFAULT_INDEX_QUERY = "CALL db.index.vector.queryNodes($index, $k, $embedding) YIELD node, score "---class Neo4jRM(Retrieve):- """- Implements a retriever that utilizes Neo4j for retrieving passages.- This class manages a connection to a Neo4j database using official Neo4j Python drivers and requires- the database credentials (username, password, URI, and optionally the database name) to be set as environment variables.- Additionally, it utilizes an embedding provider (defaulting to OpenAI's services) to compute query embeddings,- which are then used to find the most relevant nodes in the Neo4j graph based on the specified node property or custom retrieval query.-- Returns a list of passages in the form of `dspy.Prediction` objects.-- Args:- index_name (str): The name of the vector index in the Neo4j database to query against.- text_node_property (Optional[str]): The property of the node containing the text. Required if `retrieval_query` is not set.- k (Optional[int]): The default number of top passages to retrieve. Defaults to 5.- retrieval_query (Optional[str]): Custom Cypher query for retrieving passages. Required if `text_node_property` is not set.- embedding_provider (str): The provider of the embedding service. Defaults to "openai".- embedding_model (str): The model identifier for generating embeddings. Defaults to "text-embedding-ada-002".-- Examples:- Below is a code snippet showcasing how to initialize Neo4jRM with environment variables for the database connection and OpenAI as the embedding provider:-- ```python- import os-- import dspy- import openai-- os.environ["NEO4J_URI"] = "bolt://localhost:7687"- os.environ["NEO4J_USERNAME"] = "neo4j"- os.environ["NEO4J_PASSWORD"] = "password"- os.environ["OPENAI_API_KEY"] = "sk-"-- neo4j_retriever = Neo4jRM(- index_name="myIndex",- text_node_property="text",- k=10,- embedding_provider="openai",- embedding_model="text-embedding-ada-002",- )-- dspy.settings.configure(rm=neo4j_retriever)- ```-- In this example, `Neo4jRM` is configured to retrieve nodes based on the "text" property from an index named "myIndex",- using embeddings computed by OpenAI's "text-embedding-ada-002" model.- """-- def __init__(- self,- index_name: str,- text_node_property: str = None,- k: int = 5,- retrieval_query: str = None,- embedding_provider: str = "openai",- embedding_model: str = "text-embedding-ada-002",- embedding_function: Optional[Callable] = None,- ):- super().__init__(k=k)- self.index_name = index_name- self.username = os.getenv("NEO4J_USERNAME")- self.password = os.getenv("NEO4J_PASSWORD")- self.uri = os.getenv("NEO4J_URI")- self.database = os.getenv("NEO4J_DATABASE", "neo4j")- self.k = k- self.retrieval_query = retrieval_query- self.text_node_property = text_node_property- if not self.username:- raise ValueError("Environment variable NEO4J_USERNAME must be set")- if not self.password:- raise ValueError("Environment variable NEO4J_PASSWORD must be set")- if not self.uri:- raise ValueError("Environment variable NEO4J_URI must be set")- if not self.text_node_property and not self.retrieval_query:- raise ValueError("Either `text_node_property` or `retrieval_query` parameters must be defined")- try:- self.driver = GraphDatabase.driver(self.uri, auth=(self.username, self.password))- self.driver.verify_connectivity()-- except (- ServiceUnavailable,- AuthError,- ) as e:- raise ConnectionError("Failed to connect to Neo4j database") from e-- self.embedder = embedding_function or Embedder(provider=embedding_provider, model=embedding_model)-- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> Prediction:- if not isinstance(query_or_queries, list):- query_or_queries = [query_or_queries]- query_vectors = self.embedder(query_or_queries)- contents = []- retrieval_query = self.retrieval_query or f"RETURN node.{self.text_node_property} AS text, score"- for vector in query_vectors:- records, _, _ = self.driver.execute_query(- DEFAULT_INDEX_QUERY + retrieval_query,- {"embedding": vector, "index": self.index_name, "k": k or self.k},- database_=self.database,- )- contents.extend([{"passage": dotdict({"long_text": r["text"]}), "score": r["score"]} for r in records])- sorted_passages = sorted(- contents,- key=lambda x: x["score"],- reverse=True,- )[: k or self.k]- return [el["passage"] for el in sorted_passages]diff --git a/dspy/retrieve/pgvector_rm.py b/dspy/retrieve/pgvector_rm.pydeleted file mode 100644index b05767c4a..000000000--- a/dspy/retrieve/pgvector_rm.py+++ /dev/null@@ -1,158 +0,0 @@-import warnings-from typing import Callable, Optional--import dspy--try:- import psycopg2- from pgvector.psycopg2 import register_vector- from psycopg2 import sql-except ImportError:- raise ImportError(- "The 'pgvector' extra is required to use PgVectorRM. Install it with `pip install dspy-ai[pgvector]`. Also, try `pip install pgvector psycopg2`.",- )-try:- import openai-except ImportError:- warnings.warn(- "`openai` is not installed. Install it with `pip install openai` to use OpenAI embedding models.",- category=ImportWarning,- )---class PgVectorRM(dspy.Retrieve):- """- Implements a retriever that (as the name suggests) uses pgvector to retrieve passages,- using a raw SQL query and a postgresql connection managed by psycopg2.-- It needs to register the pgvector extension with the psycopg2 connection-- Returns a list of dspy.Example objects-- Args:- db_url (str): A PostgreSQL database URL in psycopg2's DSN format- pg_table_name (Optional[str]): name of the table containing passages- openai_client (openai.OpenAI): OpenAI client to use for computing query embeddings. Either openai_client or embedding_func must be provided.- embedding_func (Callable): A function to use for computing query embeddings. Either openai_client or embedding_func must be provided.- content_field (str = "text"): Field containing the passage text. Defaults to "text"- k (Optional[int]): Default number of top passages to retrieve. Defaults to 20- embedding_field (str = "embedding"): Field containing passage embeddings. Defaults to "embedding"- fields (List[str] = ['text']): Fields to retrieve from the table. Defaults to "text"- embedding_model (str = "text-embedding-ada-002"): Field containing the OpenAI embedding model to use. Defaults to "text-embedding-ada-002"-- Examples:- Below is a code snippet that shows how to use PgVector as the default retriever-- ```python- import dspy- import openai- import psycopg2-- openai.api_key = os.environ.get("OPENAI_API_KEY", None)- openai_client = openai.OpenAI()-- llm = dspy.OpenAI(model="gpt-3.5-turbo")-- DATABASE_URL should be in the format postgresql://user:password@host/database- db_url=os.getenv("DATABASE_URL")-- retriever_model = PgVectorRM(conn, openai_client=openai_client, "paragraphs", fields=["text", "document_id"], k=20)- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use PgVector in the forward() function of a module- ```python- self.retrieve = PgVectorRM(db_url, openai_client=openai_client, "paragraphs", fields=["text", "document_id"], k=20)- ```- """-- def __init__(- self,- db_url: str,- pg_table_name: str,- openai_client: Optional[openai.OpenAI] = None,- embedding_func: Optional[Callable] = None,- k: int = 20,- embedding_field: str = "embedding",- fields: Optional[list[str]] = None,- content_field: str = "text",- embedding_model: str = "text-embedding-ada-002",- include_similarity: bool = False,- ):- """- k = 20 is the number of paragraphs to retrieve- """- assert (- openai_client or embedding_func- ), "Either openai_client or embedding_func must be provided."- self.openai_client = openai_client- self.embedding_func = embedding_func-- self.conn = psycopg2.connect(db_url)- register_vector(self.conn)- self.pg_table_name = pg_table_name- self.fields = fields or ["text"]- self.content_field = content_field- self.embedding_field = embedding_field- self.embedding_model = embedding_model- self.include_similarity = include_similarity-- super().__init__(k=k)-- def forward(self, query: str, k: int = None):- """Search with PgVector for k top passages for query using cosine similarity-- Args:- query (str): The query to search for- k (int): The number of top passages to retrieve. Defaults to the value set in the constructor.- Returns:- dspy.Prediction: an object containing the retrieved passages.- """- # Embed query- query_embedding = self._get_embeddings(query)-- retrieved_docs = []-- fields = sql.SQL(",").join([sql.Identifier(f) for f in self.fields])- if self.include_similarity:- similarity_field = sql.SQL(",") + sql.SQL(- "1 - ({embedding_field} <=> %s::vector) AS similarity",- ).format(embedding_field=sql.Identifier(self.embedding_field))- fields += similarity_field- args = (query_embedding, query_embedding, k if k else self.k)- else:- args = (query_embedding, k if k else self.k)-- sql_query = sql.SQL(- "select {fields} from {table} order by {embedding_field} <=> %s::vector limit %s",- ).format(- fields=fields,- table=sql.Identifier(self.pg_table_name),- embedding_field=sql.Identifier(self.embedding_field),- )-- with self.conn as conn:- with conn.cursor() as cur:- cur.execute(sql_query, args)- rows = cur.fetchall()- columns = [descrip[0] for descrip in cur.description]- for row in rows:- data = dict(zip(columns, row))- data["long_text"] = data[self.content_field]- retrieved_docs.append(dspy.Example(**data))- # Return Prediction- return retrieved_docs-- def _get_embeddings(self, query: str) -> list[float]:- if self.openai_client is not None:- return (- self.openai_client.embeddings.create(- model=self.embedding_model,- input=query,- encoding_format="float",- )- .data[0]- .embedding- )- else:- return self.embedding_func(query)diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.pydeleted file mode 100644index f5fe245af..000000000--- a/dspy/retrieve/pinecone_rm.py+++ /dev/null@@ -1,273 +0,0 @@-"""-Retriever model for Pinecone-Author: Dhar Rawal (@drawal1)-"""--from typing import List, Optional, Union--import backoff--from dspy import Retrieve, Prediction-from dspy.dsp.utils.settings import settings-from dspy.dsp.utils import dotdict--try:- import pinecone-except ImportError:- pinecone = None--if pinecone is None:- raise ImportError(- "The pinecone library is required to use PineconeRM. Install it with `pip install dspy-ai[pinecone]`",- )--import openai--try:- OPENAI_LEGACY = int(openai.version.__version__[0]) == 0-except Exception:- OPENAI_LEGACY = True--try:- import openai.error- ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError)-except Exception:- ERRORS = (openai.RateLimitError, openai.APIError)--class PineconeRM(Retrieve):- """- A retrieval module that uses Pinecone to return the top passages for a given query.-- Assumes that the Pinecone index has been created and populated with the following metadata:- - text: The text of the passage-- Args:- pinecone_index_name (str): The name of the Pinecone index to query against.- pinecone_api_key (str, optional): The Pinecone API key. Defaults to None.- pinecone_env (str, optional): The Pinecone environment. Defaults to None.- local_embed_model (str, optional): The local embedding model to use. A popular default is "sentence-transformers/all-mpnet-base-v2".- openai_embed_model (str, optional): The OpenAI embedding model to use. Defaults to "text-embedding-ada-002".- openai_api_key (str, optional): The API key for OpenAI. Defaults to None.- openai_org (str, optional): The organization for OpenAI. Defaults to None.- k (int, optional): The number of top passages to retrieve. Defaults to 3.-- Returns:- dspy.Prediction: An object containing the retrieved passages.-- Examples:- Below is a code snippet that shows how to use this as the default retriever:- ```python- llm = dspy.OpenAI(model="gpt-3.5-turbo")- retriever_model = PineconeRM(openai.api_key)- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use this in the forward() function of a module- ```python- self.retrieve = PineconeRM(k=num_passages)- ```- """-- def __init__(- self,- pinecone_index_name: str,- pinecone_api_key: Optional[str] = None,- pinecone_env: Optional[str] = None,- local_embed_model: Optional[str] = None,- openai_embed_model: Optional[str] = "text-embedding-ada-002",- openai_api_key: Optional[str] = None,- openai_org: Optional[str] = None,- k: int = 3,- ):- if local_embed_model is not None:- try:- import torch- from transformers import AutoModel, AutoTokenizer- except ImportError as exc:- raise ModuleNotFoundError(- "You need to install Hugging Face transformers (with torch dependencies - pip install transformers[torch]) library to use a local embedding model with PineconeRM.",- ) from exc-- self._local_embed_model = AutoModel.from_pretrained(local_embed_model)- self._local_tokenizer = AutoTokenizer.from_pretrained(local_embed_model)- self.use_local_model = True- self.device = torch.device(- 'cuda:0' if torch.cuda.is_available() else- 'mps' if torch.backends.mps.is_available()- else 'cpu',- )- elif openai_embed_model is not None:- self._openai_embed_model = openai_embed_model- self.use_local_model = False- # If not provided, defaults to env vars OPENAI_API_KEY and OPENAI_ORGANIZATION- if openai_api_key:- openai.api_key = openai_api_key- if openai_org:- openai.organization = openai_org- else:- raise ValueError(- "Either local_embed_model or openai_embed_model must be provided.",- )-- self._pinecone_index = self._init_pinecone(- pinecone_index_name, pinecone_api_key, pinecone_env,- )-- super().__init__(k=k)-- def _init_pinecone(- self,- index_name: str,- api_key: Optional[str] = None,- environment: Optional[str] = None,- dimension: Optional[int] = None,- distance_metric: Optional[str] = None,- ) -> pinecone.Index:- """Initialize pinecone and return the loaded index.-- Args:- index_name (str): The name of the index to load. If the index is not does not exist, it will be created.- api_key (str, optional): The Pinecone API key, defaults to env var PINECONE_API_KEY if not provided.- environment (str, optional): The environment (ie. `us-west1-gcp` or `gcp-starter`. Defaults to env PINECONE_ENVIRONMENT.-- Raises:- ValueError: If api_key or environment is not provided and not set as an environment variable.-- Returns:- pinecone.Index: The loaded index.- """-- # Pinecone init overrides default if kwargs are present, so we need to exclude if None- kwargs = {}- if api_key:- kwargs["api_key"] = api_key- if environment:- kwargs["environment"] = environment- pinecone.init(**kwargs)-- active_indexes = pinecone.list_indexes()- if index_name not in active_indexes:- if dimension is None and distance_metric is None:- raise ValueError(- "dimension and distance_metric must be provided since the index provided does not exist.",- )-- pinecone.create_index(- name=index_name,- dimension=dimension,- metric=distance_metric,- )-- return pinecone.Index(index_name)-- def _mean_pooling(- self,- model_output,- attention_mask,- ):- try:- import torch- except ImportError as exc:- raise ModuleNotFoundError(- "You need to install torch to use a local embedding model with PineconeRM.",- ) from exc-- token_embeddings = model_output[0] # First element of model_output contains all token embeddings- input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()- return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)-- @backoff.on_exception(- backoff.expo,- ERRORS,- max_time=settings.backoff_time,- )- def _get_embeddings(- self,- queries: List[str],- ) -> List[List[float]]:- """Return query vector after creating embedding using OpenAI-- Args:- queries (list): List of query strings to embed.-- Returns:- List[List[float]]: List of embeddings corresponding to each query.- """- try:- import torch- except ImportError as exc:- raise ModuleNotFoundError(- "You need to install torch to use a local embedding model with PineconeRM.",- ) from exc-- if not self.use_local_model:- if OPENAI_LEGACY:- embedding = openai.Embedding.create(- input=queries, model=self._openai_embed_model,- )- else:- embedding = openai.embeddings.create(- input=queries, model=self._openai_embed_model,- ).model_dump()- return [embedding["embedding"] for embedding in embedding["data"]]-- # Use local model- encoded_input = self._local_tokenizer(queries, padding=True, truncation=True, return_tensors="pt").to(self.device)- with torch.no_grad():- model_output = self._local_embed_model(**encoded_input.to(self.device))-- embeddings = self._mean_pooling(model_output, encoded_input['attention_mask'])- normalized_embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)- return normalized_embeddings.cpu().numpy().tolist()-- # we need a pooling strategy to get a single vector representation of the input- # so the default is to take the mean of the hidden states-- def forward(self, query_or_queries: Union[str, List[str]]) -> Prediction:- """Search with pinecone for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q] # Filter empty queries- embeddings = self._get_embeddings(queries)-- # For single query, just look up the top k passages- if len(queries) == 1:- results_dict = self._pinecone_index.query(- embeddings[0], top_k=self.k, include_metadata=True,- )-- # Sort results by score- sorted_results = sorted(- results_dict["matches"], key=lambda x: x.get("scores", 0.0), reverse=True,- )- passages = [result["metadata"]["text"] for result in sorted_results]- passages = [dotdict({"long_text": passage for passage in passages})]- return Prediction(passages=passages)-- # For multiple queries, query each and return the highest scoring passages- # If a passage is returned multiple times, the score is accumulated. For this reason we increase top_k by 3x- passage_scores = {}- for embedding in embeddings:- results_dict = self._pinecone_index.query(- embedding, top_k=self.k * 3, include_metadata=True,- )- for result in results_dict["matches"]:- passage_scores[result["metadata"]["text"]] = (- passage_scores.get(result["metadata"]["text"], 0.0)- + result["score"]- )-- sorted_passages = sorted(- passage_scores.items(), key=lambda x: x[1], reverse=True,- )[: self.k]- return Prediction(passages=[dotdict({"long_text": passage}) for passage, _ in sorted_passages])diff --git a/dspy/retrieve/qdrant_rm.py b/dspy/retrieve/qdrant_rm.pydeleted file mode 100644index 956adeab4..000000000--- a/dspy/retrieve/qdrant_rm.py+++ /dev/null@@ -1,116 +0,0 @@-# from collections import defaultdict-# from typing import Optional, Union--# import dspy-# from dspy.dsp.modules.sentence_vectorizer import BaseSentenceVectorizer, FastEmbedVectorizer-# from dspy.dsp.utils import dotdict--# try:-# from qdrant_client import QdrantClient, models-# except ImportError as e:-# raise ImportError(-# "The 'qdrant' extra is required to use QdrantRM. Install it with `pip install dspy-ai[qdrant]`",-# ) from e---# class QdrantRM(dspy.Retrieve):-# """A retrieval module that uses Qdrant to return the top passages for a given query.--# Args:-# qdrant_collection_name (str): The name of the Qdrant collection.-# qdrant_client (QdrantClient): An instance of `qdrant_client.QdrantClient`.-# k (int, optional): The default number of top passages to retrieve. Default: 3.-# document_field (str, optional): The key in the Qdrant payload with the content. Default: `"document"`.-# vectorizer (BaseSentenceVectorizer, optional): An implementation `BaseSentenceVectorizer`.-# Default: `FastEmbedVectorizer`.-# vector_name (str, optional): Name of the vector in the collection. Default: The first available vector name.--# Examples:-# Below is a code snippet that shows how to use Qdrant as the default retriver:-# ```python-# from qdrant_client import QdrantClient--# llm = dspy.OpenAI(model="gpt-3.5-turbo")-# qdrant_client = QdrantClient()-# retriever_model = QdrantRM("my_collection_name", qdrant_client=qdrant_client)-# dspy.settings.configure(lm=llm, rm=retriever_model)-# ```--# Below is a code snippet that shows how to use Qdrant in the forward() function of a module-# ```python-# self.retrieve = QdrantRM(question, k=num_passages, filter=filter)-# ```-# """--# def __init__(-# self,-# qdrant_collection_name: str,-# qdrant_client: QdrantClient,-# k: int = 3,-# document_field: str = "document",-# vectorizer: Optional[BaseSentenceVectorizer] = None,-# vector_name: Optional[str] = None,-# ):-# self._collection_name = qdrant_collection_name-# self._client = qdrant_client--# self._vectorizer = vectorizer or FastEmbedVectorizer(self._client.embedding_model_name)--# self._document_field = document_field--# self._vector_name = vector_name or self._get_first_vector_name()--# super().__init__(k=k)--# def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None, filter: Optional[models.Filter]=None) -> dspy.Prediction:-# """Search with Qdrant for self.k top passages for query.--# Args:-# query_or_queries (Union[str, List[str]]): The query or queries to search for.-# k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.-# filter (Optional["Filter"]): "Look only for points which satisfies this conditions". Default: None.--# Returns:-# dspy.Prediction: An object containing the retrieved passages.-# """-# queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries-# queries = [q for q in queries if q] # Filter empty queries--# vectors = self._vectorizer(queries)--# search_requests = [-# models.QueryRequest(-# query=vector,-# using=self._vector_name,-# limit=k or self.k,-# with_payload=[self._document_field],-# filter=filter,-# )-# for vector in vectors-# ]-# batch_results = self._client.query_batch_points(self._collection_name, requests=search_requests)--# passages_scores = defaultdict(float)-# for batch in batch_results:-# for result in batch.points:-# # If a passage is returned multiple times, the score is accumulated.-# document = result.payload.get(self._document_field)-# passages_scores[document] += result.score--# # Sort passages by their accumulated scores in descending order-# sorted_passages = sorted(passages_scores.items(), key=lambda x: x[1], reverse=True)[:k]--# # Wrap each sorted passage in a dotdict with 'long_text'-# return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]--# def _get_first_vector_name(self) -> Optional[str]:-# vectors = self._client.get_collection(self._collection_name).config.params.vectors--# if not isinstance(vectors, dict):-# # The collection only has the default, unnamed vector-# return None--# first_vector_name = list(vectors.keys())[0]--# # The collection has multiple vectors. Could also include the falsy unnamed vector - Empty string("")-# return first_vector_name or Nonediff --git a/dspy/retrieve/ragatouille_rm.py b/dspy/retrieve/ragatouille_rm.pydeleted file mode 100644index 54f31baa0..000000000--- a/dspy/retrieve/ragatouille_rm.py+++ /dev/null@@ -1,62 +0,0 @@-from typing import Optional, Union--import dspy-from dspy.dsp.utils.utils import dotdict--try:- from ragatouille import RAGPretrainedModel-except ImportError:- raise Exception(- "You need to install RAGAtouille library, Please use the command: pip install ragatouille",- )---class RAGatouilleRM(dspy.Retrieve):- """A retrieval model that uses RAGatouille library to return the top passages for a given query.-- Assumes that you already have an index created with RAGatouille.- Reference: https://github.com/bclavie/RAGatouille-- Args:- index_root (str): Folder path where you index is stored.- index_name (str): Name of the index you want to retrieve from.- k (int, optional): The default number of passages to retrieve. Defaults to 3.-- Examples:- Below is a code snippet that shows how to use RAGatouille index as the default retriver:- ```python- llm = dspy.OpenAI(model="gpt-3.5-turbo")- rm = RAGatouilleRM(index_root="ragatouille/colbert/indexes", index_name="my_index")- dspy.settings.configure(lm=llm, rm=rm)- ```- """-- def __init__(self, index_root: str, index_name: str, k: int = 3):- super().__init__(k=k)- self.index_path = f"{index_root}/{index_name}"- try:- self.model = RAGPretrainedModel.from_index(index_path=self.index_path)- except FileNotFoundError as e:- raise FileNotFoundError(f"Index not found: {e}")-- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]) -> dspy.Prediction:- """Search with RAGAtouille based index for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- k = k if k is not None else self.k- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q]- passages = []- for query in queries:- results = self.model.search(query=query, k=k)- passages.extend(dotdict({"long_text": d["content"]}) for d in results)- return dspy.Prediction(passages=passages)\ No newline at end of filediff --git a/dspy/retrieve/snowflake_rm.py b/dspy/retrieve/snowflake_rm.pydeleted file mode 100644index 14048197b..000000000--- a/dspy/retrieve/snowflake_rm.py+++ /dev/null@@ -1,291 +0,0 @@-import json-import re-from typing import Any, Optional, Type, Union--from pydantic import BaseModel, Field, ValidationError--import dspy-from dspy.dsp.utils import dotdict--try:- from snowflake.core import Root- from snowflake.snowpark.functions import col-except ImportError:- raise ImportError(- "The snowflake-snowpark-python library is required to use SnowflakeRM. Install it with dspy-ai[snowflake]",- )---class SnowflakeRM(dspy.Retrieve):- """A retrieval module that uses Snowflake's Cortex Search service to return the top relevant passages for a given query.-- Assumes that a Snowflake Cortex Search endpoint has been configured by the use.-- For more information on configuring the Cortex Search service, visit: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-search/cortex-search-overview-- Args:- snowflake_session (object): Snowflake Snowpark session for accessing the service.- cortex_search_service(str): Name of the Cortex Search service to be used.- snowflake_database (str): The name of the Snowflake table containing document embeddings.- snowflake_schema (str): The name of the Snowflake table containing document embeddings.- auto_filter (bool): Auto generate metadata filter based on user query and push it down prior to retrieving Cortex Search results.- k (int, optional): The default number of top passages to retrieve. Defaults to 3.- """-- def __init__(- self,- snowflake_session: object,- cortex_search_service: str,- snowflake_database: str,- snowflake_schema: str,- auto_filter=False,- k: int = 3,- max_retries=3,- ):- super().__init__(k=k)- self.k = k- self.auto_filter = auto_filter- self.max_retries = max_retries- self.cortex_search_service_name = cortex_search_service- self.client = self._fetch_cortex_service(- snowflake_session, snowflake_database, snowflake_schema, cortex_search_service- )-- if self.auto_filter:- self.sample_values = self._get_sample_values(- snowpark_session=snowflake_session, cortex_search_service=cortex_search_service- )- self.optimized_filter_gen = SmartSearch()- self.filter_lm = dspy.Snowflake(session=snowflake_session, model="mixtral-8x7b")-- def forward(- self,- query_or_queries: Union[str, list[str]],- retrieval_columns: list[str],- filter: Optional[dict] = None,- k: Optional[int] = None,- ) -> dspy.Prediction:- """Query Cortex Search endpoint for top k relevant passages.- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.- retrieval_columns (List[str]): Columns to include in response.- filter (Optional[json]):Filter query.- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- k = self.k if k is None else k- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries- queries = [q for q in queries if q]- passages = []-- for cortex_query in queries:- if self.auto_filter:- response_chunks = self._cortex_search_with_auto_filter(- cortex_search_service=self.client, query=cortex_query, columns=retrieval_columns, k=k- )-- else:- response_chunks = self._query_cortex_search(- cortex_search_service=self.client,- query=cortex_query,- columns=retrieval_columns,- filter=filter,- k=k,- )-- if len(retrieval_columns) == 1:- passages.extend(- dotdict({"long_text": passage[retrieval_columns[0]]}) for passage in response_chunks["results"]- )- else:- passages.extend(dotdict({"long_text": str(passage)}) for passage in response_chunks["results"])-- return passages-- def _fetch_cortex_service(self, snowpark_session, snowflake_database, snowflake_schema, cortex_search_service_name):- """Fetch the Cortex Search service to be used"""- snowpark_session.query_tag = {"origin": "sf_sit", "name": "dspy", "version": {"major": 1, "minor": 0}}- root = Root(snowpark_session)-- # fetch service- search_service = (- root.databases[snowflake_database]- .schemas[snowflake_schema]- .cortex_search_services[cortex_search_service_name]- )-- return search_service-- def _query_cortex_search(self, cortex_search_service, query, columns, filter, k):- """Query Cortex Search endpoint for top relevant passages .-- Args:- cortex_search_service (object): cortex search service for querying- query (str): The query or queries to search for.- columns (Optional[list]): A comma-separated list of columns to return for each relevant result in the response. These columns must be included in the source query for the service.- filter (Optional[json]): A filter object for filtering results based on data in the ATTRIBUTES columns. See Filter syntax.- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- # query service- resp = cortex_search_service.search(query=query, columns=columns, filter=filter, limit=k)-- return json.loads(resp.to_json())-- def _cortex_search_with_auto_filter(self, cortex_search_service, query, columns, k):- """Cortex Search Query with automatic metadata filter generation."""-- search_response = None- with dspy.settings.context(lm=self.filter_lm):- if self.auto_filter:- for _ in range(self.max_retries):- raw_filter_query = self.optimized_filter_gen(- query=query, attributes=str([*self.sample_values]), sample_values=str(self.sample_values)- )["answer"]-- try:- filter_query = json.loads(raw_filter_query)- search_response = self._query_cortex_search(- cortex_search_service=cortex_search_service,- query=query,- columns=columns,- filter=filter_query,- k=k,- )- break- except json.JSONDecodeError:- continue- except Exception:- continue-- else:- search_response = self._query_cortex_search(- cortex_search_service=cortex_search_service, query=query, columns=columns, filter=None, k=k- )-- return search_response-- def _get_search_attributes(self, snowpark_session, search_service_name):- df = snowpark_session.sql("SHOW CORTEX SEARCH SERVICES")- raw_atts = (- df.where(col('"name"') == search_service_name).select('"attribute_columns"').to_pandas().loc[0].values[0]- )- attribute_list = raw_atts.split(",")-- return attribute_list-- def _get_search_table(self, snowpark_session, search_service_name):- df = snowpark_session.sql("SHOW CORTEX SEARCH SERVICES")- table_def = df.where(col('"name"') == search_service_name).select('"definition"').to_pandas().loc[0].values[0]-- pattern = r"FROM\s+([\w\.]+)"- match = re.search(pattern, table_def)-- if match:- from_value = match.group(1)- return from_value- else:- print("No match found.")-- return table_def-- def _get_sample_values(self, snowpark_session, cortex_search_service, max_samples=10):- sample_values = {}- attributes = self._get_search_attributes(- snowpark_session=snowpark_session, search_service_name=cortex_search_service- )- table_name = self._get_search_table(- snowpark_session=snowpark_session, search_service_name=cortex_search_service- )-- for attribute in attributes:- query = f"""SELECT DISTINCT({attribute}) FROM {table_name} LIMIT {max_samples}"""- sample_values[attribute] = list(snowpark_session.sql(query).to_pandas()[attribute].values)-- return sample_values---class JSONFilter(BaseModel):- answer: str = Field(description="The filter_query in valid JSON format")-- @classmethod- def model_validate_json(cls, json_data: str, *, strict: bool | None = None, context: dict[str, Any] | None = None):- __tracebackhide__ = True- try:- return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)- except ValidationError:- min_length = get_min_length(cls)- for substring_length in range(len(json_data), min_length - 1, -1):- for start in range(len(json_data) - substring_length + 1):- substring = json_data[start : start + substring_length]- try:- res = cls.__pydantic_validator__.validate_json(substring, strict=strict, context=context)- return res- except ValidationError:- pass- raise ValueError("Could not find valid json")---class GenerateFilter(dspy.Signature):- """- Given a query, attributes in the data, and example values of each attribute, generate a filter in valid JSON format.- Ensure the filter only uses valid operators: @eq, @contains,@and,@or,@not- Ensure only the valid JSON is output with no other reasoning.-- ---- Query: What was the sentiment of CEOs between 2021 and 2024?- Attributes: industry,hq,date- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}- Answer: {"@or":[{"@eq":{"year":"2021"}},{"@eq":{"year":"2022"}},{"@eq":{"year":"2023"}},{"@eq":{"year":"2024"}}]}-- Query: What is the sentiment of Biotech CEO's of companies based in New York?- Attributes: industry,hq,date- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}- Answer: {"@and": [ { "@eq": { "industry"": "biotechnology" } }, { "@eq": { "HQ": "NY,US" } }]}-- Query: What is the sentiment of Biotech CEOs outside of California?- Attributes: industry,hq,date- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}- Answer: {"@and":[{ "@eq": { "industry": "biotechnology" } },{"@not":{"@eq":{"HQ":"CA,US"}}}]}-- Query: What is the sentiment of Biotech CEOs outside of California?- Attributes: industry,hq,date- Sample Values: {"industry":["biotechnology","healthcare","agriculture"],"HQ":["NY, US","CA,US","FL,US"],"date":["01/01,1999","01/01/2024"]}- Answer: {"@and":[{ "@eq": { "industry": "biotechnology" } },{"@not":{"@eq":{"HQ":"CA,US"}}}]}-- Query: What is sentiment towards ag and biotech companies based outside of the US?- Attributes: industry,hq,date- Sample Values: {"industry"":["biotechnology","healthcare","agriculture"],"COUNTRY":["United States","Ireland","Russia","Georgia","Spain"],"month":["01","02","03","06","11","12""],""year"":["2022","2023","2024"]}- Answer:{"@and": [{ "@or": [{"@eq":{ "industry": "biotechnology" } },{"@eq":{"industry":"agriculture"}}]},{ "@not": {"@eq": { "COUNTRY": "United States" } }}]}-- """-- query = dspy.InputField(desc="user query")- attributes = dspy.InputField(desc="attributes to filter on")- sample_values = dspy.InputField(desc="examples of values per attribute")- answer: JSONFilter = dspy.OutputField(- desc="filter query in valid JSON format. ONLY output the filter query in JSON, no reasoning"- )---class SmartSearch(dspy.Module):- def __init__(self):- super().__init__()- self.filter_gen = dspy.ChainOfThought(GenerateFilter)-- def forward(self, query, attributes, sample_values):- filter_query = self.filter_gen(query=query, attributes=attributes, sample_values=sample_values)-- return filter_query---def get_min_length(model: Type[BaseModel]):- min_length = 0- for key, field in model.model_fields.items():- if issubclass(field.annotation, BaseModel):- min_length += get_min_length(field.annotation)- min_length += len(key)- return min_lengthdiff --git a/dspy/retrieve/vectara_rm.py b/dspy/retrieve/vectara_rm.pydeleted file mode 100644index fc1bc9bc0..000000000--- a/dspy/retrieve/vectara_rm.py+++ /dev/null@@ -1,167 +0,0 @@-import json-import os-from collections import defaultdict-from typing import List, Optional, Union--import requests--import dspy-from dspy.dsp.utils import dotdict--START_SNIPPET = "<%START%>"-END_SNIPPET = "<%END%>"--def remove_snippet(s: str) -> str:- return s.replace(START_SNIPPET, "").replace(END_SNIPPET, "")--class VectaraRM(dspy.Retrieve):- """- A retrieval module that uses Vectara to return the top passages for a given query.-- Assumes that a Vectara corpora have been created and populated with the following payload:- - document: The text of the passage-- Args:- vectara_customer_id (str): Vectara Customer ID. defaults to VECTARA_CUSTOMER_ID environment variable- vectara_corpus_id (str): Vectara Corpus ID. defaults to VECTARA_CORPUS_ID environment variable- vectara_api_key (str): Vectara API Key. defaults to VECTARA_API_KEY environment variable- k (int, optional): The default number of top passages to retrieve. Defaults to 3.-- Examples:- Below is a code snippet that shows how to use Vectara as the default retriver:- ```python- from vectara_client import vectaraClient-- llm = dspy.OpenAI(model="gpt-3.5-turbo")- retriever_model = vectaraRM("<VECTARA_CUSTOMER_ID>", "<VECTARA_CORPUS_ID>", "<VECTARA_API_KEY>")- dspy.settings.configure(lm=llm, rm=retriever_model)- ```-- Below is a code snippet that shows how to use Vectara in the forward() function of a module- ```python- self.retrieve = vectaraRM("<VECTARA_CUSTOMER_ID>", "<VECTARA_CORPUS_ID>", "<VECTARA_API_KEY>", k=num_passages)- ```- """-- def __init__(- self,- vectara_customer_id: Optional[str] = None,- vectara_corpus_id: Optional[str] = None,- vectara_api_key: Optional[str] = None,- k: int = 5,- ):- if vectara_customer_id is None:- vectara_customer_id = os.environ.get("VECTARA_CUSTOMER_ID", "")- if vectara_corpus_id is None:- vectara_corpus_id = os.environ.get("VECTARA_CORPUS_ID", "")- if vectara_api_key is None:- vectara_api_key = os.environ.get("VECTARA_API_KEY", "")-- self._vectara_customer_id = vectara_customer_id- self._vectara_corpus_id = vectara_corpus_id- self._vectara_api_key = vectara_api_key- self._n_sentences_before = self._n_sentences_after = 2- self._vectara_timeout = 120- super().__init__(k=k)-- def _vectara_query(- self,- query: str,- limit: int = 5,- ) -> List[str]:- """Query Vectara index to get for top k matching passages.- Args:- query: query string- """- # If multiple corpus ids are provided (comma-separated), create a list of corpus keys- # otherwise by default, the `split(',')` is a no-op so retains the single corpus id- corpus_key = [- {- "customerId": self._vectara_customer_id,- "corpusId": corpus_id,- "lexicalInterpolationConfig": {"lambda": 0.025 },- } for corpus_id in self._vectara_corpus_id.split(',')- ]-- data = {- "query": [- {- "query": query,- "start": 0,- "numResults": limit,- "contextConfig": {- "sentencesBefore": self._n_sentences_before,- "sentencesAfter": self._n_sentences_after,- "startTag": START_SNIPPET,- "endTag": END_SNIPPET,- },- "corpusKey": corpus_key,- },- ],- }-- headers = {- "x-api-key": self._vectara_api_key,- "customer-id": self._vectara_customer_id,- "Content-Type": "application/json",- "X-Source": "dspy",- }-- response = requests.post(- headers=headers,- url="https://api.vectara.io/v1/query",- data=json.dumps(data),- timeout=self._vectara_timeout,- )-- if response.status_code != 200:- print(- "Query failed %s",- f"(code {response.status_code}, reason {response.reason}, details "- f"{response.text})",- )- return []-- result = response.json()- responses = result["responseSet"][0]["response"]-- res = [- {- "text": remove_snippet(x["text"]),- "score": x["score"],- } for x in responses- ]- return res-- def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:- """Search with Vectara for self.k top passages for query-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.- k (Optional[int]): The number of top passages to retrieve. Defaults to self.k.- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = (- [query_or_queries]- if isinstance(query_or_queries, str)- else query_or_queries- )- queries = [q for q in queries if q] # Filter empty queries- k = k if k is not None else self.k-- all_res = []- limit = 3*k if len(queries) > 1 else k- for query in queries:- _res = self._vectara_query(query, limit=limit)- all_res.append(_res)-- passages = defaultdict(float)- for res_list in all_res:- for res in res_list:- passages[res["text"]] += res["score"]- sorted_passages = sorted(- passages.items(), key=lambda x: x[1], reverse=True)[:k]-- return [dotdict({"long_text": passage}) for passage, _ in sorted_passages]-\ No newline at end of filediff --git a/dspy/retrieve/watson_discovery_rm.py b/dspy/retrieve/watson_discovery_rm.pydeleted file mode 100644index 47b03f804..000000000--- a/dspy/retrieve/watson_discovery_rm.py+++ /dev/null@@ -1,107 +0,0 @@-import json-from typing import Optional, Union--import requests-from requests.auth import HTTPBasicAuth--import dspy-from dspy.dsp.utils import dotdict---class WatsonDiscoveryRM(dspy.Retrieve):- """A retrieval module that uses Watson Discovery to return the top passages for a given query.-- Args:- apikey (str): apikey for authentication purposes,- url (str): endpoint URL that includes the service instance ID- version (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.- project_id (str): The Universally Unique Identifier (UUID) of the project.- collection_ids (list): An array containing the collections on which the search will be executed.- k (int, optional): The number of top passages to retrieve. Defaults to 5.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """-- def __init__(- self,- apikey: str,- url:str,- version:str,- project_id:str,- collection_ids:list=None,- k: int = 5,- ):- if collection_ids is None:- collection_ids = []- self.apikey=apikey- self.url=url,- self.version:version- self.project_id:project_id- self.collection_ids=collection_ids- self.k: k- self.query_url=url + "/v2/projects/" + project_id + "/query?version=" + version-- super().__init__(k=k)-- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:- """Search with Watson Discovery for self.k top passages for query.-- Args:- query_or_queries (Union[str, List[str]]): The query or queries to search for.- k (int, optional): The number of top passages to retrieve.-- Returns:- dspy.Prediction: An object containing the retrieved passages.- """- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries- queries = [q for q in queries if q] # Filter empty queries- k = self.k if k is None else k-- response=[]- try:- for query in queries:- payload = json.dumps({- "collection_ids": self.collection_ids,- "natural_language_query": "text:"+query,- "passages": {- "enabled": True,- "count": k,- "fields":["text"],- "characters": 500,- "find_answers": False,- "per_document": False,- },- "highlight": False,- })- headers = {'Content-Type': 'application/json'}-- discovery_results = requests.request(- "POST",- url=self.query_url,- headers=headers,- auth=HTTPBasicAuth("apikey", self.apikey),- data=payload,- )-- discovery_results.raise_for_status()-- doc_dict={}- for d in discovery_results.json()["results"]:- doc_dict[d["document_id"]]=d["title"]-- for d in discovery_results.json()["passages"]:- response.append(dotdict({- "title":doc_dict[d["document_id"]],- "long_text": doc_dict[d["document_id"]]+" | " + d["passage_text"],- "passage_score": d["passage_score"],- "document_id": d["document_id"],- "collection_id": d["collection_id"],- "start_offset": d["start_offset"],- "end_offset": d["end_offset"],- "field": d["field"],- }))-- except requests.exceptions.RequestException as err:- raise SystemExit(err) from err- return response\ No newline at end of filediff --git a/dspy/retrieve/you_rm.py b/dspy/retrieve/you_rm.pydeleted file mode 100644index cf7498b3c..000000000--- a/dspy/retrieve/you_rm.py+++ /dev/null@@ -1,114 +0,0 @@-import os-import warnings-from typing import Any, Literal, Optional, Union--import requests--import dspy-from dspy.dsp.utils import dotdict---class YouRM(dspy.Retrieve):- """Retriever for You.com's Search and News API.-- [API reference](https://documentation.you.com/api-reference/)-- Args:- ydc_api_key: you.com API key, if `YDC_API_KEY` is not set in the environment- k: If ``endpoint="search"``, the max snippets to return per search hit.- If ``endpoint="news"``, the max articles to return.- endpoint: you.com endpoints- num_web_results: The max number of web results to return, must be under 20- safesearch: Safesearch settings, one of "off", "moderate", "strict", defaults to moderate- country: Country code, ex: 'US' for United States, see API reference for more info- search_lang: (News API) Language codes, ex: 'en' for English, see API reference for more info- ui_lang: (News API) User interface language for the response, ex: 'en' for English.- See API reference for more info- spellcheck: (News API) Whether to spell check query or not, defaults to True- """-- def __init__(- self,- ydc_api_key: Optional[str] = None,- k: int = 3,- endpoint: Literal["search", "news"] = "search",- num_web_results: Optional[int] = None,- safesearch: Optional[Literal["off", "moderate", "strict"]] = None,- country: Optional[str] = None,- search_lang: Optional[str] = None,- ui_lang: Optional[str] = None,- spellcheck: Optional[bool] = None,- ):- super().__init__(k=k)-- # Data validation- if not ydc_api_key and not os.environ.get("YDC_API_KEY"):- raise RuntimeError('You must supply `ydc_api_key` or set environment variable "YDC_API_KEY"')-- if endpoint not in ("search", "news"):- raise ValueError('`endpoint` must be either "search" or "news"')-- # Raise warning if News API-specific fields are set but endpoint is not "news"- if endpoint != "news":- news_api_fields = (search_lang, ui_lang, spellcheck)- for field in news_api_fields:- if field:- warnings.warn(- (- f"News API-specific field '{field}' is set but `{endpoint=}`. "- "This will have no effect."- ),- UserWarning,- )-- self.ydc_api_key = ydc_api_key or os.environ.get("YDC_API_KEY")- self.endpoint = endpoint- self.num_web_results = num_web_results- self.safesearch = safesearch- self.country = country- self.search_lang = search_lang- self.ui_lang = ui_lang- self.spellcheck = spellcheck-- def _generate_params(self, query: str) -> dict[str, Any]:- params = {"safesearch": self.safesearch, "country": self.country}-- if self.endpoint == "search":- params.update(- query=query,- num_web_results=self.num_web_results,- )- elif self.endpoint == "news":- params.update(- q=query,- count=self.num_web_results,- search_lang=self.search_lang,- ui_lang=self.ui_lang,- spellcheck=self.spellcheck,- )-- # Remove `None` values- params = {k: v for k, v in params.items() if v is not None}- return params-- def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int] = None) -> dspy.Prediction:- k = k if k is not None else self.k-- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries- docs: list[str]- for query in queries:- headers = {"X-API-Key": self.ydc_api_key}- params = self._generate_params(query)- response = requests.get(- f"https://api.ydc-index.io/{self.endpoint}",- params=params,- headers=headers,- )- response.raise_for_status()- results = response.json()-- if self.endpoint == "search":- docs = [snippet for hits in results["hits"][:k] for snippet in hits["snippets"]]- elif self.endpoint == "news":- docs = [article["description"] for article in results["news"]["results"][:k]]- return [dotdict({"long_text": document}) for document in docs]diff --git a/dspy/retrievers/__init__.py b/dspy/retrievers/__init__.pyindex a8eb9e231..ffaf603bb 100644--- a/dspy/retrievers/__init__.py+++ b/dspy/retrievers/__init__.py@@ -1,3 +1,4 @@from dspy.retrievers.embeddings import Embeddings+from dspy.retrievers.retrieve import Retrieve-__all__ = ["Embeddings"]+__all__ = ["Embeddings", "Retrieve"]diff --git a/dspy/retrieve/databricks_rm.py b/dspy/retrievers/databricks_rm.pysimilarity index 100%rename from dspy/retrieve/databricks_rm.pyrename to dspy/retrievers/databricks_rm.pydiff --git a/dspy/retrieve/retrieve.py b/dspy/retrievers/retrieve.pysimilarity index 100%rename from dspy/retrieve/retrieve.pyrename to dspy/retrievers/retrieve.pydiff --git a/dspy/retrieve/weaviate_rm.py b/dspy/retrievers/weaviate_rm.pysimilarity index 99%rename from dspy/retrieve/weaviate_rm.pyrename to dspy/retrievers/weaviate_rm.pyindex 7d4cf436a..434afc6ec 100644--- a/dspy/retrieve/weaviate_rm.py+++ b/dspy/retrievers/weaviate_rm.py@@ -5,9 +5,10 @@ from dspy.dsp.utils import dotdictfrom dspy.primitives.prediction import Predictiontry:+ from uuid import uuid4+import weaviatefrom weaviate.util import get_valid_uuid- from uuid import uuid4except ImportError as err:raise ImportError("The 'weaviate' extra is required to use WeaviateRM. Install it with `pip install dspy-ai[weaviate]`",@@ -109,7 +110,7 @@ class WeaviateRM(dspy.Retrieve):passages.extend(dotdict({"long_text": d}) for d in parsed_results)return passages-+def get_objects(self, num_samples: int, fields: List[str]) -> List[dict]:"""Get objects from Weaviate using the cursor API."""if self._client_type == "WeaviateClient":@@ -127,7 +128,7 @@ class WeaviateRM(dspy.Retrieve):return objectselse:raise ValueError("`get_objects` is not supported for the v3 Weaviate Python client, please upgrade to v4.")-+def insert(self, new_object_properties: dict):if self._client_type == "WeaviateClient":self._weaviate_collection.data.insert(diff --git a/tests/retrieve/integration_test_pgvectorrm.py b/tests/retrieve/integration_test_pgvectorrm.pydeleted file mode 100644index be6c18f7a..000000000--- a/tests/retrieve/integration_test_pgvectorrm.py+++ /dev/null@@ -1,86 +0,0 @@-"""Instructions:-Add to dev container features:- "ghcr.io/itsmechlark/features/postgresql:1": {},- "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {}-Add to .personalization.sh:- sudo apt install -y postgresql-16-pgvector-- sudo /etc/init.d/postgresql restart-- psql -v ON_ERROR_STOP=1 --user ${PGUSER} <<EOF- create extension if not exists vector;- EOF-poetry install -E postgres-"""-from dspy.primitives.example import Example-import pytest-import psycopg2-from dspy.retrieve.pgvector_rm import PgVectorRM--DB_URL = "postgresql://postgres:password@localhost/postgres"-PG_TABLE_NAME = "test_table"---def get_pgvectorrm():- openai_client = None # Mock or use a real OpenAI client- pgvectorrm = PgVectorRM(DB_URL, PG_TABLE_NAME, openai_client=openai_client, embedding_func=lambda x: '[2,3,4]', include_similarity=True)- return pgvectorrm---def setup_pgvectorrm():- pgvectorrm = get_pgvectorrm()- conn = psycopg2.connect(DB_URL)- cursor = conn.cursor()- cursor.execute(f'DROP TABLE IF EXISTS {PG_TABLE_NAME}')- conn.commit()-- cursor.execute(f"CREATE TABLE IF NOT EXISTS {PG_TABLE_NAME} (id SERIAL PRIMARY KEY, text TEXT, embedding VECTOR)")- cursor.execute(f"INSERT INTO {PG_TABLE_NAME} (text, embedding) VALUES ('Dummy text1', '[1,2,3]'), ('Dummy text2', '[4,5,6]')")- conn.commit()-- yield pgvectorrm-- cursor.execute(f'TRUNCATE TABLE {PG_TABLE_NAME}')- conn.commit()-- cursor.close()- conn.close()---def test_pgvectorrm_retrieve(setup_pgvectorrm):- pgvectorrm = setup_pgvectorrm- query = "test query"- results = pgvectorrm(query)- assert len(results) == 2- assert results == [- Example(text='Dummy text2', similarity=0.9946115458726394),- Example(text='Dummy text1', similarity=0.9925833339709302)- ]--[email protected]("k, expected", [- (1, 1),- (2, 2),- (3, 2), # Assuming only 2 entries exist-])-def test_pgvectorrm_retrieve_diff_k(setup_pgvectorrm, k, expected):- setup_pgvectorrm.k = k- query = "test query"- results = setup_pgvectorrm(query)- assert len(results) == expected---def test_empty_table():- # Assuming setup_pgvectorrm cleans up after yielding- query = "test query"- results = get_pgvectorrm()(query)- assert len(results) == 0---def test_retrieval_without_similarity(setup_pgvectorrm):- setup_pgvectorrm.include_similarity = False- query = "test query"- results = setup_pgvectorrm(query)- # Ensure 'similarity' key is not in results- assert all('similarity' not in result for result in results)diff --git a/tests/retrieve/test_llama_index_rm.py b/tests/retrieve/test_llama_index_rm.pydeleted file mode 100644index 9a4246fa9..000000000--- a/tests/retrieve/test_llama_index_rm.py+++ /dev/null@@ -1,83 +0,0 @@-import logging--import pytest--import dspy-from dspy.datasets import HotPotQA-from dspy.utils.dummies import DummyLM--try:- from llama_index.core import Settings, VectorStoreIndex- from llama_index.core.base.base_retriever import BaseRetriever- from llama_index.core.embeddings.mock_embed_model import MockEmbedding- from llama_index.core.readers.string_iterable import StringIterableReader-- from dspy.retrieve.llama_index_rm import LlamaIndexRM--except ImportError:- logging.info("Optional dependency llama-index is not installed - skipping LlamaIndexRM tests.")---def rag_setup() -> dict:- """Builds the necessary fixtures to test LI"""- pytest.importorskip("llama_index")- dataset = HotPotQA(train_seed=1, train_size=8, eval_seed=2023, dev_size=4, test_size=0)- trainset = [x.with_inputs("question") for x in dataset.train]- devset = [x.with_inputs("question") for x in dataset.dev]- ragset = [f"Question: {x.question} Answer: {x.answer}" for x in dataset.train]- dummyset = {x.question: x.answer for x in dataset.train}-- Settings.embed_model = MockEmbedding(8)- docs = StringIterableReader().load_data(texts=ragset)- index = VectorStoreIndex.from_documents(documents=docs)- retriever = index.as_retriever()- rm = LlamaIndexRM(retriever)-- return {- "index": index,- "retriever": retriever,- "rm": rm,- "lm": DummyLM(answers=dummyset),- "trainset": trainset,- "devset": devset,- }---def test_lirm_as_rm(rag_setup):- """Test the retriever as retriever method"""- pytest.importorskip("llama_index")- retriever = rag_setup.get("retriever")- test_res_li = retriever.retrieve("At My Window was released by which American singer-songwriter?")- rm = rag_setup.get("rm")- test_res_dspy = rm.forward("At My Window was released by which American singer-songwriter?")-- assert isinstance(retriever, BaseRetriever), "Ensuring that the retriever is a LI Retriever object"- assert isinstance(test_res_li, list), "Ensuring results are a list from LI Retriever"-- assert isinstance(rm, dspy.Retrieve), "Ensuring the RM is a retriever object from dspy"- assert isinstance(test_res_dspy, list), "Ensuring the results are a list from the DSPy retriever"-- assert len(test_res_li) == len(test_res_dspy), "Rough equality check of the results"---def test_save_load_llama_index_rag(rag_setup, tmp_path):- pytest.importorskip("llama_index")-- class RAG(dspy.Module):- def __init__(self):- super().__init__()- self.retriever = dspy.Retrieve(k=3)- self.cot = dspy.ChainOfThought("question, context -> answer")-- rag = RAG()- rag.retriever.k = 4-- file_path = tmp_path / "rag.json"- rag.save(file_path)- loaded_rag = RAG()- # Before loading, the retriever k should be 3.- assert loaded_rag.retriever.k == 3- # After loading, the retriever k should be 4.- loaded_rag.load(file_path)- assert loaded_rag.retriever.k == 4