DSPy: The framework for programming—not prompting—language models
Python
feat(dspy): add support for azure ai search
HARISHKUMAR1112001f119c6ad070f
diff --git a/dspy/retrieve/azureaisearch_rm.py b/dspy/retrieve/azureaisearch_rm.pynew file mode 100644index 000000000..ed6a57b97--- /dev/null+++ b/dspy/retrieve/azureaisearch_rm.py@@ -0,0 +1,193 @@+from typing import Optional, List, Union+import dspy++from dsp.utils.utils import dotdict++try:+ from azure.core.credentials import AzureKeyCredential+ from azure.search.documents import SearchClient+ from azure.search.documents._paging import SearchItemPaged+ from azure.search.documents.models import QueryType+except ImportError:+ raise ImportError(+ "You need to install azure-search-documents library"+ "Please use the command: pip install azure-search-documents",+ )++class AzureAISearchRM(dspy.Retrieve):++ """+ A retrieval module that utilizes Azure AI Search to retrieve top passages for a given query.++ Args:+ search_service_name (str): The name of the Azure AI Search service.+ search_api_key (str): The API key for accessing the Azure AI Search service.+ search_index_name (str): The name of the search index in the Azure AI Search service.+ field_text (str): The name of the field containing text content in the search index. This field will be mapped to the "content" field in the dsp framework.+ k (int, optional): The default number of top passages to retrieve. Defaults to 3.+ semantic_ranker (bool, optional): Whether to use semantic ranking. Defaults to False.+ filter (str, optional): Additional filter query. Defaults to None.+ query_language (str, optional): The language of the query. Defaults to "en-Us".+ query_speller (str, optional): The speller mode. Defaults to "lexicon".+ use_semantic_captions (bool, optional): Whether to use semantic captions. Defaults to False.+ query_type (Optional[QueryType], optional): The type of query. Defaults to QueryType.FULL.+ semantic_configuration_name (str, optional): The name of the semantic configuration. Defaults to None.++ Examples:+ Below is a code snippet that demonstrates how to instantiate and use the AzureAISearchRM class:+ ```python+ search_service_name = "your_search_service_name"+ search_api_key = "your_search_api_key"+ search_index_name = "your_search_index_name"+ field_text = "text_content_field"++ azure_search_retriever = AzureAISearchRM(search_service_name, search_api_key, search_index_name, field_text)+ ```++ Attributes:+ search_service_name (str): The name of the Azure AI Search service.+ search_api_key (str): The API key for accessing the Azure AI Search service.+ search_index_name (str): The name of the search index in the Azure AI Search service.+ field_text (str): The name of the field containing text content in the search index.+ endpoint (str): The endpoint URL for the Azure AI Search service.+ credential (AzureKeyCredential): The Azure key credential for accessing the service.+ client (SearchClient): The Azure AI Search client instance.++ Methods:+ forward(query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:+ Search for the top passages corresponding to the given query or queries.++ azure_search_request(+ key_content: str,+ client: SearchClient,+ query: str,+ top: int,+ semantic_ranker: bool,+ filter: str,+ query_language: str,+ query_speller: str,+ use_semantic_captions: bool,+ query_type: QueryType,+ semantic_configuration_name: str+ ) -> List[dict]:+ Perform a search request to the Azure AI Search service.++ process_azure_result(results: SearchItemPaged, content_key: str, content_score: str) -> List[dict]:+ Process the results received from the Azure AI Search service and map them to the correct format.++ Raises:+ ImportError: If the required Azure AI Search libraries are not installed.++ Note:+ This class relies on the 'azure-search-documents' library for interacting with the Azure AI Search service.+ Ensure that you have the necessary permissions and correct configurations set up in Azure before using this class.+ """++ def __init__(+ self,+ search_service_name: str,+ search_api_key: str,+ search_index_name: str,+ field_text: str,+ k: int = 3,+ semantic_ranker: bool = False,+ filter: str = None,+ query_language: str = "en-Us",+ query_speller: str = "lexicon",+ use_semantic_captions: bool = False,+ query_type: Optional[QueryType] = QueryType.FULL,+ semantic_configuration_name: str = None++ ):+ self.search_service_name = search_service_name+ self.search_api_key = search_api_key+ self.search_index_name = search_index_name+ self.endpoint=f"https://{self.search_service_name}.search.windows.net"+ self.field_text = field_text # field name of the text content+ # Create a client+ self.credential = AzureKeyCredential(self.search_api_key)+ self.client = SearchClient(endpoint=self.endpoint,+ index_name=self.search_index_name,+ credential=self.credential)+ self.semantic_ranker = semantic_ranker+ self.filter = filter+ self.query_language = query_language+ self.query_speller = query_speller+ self.use_semantic_captions = use_semantic_captions+ self.query_type = query_type+ self.semantic_configuration_name = semantic_configuration_name++ super().__init__(k=k)++ def azure_search_request(self,key_content: str, client: SearchClient, query: str, top: int, semantic_ranker: bool, filter: str, query_language: str, query_speller: str, use_semantic_captions: bool, query_type: QueryType, semantic_configuration_name: str):+ """+ Search in Azure AI Search Index+ """++ # TODO: Add Support for Vector Search And Hybride Search+ if semantic_ranker:+ results = client.search(search_text=query,+ filter=filter,+ query_type=query_type,+ query_language = query_language,+ query_speller=query_speller,+ semantic_configuration_name=semantic_configuration_name,+ top=top,+ query_caption = (+ 'extractive|highlight-false'+ if use_semantic_captions+ else None+ )+ )+ else:+ results = client.search(search_text=query,top=top,filter=filter)++ sorted_results = sorted(results, key=lambda x: x['@search.score'], reverse=True)++ sorted_results = self.process_azure_result(sorted_results, key_content, key_content)++ return sorted_results++ def process_azure_result(self,results:SearchItemPaged, content_key:str, content_score: str):+ """+ process received result from Azure AI Search as dictionary array and map content and score to correct format+ """+ res = []+ for result in results:+ tmp = {}+ for key, value in result.items():+ if(key == content_key):+ tmp["text"] = value # assign content+ elif(key == content_score):+ tmp["score"] = value+ else:+ tmp[key] = value+ res.append(tmp)+ return res++ def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int]) -> dspy.Prediction:+ """+ Search with pinecone for self.k top passages for query++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.++ Returns:+ dspy.Prediction: An object containing the retrieved passages.+ """++ k = k if k is not None else self.k+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ queries = [q for q in queries if q] # Filter empty queries++ passages = []+ for query in queries:+ results = self.azure_search_request(self.field_text, self.client, query, k, self.semantic_ranker, self.filter, self.query_language, self.query_speller, self.use_semantic_captions, self.query_type, self.semantic_configuration_name)+ passages.extend(dotdict({"long_text": d['text']}) for d in results)++ return passages+\ No newline at end of file