An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Azure AI Search has optional requirements
[email protected]3922a84c4ea2
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex a078383..00bb3cb 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -13,8 +13,6 @@ from qdrant_client import QdrantClientfrom .utils import WebPageHelper-from azure.core.credentials import AzureKeyCredential-from azure.search.documents import SearchClientclass YouRM(dspy.Retrieve):def __init__(self, ydc_api_key=None, k=3, is_valid_source: Callable = None):@@ -79,6 +77,7 @@ class YouRM(dspy.Retrieve):return collected_results+class BingSearch(dspy.Retrieve):def __init__(self,@@ -1095,16 +1094,19 @@ class GoogleSearch(dspy.Retrieve):return collected_results+class AzureAISearch(dspy.Retrieve):- """Retrieve information from custom queries using Azure AI Search. General Documentation can be found at: https://learn.microsoft.com/en-us/azure/search/search-create-service-portal. Python Documentation and examples can be found at https://learn.microsoft.com/en-us/python/api/overview/azure/search-documents-readme?view=azure-python. Requires pip install azure-search-documents"""+ """Retrieve information from custom queries using Azure AI Search.+ General Documentation can be found at: https://learn.microsoft.com/en-us/azure/search/search-create-service-portal. Python Documentation and examples can be found at https://learn.microsoft.com/en-us/python/api/overview/azure/search-documents-readme?view=azure-python. Requires pip install azure-search-documents+ """def __init__(- self,- azure_ai_search_api_key=None,- azure_ai_search_url=None,- azure_ai_search_index_name=None,- k=3,- is_valid_source: Callable = None+ self,+ azure_ai_search_api_key=None,+ azure_ai_search_url=None,+ azure_ai_search_index_name=None,+ k=3,+ is_valid_source: Callable = None,):"""Params:@@ -1119,7 +1121,18 @@ class AzureAISearch(dspy.Retrieve):webpage_helper_max_threads: Maximum number of threads to use for webpage helper."""super().__init__(k=k)- if not azure_ai_search_api_key and not os.environ.get("AZURE_AI_SEARCH_API_KEY"):++ try:+ from azure.core.credentials import AzureKeyCredential+ from azure.search.documents import SearchClient+ except ImportError as err:+ raise ImportError(+ "AzureAISearch requires `pip install azure-search-documents`."+ ) from err++ if not azure_ai_search_api_key and not os.environ.get(+ "AZURE_AI_SEARCH_API_KEY"+ ):raise RuntimeError("You must supply azure_ai_search_api_key or set environment variable AZURE_AI_SEARCH_API_KEY")@@ -1127,7 +1140,7 @@ class AzureAISearch(dspy.Retrieve):self.azure_ai_search_api_key = azure_ai_search_api_keyelse:self.azure_ai_search_api_key = os.environ["AZURE_AI_SEARCH_API_KEY"]-+if not azure_ai_search_url and not os.environ.get("AZURE_AI_SEARCH_URL"):raise RuntimeError("You must supply azure_ai_search_url or set environment variable AZURE_AI_SEARCH_URL"@@ -1136,8 +1149,10 @@ class AzureAISearch(dspy.Retrieve):self.azure_ai_search_url = azure_ai_search_urlelse:self.azure_ai_search_url = os.environ["AZURE_AI_SEARCH_URL"]-- if not azure_ai_search_index_name and not os.environ.get("AZURE_AI_SEARCH_INDEX_NAME"):++ if not azure_ai_search_index_name and not os.environ.get(+ "AZURE_AI_SEARCH_INDEX_NAME"+ ):raise RuntimeError("You must supply azure_ai_search_index_name or set environment variable AZURE_AI_SEARCH_INDEX_NAME")@@ -1180,7 +1195,11 @@ class AzureAISearch(dspy.Retrieve):self.usage += len(queries)collected_results = []- client = SearchClient(self.azure_ai_search_url,self.azure_ai_search_index_name,AzureKeyCredential(self.azure_ai_search_api_key))+ client = SearchClient(+ self.azure_ai_search_url,+ self.azure_ai_search_index_name,+ AzureKeyCredential(self.azure_ai_search_api_key),+ )for query in queries:try:# https://learn.microsoft.com/en-us/python/api/azure-search-documents/azure.search.documents.searchclient?view=azure-python#azure-search-documents-searchclient-search@@ -1188,15 +1207,13 @@ class AzureAISearch(dspy.Retrieve):for result in results:document = {- "url": result['metadata_storage_path'],- "title": result['title'],+ "url": result["metadata_storage_path"],+ "title": result["title"],"description": "N/A",- "snippets": [result['chunk']]+ "snippets": [result["chunk"]],}collected_results.append(document)except Exception as e:logging.error(f"Error occurs when searching query {query}: {e}")return collected_results--diff --git a/requirements.txt b/requirements.txtindex 07e40c3..7e1a88a 100644--- a/requirements.txt+++ b/requirements.txt@@ -7,5 +7,4 @@ trafilaturalangchain-huggingfaceqdrant-clientlangchain-qdrant-numpy==1.26.4-azure-search-documents==11.5.1\ No newline at end of file+numpy==1.26.4\ No newline at end of file