An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Add Azure AI Search Support
[email protected]db59481dbeea
diff --git a/README.md b/README.mdindex e027b3c..50c8f36 100644--- a/README.md+++ b/README.md@@ -93,7 +93,7 @@ You could also install the source code which allows you to modify the behavior oCurrently, our package support:- `OpenAIModel`, `AzureOpenAIModel`, `ClaudeModel`, `VLLMClient`, `TGIClient`, `TogetherClient`, `OllamaClient`, `GoogleModel`, `DeepSeekModel`, `GroqModel` as language model components-- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, `DuckDuckGoSearchRM`, `TavilySearchRM`, `GoogleSearch` as retrieval module components+- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, `DuckDuckGoSearchRM`, `TavilySearchRM`, `GoogleSearch`, and `AzureAISearch` as retrieval module components:star2: **PRs for integrating more language models into [knowledge_storm/lm.py](knowledge_storm/lm.py) and search engines/retrievers into [knowledge_storm/rm.py](knowledge_storm/rm.py) are highly appreciated!**diff --git a/examples/storm_examples/run_storm_wiki_gpt.py b/examples/storm_examples/run_storm_wiki_gpt.pyindex ac07968..b1740a1 100644--- a/examples/storm_examples/run_storm_wiki_gpt.py+++ b/examples/storm_examples/run_storm_wiki_gpt.py@@ -20,10 +20,11 @@ args.output_dir/"""import os+from argparse import ArgumentParserfrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import OpenAIModel, AzureOpenAIModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG, AzureAISearchfrom knowledge_storm.utils import load_api_key@@ -72,6 +73,7 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.+match args.retriever:case 'bing':rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)@@ -87,8 +89,10 @@ def main(args):rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)case 'searxng':rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case 'azure_ai_search':+ rm = AzureAISearch(azure_ai_search_api_key=os.getenv('AZURE_AI_SEARCH_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", "searxng", or "azure_ai_search"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -113,7 +117,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng', 'azure_ai_search'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',@@ -138,4 +142,4 @@ if __name__ == '__main__':parser.add_argument('--remove-duplicate', action='store_true',help='If True, remove duplicate content from the article.')- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 7f029e7..a078383 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -13,6 +13,8 @@ from qdrant_client import QdrantClientfrom .utils import WebPageHelper+from azure.core.credentials import AzureKeyCredential+from azure.search.documents import SearchClientclass YouRM(dspy.Retrieve):def __init__(self, ydc_api_key=None, k=3, is_valid_source: Callable = None):@@ -77,7 +79,6 @@ class YouRM(dspy.Retrieve):return collected_results-class BingSearch(dspy.Retrieve):def __init__(self,@@ -1093,3 +1094,109 @@ class GoogleSearch(dspy.Retrieve):collected_results.append(r)return collected_results++class AzureAISearch(dspy.Retrieve):+ """Retrieve information from custom queries using Azure AI Search. General Documentation can be found at: https://learn.microsoft.com/en-us/azure/search/search-create-service-portal. Python Documentation and examples can be found at https://learn.microsoft.com/en-us/python/api/overview/azure/search-documents-readme?view=azure-python. Requires pip install azure-search-documents"""++ def __init__(+ self,+ azure_ai_search_api_key=None,+ azure_ai_search_url=None,+ azure_ai_search_index_name=None,+ k=3,+ is_valid_source: Callable = None+ ):+ """+ Params:+ azure_ai_search_api_key: Azure AI Search API key. Check out https://learn.microsoft.com/en-us/azure/search/search-security-api-keys?tabs=rest-use%2Cportal-find%2Cportal-query+ "API key" section+ azure_ai_search_url: Custom Azure AI Search Endpoint URL. Check out https://learn.microsoft.com/en-us/azure/search/search-create-service-portal#name-the-service+ azure_ai_search_index_name: Custom Azure AI Search Index Name. Check out https://learn.microsoft.com/en-us/azure/search/search-how-to-create-search-index?tabs=portal+ k: Number of top results to retrieve.+ is_valid_source: Optional function to filter valid sources.+ min_char_count: Minimum character count for the article to be considered valid.+ snippet_chunk_size: Maximum character count for each snippet.+ webpage_helper_max_threads: Maximum number of threads to use for webpage helper.+ """+ super().__init__(k=k)+ if not azure_ai_search_api_key and not os.environ.get("AZURE_AI_SEARCH_API_KEY"):+ raise RuntimeError(+ "You must supply azure_ai_search_api_key or set environment variable AZURE_AI_SEARCH_API_KEY"+ )+ elif azure_ai_search_api_key:+ self.azure_ai_search_api_key = azure_ai_search_api_key+ else:+ self.azure_ai_search_api_key = os.environ["AZURE_AI_SEARCH_API_KEY"]++ if not azure_ai_search_url and not os.environ.get("AZURE_AI_SEARCH_URL"):+ raise RuntimeError(+ "You must supply azure_ai_search_url or set environment variable AZURE_AI_SEARCH_URL"+ )+ elif azure_ai_search_url:+ self.azure_ai_search_url = azure_ai_search_url+ else:+ self.azure_ai_search_url = os.environ["AZURE_AI_SEARCH_URL"]++ if not azure_ai_search_index_name and not os.environ.get("AZURE_AI_SEARCH_INDEX_NAME"):+ raise RuntimeError(+ "You must supply azure_ai_search_index_name or set environment variable AZURE_AI_SEARCH_INDEX_NAME"+ )+ elif azure_ai_search_index_name:+ self.azure_ai_search_index_name = azure_ai_search_index_name+ else:+ self.azure_ai_search_index_name = os.environ["AZURE_AI_SEARCH_INDEX_NAME"]++ self.usage = 0++ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.+ if is_valid_source:+ self.is_valid_source = is_valid_source+ else:+ self.is_valid_source = lambda x: True++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0++ return {"AzureAISearch": usage}++ def forward(+ self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []+ ):+ """Search with Azure Open AI for self.k top passages for query or queries++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.++ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)+ collected_results = []++ client = SearchClient(self.azure_ai_search_url,self.azure_ai_search_index_name,AzureKeyCredential(self.azure_ai_search_api_key))+ for query in queries:+ try:+ # https://learn.microsoft.com/en-us/python/api/azure-search-documents/azure.search.documents.searchclient?view=azure-python#azure-search-documents-searchclient-search+ results = client.search(search_text=query, top=1)++ for result in results:+ document = {+ "url": result['metadata_storage_path'],+ "title": result['title'],+ "description": "N/A",+ "snippets": [result['chunk']]+ }+ collected_results.append(document)+ except Exception as e:+ logging.error(f"Error occurs when searching query {query}: {e}")++ return collected_results++diff --git a/requirements.txt b/requirements.txtindex 7e1a88a..07e40c3 100644--- a/requirements.txt+++ b/requirements.txt@@ -7,4 +7,5 @@ trafilaturalangchain-huggingfaceqdrant-clientlangchain-qdrant-numpy==1.26.4\ No newline at end of file+numpy==1.26.4+azure-search-documents==11.5.1\ No newline at end of file