An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
support brave search
Kevin Jiangf74b91ca8d15
diff --git a/examples/run_storm_wiki_claude.py b/examples/run_storm_wiki_claude.pyindex 64031f6..2eed68a 100644--- a/examples/run_storm_wiki_claude.py+++ b/examples/run_storm_wiki_claude.py@@ -89,7 +89,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_deepseek.py b/examples/run_storm_wiki_deepseek.pyindex 2a7b256..194f247 100644--- a/examples/run_storm_wiki_deepseek.py+++ b/examples/run_storm_wiki_deepseek.py@@ -123,7 +123,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'], required=True,+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'], required=True,help='The search engine API to use for retrieving information.')parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')diff --git a/examples/run_storm_wiki_gemini.py b/examples/run_storm_wiki_gemini.pyindex 5e9f9c8..edee64f 100644--- a/examples/run_storm_wiki_gemini.py+++ b/examples/run_storm_wiki_gemini.py@@ -91,7 +91,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_gpt.py b/examples/run_storm_wiki_gpt.pyindex 5e4dda3..998b0df 100644--- a/examples/run_storm_wiki_gpt.py+++ b/examples/run_storm_wiki_gpt.py@@ -101,7 +101,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_mistral.py b/examples/run_storm_wiki_mistral.pyindex 76f85dc..ff9684c 100644--- a/examples/run_storm_wiki_mistral.py+++ b/examples/run_storm_wiki_mistral.py@@ -147,7 +147,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_ollama.py b/examples/run_storm_wiki_ollama.pyindex 35ba99e..c21ea48 100644--- a/examples/run_storm_wiki_ollama.py+++ b/examples/run_storm_wiki_ollama.py@@ -151,7 +151,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 7953626..b69926d 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -307,9 +307,10 @@ class VectorRM(dspy.Retrieve):return collected_results+class SerperRM(dspy.Retrieve):"""Retrieve information from custom queries using Serper.dev."""-+def __init__(self, serper_search_api_key=None, query_params=None):"""Args:serper_search_api_key str: API key to run serper, can be found by creating an account on https://serper.dev/@@ -373,8 +374,8 @@ class SerperRM(dspy.Retrieve):def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str]):"""Calls the API and searches for the query passed in.--++Args:query_or_queries (Union[str, List[str]]): The query or queries to search for.exclude_urls (List[str]): Dummy parameter to match the interface. Does not have any effect.@@ -395,7 +396,7 @@ class SerperRM(dspy.Retrieve):if query == 'Queries:':continuequery_params = self.query_params-+# All available parameters can be found in the playground: https://serper.dev/playground# Sets the json value for query to be the query that is being parsed.query_params['q'] = query@@ -441,3 +442,71 @@ class SerperRM(dspy.Retrieve):continuereturn collected_results+++class BraveRM(dspy.Retrieve):+ def __init__(self, brave_search_api_key=None, k=3, is_valid_source: Callable = None):+ super().__init__(k=k)+ if not brave_search_api_key and not os.environ.get("BRAVE_API_KEY"):+ raise RuntimeError("You must supply brave_search_api_key or set environment variable BRAVE_API_KEY")+ elif brave_search_api_key:+ self.brave_search_api_key = brave_search_api_key+ else:+ self.brave_search_api_key = os.environ["BRAVE_API_KEY"]+ self.usage = 0++ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.+ if is_valid_source:+ self.is_valid_source = is_valid_source+ else:+ self.is_valid_source = lambda x: True++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0++ return {'BraveRM': usage}++ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):+ """Search with api.search.brave.com for self.k top passages for query or queries++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.++ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)+ collected_results = []+ for query in queries:+ try:+ headers = {+ "Accept": "application/json",+ "Accept-Encoding": "gzip",+ "X-Subscription-Token": self.brave_search_api_key+ }+ response = requests.get(+ f"https://api.search.brave.com/res/v1/web/search?result_filter=web&q={query}",+ headers=headers,+ ).json()+ results = response.get('web', {}).get('results', [])++ for result in results:+ collected_results.extend(+ {+ 'snippets': result.get('extra_snippets', []),+ 'title': result.get('title'),+ 'url': result.get('url'),+ 'description': result.get('description'),+ }+ )+ except Exception as e:+ logging.error(f'Error occurs when searching query {query}: {e}')++ return collected_results