Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

support brave search

Kevin Jiangf74b91ca8d15
diff --git a/examples/run_storm_wiki_claude.py b/examples/run_storm_wiki_claude.py
index 64031f6..2eed68a 100644
--- a/examples/run_storm_wiki_claude.py
+++ b/examples/run_storm_wiki_claude.py
@@ -89,7 +89,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_deepseek.py b/examples/run_storm_wiki_deepseek.py
index 2a7b256..194f247 100644
--- a/examples/run_storm_wiki_deepseek.py
+++ b/examples/run_storm_wiki_deepseek.py
@@ -123,7 +123,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'], required=True,
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'], required=True,
help='The search engine API to use for retrieving information.')
parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',
help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')
diff --git a/examples/run_storm_wiki_gemini.py b/examples/run_storm_wiki_gemini.py
index 5e9f9c8..edee64f 100644
--- a/examples/run_storm_wiki_gemini.py
+++ b/examples/run_storm_wiki_gemini.py
@@ -91,7 +91,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_gpt.py b/examples/run_storm_wiki_gpt.py
index 5e4dda3..998b0df 100644
--- a/examples/run_storm_wiki_gpt.py
+++ b/examples/run_storm_wiki_gpt.py
@@ -101,7 +101,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_mistral.py b/examples/run_storm_wiki_mistral.py
index 76f85dc..ff9684c 100644
--- a/examples/run_storm_wiki_mistral.py
+++ b/examples/run_storm_wiki_mistral.py
@@ -147,7 +147,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_ollama.py b/examples/run_storm_wiki_ollama.py
index 35ba99e..c21ea48 100644
--- a/examples/run_storm_wiki_ollama.py
+++ b/examples/run_storm_wiki_ollama.py
@@ -151,7 +151,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.py
index 7953626..b69926d 100644
--- a/knowledge_storm/rm.py
+++ b/knowledge_storm/rm.py
@@ -307,9 +307,10 @@ class VectorRM(dspy.Retrieve):
return collected_results
+
class SerperRM(dspy.Retrieve):
"""Retrieve information from custom queries using Serper.dev."""
-
+
def __init__(self, serper_search_api_key=None, query_params=None):
"""Args:
serper_search_api_key str: API key to run serper, can be found by creating an account on https://serper.dev/
@@ -373,8 +374,8 @@ class SerperRM(dspy.Retrieve):
def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str]):
"""
Calls the API and searches for the query passed in.
-
-
+
+
Args:
query_or_queries (Union[str, List[str]]): The query or queries to search for.
exclude_urls (List[str]): Dummy parameter to match the interface. Does not have any effect.
@@ -395,7 +396,7 @@ class SerperRM(dspy.Retrieve):
if query == 'Queries:':
continue
query_params = self.query_params
-
+
# All available parameters can be found in the playground: https://serper.dev/playground
# Sets the json value for query to be the query that is being parsed.
query_params['q'] = query
@@ -441,3 +442,71 @@ class SerperRM(dspy.Retrieve):
continue
return collected_results
+
+
+class BraveRM(dspy.Retrieve):
+ def __init__(self, brave_search_api_key=None, k=3, is_valid_source: Callable = None):
+ super().__init__(k=k)
+ if not brave_search_api_key and not os.environ.get("BRAVE_API_KEY"):
+ raise RuntimeError("You must supply brave_search_api_key or set environment variable BRAVE_API_KEY")
+ elif brave_search_api_key:
+ self.brave_search_api_key = brave_search_api_key
+ else:
+ self.brave_search_api_key = os.environ["BRAVE_API_KEY"]
+ self.usage = 0
+
+ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+
+ return {'BraveRM': usage}
+
+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):
+ """Search with api.search.brave.com for self.k top passages for query or queries
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+ collected_results = []
+ for query in queries:
+ try:
+ headers = {
+ "Accept": "application/json",
+ "Accept-Encoding": "gzip",
+ "X-Subscription-Token": self.brave_search_api_key
+ }
+ response = requests.get(
+ f"https://api.search.brave.com/res/v1/web/search?result_filter=web&q={query}",
+ headers=headers,
+ ).json()
+ results = response.get('web', {}).get('results', [])
+
+ for result in results:
+ collected_results.extend(
+ {
+ 'snippets': result.get('extra_snippets', []),
+ 'title': result.get('title'),
+ 'url': result.get('url'),
+ 'description': result.get('description'),
+ }
+ )
+ except Exception as e:
+ logging.error(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results