Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

Merge branch 'main' into feature/add-support-for-searxng

Yijia Shao231c34bf51f9
diff --cc knowledge_storm/rm.py
index a099ca8,4c438c4..2b4e8a8
--- a/knowledge_storm/rm.py
+++ b/knowledge_storm/rm.py
@@@ -542,69 -443,69 +443,137 @@@ class SerperRM(dspy.Retrieve)
return collected_results
+ class BraveRM(dspy.Retrieve):
+ def __init__(self, brave_search_api_key=None, k=3, is_valid_source: Callable = None):
+ super().__init__(k=k)
+ if not brave_search_api_key and not os.environ.get("BRAVE_API_KEY"):
+ raise RuntimeError("You must supply brave_search_api_key or set environment variable BRAVE_API_KEY")
+ elif brave_search_api_key:
+ self.brave_search_api_key = brave_search_api_key
+ else:
+ self.brave_search_api_key = os.environ["BRAVE_API_KEY"]
+ self.usage = 0
+
+ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+
+ return {'BraveRM': usage}
+
+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):
+ """Search with api.search.brave.com for self.k top passages for query or queries
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+ collected_results = []
+ for query in queries:
+ try:
+ headers = {
+ "Accept": "application/json",
+ "Accept-Encoding": "gzip",
+ "X-Subscription-Token": self.brave_search_api_key
+ }
+ response = requests.get(
+ f"https://api.search.brave.com/res/v1/web/search?result_filter=web&q={query}",
+ headers=headers,
+ ).json()
+ results = response.get('web', {}).get('results', [])
+
+ for result in results:
+ collected_results.append(
+ {
+ 'snippets': result.get('extra_snippets', []),
+ 'title': result.get('title'),
+ 'url': result.get('url'),
+ 'description': result.get('description'),
+ }
+ )
+ except Exception as e:
+ logging.error(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results
++
++
+class SearXNG(dspy.Retrieve):
+ def __init__(self, searxng_api_url, searxng_api_key=None, k=3, is_valid_source: Callable = None):
+ """Initialize the SearXNG search retriever.
+ Please set up SearXNG according to https://docs.searxng.org/index.html.
+
+ Args:
+ searxng_api_url (str): The URL of the SearXNG API. Consult SearXNG documentation for details.
+ searxng_api_key (str, optional): The API key for the SearXNG API. Defaults to None. Consult SearXNG documentation for details.
+ k (int, optional): The number of top passages to retrieve. Defaults to 3.
+ is_valid_source (Callable, optional): A function that takes a URL and returns a boolean indicating if the
+ source is valid. Defaults to None.
+ """
+ super().__init__(k=k)
+ if not searxng_api_url:
+ raise RuntimeError("You must supply searxng_api_url")
+ self.searxng_api_url = searxng_api_url
+ self.searxng_api_key = searxng_api_key
+ self.usage = 0
+
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+ return {'SearXNG': usage}
+
+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):
+ """Search with SearxNG for self.k top passages for query or queries
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+ collected_results = []
+ headers = {"Authorization": f"Bearer {self.searxng_api_key}"} if self.searxng_api_key else {}
+
+ for query in queries:
+ try:
+ params = {"q": query, "format": "json"}
+ response = requests.get(self.searxng_api_url, headers=headers, params=params)
+ results = response.json()
+
+ for r in results['results']:
+ if self.is_valid_source(r['url']) and r['url'] not in exclude_urls:
+ collected_results.append({
+ 'description': r.get('content', ''),
+ 'snippets': [r.get('content', '')],
+ 'title': r.get('title', ''),
+ 'url': r['url']
+ })
+ except Exception as e:
+ logging.error(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results