Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

Updated to pull latest changes from main

zenith110525e66036262
diff --cc knowledge_storm/rm.py
index 743428d,2b4e8a8..0fc9d7f
--- a/knowledge_storm/rm.py
+++ b/knowledge_storm/rm.py
@@@ -509,119 -510,70 +509,185 @@@ class BraveRM(dspy.Retrieve)
return collected_results
-
+ class SearXNG(dspy.Retrieve):
+ def __init__(self, searxng_api_url, searxng_api_key=None, k=3, is_valid_source: Callable = None):
+ """Initialize the SearXNG search retriever.
+ Please set up SearXNG according to https://docs.searxng.org/index.html.
+
+ Args:
+ searxng_api_url (str): The URL of the SearXNG API. Consult SearXNG documentation for details.
+ searxng_api_key (str, optional): The API key for the SearXNG API. Defaults to None. Consult SearXNG documentation for details.
+ k (int, optional): The number of top passages to retrieve. Defaults to 3.
+ is_valid_source (Callable, optional): A function that takes a URL and returns a boolean indicating if the
+ source is valid. Defaults to None.
+ """
+ super().__init__(k=k)
+ if not searxng_api_url:
+ raise RuntimeError("You must supply searxng_api_url")
+ self.searxng_api_url = searxng_api_url
+ self.searxng_api_key = searxng_api_key
+ self.usage = 0
+
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+ return {'SearXNG': usage}
+
+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):
+ """Search with SearxNG for self.k top passages for query or queries
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+ collected_results = []
+ headers = {"Authorization": f"Bearer {self.searxng_api_key}"} if self.searxng_api_key else {}
+
+ for query in queries:
+ try:
+ params = {"q": query, "format": "json"}
+ response = requests.get(self.searxng_api_url, headers=headers, params=params)
+ results = response.json()
+
+ for r in results['results']:
+ if self.is_valid_source(r['url']) and r['url'] not in exclude_urls:
+ collected_results.append({
+ 'description': r.get('content', ''),
+ 'snippets': [r.get('content', '')],
+ 'title': r.get('title', ''),
+ 'url': r['url']
+ })
+ except Exception as e:
+ logging.error(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results
+
+class DuckDuckGoSearchRM(dspy.Retrieve):
+ """Retrieve information from custom queries using DuckDuckGo."""
+ def __init__(
+ self,
+ k: int =3,
+ is_valid_source: Callable = None,
+ min_char_count: int = 150,
+ snippet_chunk_size: int = 1000,
+ webpage_helper_max_threads=10,
+ safe_search: str = 'On',
+ region: str = 'us-en'
+ ):
+ """
+ Params:
+ min_char_count: Minimum character count for the article to be considered valid.
+ snippet_chunk_size: Maximum character count for each snippet.
+ webpage_helper_max_threads: Maximum number of threads to use for webpage helper.
+ **kwargs: Additional parameters for the OpenAI API.
+ """
+ super().__init__(k=k)
+ try:
+ from duckduckgo_search import DDGS
+ except ImportError as err:
+ raise ImportError("Duckduckgo requires `pip install duckduckgo_search`.") from err
+ self.k = k
+ self.webpage_helper = WebPageHelper(
+ min_char_count=min_char_count,
+ snippet_chunk_size=snippet_chunk_size,
+ max_thread_num=webpage_helper_max_threads,
+ )
+ self.usage = 0
+ # All params for search can be found here:
+ # https://duckduckgo.com/duckduckgo-help-pages/settings/params/
+
+ # Sets the backend to be api
+ self.duck_duck_go_backend = 'api'
+
+ # Only gets safe search results
+ self.duck_duck_go_safe_search = safe_search
+
+ # Specifies the region that the search will use
+ self.duck_duck_go_region = region
+
+
+ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+ # Import the duckduckgo search library found here: https://github.com/deedy5/duckduckgo_search
+ self.ddgs = DDGS()
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+ return {'DuckDuckGoRM': usage}
+
+ def forward(
+ self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []
+ ):
+ """Search with DuckDuckGoSearch for self.k top passages for query or queries
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+
+ collected_results = []
+
+ for query in queries:
+ # list of dicts that will be parsed to return
+ results = self.ddgs.text(
+ query, max_results=self.k, backend=self.duck_duck_go_backend
+ )
+
+ for d in results:
+ # assert d is dict
+ if not isinstance(d, dict):
+ print(f'Invalid result: {d}\n')
+ continue
+
+ try:
+ # ensure keys are present
+ url = d.get('href', None)
+ title = d.get('title', None)
+ description = d.get('description', title)
+ snippets = [d.get('body', None)]
+
+ # raise exception of missing key(s)
+ if not all([url, title, description, snippets]):
+ raise ValueError(f'Missing key(s) in result: {d}')
+ if self.is_valid_source(url) and url not in exclude_urls:
+ result = {
+ 'url': url,
+ 'title': title,
+ 'description': description,
+ 'snippets': snippets,
+ }
+ collected_results.append(result)
+ else:
+ print(f'invalid source {url} or url in exclude_urls')
+ except Exception as e:
+ print(f'Error occurs when processing {result=}: {e}\n')
+ print(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results
+