An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Merge branch 'main' into feature/add-support-for-searxng
Yijia Shao231c34bf51f9
diff --cc knowledge_storm/rm.pyindex a099ca8,4c438c4..2b4e8a8--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@@ -542,69 -443,69 +443,137 @@@ class SerperRM(dspy.Retrieve)return collected_results+ class BraveRM(dspy.Retrieve):+ def __init__(self, brave_search_api_key=None, k=3, is_valid_source: Callable = None):+ super().__init__(k=k)+ if not brave_search_api_key and not os.environ.get("BRAVE_API_KEY"):+ raise RuntimeError("You must supply brave_search_api_key or set environment variable BRAVE_API_KEY")+ elif brave_search_api_key:+ self.brave_search_api_key = brave_search_api_key+ else:+ self.brave_search_api_key = os.environ["BRAVE_API_KEY"]+ self.usage = 0++ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.+ if is_valid_source:+ self.is_valid_source = is_valid_source+ else:+ self.is_valid_source = lambda x: True++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0++ return {'BraveRM': usage}++ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):+ """Search with api.search.brave.com for self.k top passages for query or queries++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.++ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)+ collected_results = []+ for query in queries:+ try:+ headers = {+ "Accept": "application/json",+ "Accept-Encoding": "gzip",+ "X-Subscription-Token": self.brave_search_api_key+ }+ response = requests.get(+ f"https://api.search.brave.com/res/v1/web/search?result_filter=web&q={query}",+ headers=headers,+ ).json()+ results = response.get('web', {}).get('results', [])++ for result in results:+ collected_results.append(+ {+ 'snippets': result.get('extra_snippets', []),+ 'title': result.get('title'),+ 'url': result.get('url'),+ 'description': result.get('description'),+ }+ )+ except Exception as e:+ logging.error(f'Error occurs when searching query {query}: {e}')++ return collected_results+++++class SearXNG(dspy.Retrieve):+ def __init__(self, searxng_api_url, searxng_api_key=None, k=3, is_valid_source: Callable = None):+ """Initialize the SearXNG search retriever.+ Please set up SearXNG according to https://docs.searxng.org/index.html.++ Args:+ searxng_api_url (str): The URL of the SearXNG API. Consult SearXNG documentation for details.+ searxng_api_key (str, optional): The API key for the SearXNG API. Defaults to None. Consult SearXNG documentation for details.+ k (int, optional): The number of top passages to retrieve. Defaults to 3.+ is_valid_source (Callable, optional): A function that takes a URL and returns a boolean indicating if the+ source is valid. Defaults to None.+ """+ super().__init__(k=k)+ if not searxng_api_url:+ raise RuntimeError("You must supply searxng_api_url")+ self.searxng_api_url = searxng_api_url+ self.searxng_api_key = searxng_api_key+ self.usage = 0++ if is_valid_source:+ self.is_valid_source = is_valid_source+ else:+ self.is_valid_source = lambda x: True++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0+ return {'SearXNG': usage}++ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):+ """Search with SearxNG for self.k top passages for query or queries++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.++ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)+ collected_results = []+ headers = {"Authorization": f"Bearer {self.searxng_api_key}"} if self.searxng_api_key else {}++ for query in queries:+ try:+ params = {"q": query, "format": "json"}+ response = requests.get(self.searxng_api_url, headers=headers, params=params)+ results = response.json()++ for r in results['results']:+ if self.is_valid_source(r['url']) and r['url'] not in exclude_urls:+ collected_results.append({+ 'description': r.get('content', ''),+ 'snippets': [r.get('content', '')],+ 'title': r.get('title', ''),+ 'url': r['url']+ })+ except Exception as e:+ logging.error(f'Error occurs when searching query {query}: {e}')++ return collected_results