An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Support backoff and retry when encountering `duckduckgo_search.exceptions.RatelimitException`.
shaoyijiad50511f76a70
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 7d4d696..dfcee5e 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -1,8 +1,11 @@import loggingimport osfrom typing import Callable, Union, List++import backoffimport dspyimport requests+from dsp import backoff_hdlr, giveup_hdlrfrom langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_qdrant import Qdrant@@ -684,6 +687,20 @@ class DuckDuckGoSearchRM(dspy.Retrieve):self.usage = 0return {"DuckDuckGoRM": usage}+ @backoff.on_exception(+ backoff.expo,+ (Exception,),+ max_time=1000,+ max_tries=8,+ on_backoff=backoff_hdlr,+ giveup=giveup_hdlr,+ )+ def request(self, query: str):+ results = self.ddgs.text(+ query, max_results=self.k, backend=self.duck_duck_go_backend+ )+ return results+def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):@@ -705,9 +722,7 @@ class DuckDuckGoSearchRM(dspy.Retrieve):for query in queries:# list of dicts that will be parsed to return- results = self.ddgs.text(- query, max_results=self.k, backend=self.duck_duck_go_backend- )+ results = self.request(query)for d in results:# assert d is dict