An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Auto-sync-2024-26-04-04-11-33
Yucheng-Jiang80f1734f5a6d
diff --git a/src/lm.py b/src/lm.pyindex 9c38f40..8dfdc8a 100644--- a/src/lm.py+++ b/src/lm.py@@ -4,7 +4,6 @@ import randomimport threadingfrom typing import Optional, Literal, Any-import anthropicimport backoffimport dspyimport requests@@ -109,14 +108,14 @@ class ClaudeModel(dspy.dsp.modules.lm.LM):def __init__(self,- model: str = "claude-3-opus-20240229",+ model: str,api_key: Optional[str] = None,api_base: Optional[str] = None,**kwargs,):super().__init__(model)try:- from anthropic import Anthropic+ from anthropic import Anthropic, RateLimitErrorexcept ImportError as err:raise ImportError("Claude requires `pip install anthropic`.") from err@@ -189,7 +188,7 @@ class ClaudeModel(dspy.dsp.modules.lm.LM):@backoff.on_exception(backoff.expo,- (anthropic.RateLimitError),+ (RateLimitError,),max_time=1000,max_tries=8,on_backoff=backoff_hdlr,diff --git a/src/storm_wiki/modules/knowledge_curation.py b/src/storm_wiki/modules/knowledge_curation.pyindex 16f0de3..4fe7f15 100644--- a/src/storm_wiki/modules/knowledge_curation.py+++ b/src/storm_wiki/modules/knowledge_curation.py@@ -9,9 +9,14 @@ from interface import KnowledgeCurationModule, Retrieverfrom storm_wiki.modules.callback import BaseCallbackHandlerfrom storm_wiki.modules.persona_generator import StormPersonaGeneratorfrom storm_wiki.modules.storm_dataclass import DialogueTurn, StormInformationTable, StormInformation-from streamlit.runtime.scriptrunner import add_script_run_ctxfrom utils import ArticleTextProcessing+try:+ from streamlit.runtime.scriptrunner import add_script_run_ctx+ streamlit_connection = True+except ImportError as err:+ streamlit_connection = False+script_dir = os.path.dirname(os.path.abspath(__file__))@@ -260,9 +265,11 @@ class StormKnowledgeCurationModule(KnowledgeCurationModule):with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:future_to_persona = {executor.submit(run_conv, persona): persona for persona in considered_personas}- # Ensure the logging context is correct when connecting with Streamlit frontend.- for t in executor._threads:- add_script_run_ctx(t)++ if streamlit_connection:+ # Ensure the logging context is correct when connecting with Streamlit frontend.+ for t in executor._threads:+ add_script_run_ctx(t)for future in as_completed(future_to_persona):persona = future_to_persona[future]