An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Adjust directory structure: src/knowledge_storm -> knowledge_storm.
shaoyijia9cf9eefdd04a
diff --git a/src/knowledge_storm/__init__.py b/knowledge_storm/__init__.pysimilarity index 100%rename from src/knowledge_storm/__init__.pyrename to knowledge_storm/__init__.pydiff --git a/src/knowledge_storm/interface.py b/knowledge_storm/interface.pysimilarity index 100%rename from src/knowledge_storm/interface.pyrename to knowledge_storm/interface.pydiff --git a/src/knowledge_storm/lm.py b/knowledge_storm/lm.pysimilarity index 91%rename from src/knowledge_storm/lm.pyrename to knowledge_storm/lm.pyindex 60dff4d..886137f 100644--- a/src/knowledge_storm/lm.py+++ b/knowledge_storm/lm.py@@ -25,13 +25,10 @@ class OpenAIModel(dspy.OpenAI):self,model: str = "gpt-3.5-turbo-instruct",api_key: Optional[str] = None,- api_provider: Literal["openai", "azure"] = "openai",- api_base: Optional[str] = None,model_type: Literal["chat", "text"] = None,**kwargs):- super().__init__(model=model, api_key=api_key, api_provider=api_provider, api_base=api_base,- model_type=model_type, **kwargs)+ super().__init__(model=model, api_key=api_key, model_type=model_type, **kwargs)self._token_usage_lock = threading.Lock()self.prompt_tokens = 0self.completion_tokens = 0@@ -108,6 +105,46 @@ class OpenAIModel(dspy.OpenAI):return completions+class AzureOpenAIModel(dspy.AzureOpenAI):+ """A wrapper class for dspy.AzureOpenAI."""+ def __init__(+ self,+ api_base: str,+ api_version: str,+ model: str = "gpt-3.5-turbo-instruct",+ api_key: Optional[str] = None,+ model_type: Literal["chat", "text"] = "chat",+ **kwargs,+ ):+ super().__init__(+ api_base=api_base, api_version=api_version, model=model, api_key=api_key, model_type=model_type, **kwargs)+ self._token_usage_lock = threading.Lock()+ self.prompt_tokens = 0+ self.completion_tokens = 0++ def log_usage(self, response):+ """Log the total tokens from the OpenAI API response.+ Override log_usage() in dspy.AzureOpenAI for tracking accumulated token usage."""+ usage_data = response.get('usage')+ if usage_data:+ with self._token_usage_lock:+ self.prompt_tokens += usage_data.get('prompt_tokens', 0)+ self.completion_tokens += usage_data.get('completion_tokens', 0)++ def get_usage_and_reset(self):+ """Get the total tokens used and reset the token usage."""+ usage = {+ self.kwargs.get('model') or self.kwargs.get('engine'):+ {'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens}+ }+ self.prompt_tokens = 0+ self.completion_tokens = 0++ return usage++++class ClaudeModel(dspy.dsp.modules.lm.LM):"""Copied from dspy/dsp/modules/anthropic.py with the addition of tracking token usage."""diff --git a/src/knowledge_storm/rm.py b/knowledge_storm/rm.pysimilarity index 100%rename from src/knowledge_storm/rm.pyrename to knowledge_storm/rm.pydiff --git a/src/knowledge_storm/storm_wiki/__init__.py b/knowledge_storm/storm_wiki/__init__.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/__init__.pyrename to knowledge_storm/storm_wiki/__init__.pydiff --git a/src/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/engine.pyrename to knowledge_storm/storm_wiki/engine.pydiff --git a/src/knowledge_storm/storm_wiki/modules/__init__.py b/knowledge_storm/storm_wiki/modules/__init__.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/__init__.pyrename to knowledge_storm/storm_wiki/modules/__init__.pydiff --git a/src/knowledge_storm/storm_wiki/modules/article_generation.py b/knowledge_storm/storm_wiki/modules/article_generation.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/article_generation.pyrename to knowledge_storm/storm_wiki/modules/article_generation.pydiff --git a/src/knowledge_storm/storm_wiki/modules/article_polish.py b/knowledge_storm/storm_wiki/modules/article_polish.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/article_polish.pyrename to knowledge_storm/storm_wiki/modules/article_polish.pydiff --git a/src/knowledge_storm/storm_wiki/modules/callback.py b/knowledge_storm/storm_wiki/modules/callback.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/callback.pyrename to knowledge_storm/storm_wiki/modules/callback.pydiff --git a/src/knowledge_storm/storm_wiki/modules/knowledge_curation.py b/knowledge_storm/storm_wiki/modules/knowledge_curation.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/knowledge_curation.pyrename to knowledge_storm/storm_wiki/modules/knowledge_curation.pydiff --git a/src/knowledge_storm/storm_wiki/modules/outline_generation.py b/knowledge_storm/storm_wiki/modules/outline_generation.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/outline_generation.pyrename to knowledge_storm/storm_wiki/modules/outline_generation.pydiff --git a/src/knowledge_storm/storm_wiki/modules/persona_generator.py b/knowledge_storm/storm_wiki/modules/persona_generator.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/persona_generator.pyrename to knowledge_storm/storm_wiki/modules/persona_generator.pydiff --git a/src/knowledge_storm/storm_wiki/modules/retriever.py b/knowledge_storm/storm_wiki/modules/retriever.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/retriever.pyrename to knowledge_storm/storm_wiki/modules/retriever.pydiff --git a/src/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.pysimilarity index 100%rename from src/knowledge_storm/storm_wiki/modules/storm_dataclass.pyrename to knowledge_storm/storm_wiki/modules/storm_dataclass.pydiff --git a/src/knowledge_storm/utils.py b/knowledge_storm/utils.pysimilarity index 99%rename from src/knowledge_storm/utils.pyrename to knowledge_storm/utils.pyindex cc1a6e5..5cf6f45 100644--- a/src/knowledge_storm/utils.py+++ b/knowledge_storm/utils.py@@ -1,5 +1,6 @@import concurrent.futuresimport json+import loggingimport osimport pickleimport re@@ -11,6 +12,8 @@ import tomlfrom langchain_text_splitters import RecursiveCharacterTextSplitterfrom trafilatura import extract+logging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.+def load_api_key(toml_file_path):try:diff --git a/setup.py b/setup.pyindex 00eb7a7..120927b 100644--- a/setup.py+++ b/setup.py@@ -24,8 +24,7 @@ setup(long_description_content_type="text/markdown",url="https://github.com/stanford-oval/storm",license="MIT License",- package_dir={"": "src"},- packages=find_packages("src"),+ packages=find_packages(),classifiers=["Development Status :: 3 - Alpha","License :: OSI Approved :: MIT License",