Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

Adjust directory structure: src/knowledge_storm -> knowledge_storm.

shaoyijia9cf9eefdd04a
diff --git a/src/knowledge_storm/__init__.py b/knowledge_storm/__init__.py
similarity index 100%
rename from src/knowledge_storm/__init__.py
rename to knowledge_storm/__init__.py
diff --git a/src/knowledge_storm/interface.py b/knowledge_storm/interface.py
similarity index 100%
rename from src/knowledge_storm/interface.py
rename to knowledge_storm/interface.py
diff --git a/src/knowledge_storm/lm.py b/knowledge_storm/lm.py
similarity index 91%
rename from src/knowledge_storm/lm.py
rename to knowledge_storm/lm.py
index 60dff4d..886137f 100644
--- a/src/knowledge_storm/lm.py
+++ b/knowledge_storm/lm.py
@@ -25,13 +25,10 @@ class OpenAIModel(dspy.OpenAI):
self,
model: str = "gpt-3.5-turbo-instruct",
api_key: Optional[str] = None,
- api_provider: Literal["openai", "azure"] = "openai",
- api_base: Optional[str] = None,
model_type: Literal["chat", "text"] = None,
**kwargs
):
- super().__init__(model=model, api_key=api_key, api_provider=api_provider, api_base=api_base,
- model_type=model_type, **kwargs)
+ super().__init__(model=model, api_key=api_key, model_type=model_type, **kwargs)
self._token_usage_lock = threading.Lock()
self.prompt_tokens = 0
self.completion_tokens = 0
@@ -108,6 +105,46 @@ class OpenAIModel(dspy.OpenAI):
return completions
+class AzureOpenAIModel(dspy.AzureOpenAI):
+ """A wrapper class for dspy.AzureOpenAI."""
+ def __init__(
+ self,
+ api_base: str,
+ api_version: str,
+ model: str = "gpt-3.5-turbo-instruct",
+ api_key: Optional[str] = None,
+ model_type: Literal["chat", "text"] = "chat",
+ **kwargs,
+ ):
+ super().__init__(
+ api_base=api_base, api_version=api_version, model=model, api_key=api_key, model_type=model_type, **kwargs)
+ self._token_usage_lock = threading.Lock()
+ self.prompt_tokens = 0
+ self.completion_tokens = 0
+
+ def log_usage(self, response):
+ """Log the total tokens from the OpenAI API response.
+ Override log_usage() in dspy.AzureOpenAI for tracking accumulated token usage."""
+ usage_data = response.get('usage')
+ if usage_data:
+ with self._token_usage_lock:
+ self.prompt_tokens += usage_data.get('prompt_tokens', 0)
+ self.completion_tokens += usage_data.get('completion_tokens', 0)
+
+ def get_usage_and_reset(self):
+ """Get the total tokens used and reset the token usage."""
+ usage = {
+ self.kwargs.get('model') or self.kwargs.get('engine'):
+ {'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens}
+ }
+ self.prompt_tokens = 0
+ self.completion_tokens = 0
+
+ return usage
+
+
+
+
class ClaudeModel(dspy.dsp.modules.lm.LM):
"""Copied from dspy/dsp/modules/anthropic.py with the addition of tracking token usage."""
diff --git a/src/knowledge_storm/rm.py b/knowledge_storm/rm.py
similarity index 100%
rename from src/knowledge_storm/rm.py
rename to knowledge_storm/rm.py
diff --git a/src/knowledge_storm/storm_wiki/__init__.py b/knowledge_storm/storm_wiki/__init__.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/__init__.py
rename to knowledge_storm/storm_wiki/__init__.py
diff --git a/src/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/engine.py
rename to knowledge_storm/storm_wiki/engine.py
diff --git a/src/knowledge_storm/storm_wiki/modules/__init__.py b/knowledge_storm/storm_wiki/modules/__init__.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/__init__.py
rename to knowledge_storm/storm_wiki/modules/__init__.py
diff --git a/src/knowledge_storm/storm_wiki/modules/article_generation.py b/knowledge_storm/storm_wiki/modules/article_generation.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/article_generation.py
rename to knowledge_storm/storm_wiki/modules/article_generation.py
diff --git a/src/knowledge_storm/storm_wiki/modules/article_polish.py b/knowledge_storm/storm_wiki/modules/article_polish.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/article_polish.py
rename to knowledge_storm/storm_wiki/modules/article_polish.py
diff --git a/src/knowledge_storm/storm_wiki/modules/callback.py b/knowledge_storm/storm_wiki/modules/callback.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/callback.py
rename to knowledge_storm/storm_wiki/modules/callback.py
diff --git a/src/knowledge_storm/storm_wiki/modules/knowledge_curation.py b/knowledge_storm/storm_wiki/modules/knowledge_curation.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/knowledge_curation.py
rename to knowledge_storm/storm_wiki/modules/knowledge_curation.py
diff --git a/src/knowledge_storm/storm_wiki/modules/outline_generation.py b/knowledge_storm/storm_wiki/modules/outline_generation.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/outline_generation.py
rename to knowledge_storm/storm_wiki/modules/outline_generation.py
diff --git a/src/knowledge_storm/storm_wiki/modules/persona_generator.py b/knowledge_storm/storm_wiki/modules/persona_generator.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/persona_generator.py
rename to knowledge_storm/storm_wiki/modules/persona_generator.py
diff --git a/src/knowledge_storm/storm_wiki/modules/retriever.py b/knowledge_storm/storm_wiki/modules/retriever.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/retriever.py
rename to knowledge_storm/storm_wiki/modules/retriever.py
diff --git a/src/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.py
similarity index 100%
rename from src/knowledge_storm/storm_wiki/modules/storm_dataclass.py
rename to knowledge_storm/storm_wiki/modules/storm_dataclass.py
diff --git a/src/knowledge_storm/utils.py b/knowledge_storm/utils.py
similarity index 99%
rename from src/knowledge_storm/utils.py
rename to knowledge_storm/utils.py
index cc1a6e5..5cf6f45 100644
--- a/src/knowledge_storm/utils.py
+++ b/knowledge_storm/utils.py
@@ -1,5 +1,6 @@
import concurrent.futures
import json
+import logging
import os
import pickle
import re
@@ -11,6 +12,8 @@ import toml
from langchain_text_splitters import RecursiveCharacterTextSplitter
from trafilatura import extract
+logging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.
+
def load_api_key(toml_file_path):
try:
diff --git a/setup.py b/setup.py
index 00eb7a7..120927b 100644
--- a/setup.py
+++ b/setup.py
@@ -24,8 +24,7 @@ setup(
long_description_content_type="text/markdown",
url="https://github.com/stanford-oval/storm",
license="MIT License",
- package_dir={"": "src"},
- packages=find_packages("src"),
+ packages=find_packages(),
classifiers=[
"Development Status :: 3 - Alpha",
"License :: OSI Approved :: MIT License",