Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

Wrap src into a python package.

shaoyijia885a7d1cdcff
diff --git a/requirements.txt b/requirements.txt
index b1bdfa5..8ac1b95 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,27 +1,7 @@
dspy_ai==2.4.9
-streamlit==1.31.1
wikipedia==1.4.0
-streamlit_authenticator==0.2.3
-streamlit_oauth==0.1.8
-streamlit-card
-google-cloud==0.34.0
-google-cloud-vision==3.5.0
-google-cloud-storage==2.14.0
sentence_transformers
toml
-markdown
-unidecode
-extra-streamlit-components==0.1.60
-google-cloud-firestore==2.14.0
-firebase-admin==6.4.0
-streamlit_extras
-streamlit_cookies_manager
-deprecation==2.1.0
-st-pages==0.4.5
-streamlit-float
-streamlit-option-menu
-sentry-sdk
-pdfkit==1.0.0
langchain-text-splitters
trafilatura
langchain-huggingface
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..00eb7a7
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,40 @@
+import re
+
+from setuptools import setup, find_packages
+
+# Read the content of the README file
+with open("README.md", encoding="utf-8") as f:
+ long_description = f.read()
+ # Remove p tags.
+ pattern = re.compile(r'<p.*?>.*?</p>', re.DOTALL)
+ long_description = re.sub(pattern, '', long_description)
+
+# Read the content of the requirements.txt file
+with open("requirements.txt", encoding="utf-8") as f:
+ requirements = f.read().splitlines()
+
+
+setup(
+ name="knowledge-storm",
+ version="0.2.1",
+ author="Yijia Shao, Yucheng Jiang",
+ description="STORM: A language model-powered knowledge curation engine.",
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ url="https://github.com/stanford-oval/storm",
+ license="MIT License",
+ package_dir={"": "src"},
+ packages=find_packages("src"),
+ classifiers=[
+ "Development Status :: 3 - Alpha",
+ "License :: OSI Approved :: MIT License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ ],
+ python_requires='>=3.9',
+ install_requires=requirements,
+)
diff --git a/src/knowledge_storm/__init__.py b/src/knowledge_storm/__init__.py
new file mode 100644
index 0000000..f1fd18e
--- /dev/null
+++ b/src/knowledge_storm/__init__.py
@@ -0,0 +1,5 @@
+from .storm_wiki.engine import (
+ STORMWikiLMConfigs,
+ STORMWikiRunnerArguments,
+ STORMWikiRunner
+)
diff --git a/src/interface.py b/src/knowledge_storm/interface.py
similarity index 100%
rename from src/interface.py
rename to src/knowledge_storm/interface.py
diff --git a/src/lm.py b/src/knowledge_storm/lm.py
similarity index 100%
rename from src/lm.py
rename to src/knowledge_storm/lm.py
diff --git a/src/rm.py b/src/knowledge_storm/rm.py
similarity index 99%
rename from src/rm.py
rename to src/knowledge_storm/rm.py
index 5126aa5..86f5970 100644
--- a/src/rm.py
+++ b/src/knowledge_storm/rm.py
@@ -5,13 +5,13 @@ from typing import Callable, Union, List
import dspy
import pandas as pd
import requests
-from langchain_huggingface import HuggingFaceEmbeddings
from langchain_core.documents import Document
+from langchain_huggingface import HuggingFaceEmbeddings
from langchain_qdrant import Qdrant
from qdrant_client import QdrantClient, models
from tqdm import tqdm
-from utils import WebPageHelper
+from .utils import WebPageHelper
class YouRM(dspy.Retrieve):
diff --git a/src/storm_wiki/__init__.py b/src/knowledge_storm/storm_wiki/__init__.py
similarity index 100%
rename from src/storm_wiki/__init__.py
rename to src/knowledge_storm/storm_wiki/__init__.py
diff --git a/src/storm_wiki/engine.py b/src/knowledge_storm/storm_wiki/engine.py
similarity index 85%
rename from src/storm_wiki/engine.py
rename to src/knowledge_storm/storm_wiki/engine.py
index 4191d35..e0c8dfc 100644
--- a/src/storm_wiki/engine.py
+++ b/src/knowledge_storm/storm_wiki/engine.py
@@ -5,17 +5,18 @@ from dataclasses import dataclass, field
from typing import Union, Literal, Optional
import dspy
-from interface import Engine, LMConfigs
-from lm import OpenAIModel
-from storm_wiki.modules.article_generation import StormArticleGenerationModule
-from storm_wiki.modules.article_polish import StormArticlePolishingModule
-from storm_wiki.modules.callback import BaseCallbackHandler
-from storm_wiki.modules.knowledge_curation import StormKnowledgeCurationModule
-from storm_wiki.modules.outline_generation import StormOutlineGenerationModule
-from storm_wiki.modules.persona_generator import StormPersonaGenerator
-from storm_wiki.modules.retriever import StormRetriever
-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle
-from utils import FileIOHelper, makeStringRed
+
+from .modules.article_generation import StormArticleGenerationModule
+from .modules.article_polish import StormArticlePolishingModule
+from .modules.callback import BaseCallbackHandler
+from .modules.knowledge_curation import StormKnowledgeCurationModule
+from .modules.outline_generation import StormOutlineGenerationModule
+from .modules.persona_generator import StormPersonaGenerator
+from .modules.retriever import StormRetriever
+from .modules.storm_dataclass import StormInformationTable, StormArticle
+from ..interface import Engine, LMConfigs
+from ..lm import OpenAIModel
+from ..utils import FileIOHelper, makeStringRed
class STORMWikiLMConfigs(LMConfigs):
@@ -233,16 +234,20 @@ class STORMWikiRunner(Engine):
f.write(json.dumps(call) + '\n')
def _load_information_table_from_local_fs(self, information_table_local_path):
- assert os.path.exists(information_table_local_path), makeStringRed(f"{information_table_local_path} not exists. Please set --do-research argument to prepare the conversation_log.json for this topic.")
+ assert os.path.exists(information_table_local_path), makeStringRed(
+ f"{information_table_local_path} not exists. Please set --do-research argument to prepare the conversation_log.json for this topic.")
return StormInformationTable.from_conversation_log_file(information_table_local_path)
-
+
def _load_outline_from_local_fs(self, topic, outline_local_path):
- assert os.path.exists(outline_local_path), makeStringRed(f"{outline_local_path} not exists. Please set --do-generate-outline argument to prepare the storm_gen_outline.txt for this topic.")
+ assert os.path.exists(outline_local_path), makeStringRed(
+ f"{outline_local_path} not exists. Please set --do-generate-outline argument to prepare the storm_gen_outline.txt for this topic.")
return StormArticle.from_outline_file(topic=topic, file_path=outline_local_path)
def _load_draft_article_from_local_fs(self, topic, draft_article_path, url_to_info_path):
- assert os.path.exists(draft_article_path), makeStringRed(f"{draft_article_path} not exists. Please set --do-generate-article argument to prepare the storm_gen_article.txt for this topic.")
- assert os.path.exists(url_to_info_path), makeStringRed(f"{url_to_info_path} not exists. Please set --do-generate-article argument to prepare the url_to_info.json for this topic.")
+ assert os.path.exists(draft_article_path), makeStringRed(
+ f"{draft_article_path} not exists. Please set --do-generate-article argument to prepare the storm_gen_article.txt for this topic.")
+ assert os.path.exists(url_to_info_path), makeStringRed(
+ f"{url_to_info_path} not exists. Please set --do-generate-article argument to prepare the url_to_info.json for this topic.")
article_text = FileIOHelper.load_str(draft_article_path)
references = FileIOHelper.load_json(url_to_info_path)
return StormArticle.from_string(topic_name=topic, article_text=article_text, references=references)
@@ -274,7 +279,8 @@ class STORMWikiRunner(Engine):
callback_handler: A callback handler to handle the intermediate results.
"""
assert do_research or do_generate_outline or do_generate_article or do_polish_article, \
- makeStringRed("No action is specified. Please set at least one of --do-research, --do-generate-outline, --do-generate-article, --do-polish-article")
+ makeStringRed(
+ "No action is specified. Please set at least one of --do-research, --do-generate-outline, --do-generate-article, --do-polish-article")
self.topic = topic
self.article_dir_name = topic.replace(' ', '_').replace('/', '_')
@@ -291,7 +297,8 @@ class STORMWikiRunner(Engine):
if do_generate_outline:
# load information table if it's not initialized
if information_table is None:
- information_table = self._load_information_table_from_local_fs(os.path.join(self.article_output_dir, 'conversation_log.json'))
+ information_table = self._load_information_table_from_local_fs(
+ os.path.join(self.article_output_dir, 'conversation_log.json'))
outline = self.run_outline_generation_module(information_table=information_table,
callback_handler=callback_handler)
@@ -299,9 +306,12 @@ class STORMWikiRunner(Engine):
draft_article: StormArticle = None
if do_generate_article:
if information_table is None:
- information_table = self._load_information_table_from_local_fs(os.path.join(self.article_output_dir, 'conversation_log.json'))
+ information_table = self._load_information_table_from_local_fs(
+ os.path.join(self.article_output_dir, 'conversation_log.json'))
if outline is None:
- outline = self._load_outline_from_local_fs(topic=topic, outline_local_path=os.path.join(self.article_output_dir, 'storm_gen_outline.txt'))
+ outline = self._load_outline_from_local_fs(topic=topic,
+ outline_local_path=os.path.join(self.article_output_dir,
+ 'storm_gen_outline.txt'))
draft_article = self.run_article_generation_module(outline=outline,
information_table=information_table,
callback_handler=callback_handler)
@@ -311,5 +321,7 @@ class STORMWikiRunner(Engine):
if draft_article is None:
draft_article_path = os.path.join(self.article_output_dir, 'storm_gen_article.txt')
url_to_info_path = os.path.join(self.article_output_dir, 'url_to_info.json')
- draft_article = self._load_draft_article_from_local_fs(topic=topic, draft_article_path=draft_article_path, url_to_info_path=url_to_info_path)
+ draft_article = self._load_draft_article_from_local_fs(topic=topic,
+ draft_article_path=draft_article_path,
+ url_to_info_path=url_to_info_path)
self.run_article_polishing_module(draft_article=draft_article, remove_duplicate=remove_duplicate)
diff --git a/src/storm_wiki/modules/__init__.py b/src/knowledge_storm/storm_wiki/modules/__init__.py
similarity index 74%
rename from src/storm_wiki/modules/__init__.py
rename to src/knowledge_storm/storm_wiki/modules/__init__.py
index 51ee012..9419a31 100644
--- a/src/storm_wiki/modules/__init__.py
+++ b/src/knowledge_storm/storm_wiki/modules/__init__.py
@@ -1,4 +1,4 @@
from .knowledge_curation import *
from .persona_generator import *
from .retriever import *
-from .storm_dataclass import *
\ No newline at end of file
+from .storm_dataclass import *
diff --git a/src/storm_wiki/modules/article_generation.py b/src/knowledge_storm/storm_wiki/modules/article_generation.py
similarity index 96%
rename from src/storm_wiki/modules/article_generation.py
rename to src/knowledge_storm/storm_wiki/modules/article_generation.py
index 0dfb76d..a114b3e 100644
--- a/src/storm_wiki/modules/article_generation.py
+++ b/src/knowledge_storm/storm_wiki/modules/article_generation.py
@@ -5,10 +5,11 @@ from concurrent.futures import as_completed
from typing import List, Union
import dspy
-from interface import ArticleGenerationModule
-from storm_wiki.modules.callback import BaseCallbackHandler
-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle, StormInformation
-from utils import ArticleTextProcessing
+
+from .callback import BaseCallbackHandler
+from .storm_dataclass import StormInformationTable, StormArticle, StormInformation
+from ...interface import ArticleGenerationModule
+from ...utils import ArticleTextProcessing
class StormArticleGenerationModule(ArticleGenerationModule):
diff --git a/src/storm_wiki/modules/article_polish.py b/src/knowledge_storm/storm_wiki/modules/article_polish.py
similarity index 96%
rename from src/storm_wiki/modules/article_polish.py
rename to src/knowledge_storm/storm_wiki/modules/article_polish.py
index 5f38f05..b70bb83 100644
--- a/src/storm_wiki/modules/article_polish.py
+++ b/src/knowledge_storm/storm_wiki/modules/article_polish.py
@@ -2,9 +2,10 @@ import copy
from typing import Union
import dspy
-from interface import ArticlePolishingModule
-from storm_wiki.modules.storm_dataclass import StormArticle
-from utils import ArticleTextProcessing
+
+from .storm_dataclass import StormArticle
+from ...interface import ArticlePolishingModule
+from ...utils import ArticleTextProcessing
class StormArticlePolishingModule(ArticlePolishingModule):
diff --git a/src/storm_wiki/modules/callback.py b/src/knowledge_storm/storm_wiki/modules/callback.py
similarity index 98%
rename from src/storm_wiki/modules/callback.py
rename to src/knowledge_storm/storm_wiki/modules/callback.py
index 945a45d..a4b702d 100644
--- a/src/storm_wiki/modules/callback.py
+++ b/src/knowledge_storm/storm_wiki/modules/callback.py
@@ -31,4 +31,4 @@ class BaseCallbackHandler:
def on_outline_refinement_end(self, outline: str, **kwargs):
"""Run when the outline refinement finishes."""
- pass
\ No newline at end of file
+ pass
diff --git a/src/storm_wiki/modules/knowledge_curation.py b/src/knowledge_storm/storm_wiki/modules/knowledge_curation.py
similarity index 97%
rename from src/storm_wiki/modules/knowledge_curation.py
rename to src/knowledge_storm/storm_wiki/modules/knowledge_curation.py
index 4fe7f15..8e881c6 100644
--- a/src/storm_wiki/modules/knowledge_curation.py
+++ b/src/knowledge_storm/storm_wiki/modules/knowledge_curation.py
@@ -5,14 +5,16 @@ from concurrent.futures import as_completed
from typing import Union, List, Tuple, Optional, Dict
import dspy
-from interface import KnowledgeCurationModule, Retriever
-from storm_wiki.modules.callback import BaseCallbackHandler
-from storm_wiki.modules.persona_generator import StormPersonaGenerator
-from storm_wiki.modules.storm_dataclass import DialogueTurn, StormInformationTable, StormInformation
-from utils import ArticleTextProcessing
+
+from .callback import BaseCallbackHandler
+from .persona_generator import StormPersonaGenerator
+from .storm_dataclass import DialogueTurn, StormInformationTable, StormInformation
+from ...interface import KnowledgeCurationModule, Retriever
+from ...utils import ArticleTextProcessing
try:
from streamlit.runtime.scriptrunner import add_script_run_ctx
+
streamlit_connection = True
except ImportError as err:
streamlit_connection = False
diff --git a/src/storm_wiki/modules/outline_generation.py b/src/knowledge_storm/storm_wiki/modules/outline_generation.py
similarity index 96%
rename from src/storm_wiki/modules/outline_generation.py
rename to src/knowledge_storm/storm_wiki/modules/outline_generation.py
index 2b09d52..1f45b1c 100644
--- a/src/storm_wiki/modules/outline_generation.py
+++ b/src/knowledge_storm/storm_wiki/modules/outline_generation.py
@@ -1,10 +1,11 @@
from typing import Union, Optional, Tuple
import dspy
-from interface import OutlineGenerationModule
-from storm_wiki.modules.callback import BaseCallbackHandler
-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle
-from utils import ArticleTextProcessing
+
+from .callback import BaseCallbackHandler
+from .storm_dataclass import StormInformationTable, StormArticle
+from ...interface import OutlineGenerationModule
+from ...utils import ArticleTextProcessing
class StormOutlineGenerationModule(OutlineGenerationModule):
diff --git a/src/storm_wiki/modules/persona_generator.py b/src/knowledge_storm/storm_wiki/modules/persona_generator.py
similarity index 100%
rename from src/storm_wiki/modules/persona_generator.py
rename to src/knowledge_storm/storm_wiki/modules/persona_generator.py
diff --git a/src/storm_wiki/modules/internet_source_restrictions.json b/src/knowledge_storm/storm_wiki/modules/retriever.py
similarity index 72%
rename from src/storm_wiki/modules/internet_source_restrictions.json
rename to src/knowledge_storm/storm_wiki/modules/retriever.py
index 0a71e1a..179ae99 100644
--- a/src/storm_wiki/modules/internet_source_restrictions.json
+++ b/src/knowledge_storm/storm_wiki/modules/retriever.py
@@ -1,5 +1,15 @@
-{
- "generally_unreliable": [
+from typing import Union, List
+from urllib.parse import urlparse
+
+import dspy
+
+from .storm_dataclass import StormInformation
+from ...interface import Retriever, Information
+from ...utils import ArticleTextProcessing
+
+# Internet source restrictions according to Wikipedia standard:
+# https://en.wikipedia.org/wiki/Wikipedia:Reliable_sources/Perennial_sources
+GENERALLY_UNRELIABLE = {
"112_Ukraine",
"Ad_Fontes_Media",
"AlterNet",
@@ -139,9 +149,8 @@
"WordPress.com",
"Worldometer",
"YouTube",
- "ZDNet"
- ],
- "deprecated": [
+ "ZDNet"}
+DEPRECATED = {
"Al_Mayadeen",
"ANNA_News",
"Baidu_Baike",
@@ -189,8 +198,8 @@
"Voltaire_Network",
"WorldNetDaily",
"Zero_Hedge"
- ],
- "blacklisted": [
+}
+BLACKLISTED = {
"Advameg",
"bestgore.com",
"Breitbart_News",
@@ -210,5 +219,32 @@
"Swarajya",
"Veterans_Today",
"ZoomInfo"
- ]
}
+
+
+def is_valid_wikipedia_source(url):
+ parsed_url = urlparse(url)
+ # Check if the URL is from a reliable domain
+ combined_set = GENERALLY_UNRELIABLE | DEPRECATED | BLACKLISTED
+ for domain in combined_set:
+ if domain in parsed_url.netloc:
+ return False
+
+ return True
+
+
+class StormRetriever(Retriever):
+ def __init__(self, rm: dspy.Retrieve, k=3):
+ super().__init__(search_top_k=k)
+ self._rm = rm
+ if hasattr(rm, 'is_valid_source'):
+ rm.is_valid_source = is_valid_wikipedia_source
+
+ def retrieve(self, query: Union[str, List[str]], exclude_urls: List[str] = []) -> List[Information]:
+ retrieved_data_list = self._rm(query_or_queries=query, exclude_urls=exclude_urls)
+ for data in retrieved_data_list:
+ for i in range(len(data['snippets'])):
+ # STORM generate the article with citations. We do not consider multi-hop citations.
+ # Remove citations in the source to avoid confusion.
+ data['snippets'][i] = ArticleTextProcessing.remove_citations(data['snippets'][i])
+ return [StormInformation.from_dict(data) for data in retrieved_data_list]
diff --git a/src/storm_wiki/modules/storm_dataclass.py b/src/knowledge_storm/storm_wiki/modules/storm_dataclass.py
similarity index 99%
rename from src/storm_wiki/modules/storm_dataclass.py
rename to src/knowledge_storm/storm_wiki/modules/storm_dataclass.py
index d75760c..4f54ec4 100644
--- a/src/storm_wiki/modules/storm_dataclass.py
+++ b/src/knowledge_storm/storm_wiki/modules/storm_dataclass.py
@@ -4,10 +4,11 @@ from collections import OrderedDict
from typing import Union, Optional, Any, List, Tuple, Dict
import numpy as np
-from interface import Information, InformationTable, Article, ArticleSectionNode
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
-from utils import ArticleTextProcessing, FileIOHelper
+
+from ...interface import Information, InformationTable, Article, ArticleSectionNode
+from ...utils import ArticleTextProcessing, FileIOHelper
class StormInformation(Information):
diff --git a/src/utils.py b/src/knowledge_storm/utils.py
similarity index 100%
rename from src/utils.py
rename to src/knowledge_storm/utils.py
diff --git a/src/storm_wiki/modules/retriever.py b/src/storm_wiki/modules/retriever.py
deleted file mode 100644
index 79cc206..0000000
--- a/src/storm_wiki/modules/retriever.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import json
-import os
-from typing import Union, List
-from urllib.parse import urlparse
-
-import dspy
-import storm_wiki.modules.storm_dataclass as storm_dataclass
-from interface import Retriever, Information
-from rm import YouRM
-from utils import ArticleTextProcessing
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-with open(os.path.join(SCRIPT_DIR, 'internet_source_restrictions.json')) as f:
- domain_restriction_dict = json.load(f)
- GENERALLY_UNRELIABLE = set(domain_restriction_dict["generally_unreliable"])
- DEPRECATED = set(domain_restriction_dict["deprecated"])
- BLACKLISTED = set(domain_restriction_dict["blacklisted"])
-
-
-def is_valid_wikipedia_source(url):
- parsed_url = urlparse(url)
- # Check if the URL is from a reliable domain
- combined_set = GENERALLY_UNRELIABLE | DEPRECATED | BLACKLISTED
- for domain in combined_set:
- if domain in parsed_url.netloc:
- return False
-
- return True
-
-
-class StormRetriever(Retriever):
- def __init__(self, rm: dspy.Retrieve, k=3):
- super().__init__(search_top_k=k)
- self._rm = rm
- if hasattr(rm, 'is_valid_source'):
- rm.is_valid_source = is_valid_wikipedia_source
-
- def retrieve(self, query: Union[str, List[str]], exclude_urls: List[str] = []) -> List[Information]:
- retrieved_data_list = self._rm(query_or_queries=query, exclude_urls=exclude_urls)
- for data in retrieved_data_list:
- for i in range(len(data['snippets'])):
- # STORM generate the article with citations. We do not consider multi-hop citations.
- # Remove citations in the source to avoid confusion.
- data['snippets'][i] = ArticleTextProcessing.remove_citations(data['snippets'][i])
- return [storm_dataclass.StormInformation.from_dict(data) for data in retrieved_data_list]