An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Wrap src into a python package.
shaoyijia885a7d1cdcff
diff --git a/requirements.txt b/requirements.txtindex b1bdfa5..8ac1b95 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,27 +1,7 @@dspy_ai==2.4.9-streamlit==1.31.1wikipedia==1.4.0-streamlit_authenticator==0.2.3-streamlit_oauth==0.1.8-streamlit-card-google-cloud==0.34.0-google-cloud-vision==3.5.0-google-cloud-storage==2.14.0sentence_transformerstoml-markdown-unidecode-extra-streamlit-components==0.1.60-google-cloud-firestore==2.14.0-firebase-admin==6.4.0-streamlit_extras-streamlit_cookies_manager-deprecation==2.1.0-st-pages==0.4.5-streamlit-float-streamlit-option-menu-sentry-sdk-pdfkit==1.0.0langchain-text-splitterstrafilaturalangchain-huggingfacediff --git a/setup.py b/setup.pynew file mode 100644index 0000000..00eb7a7--- /dev/null+++ b/setup.py@@ -0,0 +1,40 @@+import re++from setuptools import setup, find_packages++# Read the content of the README file+with open("README.md", encoding="utf-8") as f:+ long_description = f.read()+ # Remove p tags.+ pattern = re.compile(r'<p.*?>.*?</p>', re.DOTALL)+ long_description = re.sub(pattern, '', long_description)++# Read the content of the requirements.txt file+with open("requirements.txt", encoding="utf-8") as f:+ requirements = f.read().splitlines()+++setup(+ name="knowledge-storm",+ version="0.2.1",+ author="Yijia Shao, Yucheng Jiang",+ author_email="[email protected], [email protected]",+ description="STORM: A language model-powered knowledge curation engine.",+ long_description=long_description,+ long_description_content_type="text/markdown",+ url="https://github.com/stanford-oval/storm",+ license="MIT License",+ package_dir={"": "src"},+ packages=find_packages("src"),+ classifiers=[+ "Development Status :: 3 - Alpha",+ "License :: OSI Approved :: MIT License",+ "Operating System :: OS Independent",+ "Programming Language :: Python :: 3",+ "Programming Language :: Python :: 3.9",+ "Programming Language :: Python :: 3.10",+ "Programming Language :: Python :: 3.11",+ ],+ python_requires='>=3.9',+ install_requires=requirements,+)diff --git a/src/knowledge_storm/__init__.py b/src/knowledge_storm/__init__.pynew file mode 100644index 0000000..f1fd18e--- /dev/null+++ b/src/knowledge_storm/__init__.py@@ -0,0 +1,5 @@+from .storm_wiki.engine import (+ STORMWikiLMConfigs,+ STORMWikiRunnerArguments,+ STORMWikiRunner+)diff --git a/src/interface.py b/src/knowledge_storm/interface.pysimilarity index 100%rename from src/interface.pyrename to src/knowledge_storm/interface.pydiff --git a/src/lm.py b/src/knowledge_storm/lm.pysimilarity index 100%rename from src/lm.pyrename to src/knowledge_storm/lm.pydiff --git a/src/rm.py b/src/knowledge_storm/rm.pysimilarity index 99%rename from src/rm.pyrename to src/knowledge_storm/rm.pyindex 5126aa5..86f5970 100644--- a/src/rm.py+++ b/src/knowledge_storm/rm.py@@ -5,13 +5,13 @@ from typing import Callable, Union, Listimport dspyimport pandas as pdimport requests-from langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_core.documents import Document+from langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_qdrant import Qdrantfrom qdrant_client import QdrantClient, modelsfrom tqdm import tqdm-from utils import WebPageHelper+from .utils import WebPageHelperclass YouRM(dspy.Retrieve):diff --git a/src/storm_wiki/__init__.py b/src/knowledge_storm/storm_wiki/__init__.pysimilarity index 100%rename from src/storm_wiki/__init__.pyrename to src/knowledge_storm/storm_wiki/__init__.pydiff --git a/src/storm_wiki/engine.py b/src/knowledge_storm/storm_wiki/engine.pysimilarity index 85%rename from src/storm_wiki/engine.pyrename to src/knowledge_storm/storm_wiki/engine.pyindex 4191d35..e0c8dfc 100644--- a/src/storm_wiki/engine.py+++ b/src/knowledge_storm/storm_wiki/engine.py@@ -5,17 +5,18 @@ from dataclasses import dataclass, fieldfrom typing import Union, Literal, Optionalimport dspy-from interface import Engine, LMConfigs-from lm import OpenAIModel-from storm_wiki.modules.article_generation import StormArticleGenerationModule-from storm_wiki.modules.article_polish import StormArticlePolishingModule-from storm_wiki.modules.callback import BaseCallbackHandler-from storm_wiki.modules.knowledge_curation import StormKnowledgeCurationModule-from storm_wiki.modules.outline_generation import StormOutlineGenerationModule-from storm_wiki.modules.persona_generator import StormPersonaGenerator-from storm_wiki.modules.retriever import StormRetriever-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle-from utils import FileIOHelper, makeStringRed++from .modules.article_generation import StormArticleGenerationModule+from .modules.article_polish import StormArticlePolishingModule+from .modules.callback import BaseCallbackHandler+from .modules.knowledge_curation import StormKnowledgeCurationModule+from .modules.outline_generation import StormOutlineGenerationModule+from .modules.persona_generator import StormPersonaGenerator+from .modules.retriever import StormRetriever+from .modules.storm_dataclass import StormInformationTable, StormArticle+from ..interface import Engine, LMConfigs+from ..lm import OpenAIModel+from ..utils import FileIOHelper, makeStringRedclass STORMWikiLMConfigs(LMConfigs):@@ -233,16 +234,20 @@ class STORMWikiRunner(Engine):f.write(json.dumps(call) + '\n')def _load_information_table_from_local_fs(self, information_table_local_path):- assert os.path.exists(information_table_local_path), makeStringRed(f"{information_table_local_path} not exists. Please set --do-research argument to prepare the conversation_log.json for this topic.")+ assert os.path.exists(information_table_local_path), makeStringRed(+ f"{information_table_local_path} not exists. Please set --do-research argument to prepare the conversation_log.json for this topic.")return StormInformationTable.from_conversation_log_file(information_table_local_path)-+def _load_outline_from_local_fs(self, topic, outline_local_path):- assert os.path.exists(outline_local_path), makeStringRed(f"{outline_local_path} not exists. Please set --do-generate-outline argument to prepare the storm_gen_outline.txt for this topic.")+ assert os.path.exists(outline_local_path), makeStringRed(+ f"{outline_local_path} not exists. Please set --do-generate-outline argument to prepare the storm_gen_outline.txt for this topic.")return StormArticle.from_outline_file(topic=topic, file_path=outline_local_path)def _load_draft_article_from_local_fs(self, topic, draft_article_path, url_to_info_path):- assert os.path.exists(draft_article_path), makeStringRed(f"{draft_article_path} not exists. Please set --do-generate-article argument to prepare the storm_gen_article.txt for this topic.")- assert os.path.exists(url_to_info_path), makeStringRed(f"{url_to_info_path} not exists. Please set --do-generate-article argument to prepare the url_to_info.json for this topic.")+ assert os.path.exists(draft_article_path), makeStringRed(+ f"{draft_article_path} not exists. Please set --do-generate-article argument to prepare the storm_gen_article.txt for this topic.")+ assert os.path.exists(url_to_info_path), makeStringRed(+ f"{url_to_info_path} not exists. Please set --do-generate-article argument to prepare the url_to_info.json for this topic.")article_text = FileIOHelper.load_str(draft_article_path)references = FileIOHelper.load_json(url_to_info_path)return StormArticle.from_string(topic_name=topic, article_text=article_text, references=references)@@ -274,7 +279,8 @@ class STORMWikiRunner(Engine):callback_handler: A callback handler to handle the intermediate results."""assert do_research or do_generate_outline or do_generate_article or do_polish_article, \- makeStringRed("No action is specified. Please set at least one of --do-research, --do-generate-outline, --do-generate-article, --do-polish-article")+ makeStringRed(+ "No action is specified. Please set at least one of --do-research, --do-generate-outline, --do-generate-article, --do-polish-article")self.topic = topicself.article_dir_name = topic.replace(' ', '_').replace('/', '_')@@ -291,7 +297,8 @@ class STORMWikiRunner(Engine):if do_generate_outline:# load information table if it's not initializedif information_table is None:- information_table = self._load_information_table_from_local_fs(os.path.join(self.article_output_dir, 'conversation_log.json'))+ information_table = self._load_information_table_from_local_fs(+ os.path.join(self.article_output_dir, 'conversation_log.json'))outline = self.run_outline_generation_module(information_table=information_table,callback_handler=callback_handler)@@ -299,9 +306,12 @@ class STORMWikiRunner(Engine):draft_article: StormArticle = Noneif do_generate_article:if information_table is None:- information_table = self._load_information_table_from_local_fs(os.path.join(self.article_output_dir, 'conversation_log.json'))+ information_table = self._load_information_table_from_local_fs(+ os.path.join(self.article_output_dir, 'conversation_log.json'))if outline is None:- outline = self._load_outline_from_local_fs(topic=topic, outline_local_path=os.path.join(self.article_output_dir, 'storm_gen_outline.txt'))+ outline = self._load_outline_from_local_fs(topic=topic,+ outline_local_path=os.path.join(self.article_output_dir,+ 'storm_gen_outline.txt'))draft_article = self.run_article_generation_module(outline=outline,information_table=information_table,callback_handler=callback_handler)@@ -311,5 +321,7 @@ class STORMWikiRunner(Engine):if draft_article is None:draft_article_path = os.path.join(self.article_output_dir, 'storm_gen_article.txt')url_to_info_path = os.path.join(self.article_output_dir, 'url_to_info.json')- draft_article = self._load_draft_article_from_local_fs(topic=topic, draft_article_path=draft_article_path, url_to_info_path=url_to_info_path)+ draft_article = self._load_draft_article_from_local_fs(topic=topic,+ draft_article_path=draft_article_path,+ url_to_info_path=url_to_info_path)self.run_article_polishing_module(draft_article=draft_article, remove_duplicate=remove_duplicate)diff --git a/src/storm_wiki/modules/__init__.py b/src/knowledge_storm/storm_wiki/modules/__init__.pysimilarity index 74%rename from src/storm_wiki/modules/__init__.pyrename to src/knowledge_storm/storm_wiki/modules/__init__.pyindex 51ee012..9419a31 100644--- a/src/storm_wiki/modules/__init__.py+++ b/src/knowledge_storm/storm_wiki/modules/__init__.py@@ -1,4 +1,4 @@from .knowledge_curation import *from .persona_generator import *from .retriever import *-from .storm_dataclass import *\ No newline at end of file+from .storm_dataclass import *diff --git a/src/storm_wiki/modules/article_generation.py b/src/knowledge_storm/storm_wiki/modules/article_generation.pysimilarity index 96%rename from src/storm_wiki/modules/article_generation.pyrename to src/knowledge_storm/storm_wiki/modules/article_generation.pyindex 0dfb76d..a114b3e 100644--- a/src/storm_wiki/modules/article_generation.py+++ b/src/knowledge_storm/storm_wiki/modules/article_generation.py@@ -5,10 +5,11 @@ from concurrent.futures import as_completedfrom typing import List, Unionimport dspy-from interface import ArticleGenerationModule-from storm_wiki.modules.callback import BaseCallbackHandler-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle, StormInformation-from utils import ArticleTextProcessing++from .callback import BaseCallbackHandler+from .storm_dataclass import StormInformationTable, StormArticle, StormInformation+from ...interface import ArticleGenerationModule+from ...utils import ArticleTextProcessingclass StormArticleGenerationModule(ArticleGenerationModule):diff --git a/src/storm_wiki/modules/article_polish.py b/src/knowledge_storm/storm_wiki/modules/article_polish.pysimilarity index 96%rename from src/storm_wiki/modules/article_polish.pyrename to src/knowledge_storm/storm_wiki/modules/article_polish.pyindex 5f38f05..b70bb83 100644--- a/src/storm_wiki/modules/article_polish.py+++ b/src/knowledge_storm/storm_wiki/modules/article_polish.py@@ -2,9 +2,10 @@ import copyfrom typing import Unionimport dspy-from interface import ArticlePolishingModule-from storm_wiki.modules.storm_dataclass import StormArticle-from utils import ArticleTextProcessing++from .storm_dataclass import StormArticle+from ...interface import ArticlePolishingModule+from ...utils import ArticleTextProcessingclass StormArticlePolishingModule(ArticlePolishingModule):diff --git a/src/storm_wiki/modules/callback.py b/src/knowledge_storm/storm_wiki/modules/callback.pysimilarity index 98%rename from src/storm_wiki/modules/callback.pyrename to src/knowledge_storm/storm_wiki/modules/callback.pyindex 945a45d..a4b702d 100644--- a/src/storm_wiki/modules/callback.py+++ b/src/knowledge_storm/storm_wiki/modules/callback.py@@ -31,4 +31,4 @@ class BaseCallbackHandler:def on_outline_refinement_end(self, outline: str, **kwargs):"""Run when the outline refinement finishes."""- pass\ No newline at end of file+ passdiff --git a/src/storm_wiki/modules/knowledge_curation.py b/src/knowledge_storm/storm_wiki/modules/knowledge_curation.pysimilarity index 97%rename from src/storm_wiki/modules/knowledge_curation.pyrename to src/knowledge_storm/storm_wiki/modules/knowledge_curation.pyindex 4fe7f15..8e881c6 100644--- a/src/storm_wiki/modules/knowledge_curation.py+++ b/src/knowledge_storm/storm_wiki/modules/knowledge_curation.py@@ -5,14 +5,16 @@ from concurrent.futures import as_completedfrom typing import Union, List, Tuple, Optional, Dictimport dspy-from interface import KnowledgeCurationModule, Retriever-from storm_wiki.modules.callback import BaseCallbackHandler-from storm_wiki.modules.persona_generator import StormPersonaGenerator-from storm_wiki.modules.storm_dataclass import DialogueTurn, StormInformationTable, StormInformation-from utils import ArticleTextProcessing++from .callback import BaseCallbackHandler+from .persona_generator import StormPersonaGenerator+from .storm_dataclass import DialogueTurn, StormInformationTable, StormInformation+from ...interface import KnowledgeCurationModule, Retriever+from ...utils import ArticleTextProcessingtry:from streamlit.runtime.scriptrunner import add_script_run_ctx+streamlit_connection = Trueexcept ImportError as err:streamlit_connection = Falsediff --git a/src/storm_wiki/modules/outline_generation.py b/src/knowledge_storm/storm_wiki/modules/outline_generation.pysimilarity index 96%rename from src/storm_wiki/modules/outline_generation.pyrename to src/knowledge_storm/storm_wiki/modules/outline_generation.pyindex 2b09d52..1f45b1c 100644--- a/src/storm_wiki/modules/outline_generation.py+++ b/src/knowledge_storm/storm_wiki/modules/outline_generation.py@@ -1,10 +1,11 @@from typing import Union, Optional, Tupleimport dspy-from interface import OutlineGenerationModule-from storm_wiki.modules.callback import BaseCallbackHandler-from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle-from utils import ArticleTextProcessing++from .callback import BaseCallbackHandler+from .storm_dataclass import StormInformationTable, StormArticle+from ...interface import OutlineGenerationModule+from ...utils import ArticleTextProcessingclass StormOutlineGenerationModule(OutlineGenerationModule):diff --git a/src/storm_wiki/modules/persona_generator.py b/src/knowledge_storm/storm_wiki/modules/persona_generator.pysimilarity index 100%rename from src/storm_wiki/modules/persona_generator.pyrename to src/knowledge_storm/storm_wiki/modules/persona_generator.pydiff --git a/src/storm_wiki/modules/internet_source_restrictions.json b/src/knowledge_storm/storm_wiki/modules/retriever.pysimilarity index 72%rename from src/storm_wiki/modules/internet_source_restrictions.jsonrename to src/knowledge_storm/storm_wiki/modules/retriever.pyindex 0a71e1a..179ae99 100644--- a/src/storm_wiki/modules/internet_source_restrictions.json+++ b/src/knowledge_storm/storm_wiki/modules/retriever.py@@ -1,5 +1,15 @@-{- "generally_unreliable": [+from typing import Union, List+from urllib.parse import urlparse++import dspy++from .storm_dataclass import StormInformation+from ...interface import Retriever, Information+from ...utils import ArticleTextProcessing++# Internet source restrictions according to Wikipedia standard:+# https://en.wikipedia.org/wiki/Wikipedia:Reliable_sources/Perennial_sources+GENERALLY_UNRELIABLE = {"112_Ukraine","Ad_Fontes_Media","AlterNet",@@ -139,9 +149,8 @@"WordPress.com","Worldometer","YouTube",- "ZDNet"- ],- "deprecated": [+ "ZDNet"}+DEPRECATED = {"Al_Mayadeen","ANNA_News","Baidu_Baike",@@ -189,8 +198,8 @@"Voltaire_Network","WorldNetDaily","Zero_Hedge"- ],- "blacklisted": [+}+BLACKLISTED = {"Advameg","bestgore.com","Breitbart_News",@@ -210,5 +219,32 @@"Swarajya","Veterans_Today","ZoomInfo"- ]}+++def is_valid_wikipedia_source(url):+ parsed_url = urlparse(url)+ # Check if the URL is from a reliable domain+ combined_set = GENERALLY_UNRELIABLE | DEPRECATED | BLACKLISTED+ for domain in combined_set:+ if domain in parsed_url.netloc:+ return False++ return True+++class StormRetriever(Retriever):+ def __init__(self, rm: dspy.Retrieve, k=3):+ super().__init__(search_top_k=k)+ self._rm = rm+ if hasattr(rm, 'is_valid_source'):+ rm.is_valid_source = is_valid_wikipedia_source++ def retrieve(self, query: Union[str, List[str]], exclude_urls: List[str] = []) -> List[Information]:+ retrieved_data_list = self._rm(query_or_queries=query, exclude_urls=exclude_urls)+ for data in retrieved_data_list:+ for i in range(len(data['snippets'])):+ # STORM generate the article with citations. We do not consider multi-hop citations.+ # Remove citations in the source to avoid confusion.+ data['snippets'][i] = ArticleTextProcessing.remove_citations(data['snippets'][i])+ return [StormInformation.from_dict(data) for data in retrieved_data_list]diff --git a/src/storm_wiki/modules/storm_dataclass.py b/src/knowledge_storm/storm_wiki/modules/storm_dataclass.pysimilarity index 99%rename from src/storm_wiki/modules/storm_dataclass.pyrename to src/knowledge_storm/storm_wiki/modules/storm_dataclass.pyindex d75760c..4f54ec4 100644--- a/src/storm_wiki/modules/storm_dataclass.py+++ b/src/knowledge_storm/storm_wiki/modules/storm_dataclass.py@@ -4,10 +4,11 @@ from collections import OrderedDictfrom typing import Union, Optional, Any, List, Tuple, Dictimport numpy as np-from interface import Information, InformationTable, Article, ArticleSectionNodefrom sentence_transformers import SentenceTransformerfrom sklearn.metrics.pairwise import cosine_similarity-from utils import ArticleTextProcessing, FileIOHelper++from ...interface import Information, InformationTable, Article, ArticleSectionNode+from ...utils import ArticleTextProcessing, FileIOHelperclass StormInformation(Information):diff --git a/src/utils.py b/src/knowledge_storm/utils.pysimilarity index 100%rename from src/utils.pyrename to src/knowledge_storm/utils.pydiff --git a/src/storm_wiki/modules/retriever.py b/src/storm_wiki/modules/retriever.pydeleted file mode 100644index 79cc206..0000000--- a/src/storm_wiki/modules/retriever.py+++ /dev/null@@ -1,45 +0,0 @@-import json-import os-from typing import Union, List-from urllib.parse import urlparse--import dspy-import storm_wiki.modules.storm_dataclass as storm_dataclass-from interface import Retriever, Information-from rm import YouRM-from utils import ArticleTextProcessing--SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))-with open(os.path.join(SCRIPT_DIR, 'internet_source_restrictions.json')) as f:- domain_restriction_dict = json.load(f)- GENERALLY_UNRELIABLE = set(domain_restriction_dict["generally_unreliable"])- DEPRECATED = set(domain_restriction_dict["deprecated"])- BLACKLISTED = set(domain_restriction_dict["blacklisted"])---def is_valid_wikipedia_source(url):- parsed_url = urlparse(url)- # Check if the URL is from a reliable domain- combined_set = GENERALLY_UNRELIABLE | DEPRECATED | BLACKLISTED- for domain in combined_set:- if domain in parsed_url.netloc:- return False-- return True---class StormRetriever(Retriever):- def __init__(self, rm: dspy.Retrieve, k=3):- super().__init__(search_top_k=k)- self._rm = rm- if hasattr(rm, 'is_valid_source'):- rm.is_valid_source = is_valid_wikipedia_source-- def retrieve(self, query: Union[str, List[str]], exclude_urls: List[str] = []) -> List[Information]:- retrieved_data_list = self._rm(query_or_queries=query, exclude_urls=exclude_urls)- for data in retrieved_data_list:- for i in range(len(data['snippets'])):- # STORM generate the article with citations. We do not consider multi-hop citations.- # Remove citations in the source to avoid confusion.- data['snippets'][i] = ArticleTextProcessing.remove_citations(data['snippets'][i])- return [storm_dataclass.StormInformation.from_dict(data) for data in retrieved_data_list]