Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

fix code style with black

Yucheng-Jiang5fd4bc9bde2e
diff --git a/examples/costorm_examples/run_costorm_gpt.py b/examples/costorm_examples/run_costorm_gpt.py
index 9046bd5..4013891 100644
--- a/examples/costorm_examples/run_costorm_gpt.py
+++ b/examples/costorm_examples/run_costorm_gpt.py
@@ -16,51 +16,83 @@ args.output_dir/
import os
import json
from argparse import ArgumentParser
-from knowledge_storm.collaborative_storm.engine import CollaborativeStormLMConfigs, RunnerArgument, CoStormRunner
-from knowledge_storm.collaborative_storm.modules.callback import LocalConsolePrintCallBackHandler
+from knowledge_storm.collaborative_storm.engine import (
+ CollaborativeStormLMConfigs,
+ RunnerArgument,
+ CoStormRunner,
+)
+from knowledge_storm.collaborative_storm.modules.callback import (
+ LocalConsolePrintCallBackHandler,
+)
from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel
from knowledge_storm.logging_wrapper import LoggingWrapper
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_config: CollaborativeStormLMConfigs = CollaborativeStormLMConfigs()
- openai_kwargs = {
- "api_key": os.getenv("OPENAI_API_KEY"),
- "api_provider": "openai",
- "temperature": 1.0,
- "top_p": 0.9,
- "api_base": None,
- } if os.getenv('OPENAI_API_TYPE') == 'openai' else {
- "api_key": os.getenv("AZURE_API_KEY"),
- "temperature": 1.0,
- "top_p": 0.9,
- "api_base": os.getenv("AZURE_API_BASE"),
- "api_version": os.getenv("AZURE_API_VERSION"),
- }
-
- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel
+ openai_kwargs = (
+ {
+ "api_key": os.getenv("OPENAI_API_KEY"),
+ "api_provider": "openai",
+ "temperature": 1.0,
+ "top_p": 0.9,
+ "api_base": None,
+ }
+ if os.getenv("OPENAI_API_TYPE") == "openai"
+ else {
+ "api_key": os.getenv("AZURE_API_KEY"),
+ "temperature": 1.0,
+ "top_p": 0.9,
+ "api_base": os.getenv("AZURE_API_BASE"),
+ "api_version": os.getenv("AZURE_API_VERSION"),
+ }
+ )
+
+ ModelClass = (
+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel
+ )
# If you are using Azure service, make sure the model name matches your own deployed model name.
# The default name here is only used for demonstration and may not match your case.
- gpt_4o_mini_model_name = 'gpt-4o-mini'
- gpt_4o_model_name = 'gpt-4o'
- if os.getenv('OPENAI_API_TYPE') == 'azure':
- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')
- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')
+ gpt_4o_mini_model_name = "gpt-4o-mini"
+ gpt_4o_model_name = "gpt-4o"
+ if os.getenv("OPENAI_API_TYPE") == "azure":
+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")
+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")
# STORM is a LM system so different components can be powered by different models.
# For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm
# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models
# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm
# which is responsible for generating sections with citations.
- question_answering_lm = ModelClass(model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs)
- discourse_manage_lm = ModelClass(model=gpt_4o_model_name, max_tokens=500, **openai_kwargs)
- utterance_polishing_lm = ModelClass(model=gpt_4o_model_name, max_tokens=2000, **openai_kwargs)
- warmstart_outline_gen_lm = ModelClass(model=gpt_4o_model_name, max_tokens=500, **openai_kwargs)
- question_asking_lm = ModelClass(model=gpt_4o_model_name, max_tokens=300, **openai_kwargs)
- knowledge_base_lm = ModelClass(model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs)
+ question_answering_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs
+ )
+ discourse_manage_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=500, **openai_kwargs
+ )
+ utterance_polishing_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=2000, **openai_kwargs
+ )
+ warmstart_outline_gen_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=500, **openai_kwargs
+ )
+ question_asking_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=300, **openai_kwargs
+ )
+ knowledge_base_lm = ModelClass(
+ model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs
+ )
lm_config.set_question_answering_lm(question_answering_lm)
lm_config.set_discourse_manage_lm(discourse_manage_lm)
@@ -69,7 +101,7 @@ def main(args):
lm_config.set_question_asking_lm(question_asking_lm)
lm_config.set_knowledge_base_lm(knowledge_base_lm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner_argument = RunnerArgument(
topic=topic,
retrieve_top_k=args.retrieve_top_k,
@@ -83,49 +115,76 @@ def main(args):
max_thread_num=args.max_thread_num,
max_num_round_table_experts=args.max_num_round_table_experts,
moderator_override_N_consecutive_answering_turn=args.moderator_override_N_consecutive_answering_turn,
- node_expansion_trigger_count=args.node_expansion_trigger_count)
+ node_expansion_trigger_count=args.node_expansion_trigger_count,
+ )
logging_wrapper = LoggingWrapper(lm_config)
- callback_handler = LocalConsolePrintCallBackHandler() if args.enable_log_print else None
+ callback_handler = (
+ LocalConsolePrintCallBackHandler() if args.enable_log_print else None
+ )
# Co-STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=runner_argument.retrieve_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=runner_argument.retrieve_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=runner_argument.retrieve_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=runner_argument.retrieve_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=runner_argument.retrieve_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=runner_argument.retrieve_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=runner_argument.retrieve_top_k,
+ )
+ case "you":
+ rm = YouRM(
+ ydc_api_key=os.getenv("YDC_API_KEY"), k=runner_argument.retrieve_top_k
+ )
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=runner_argument.retrieve_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=runner_argument.retrieve_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=runner_argument.retrieve_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"),
+ k=runner_argument.retrieve_top_k,
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
- costorm_runner = CoStormRunner(lm_config=lm_config,
- runner_argument=runner_argument,
- logging_wrapper=logging_wrapper,
- rm=rm,
- callback_handler=callback_handler)
+ costorm_runner = CoStormRunner(
+ lm_config=lm_config,
+ runner_argument=runner_argument,
+ logging_wrapper=logging_wrapper,
+ rm=rm,
+ callback_handler=callback_handler,
+ )
# warm start the system
costorm_runner.warm_start()
# Below is an example of how users may interact with Co-STORM to seek information together
# In actual deployment, we suggest allowing the user to decide whether to observe the agent utterance or inject a turn
-
+
# observing Co-STORM LLM agent utterance for 5 turns
for _ in range(1):
conv_turn = costorm_runner.step()
print(f"**{conv_turn.role}**: {conv_turn.utterance}\n")
-
+
# active engaging by injecting your utterance
- your_utterance = input('Your utterance: ')
+ your_utterance = input("Your utterance: ")
costorm_runner.step(user_utterance=your_utterance)
# continue observing
@@ -154,93 +213,105 @@ def main(args):
json.dump(log_dump, f, indent=2)
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/co-storm',
- help='Directory to store the outputs.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/co-storm",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
# hyperparameters for co-storm
parser.add_argument(
- '--retrieve_top_k',
+ "--retrieve_top_k",
type=int,
default=10,
- help='Retrieve top k results for each query in retriever.'
+ help="Retrieve top k results for each query in retriever.",
)
parser.add_argument(
- '--max_search_queries',
+ "--max_search_queries",
type=int,
default=2,
- help='Maximum number of search queries to consider for each question.'
+ help="Maximum number of search queries to consider for each question.",
)
parser.add_argument(
- '--total_conv_turn',
+ "--total_conv_turn",
type=int,
default=20,
- help='Maximum number of turns in conversation.'
+ help="Maximum number of turns in conversation.",
)
parser.add_argument(
- '--max_search_thread',
+ "--max_search_thread",
type=int,
default=5,
- help='Maximum number of parallel threads for retriever.'
+ help="Maximum number of parallel threads for retriever.",
)
parser.add_argument(
- '--max_search_queries_per_turn',
+ "--max_search_queries_per_turn",
type=int,
default=3,
- help='Maximum number of search queries to consider in each turn.'
+ help="Maximum number of search queries to consider in each turn.",
)
parser.add_argument(
- '--warmstart_max_num_experts',
+ "--warmstart_max_num_experts",
type=int,
default=3,
- help='Max number of experts in perspective-guided QA during warm start.'
+ help="Max number of experts in perspective-guided QA during warm start.",
)
parser.add_argument(
- '--warmstart_max_turn_per_experts',
+ "--warmstart_max_turn_per_experts",
type=int,
default=2,
- help='Max number of turns per perspective during warm start.'
+ help="Max number of turns per perspective during warm start.",
)
parser.add_argument(
- '--warmstart_max_thread',
+ "--warmstart_max_thread",
type=int,
default=3,
- help='Max number of threads for parallel perspective-guided QA during warm start.'
+ help="Max number of threads for parallel perspective-guided QA during warm start.",
)
parser.add_argument(
- '--max_thread_num',
+ "--max_thread_num",
type=int,
default=10,
- help=("Maximum number of threads to use. "
- "Consider reducing it if you keep getting 'Exceed rate limit' errors when calling the LM API.")
+ help=(
+ "Maximum number of threads to use. "
+ "Consider reducing it if you keep getting 'Exceed rate limit' errors when calling the LM API."
+ ),
)
parser.add_argument(
- '--max_num_round_table_experts',
+ "--max_num_round_table_experts",
type=int,
default=2,
- help='Max number of active experts in round table discussion.'
+ help="Max number of active experts in round table discussion.",
)
parser.add_argument(
- '--moderator_override_N_consecutive_answering_turn',
+ "--moderator_override_N_consecutive_answering_turn",
type=int,
default=3,
- help=('Number of consecutive expert answering turns before the moderator overrides the conversation.')
+ help=(
+ "Number of consecutive expert answering turns before the moderator overrides the conversation."
+ ),
)
parser.add_argument(
- '--node_expansion_trigger_count',
+ "--node_expansion_trigger_count",
type=int,
default=10,
- help='Trigger node expansion for nodes that contain more than N snippets.'
+ help="Trigger node expansion for nodes that contain more than N snippets.",
)
# Boolean flags
parser.add_argument(
- '--enable_log_print',
- action='store_true',
- help='If set, enable console log print.'
+ "--enable_log_print",
+ action="store_true",
+ help="If set, enable console log print.",
)
main(parser.parse_args())
diff --git a/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py b/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py
index 926192b..f3dfe51 100644
--- a/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py
+++ b/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py
@@ -9,21 +9,28 @@ import pandas as pd
if __name__ == "__main__":
parser = ArgumentParser()
- parser.add_argument("--input-path", type=str, help="Path to arxiv_data_210930-054931.csv.")
- parser.add_argument("--output-path", type=str,
- help="Path to store the csv file that is compatible with VectorRM.")
+ parser.add_argument(
+ "--input-path", type=str, help="Path to arxiv_data_210930-054931.csv."
+ )
+ parser.add_argument(
+ "--output-path",
+ type=str,
+ help="Path to store the csv file that is compatible with VectorRM.",
+ )
args = parser.parse_args()
df = pd.read_csv(args.input_path)
- print(f'The original dataset has {len(df)} samples.')
+ print(f"The original dataset has {len(df)} samples.")
# Downsample the dataset.
- df = df[df['terms'] == "['cs.CV']"]
+ df = df[df["terms"] == "['cs.CV']"]
# Reformat the dataset to match the VectorRM input format.
df.rename(columns={"abstracts": "content", "titles": "title"}, inplace=True)
- df['url'] = ['uid_' + str(idx) for idx in range(len(df))] # Ensure the url is unique.
- df['description'] = ''
+ df["url"] = [
+ "uid_" + str(idx) for idx in range(len(df))
+ ] # Ensure the url is unique.
+ df["description"] = ""
- print(f'The downsampled dataset has {len(df)} samples.')
- df.to_csv(args.output_path, index=False)
\ No newline at end of file
+ print(f"The downsampled dataset has {len(df)} samples.")
+ df.to_csv(args.output_path, index=False)
diff --git a/examples/storm_examples/run_storm_wiki_claude.py b/examples/storm_examples/run_storm_wiki_claude.py
index 1da12c0..10f6e36 100644
--- a/examples/storm_examples/run_storm_wiki_claude.py
+++ b/examples/storm_examples/run_storm_wiki_claude.py
@@ -19,19 +19,31 @@ args.output_dir/
import os
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import ClaudeModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
claude_kwargs = {
- 'api_key': os.getenv("ANTHROPIC_API_KEY"),
- 'temperature': 1.0,
- 'top_p': 0.9
+ "api_key": os.getenv("ANTHROPIC_API_KEY"),
+ "temperature": 1.0,
+ "top_p": 0.9,
}
# STORM is a LM system so different components can be powered by different models.
@@ -39,11 +51,21 @@ def main(args):
# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models
# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm
# which is responsible for generating sections with citations.
- conv_simulator_lm = ClaudeModel(model='claude-3-haiku-20240307', max_tokens=500, **claude_kwargs)
- question_asker_lm = ClaudeModel(model='claude-3-sonnet-20240229', max_tokens=500, **claude_kwargs)
- outline_gen_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=400, **claude_kwargs)
- article_gen_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=700, **claude_kwargs)
- article_polish_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=4000, **claude_kwargs)
+ conv_simulator_lm = ClaudeModel(
+ model="claude-3-haiku-20240307", max_tokens=500, **claude_kwargs
+ )
+ question_asker_lm = ClaudeModel(
+ model="claude-3-sonnet-20240229", max_tokens=500, **claude_kwargs
+ )
+ outline_gen_lm = ClaudeModel(
+ model="claude-3-opus-20240229", max_tokens=400, **claude_kwargs
+ )
+ article_gen_lm = ClaudeModel(
+ model="claude-3-opus-20240229", max_tokens=700, **claude_kwargs
+ )
+ article_polish_lm = ClaudeModel(
+ model="claude-3-opus-20240229", max_tokens=4000, **claude_kwargs
+ )
lm_configs.set_conv_simulator_lm(conv_simulator_lm)
lm_configs.set_question_asker_lm(question_asker_lm)
@@ -62,26 +84,45 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
-
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
+
runner = STORMWikiRunner(engine_args, lm_configs, rm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -93,38 +134,81 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/claude',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/claude",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
- main(parser.parse_args())
\ No newline at end of file
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_deepseek.py b/examples/storm_examples/run_storm_wiki_deepseek.py
index e4cf5fc..e831797 100644
--- a/examples/storm_examples/run_storm_wiki_deepseek.py
+++ b/examples/storm_examples/run_storm_wiki_deepseek.py
@@ -22,9 +22,21 @@ import re
import logging
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import DeepSeekModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
@@ -34,10 +46,10 @@ def sanitize_topic(topic):
Remove or replace characters that are not allowed in file names.
"""
# Replace spaces with underscores
- topic = topic.replace(' ', '_')
+ topic = topic.replace(" ", "_")
# Remove any character that isn't alphanumeric, underscore, or hyphen
- topic = re.sub(r'[^a-zA-Z0-9_-]', '', topic)
+ topic = re.sub(r"[^a-zA-Z0-9_-]", "", topic)
# Ensure the topic isn't empty after sanitization
if not topic:
@@ -47,29 +59,37 @@ def sanitize_topic(topic):
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
logger = logging.getLogger(__name__)
# Ensure DEEPSEEK_API_KEY is set
if not os.getenv("DEEPSEEK_API_KEY"):
- raise ValueError("DEEPSEEK_API_KEY environment variable is not set. Please set it in your secrets.toml file.")
+ raise ValueError(
+ "DEEPSEEK_API_KEY environment variable is not set. Please set it in your secrets.toml file."
+ )
deepseek_kwargs = {
- 'api_key': os.getenv("DEEPSEEK_API_KEY"),
- 'api_base': os.getenv("DEEPSEEK_API_BASE", "https://api.deepseek.com"),
- 'temperature': args.temperature,
- 'top_p': args.top_p,
+ "api_key": os.getenv("DEEPSEEK_API_KEY"),
+ "api_base": os.getenv("DEEPSEEK_API_BASE", "https://api.deepseek.com"),
+ "temperature": args.temperature,
+ "top_p": args.top_p,
}
# DeepSeek offers two main models: 'deepseek-chat' for general tasks and 'deepseek-coder' for coding tasks
# Users can choose the appropriate model based on their needs
- conv_simulator_lm = DeepSeekModel(model=args.model, max_tokens=500, **deepseek_kwargs)
- question_asker_lm = DeepSeekModel(model=args.model, max_tokens=500, **deepseek_kwargs)
+ conv_simulator_lm = DeepSeekModel(
+ model=args.model, max_tokens=500, **deepseek_kwargs
+ )
+ question_asker_lm = DeepSeekModel(
+ model=args.model, max_tokens=500, **deepseek_kwargs
+ )
outline_gen_lm = DeepSeekModel(model=args.model, max_tokens=400, **deepseek_kwargs)
article_gen_lm = DeepSeekModel(model=args.model, max_tokens=700, **deepseek_kwargs)
- article_polish_lm = DeepSeekModel(model=args.model, max_tokens=4000, **deepseek_kwargs)
+ article_polish_lm = DeepSeekModel(
+ model=args.model, max_tokens=4000, **deepseek_kwargs
+ )
lm_configs.set_conv_simulator_lm(conv_simulator_lm)
lm_configs.set_question_asker_lm(question_asker_lm)
@@ -88,26 +108,45 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
sanitized_topic = sanitize_topic(topic)
try:
@@ -126,44 +165,94 @@ def main(args):
raise
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/deepseek',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
- parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',
- help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')
- parser.add_argument('--temperature', type=float, default=1.0,
- help='Sampling temperature to use.')
- parser.add_argument('--top_p', type=float, default=0.9,
- help='Top-p sampling parameter.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/deepseek",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
+ parser.add_argument(
+ "--model",
+ type=str,
+ choices=["deepseek-chat", "deepseek-coder"],
+ default="deepseek-chat",
+ help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.',
+ )
+ parser.add_argument(
+ "--temperature", type=float, default=1.0, help="Sampling temperature to use."
+ )
+ parser.add_argument(
+ "--top_p", type=float, default=0.9, help="Top-p sampling parameter."
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
- main(parser.parse_args())
\ No newline at end of file
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_gemini.py b/examples/storm_examples/run_storm_wiki_gemini.py
index d574062..947832f 100644
--- a/examples/storm_examples/run_storm_wiki_gemini.py
+++ b/examples/storm_examples/run_storm_wiki_gemini.py
@@ -19,18 +19,31 @@ args.output_dir/
import os
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import GoogleModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
+
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
gemini_kwargs = {
- 'api_key': os.getenv("GOOGLE_API_KEY"),
- 'temperature': 1.0,
- 'top_p': 0.9
+ "api_key": os.getenv("GOOGLE_API_KEY"),
+ "temperature": 1.0,
+ "top_p": 0.9,
}
# STORM is a LM system so different components can be powered by different models.
@@ -40,11 +53,21 @@ def main(args):
# which is responsible for generating sections with citations.
# To check out available Google models, see:
# https://ai.google.dev/gemini-api/docs/get-started/tutorial?lang=python#list_models
- conv_simulator_lm = GoogleModel(model='models/gemini-1.5-flash', max_tokens=500, **gemini_kwargs)
- question_asker_lm = GoogleModel(model='models/gemini-1.5-flash', max_tokens=500, **gemini_kwargs)
- outline_gen_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=400, **gemini_kwargs)
- article_gen_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=700, **gemini_kwargs)
- article_polish_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=4000, **gemini_kwargs)
+ conv_simulator_lm = GoogleModel(
+ model="models/gemini-1.5-flash", max_tokens=500, **gemini_kwargs
+ )
+ question_asker_lm = GoogleModel(
+ model="models/gemini-1.5-flash", max_tokens=500, **gemini_kwargs
+ )
+ outline_gen_lm = GoogleModel(
+ model="models/gemini-1.5-pro-exp-0801", max_tokens=400, **gemini_kwargs
+ )
+ article_gen_lm = GoogleModel(
+ model="models/gemini-1.5-pro-exp-0801", max_tokens=700, **gemini_kwargs
+ )
+ article_polish_lm = GoogleModel(
+ model="models/gemini-1.5-pro-exp-0801", max_tokens=4000, **gemini_kwargs
+ )
lm_configs.set_conv_simulator_lm(conv_simulator_lm)
lm_configs.set_question_asker_lm(question_asker_lm)
@@ -63,26 +86,45 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -94,38 +136,81 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/gemini',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/gemini",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_gpt.py b/examples/storm_examples/run_storm_wiki_gpt.py
index b1740a1..e96c680 100644
--- a/examples/storm_examples/run_storm_wiki_gpt.py
+++ b/examples/storm_examples/run_storm_wiki_gpt.py
@@ -22,40 +22,63 @@ args.output_dir/
import os
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG, AzureAISearch
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+ AzureAISearch,
+)
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
openai_kwargs = {
- 'api_key': os.getenv("OPENAI_API_KEY"),
- 'temperature': 1.0,
- 'top_p': 0.9,
+ "api_key": os.getenv("OPENAI_API_KEY"),
+ "temperature": 1.0,
+ "top_p": 0.9,
}
- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel
+ ModelClass = (
+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel
+ )
# If you are using Azure service, make sure the model name matches your own deployed model name.
# The default name here is only used for demonstration and may not match your case.
- gpt_35_model_name = 'gpt-3.5-turbo' if os.getenv('OPENAI_API_TYPE') == 'openai' else 'gpt-35-turbo'
- gpt_4_model_name = 'gpt-4o'
- if os.getenv('OPENAI_API_TYPE') == 'azure':
- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')
- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')
+ gpt_35_model_name = (
+ "gpt-3.5-turbo" if os.getenv("OPENAI_API_TYPE") == "openai" else "gpt-35-turbo"
+ )
+ gpt_4_model_name = "gpt-4o"
+ if os.getenv("OPENAI_API_TYPE") == "azure":
+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")
+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")
# STORM is a LM system so different components can be powered by different models.
# For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm
# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models
# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm
# which is responsible for generating sections with citations.
- conv_simulator_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)
- question_asker_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)
+ conv_simulator_lm = ModelClass(
+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs
+ )
+ question_asker_lm = ModelClass(
+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs
+ )
outline_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=400, **openai_kwargs)
article_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=700, **openai_kwargs)
- article_polish_lm = ModelClass(model=gpt_4_model_name, max_tokens=4000, **openai_kwargs)
+ article_polish_lm = ModelClass(
+ model=gpt_4_model_name, max_tokens=4000, **openai_kwargs
+ )
lm_configs.set_conv_simulator_lm(conv_simulator_lm)
lm_configs.set_question_asker_lm(question_asker_lm)
@@ -75,28 +98,50 @@ def main(args):
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
- case 'azure_ai_search':
- rm = AzureAISearch(azure_ai_search_api_key=os.getenv('AZURE_AI_SEARCH_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
+ case "azure_ai_search":
+ rm = AzureAISearch(
+ azure_ai_search_api_key=os.getenv("AZURE_AI_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", "searxng", or "azure_ai_search"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", "searxng", or "azure_ai_search"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -108,38 +153,90 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/gpt',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng', 'azure_ai_search'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/gpt",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=[
+ "bing",
+ "you",
+ "brave",
+ "serper",
+ "duckduckgo",
+ "tavily",
+ "searxng",
+ "azure_ai_search",
+ ],
+ help="The search engine API to use for retrieving information.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py b/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py
index 8fa4d0d..77c9dbf 100644
--- a/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py
+++ b/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py
@@ -29,7 +29,11 @@ args.output_dir/
import os
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.rm import VectorRM
from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel
from knowledge_storm.utils import load_api_key, QdrantVectorStoreManager
@@ -37,35 +41,45 @@ from knowledge_storm.utils import load_api_key, QdrantVectorStoreManager
def main(args):
# Load API key from the specified toml file path
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
# Initialize the language model configurations
engine_lm_configs = STORMWikiLMConfigs()
openai_kwargs = {
- 'api_key': os.getenv("OPENAI_API_KEY"),
- 'temperature': 1.0,
- 'top_p': 0.9,
+ "api_key": os.getenv("OPENAI_API_KEY"),
+ "temperature": 1.0,
+ "top_p": 0.9,
}
- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel
+ ModelClass = (
+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel
+ )
# If you are using Azure service, make sure the model name matches your own deployed model name.
# The default name here is only used for demonstration and may not match your case.
- gpt_35_model_name = 'gpt-3.5-turbo' if os.getenv('OPENAI_API_TYPE') == 'openai' else 'gpt-35-turbo'
- gpt_4_model_name = 'gpt-4o'
- if os.getenv('OPENAI_API_TYPE') == 'azure':
- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')
- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')
+ gpt_35_model_name = (
+ "gpt-3.5-turbo" if os.getenv("OPENAI_API_TYPE") == "openai" else "gpt-35-turbo"
+ )
+ gpt_4_model_name = "gpt-4o"
+ if os.getenv("OPENAI_API_TYPE") == "azure":
+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")
+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")
# STORM is a LM system so different components can be powered by different models.
- # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm
+ # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm
# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models
# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm
# which is responsible for generating sections with citations.
- conv_simulator_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)
- question_asker_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)
+ conv_simulator_lm = ModelClass(
+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs
+ )
+ question_asker_lm = ModelClass(
+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs
+ )
outline_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=400, **openai_kwargs)
article_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=700, **openai_kwargs)
- article_polish_lm = ModelClass(model=gpt_4_model_name, max_tokens=4000, **openai_kwargs)
+ article_polish_lm = ModelClass(
+ model=gpt_4_model_name, max_tokens=4000, **openai_kwargs
+ )
engine_lm_configs.set_conv_simulator_lm(conv_simulator_lm)
engine_lm_configs.set_question_asker_lm(question_asker_lm)
@@ -85,43 +99,49 @@ def main(args):
# Create / update the vector store with the documents in the csv file
if args.csv_file_path:
kwargs = {
- 'file_path': args.csv_file_path,
- 'content_column': 'content',
- 'title_column': 'title',
- 'url_column': 'url',
- 'desc_column': 'description',
- 'batch_size': args.embed_batch_size,
- 'vector_db_mode': args.vector_db_mode,
- 'collection_name': args.collection_name,
- 'embedding_model': args.embedding_model,
- 'device': args.device,
+ "file_path": args.csv_file_path,
+ "content_column": "content",
+ "title_column": "title",
+ "url_column": "url",
+ "desc_column": "description",
+ "batch_size": args.embed_batch_size,
+ "vector_db_mode": args.vector_db_mode,
+ "collection_name": args.collection_name,
+ "embedding_model": args.embedding_model,
+ "device": args.device,
}
- if args.vector_db_mode == 'offline':
+ if args.vector_db_mode == "offline":
QdrantVectorStoreManager.create_or_update_vector_store(
- vector_store_path=args.offline_vector_db_dir,
- **kwargs
+ vector_store_path=args.offline_vector_db_dir, **kwargs
)
- elif args.vector_db_mode == 'online':
+ elif args.vector_db_mode == "online":
QdrantVectorStoreManager.create_or_update_vector_store(
url=args.online_vector_db_url,
- api_key=os.getenv('QDRANT_API_KEY'),
+ api_key=os.getenv("QDRANT_API_KEY"),
**kwargs
)
# Setup VectorRM to retrieve information from your own data
- rm = VectorRM(collection_name=args.collection_name, embedding_model=args.embedding_model, device=args.device, k=engine_args.search_top_k)
+ rm = VectorRM(
+ collection_name=args.collection_name,
+ embedding_model=args.embedding_model,
+ device=args.device,
+ k=engine_args.search_top_k,
+ )
# initialize the vector store, either online (store the db on Qdrant server) or offline (store the db locally):
- if args.vector_db_mode == 'offline':
+ if args.vector_db_mode == "offline":
rm.init_offline_vector_db(vector_store_path=args.offline_vector_db_dir)
- elif args.vector_db_mode == 'online':
- rm.init_online_vector_db(url=args.online_vector_db_url, api_key=os.getenv('QDRANT_API_KEY'))
+ elif args.vector_db_mode == "online":
+ rm.init_online_vector_db(
+ url=args.online_vector_db_url, api_key=os.getenv("QDRANT_API_KEY")
+ )
# Initialize the STORM Wiki Runner
runner = STORMWikiRunner(engine_args, engine_lm_configs, rm)
# run the pipeline
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -136,50 +156,120 @@ def main(args):
if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/gpt_retrieval',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/gpt_retrieval",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
# provide local corpus and set up vector db
- parser.add_argument('--collection-name', type=str, default="my_documents",
- help='The collection name for vector store.')
- parser.add_argument('--embedding_model', type=str, default="BAAI/bge-m3",
- help='The collection name for vector store.')
- parser.add_argument('--device', type=str, default="mps",
- help='The device used to run the retrieval model (mps, cuda, cpu, etc).')
- parser.add_argument('--vector-db-mode', type=str, choices=['offline', 'online'],
- help='The mode of the Qdrant vector store (offline or online).')
- parser.add_argument('--offline-vector-db-dir', type=str, default='./vector_store',
- help='If use offline mode, please provide the directory to store the vector store.')
- parser.add_argument('--online-vector-db-url', type=str,
- help='If use online mode, please provide the url of the Qdrant server.')
- parser.add_argument('--csv-file-path', type=str, default=None,
- help='The path of the custom document corpus in CSV format. The CSV file should include '
- 'content, title, url, and description columns.')
- parser.add_argument('--embed-batch-size', type=int, default=64,
- help='Batch size for embedding the documents in the csv file.')
+ parser.add_argument(
+ "--collection-name",
+ type=str,
+ default="my_documents",
+ help="The collection name for vector store.",
+ )
+ parser.add_argument(
+ "--embedding_model",
+ type=str,
+ default="BAAI/bge-m3",
+ help="The collection name for vector store.",
+ )
+ parser.add_argument(
+ "--device",
+ type=str,
+ default="mps",
+ help="The device used to run the retrieval model (mps, cuda, cpu, etc).",
+ )
+ parser.add_argument(
+ "--vector-db-mode",
+ type=str,
+ choices=["offline", "online"],
+ help="The mode of the Qdrant vector store (offline or online).",
+ )
+ parser.add_argument(
+ "--offline-vector-db-dir",
+ type=str,
+ default="./vector_store",
+ help="If use offline mode, please provide the directory to store the vector store.",
+ )
+ parser.add_argument(
+ "--online-vector-db-url",
+ type=str,
+ help="If use online mode, please provide the url of the Qdrant server.",
+ )
+ parser.add_argument(
+ "--csv-file-path",
+ type=str,
+ default=None,
+ help="The path of the custom document corpus in CSV format. The CSV file should include "
+ "content, title, url, and description columns.",
+ )
+ parser.add_argument(
+ "--embed-batch-size",
+ type=int,
+ default=64,
+ help="Batch size for embedding the documents in the csv file.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
- main(parser.parse_args())
\ No newline at end of file
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_groq.py b/examples/storm_examples/run_storm_wiki_groq.py
index 0dcaadb..de5f6ae 100644
--- a/examples/storm_examples/run_storm_wiki_groq.py
+++ b/examples/storm_examples/run_storm_wiki_groq.py
@@ -21,12 +21,24 @@ import os
import re
from argparse import ArgumentParser
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
# Now import lm directly
import lm
from lm import GroqModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
@@ -36,10 +48,10 @@ def sanitize_topic(topic):
Remove or replace characters that are not allowed in file names.
"""
# Replace spaces with underscores
- topic = topic.replace(' ', '_')
+ topic = topic.replace(" ", "_")
# Remove any character that isn't alphanumeric, underscore, or hyphen
- topic = re.sub(r'[^a-zA-Z0-9_-]', '', topic)
+ topic = re.sub(r"[^a-zA-Z0-9_-]", "", topic)
# Ensure the topic isn't empty after sanitization
if not topic:
@@ -49,26 +61,34 @@ def sanitize_topic(topic):
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
# Ensure GROQ_API_KEY is set
if not os.getenv("GROQ_API_KEY"):
- raise ValueError("GROQ_API_KEY environment variable is not set. Please set it in your secrets.toml file.")
+ raise ValueError(
+ "GROQ_API_KEY environment variable is not set. Please set it in your secrets.toml file."
+ )
groq_kwargs = {
- 'api_key': os.getenv("GROQ_API_KEY"),
- 'api_base': "https://api.groq.com/openai/v1",
- 'temperature': args.temperature,
- 'top_p': args.top_p,
+ "api_key": os.getenv("GROQ_API_KEY"),
+ "api_base": "https://api.groq.com/openai/v1",
+ "temperature": args.temperature,
+ "top_p": args.top_p,
}
# Groq currently offers the "llama3-70b-8192" model with generous free API credits and the llama3.1 family of models as a preview for paying customers
- conv_simulator_lm = GroqModel(model="llama3-70b-8192", max_tokens=500, **groq_kwargs)
- question_asker_lm = GroqModel(model="llama3-70b-8192", max_tokens=500, **groq_kwargs)
+ conv_simulator_lm = GroqModel(
+ model="llama3-70b-8192", max_tokens=500, **groq_kwargs
+ )
+ question_asker_lm = GroqModel(
+ model="llama3-70b-8192", max_tokens=500, **groq_kwargs
+ )
outline_gen_lm = GroqModel(model="llama3-70b-8192", max_tokens=400, **groq_kwargs)
article_gen_lm = GroqModel(model="llama3-70b-8192", max_tokens=700, **groq_kwargs)
- article_polish_lm = GroqModel(model="llama3-70b-8192", max_tokens=4000, **groq_kwargs)
+ article_polish_lm = GroqModel(
+ model="llama3-70b-8192", max_tokens=4000, **groq_kwargs
+ )
lm_configs.set_conv_simulator_lm(conv_simulator_lm)
lm_configs.set_question_asker_lm(question_asker_lm)
@@ -87,26 +107,45 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
- topic = input('Topic: ')
+ topic = input("Topic: ")
sanitized_topic = sanitize_topic(topic)
try:
@@ -125,42 +164,87 @@ def main(args):
raise
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--output-dir', type=str, default='./results/groq',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
- parser.add_argument('--temperature', type=float, default=1.0,
- help='Sampling temperature to use.')
- parser.add_argument('--top_p', type=float, default=0.9,
- help='Top-p sampling parameter.')
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/groq",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
+ parser.add_argument(
+ "--temperature", type=float, default=1.0, help="Sampling temperature to use."
+ )
+ parser.add_argument(
+ "--top_p", type=float, default=0.9, help="Top-p sampling parameter."
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
- main(parser.parse_args())
\ No newline at end of file
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_mistral.py b/examples/storm_examples/run_storm_wiki_mistral.py
index ceabd47..1b52d7f 100644
--- a/examples/storm_examples/run_storm_wiki_mistral.py
+++ b/examples/storm_examples/run_storm_wiki_mistral.py
@@ -20,21 +20,35 @@ from argparse import ArgumentParser
from dspy import Example
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import VLLMClient
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
mistral_kwargs = {
"model": "mistralai/Mistral-7B-Instruct-v0.2",
"port": args.port,
"url": args.url,
- "stop": ('\n\n---',) # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.
+ "stop": (
+ "\n\n---",
+ ), # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.
}
conv_simulator_lm = VLLMClient(max_tokens=500, **mistral_kwargs)
@@ -60,22 +74,41 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -87,26 +120,28 @@ def main(args):
find_related_topic_example = Example(
topic="Knowledge Curation",
related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"
- "https://en.wikipedia.org/wiki/Information_science\n"
- "https://en.wikipedia.org/wiki/Library_science\n"
+ "https://en.wikipedia.org/wiki/Information_science\n"
+ "https://en.wikipedia.org/wiki/Library_science\n",
)
gen_persona_example = Example(
topic="Knowledge Curation",
examples="Title: Knowledge management\n"
- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
- " Knowledge protection methods\n Formal methods\n Informal methods\n"
- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
+ " Knowledge protection methods\n Formal methods\n Informal methods\n"
+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
personas="1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge curation. They will provide context on how knowledge curation has changed over time and its impact on modern practices.\n"
- "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"
- "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"
- "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"
- "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm."
+ "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"
+ "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"
+ "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"
+ "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm.",
)
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [
- find_related_topic_example]
+ find_related_topic_example
+ ]
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [
- gen_persona_example]
+ gen_persona_example
+ ]
# A trade-off of adding one-shot example is that it will increase the input length of the prompt. Also, some
# examples may be very long (e.g., an example for writing a section based on the given information), which may
@@ -118,24 +153,28 @@ def main(args):
topic="Example Topic",
conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",
old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"
- "# Section 2\n## Subsection 1\n## Subsection 2\n"
- "# Section 3",
+ "# Section 2\n## Subsection 1\n## Subsection 2\n"
+ "# Section 3",
outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"
- "# New Section 2\n"
- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"
+ "# New Section 2\n"
+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",
)
- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]
+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [
+ write_page_outline_example
+ ]
write_section_example = Example(
info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",
topic="Example Topic",
section="Example Section",
output="# Example Topic\n## Subsection 1\n"
- "This is an example sentence [1]. This is another example sentence [2][3].\n"
- "## Subsection 2\nThis is one more example sentence [1]."
+ "This is an example sentence [1]. This is another example sentence [2][3].\n"
+ "## Subsection 2\nThis is one more example sentence [1].",
)
- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]
+ runner.storm_article_generation.section_gen.write_section.demos = [
+ write_section_example
+ ]
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -147,42 +186,87 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--url', type=str, default='http://localhost',
- help='URL of the VLLM server.')
- parser.add_argument('--port', type=int, default=8000,
- help='Port of the VLLM server.')
- parser.add_argument('--output-dir', type=str, default='./results/mistral_7b',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--url", type=str, default="http://localhost", help="URL of the VLLM server."
+ )
+ parser.add_argument(
+ "--port", type=int, default=8000, help="Port of the VLLM server."
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/mistral_7b",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
- main(parser.parse_args())
\ No newline at end of file
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_ollama.py b/examples/storm_examples/run_storm_wiki_ollama.py
index 465e065..11d6d93 100644
--- a/examples/storm_examples/run_storm_wiki_ollama.py
+++ b/examples/storm_examples/run_storm_wiki_ollama.py
@@ -22,20 +22,34 @@ from argparse import ArgumentParser
from dspy import Example
from knowledge_storm.lm import OllamaClient
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm.rm import (
+ YouRM,
+ BingSearch,
+ BraveRM,
+ SerperRM,
+ DuckDuckGoSearchRM,
+ TavilySearchRM,
+ SearXNG,
+)
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
ollama_kwargs = {
"model": args.model,
"port": args.port,
"url": args.url,
- "stop": ('\n\n---',) # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.
+ "stop": (
+ "\n\n---",
+ ), # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.
}
conv_simulator_lm = OllamaClient(max_tokens=500, **ollama_kwargs)
@@ -61,22 +75,41 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
match args.retriever:
- case 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- case 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- case 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- case 'duckduckgo':
- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
- case 'serper':
- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
- case 'tavily':
- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
- case 'searxng':
- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case "bing":
+ rm = BingSearch(
+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "you":
+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)
+ case "brave":
+ rm = BraveRM(
+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),
+ k=engine_args.search_top_k,
+ )
+ case "duckduckgo":
+ rm = DuckDuckGoSearchRM(
+ k=engine_args.search_top_k, safe_search="On", region="us-en"
+ )
+ case "serper":
+ rm = SerperRM(
+ serper_search_api_key=os.getenv("SERPER_API_KEY"),
+ query_params={"autocorrect": True, "num": 10, "page": 1},
+ )
+ case "tavily":
+ rm = TavilySearchRM(
+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),
+ k=engine_args.search_top_k,
+ include_raw_content=True,
+ )
+ case "searxng":
+ rm = SearXNG(
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k
+ )
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
+ raise ValueError(
+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -88,26 +121,28 @@ def main(args):
find_related_topic_example = Example(
topic="Knowledge Curation",
related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"
- "https://en.wikipedia.org/wiki/Information_science\n"
- "https://en.wikipedia.org/wiki/Library_science\n"
+ "https://en.wikipedia.org/wiki/Information_science\n"
+ "https://en.wikipedia.org/wiki/Library_science\n",
)
gen_persona_example = Example(
topic="Knowledge Curation",
examples="Title: Knowledge management\n"
- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
- " Knowledge protection methods\n Formal methods\n Informal methods\n"
- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
+ " Knowledge protection methods\n Formal methods\n Informal methods\n"
+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
personas="1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge curation. They will provide context on how knowledge curation has changed over time and its impact on modern practices.\n"
- "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"
- "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"
- "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"
- "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm."
+ "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"
+ "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"
+ "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"
+ "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm.",
)
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [
- find_related_topic_example]
+ find_related_topic_example
+ ]
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [
- gen_persona_example]
+ gen_persona_example
+ ]
# A trade-off of adding one-shot example is that it will increase the input length of the prompt. Also, some
# examples may be very long (e.g., an example for writing a section based on the given information), which may
@@ -119,24 +154,28 @@ def main(args):
topic="Example Topic",
conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",
old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"
- "# Section 2\n## Subsection 1\n## Subsection 2\n"
- "# Section 3",
+ "# Section 2\n## Subsection 1\n## Subsection 2\n"
+ "# Section 3",
outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"
- "# New Section 2\n"
- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"
+ "# New Section 2\n"
+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",
)
- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]
+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [
+ write_page_outline_example
+ ]
write_section_example = Example(
info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",
topic="Example Topic",
section="Example Section",
output="# Example Topic\n## Subsection 1\n"
- "This is an example sentence [1]. This is another example sentence [2][3].\n"
- "## Subsection 2\nThis is one more example sentence [1]."
+ "This is an example sentence [1]. This is another example sentence [2][3].\n"
+ "## Subsection 2\nThis is one more example sentence [1].",
)
- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]
+ runner.storm_article_generation.section_gen.write_section.demos = [
+ write_section_example
+ ]
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -148,44 +187,90 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--url', type=str, default='http://localhost',
- help='URL of the Ollama server.')
- parser.add_argument('--port', type=int, default=11434,
- help='Port of the Ollama server.')
- parser.add_argument('--model', type=str, default='llama3:latest',
- help='Model of the Ollama server.')
- parser.add_argument('--output-dir', type=str, default='./results/ollama',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
- help='The search engine API to use for retrieving information.')
+ parser.add_argument(
+ "--url", type=str, default="http://localhost", help="URL of the Ollama server."
+ )
+ parser.add_argument(
+ "--port", type=int, default=11434, help="Port of the Ollama server."
+ )
+ parser.add_argument(
+ "--model", type=str, default="llama3:latest", help="Model of the Ollama server."
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/ollama",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
- main(parser.parse_args())
\ No newline at end of file
+ main(parser.parse_args())
diff --git a/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py b/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py
index a4be362..725a68b 100644
--- a/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py
+++ b/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py
@@ -3,21 +3,25 @@ from argparse import ArgumentParser
from dspy import Example
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import OllamaClient
from knowledge_storm.rm import SearXNG
from knowledge_storm.utils import load_api_key
def main(args):
- load_api_key(toml_file_path='secrets.toml')
+ load_api_key(toml_file_path="secrets.toml")
lm_configs = STORMWikiLMConfigs()
ollama_kwargs = {
"model": args.model,
"port": args.port,
"url": args.url,
- "stop": ('\n\n---',)
+ "stop": ("\n\n---",),
}
conv_simulator_lm = OllamaClient(max_tokens=500, **ollama_kwargs)
@@ -40,23 +44,27 @@ def main(args):
max_thread_num=args.max_thread_num,
)
- rm = SearXNG(searxng_api_url=args.searxng_api_url, searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ rm = SearXNG(
+ searxng_api_url=args.searxng_api_url,
+ searxng_api_key=os.getenv("SEARXNG_API_KEY"),
+ k=engine_args.search_top_k,
+ )
runner = STORMWikiRunner(engine_args, lm_configs, rm)
find_related_topic_example = Example(
topic="Knowledge Curation",
related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"
- "https://en.wikipedia.org/wiki/Information_science\n"
- "https://en.wikipedia.org/wiki/Library_science\n"
+ "https://en.wikipedia.org/wiki/Information_science\n"
+ "https://en.wikipedia.org/wiki/Library_science\n",
)
gen_persona_example = Example(
topic="Knowledge Curation",
examples="Title: Knowledge management\n"
- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
- " Knowledge protection methods\n Formal methods\n Informal methods\n"
- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"
+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"
+ " Knowledge protection methods\n Formal methods\n Informal methods\n"
+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",
personas=(
"1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge "
"curation. They will provide context on how knowledge curation has changed over time and its impact on "
@@ -69,35 +77,41 @@ def main(args):
"such as common features of content management systems.\n"
"5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and "
"the transition of these practices into the digital realm."
- )
+ ),
)
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [
- find_related_topic_example]
+ find_related_topic_example
+ ]
runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [
- gen_persona_example]
+ gen_persona_example
+ ]
write_page_outline_example = Example(
topic="Example Topic",
conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",
old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"
- "# Section 2\n## Subsection 1\n## Subsection 2\n"
- "# Section 3",
+ "# Section 2\n## Subsection 1\n## Subsection 2\n"
+ "# Section 3",
outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"
- "# New Section 2\n"
- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"
+ "# New Section 2\n"
+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",
)
- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]
+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [
+ write_page_outline_example
+ ]
write_section_example = Example(
info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",
topic="Example Topic",
section="Example Section",
output="# Example Topic\n## Subsection 1\n"
- "This is an example sentence [1]. This is another example sentence [2][3].\n"
- "## Subsection 2\nThis is one more example sentence [1]."
+ "This is an example sentence [1]. This is another example sentence [2][3].\n"
+ "## Subsection 2\nThis is one more example sentence [1].",
)
- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]
+ runner.storm_article_generation.section_gen.write_section.demos = [
+ write_section_example
+ ]
- topic = input('Topic: ')
+ topic = input("Topic: ")
runner.run(
topic=topic,
do_research=args.do_research,
@@ -109,46 +123,93 @@ def main(args):
runner.summary()
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
# global arguments
- parser.add_argument('--url', type=str, default='http://localhost',
- help='URL of the Ollama server.')
- parser.add_argument('--port', type=int, default=11434,
- help='Port of the Ollama server.')
- parser.add_argument('--model', type=str, default='llama3:latest',
- help='Model of the Ollama server.')
- parser.add_argument('--output-dir', type=str, default='./results/ollama',
- help='Directory to store the outputs.')
- parser.add_argument('--max-thread-num', type=int, default=3,
- help='Maximum number of threads to use. The information seeking part and the article generation'
- 'part can speed up by using multiple threads. Consider reducing it if keep getting '
- '"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['searxng'],
- help='The search engine API to use for retrieving information.')
- parser.add_argument('--searxng-api-url', type=str, required=True,
- help='URL of the SearXNG API.')
+ parser.add_argument(
+ "--url", type=str, default="http://localhost", help="URL of the Ollama server."
+ )
+ parser.add_argument(
+ "--port", type=int, default=11434, help="Port of the Ollama server."
+ )
+ parser.add_argument(
+ "--model", type=str, default="llama3:latest", help="Model of the Ollama server."
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default="./results/ollama",
+ help="Directory to store the outputs.",
+ )
+ parser.add_argument(
+ "--max-thread-num",
+ type=int,
+ default=3,
+ help="Maximum number of threads to use. The information seeking part and the article generation"
+ "part can speed up by using multiple threads. Consider reducing it if keep getting "
+ '"Exceed rate limit" error when calling LM API.',
+ )
+ parser.add_argument(
+ "--retriever",
+ type=str,
+ choices=["searxng"],
+ help="The search engine API to use for retrieving information.",
+ )
+ parser.add_argument(
+ "--searxng-api-url", type=str, required=True, help="URL of the SearXNG API."
+ )
# stage of the pipeline
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-outline', action='store_true',
- help='If True, generate an outline for the topic; otherwise, load the results.')
- parser.add_argument('--do-generate-article', action='store_true',
- help='If True, generate an article for the topic; otherwise, load the results.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
+ parser.add_argument(
+ "--do-research",
+ action="store_true",
+ help="If True, simulate conversation to research the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-outline",
+ action="store_true",
+ help="If True, generate an outline for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-generate-article",
+ action="store_true",
+ help="If True, generate an article for the topic; otherwise, load the results.",
+ )
+ parser.add_argument(
+ "--do-polish-article",
+ action="store_true",
+ help="If True, polish the article by adding a summarization section and (optionally) removing "
+ "duplicate content.",
+ )
# hyperparameters for the pre-writing stage
- parser.add_argument('--max-conv-turn', type=int, default=3,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=3,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
+ parser.add_argument(
+ "--max-conv-turn",
+ type=int,
+ default=3,
+ help="Maximum number of questions in conversational question asking.",
+ )
+ parser.add_argument(
+ "--max-perspective",
+ type=int,
+ default=3,
+ help="Maximum number of perspectives to consider in perspective-guided question asking.",
+ )
+ parser.add_argument(
+ "--search-top-k",
+ type=int,
+ default=3,
+ help="Top k search results to consider for each search query.",
+ )
# hyperparameters for the writing stage
- parser.add_argument('--retrieve-top-k', type=int, default=3,
- help='Top k collected references for each section title.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
+ parser.add_argument(
+ "--retrieve-top-k",
+ type=int,
+ default=3,
+ help="Top k collected references for each section title.",
+ )
+ parser.add_argument(
+ "--remove-duplicate",
+ action="store_true",
+ help="If True, remove duplicate content from the article.",
+ )
main(parser.parse_args())
diff --git a/frontend/demo_light/demo_util.py b/frontend/demo_light/demo_util.py
index 22258dd..cff1d9f 100644
--- a/frontend/demo_light/demo_util.py
+++ b/frontend/demo_light/demo_util.py
@@ -13,7 +13,11 @@ import streamlit as st
# Uncomment the following lines:
# import sys
# sys.path.append('../../')
-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
+from knowledge_storm import (
+ STORMWikiRunnerArguments,
+ STORMWikiRunner,
+ STORMWikiLMConfigs,
+)
from knowledge_storm.lm import OpenAIModel
from knowledge_storm.rm import YouRM
from knowledge_storm.storm_wiki.modules.callback import BaseCallbackHandler
@@ -21,7 +25,7 @@ from knowledge_storm.utils import truncate_filename
from stoc import stoc
-class DemoFileIOHelper():
+class DemoFileIOHelper:
@staticmethod
def read_structure_to_dict(articles_root_path):
"""
@@ -107,8 +111,10 @@ class DemoFileIOHelper():
file_path (str): The path to the file.
modification_time_string (str): The desired modification time in 'YYYY-MM-DD HH:MM:SS' format.
"""
- california_tz = pytz.timezone('America/Los_Angeles')
- modification_time = datetime.datetime.strptime(modification_time_string, '%Y-%m-%d %H:%M:%S')
+ california_tz = pytz.timezone("America/Los_Angeles")
+ modification_time = datetime.datetime.strptime(
+ modification_time_string, "%Y-%m-%d %H:%M:%S"
+ )
modification_time = california_tz.localize(modification_time)
modification_time_utc = modification_time.astimezone(datetime.timezone.utc)
modification_timestamp = modification_time_utc.timestamp()
@@ -125,7 +131,7 @@ class DemoFileIOHelper():
Returns:
str: The latest file's modification time in 'YYYY-MM-DD HH:MM:SS' format.
"""
- california_tz = pytz.timezone('America/Los_Angeles')
+ california_tz = pytz.timezone("America/Los_Angeles")
latest_mod_time = None
file_paths = []
@@ -138,17 +144,26 @@ class DemoFileIOHelper():
for file_path in file_paths:
modification_timestamp = os.path.getmtime(file_path)
- modification_time_utc = datetime.datetime.utcfromtimestamp(modification_timestamp)
- modification_time_utc = modification_time_utc.replace(tzinfo=datetime.timezone.utc)
- modification_time_california = modification_time_utc.astimezone(california_tz)
-
- if latest_mod_time is None or modification_time_california > latest_mod_time:
+ modification_time_utc = datetime.datetime.utcfromtimestamp(
+ modification_timestamp
+ )
+ modification_time_utc = modification_time_utc.replace(
+ tzinfo=datetime.timezone.utc
+ )
+ modification_time_california = modification_time_utc.astimezone(
+ california_tz
+ )
+
+ if (
+ latest_mod_time is None
+ or modification_time_california > latest_mod_time
+ ):
latest_mod_time = modification_time_california
if latest_mod_time is not None:
- return latest_mod_time.strftime('%Y-%m-%d %H:%M:%S')
+ return latest_mod_time.strftime("%Y-%m-%d %H:%M:%S")
else:
- return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@staticmethod
def assemble_article_data(article_file_path_dict):
@@ -171,25 +186,44 @@ class DemoFileIOHelper():
if neither the raw nor polished article text exists in the
provided file paths.
"""
- if "storm_gen_article.txt" in article_file_path_dict or "storm_gen_article_polished.txt" in article_file_path_dict:
- full_article_name = "storm_gen_article_polished.txt" if "storm_gen_article_polished.txt" in article_file_path_dict else "storm_gen_article.txt"
- article_data = {"article": DemoTextProcessingHelper.parse(
- DemoFileIOHelper.read_txt_file(article_file_path_dict[full_article_name]))}
+ if (
+ "storm_gen_article.txt" in article_file_path_dict
+ or "storm_gen_article_polished.txt" in article_file_path_dict
+ ):
+ full_article_name = (
+ "storm_gen_article_polished.txt"
+ if "storm_gen_article_polished.txt" in article_file_path_dict
+ else "storm_gen_article.txt"
+ )
+ article_data = {
+ "article": DemoTextProcessingHelper.parse(
+ DemoFileIOHelper.read_txt_file(
+ article_file_path_dict[full_article_name]
+ )
+ )
+ }
if "url_to_info.json" in article_file_path_dict:
article_data["citations"] = _construct_citation_dict_from_search_result(
- DemoFileIOHelper.read_json_file(article_file_path_dict["url_to_info.json"]))
+ DemoFileIOHelper.read_json_file(
+ article_file_path_dict["url_to_info.json"]
+ )
+ )
if "conversation_log.json" in article_file_path_dict:
article_data["conversation_log"] = DemoFileIOHelper.read_json_file(
- article_file_path_dict["conversation_log.json"])
+ article_file_path_dict["conversation_log.json"]
+ )
return article_data
return None
-class DemoTextProcessingHelper():
-
+class DemoTextProcessingHelper:
@staticmethod
def remove_citations(sent):
- return re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent)).replace(" |", "").replace("]", "")
+ return (
+ re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent))
+ .replace(" |", "")
+ .replace("]", "")
+ )
@staticmethod
def parse_conversation_history(json_data):
@@ -199,43 +233,54 @@ class DemoTextProcessingHelper():
"""
parsed_data = []
for persona_conversation_data in json_data:
- if ': ' in persona_conversation_data["perspective"]:
- name, description = persona_conversation_data["perspective"].split(": ", 1)
- elif '- ' in persona_conversation_data["perspective"]:
- name, description = persona_conversation_data["perspective"].split("- ", 1)
+ if ": " in persona_conversation_data["perspective"]:
+ name, description = persona_conversation_data["perspective"].split(
+ ": ", 1
+ )
+ elif "- " in persona_conversation_data["perspective"]:
+ name, description = persona_conversation_data["perspective"].split(
+ "- ", 1
+ )
else:
name, description = "", persona_conversation_data["perspective"]
cur_conversation = []
for dialogue_turn in persona_conversation_data["dlg_turns"]:
- cur_conversation.append({"role": "user", "content": dialogue_turn["user_utterance"]})
cur_conversation.append(
- {"role": "assistant",
- "content": DemoTextProcessingHelper.remove_citations(dialogue_turn["agent_utterance"])})
+ {"role": "user", "content": dialogue_turn["user_utterance"]}
+ )
+ cur_conversation.append(
+ {
+ "role": "assistant",
+ "content": DemoTextProcessingHelper.remove_citations(
+ dialogue_turn["agent_utterance"]
+ ),
+ }
+ )
parsed_data.append((name, description, cur_conversation))
return parsed_data
@staticmethod
def parse(text):
regex = re.compile(r']:\s+"(.*?)"\s+http')
- text = regex.sub(']: http', text)
+ text = regex.sub("]: http", text)
return text
@staticmethod
def add_markdown_indentation(input_string):
- lines = input_string.split('\n')
+ lines = input_string.split("\n")
processed_lines = [""]
for line in lines:
num_hashes = 0
for char in line:
- if char == '#':
+ if char == "#":
num_hashes += 1
else:
break
num_hashes -= 1
num_spaces = 4 * num_hashes
- new_line = ' ' * num_spaces + line
+ new_line = " " * num_spaces + line
processed_lines.append(new_line)
- return '\n'.join(processed_lines)
+ return "\n".join(processed_lines)
@staticmethod
def get_current_time_string():
@@ -245,13 +290,15 @@ class DemoTextProcessingHelper():
Returns:
str: The current California time in 'YYYY-MM-DD HH:MM:SS' format.
"""
- california_tz = pytz.timezone('America/Los_Angeles')
+ california_tz = pytz.timezone("America/Los_Angeles")
utc_now = datetime.datetime.now(datetime.timezone.utc)
california_now = utc_now.astimezone(california_tz)
- return california_now.strftime('%Y-%m-%d %H:%M:%S')
+ return california_now.strftime("%Y-%m-%d %H:%M:%S")
@staticmethod
- def compare_time_strings(time_string1, time_string2, time_format='%Y-%m-%d %H:%M:%S'):
+ def compare_time_strings(
+ time_string1, time_string2, time_format="%Y-%m-%d %H:%M:%S"
+ ):
"""
Compares two time strings to determine if they represent the same point in time.
@@ -273,13 +320,13 @@ class DemoTextProcessingHelper():
@staticmethod
def add_inline_citation_link(article_text, citation_dict):
# Regular expression to find citations like [i]
- pattern = r'\[(\d+)\]'
+ pattern = r"\[(\d+)\]"
# Function to replace each citation with its Markdown link
def replace_with_link(match):
i = match.group(1)
- url = citation_dict.get(int(i), {}).get('url', '#')
- return f'[[{i}]]({url})'
+ url = citation_dict.get(int(i), {}).get("url", "#")
+ return f"[[{i}]]({url})"
# Replace all citations in the text with Markdown links
return re.sub(pattern, replace_with_link, article_text)
@@ -292,26 +339,34 @@ class DemoTextProcessingHelper():
level = line.count("#")
title = line.strip("# ").strip()
anchor = title.lower().replace(" ", "-").replace(".", "")
- toc.append(f"<li style='margin-left: {20 * (level - 1)}px;'><a href='#{anchor}'>{title}</a></li>")
+ toc.append(
+ f"<li style='margin-left: {20 * (level - 1)}px;'><a href='#{anchor}'>{title}</a></li>"
+ )
return "<ul>" + "".join(toc) + "</ul>"
@staticmethod
def construct_bibliography_from_url_to_info(url_to_info):
bibliography_list = []
- sorted_url_to_unified_index = dict(sorted(url_to_info['url_to_unified_index'].items(),
- key=lambda item: item[1]))
+ sorted_url_to_unified_index = dict(
+ sorted(
+ url_to_info["url_to_unified_index"].items(), key=lambda item: item[1]
+ )
+ )
for url, index in sorted_url_to_unified_index.items():
- title = url_to_info['url_to_info'][url]['title']
+ title = url_to_info["url_to_info"][url]["title"]
bibliography_list.append(f"[{index}]: [{title}]({url})")
bibliography_string = "\n\n".join(bibliography_list)
return f"# References\n\n{bibliography_string}"
-class DemoUIHelper():
+class DemoUIHelper:
def st_markdown_adjust_size(content, font_size=20):
- st.markdown(f"""
+ st.markdown(
+ f"""
<span style='font-size: {font_size}px;'>{content}</span>
- """, unsafe_allow_html=True)
+ """,
+ unsafe_allow_html=True,
+ )
@staticmethod
def get_article_card_UI_style(boarder_color="#9AD8E1"):
@@ -326,7 +381,7 @@ class DemoUIHelper():
"border-radius": "5px",
"border-left": f"0.5rem solid {boarder_color}",
"box-shadow": "0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15)",
- "margin": "0px"
+ "margin": "0px",
},
"title": {
"white-space": "nowrap",
@@ -336,7 +391,7 @@ class DemoUIHelper():
"color": "rgb(49, 51, 63)",
"text-align": "left",
"width": "95%",
- "font-weight": "normal"
+ "font-weight": "normal",
},
"text": {
"white-space": "nowrap",
@@ -345,11 +400,9 @@ class DemoUIHelper():
"font-size": "25px",
"color": "rgb(49, 51, 63)",
"text-align": "left",
- "width": "95%"
+ "width": "95%",
},
- "filter": {
- "background-color": "rgba(0, 0, 0, 0)"
- }
+ "filter": {"background-color": "rgba(0, 0, 0, 0)"},
}
@staticmethod
@@ -373,7 +426,8 @@ class DemoUIHelper():
line-height: 1.5; /* Adjust this value as needed */
}
</style>
- """, unsafe_allow_html=True
+ """,
+ unsafe_allow_html=True,
)
@staticmethod
@@ -405,10 +459,12 @@ def _construct_citation_dict_from_search_result(search_results):
if search_results is None:
return None
citation_dict = {}
- for url, index in search_results['url_to_unified_index'].items():
- citation_dict[index] = {'url': url,
- 'title': search_results['url_to_info'][url]['title'],
- 'snippets': search_results['url_to_info'][url]['snippets']}
+ for url, index in search_results["url_to_unified_index"].items():
+ citation_dict[index] = {
+ "url": url,
+ "title": search_results["url_to_info"][url]["title"],
+ "snippets": search_results["url_to_info"][url]["snippets"],
+ }
return citation_dict
@@ -416,10 +472,14 @@ def _display_main_article_text(article_text, citation_dict, table_content_sideba
# Post-process the generated article for better display.
if "Write the lead section:" in article_text:
article_text = article_text[
- article_text.find("Write the lead section:") + len("Write the lead section:"):]
- if article_text[0] == '#':
- article_text = '\n'.join(article_text.split('\n')[1:])
- article_text = DemoTextProcessingHelper.add_inline_citation_link(article_text, citation_dict)
+ article_text.find("Write the lead section:")
+ + len("Write the lead section:") :
+ ]
+ if article_text[0] == "#":
+ article_text = "\n".join(article_text.split("\n")[1:])
+ article_text = DemoTextProcessingHelper.add_inline_citation_link(
+ article_text, citation_dict
+ )
# '$' needs to be changed to '\$' to avoid being interpreted as LaTeX in st.markdown()
article_text = article_text.replace("$", "\\$")
stoc.from_markdown(article_text, table_content_sidebar)
@@ -430,10 +490,10 @@ def _display_references(citation_dict):
reference_list = [f"reference [{i}]" for i in range(1, len(citation_dict) + 1)]
selected_key = st.selectbox("Select a reference", reference_list)
citation_val = citation_dict[reference_list.index(selected_key) + 1]
- citation_val['title'] = citation_val['title'].replace("$", "\\$")
+ citation_val["title"] = citation_val["title"].replace("$", "\\$")
st.markdown(f"**Title:** {citation_val['title']}")
st.markdown(f"**Url:** {citation_val['url']}")
- snippets = '\n\n'.join(citation_val['snippets']).replace("$", "\\$")
+ snippets = "\n\n".join(citation_val["snippets"]).replace("$", "\\$")
st.markdown(f"**Highlights:**\n\n {snippets}")
else:
st.markdown("**No references available**")
@@ -444,7 +504,9 @@ def _display_persona_conversations(conversation_log):
Display persona conversation in dialogue UI
"""
# get personas list as (persona_name, persona_description, dialogue turns list) tuple
- parsed_conversation_history = DemoTextProcessingHelper.parse_conversation_history(conversation_log)
+ parsed_conversation_history = DemoTextProcessingHelper.parse_conversation_history(
+ conversation_log
+ )
# construct tabs for each persona conversation
persona_tabs = st.tabs([name for (name, _, _) in parsed_conversation_history])
for idx, persona_tab in enumerate(persona_tabs):
@@ -453,7 +515,7 @@ def _display_persona_conversations(conversation_log):
st.info(parsed_conversation_history[idx][1])
# show user / agent utterance in dialogue UI
for message in parsed_conversation_history[idx][2]:
- message['content'] = message['content'].replace("$", "\\$")
+ message["content"] = message["content"].replace("$", "\\$")
with st.chat_message(message["role"]):
if message["role"] == "user":
st.markdown(f"**{message['content']}**")
@@ -461,14 +523,22 @@ def _display_persona_conversations(conversation_log):
st.markdown(message["content"])
-def _display_main_article(selected_article_file_path_dict, show_reference=True, show_conversation=True):
- article_data = DemoFileIOHelper.assemble_article_data(selected_article_file_path_dict)
+def _display_main_article(
+ selected_article_file_path_dict, show_reference=True, show_conversation=True
+):
+ article_data = DemoFileIOHelper.assemble_article_data(
+ selected_article_file_path_dict
+ )
with st.container(height=1000, border=True):
- table_content_sidebar = st.sidebar.expander("**Table of contents**", expanded=True)
- _display_main_article_text(article_text=article_data.get("article", ""),
- citation_dict=article_data.get("citations", {}),
- table_content_sidebar=table_content_sidebar)
+ table_content_sidebar = st.sidebar.expander(
+ "**Table of contents**", expanded=True
+ )
+ _display_main_article_text(
+ article_text=article_data.get("article", ""),
+ citation_dict=article_data.get("citations", {}),
+ table_content_sidebar=table_content_sidebar,
+ )
# display reference panel
if show_reference and "citations" in article_data:
@@ -479,9 +549,12 @@ def _display_main_article(selected_article_file_path_dict, show_reference=True,
# display conversation history
if show_conversation and "conversation_log" in article_data:
with st.expander(
- "**STORM** is powered by a knowledge agent that proactively research a given topic by asking good questions coming from different perspectives.\n\n"
- ":sunglasses: Click here to view the agent's brain**STORM**ing process!"):
- _display_persona_conversations(conversation_log=article_data.get("conversation_log", {}))
+ "**STORM** is powered by a knowledge agent that proactively research a given topic by asking good questions coming from different perspectives.\n\n"
+ ":sunglasses: Click here to view the agent's brain**STORM**ing process!"
+ ):
+ _display_persona_conversations(
+ conversation_log=article_data.get("conversation_log", {})
+ )
def get_demo_dir():
@@ -492,7 +565,11 @@ def clear_other_page_session_state(page_index: Optional[int]):
if page_index is None:
keys_to_delete = [key for key in st.session_state if key.startswith("page")]
else:
- keys_to_delete = [key for key in st.session_state if key.startswith("page") and f"page{page_index}" not in key]
+ keys_to_delete = [
+ key
+ for key in st.session_state
+ if key.startswith("page") and f"page{page_index}" not in key
+ ]
for key in set(keys_to_delete):
del st.session_state[key]
@@ -504,29 +581,44 @@ def set_storm_runner():
# configure STORM runner
llm_configs = STORMWikiLMConfigs()
- llm_configs.init_openai_model(openai_api_key=st.secrets['OPENAI_API_KEY'], openai_type='openai')
- llm_configs.set_question_asker_lm(OpenAIModel(model='gpt-4-1106-preview', api_key=st.secrets['OPENAI_API_KEY'],
- api_provider='openai',
- max_tokens=500, temperature=1.0, top_p=0.9))
+ llm_configs.init_openai_model(
+ openai_api_key=st.secrets["OPENAI_API_KEY"], openai_type="openai"
+ )
+ llm_configs.set_question_asker_lm(
+ OpenAIModel(
+ model="gpt-4-1106-preview",
+ api_key=st.secrets["OPENAI_API_KEY"],
+ api_provider="openai",
+ max_tokens=500,
+ temperature=1.0,
+ top_p=0.9,
+ )
+ )
engine_args = STORMWikiRunnerArguments(
output_dir=current_working_dir,
max_conv_turn=3,
max_perspective=3,
search_top_k=3,
- retrieve_top_k=5
+ retrieve_top_k=5,
)
- rm = YouRM(ydc_api_key=st.secrets['YDC_API_KEY'], k=engine_args.search_top_k)
+ rm = YouRM(ydc_api_key=st.secrets["YDC_API_KEY"], k=engine_args.search_top_k)
runner = STORMWikiRunner(engine_args, llm_configs, rm)
st.session_state["runner"] = runner
-def display_article_page(selected_article_name, selected_article_file_path_dict,
- show_title=True, show_main_article=True):
+def display_article_page(
+ selected_article_name,
+ selected_article_file_path_dict,
+ show_title=True,
+ show_main_article=True,
+):
if show_title:
- st.markdown(f"<h2 style='text-align: center;'>{selected_article_name.replace('_', ' ')}</h2>",
- unsafe_allow_html=True)
+ st.markdown(
+ f"<h2 style='text-align: center;'>{selected_article_name.replace('_', ' ')}</h2>",
+ unsafe_allow_html=True,
+ )
if show_main_article:
_display_main_article(selected_article_file_path_dict)
@@ -537,20 +629,25 @@ class StreamlitCallbackHandler(BaseCallbackHandler):
self.status_container = status_container
def on_identify_perspective_start(self, **kwargs):
- self.status_container.info('Start identifying different perspectives for researching the topic.')
+ self.status_container.info(
+ "Start identifying different perspectives for researching the topic."
+ )
def on_identify_perspective_end(self, perspectives: list[str], **kwargs):
perspective_list = "\n- ".join(perspectives)
- self.status_container.success(f'Finish identifying perspectives. Will now start gathering information'
- f' from the following perspectives:\n- {perspective_list}')
+ self.status_container.success(
+ f"Finish identifying perspectives. Will now start gathering information"
+ f" from the following perspectives:\n- {perspective_list}"
+ )
def on_information_gathering_start(self, **kwargs):
- self.status_container.info('Start browsing the Internet.')
+ self.status_container.info("Start browsing the Internet.")
def on_dialogue_turn_end(self, dlg_turn, **kwargs):
urls = list(set([r.url for r in dlg_turn.search_results]))
for url in urls:
- self.status_container.markdown(f"""
+ self.status_container.markdown(
+ f"""
<style>
.small-font {{
font-size: 14px;
@@ -559,16 +656,22 @@ class StreamlitCallbackHandler(BaseCallbackHandler):
}}
</style>
<div class="small-font">Finish browsing <a href="{url}" class="small-font" target="_blank">{url}</a>.</div>
- """, unsafe_allow_html=True)
+ """,
+ unsafe_allow_html=True,
+ )
def on_information_gathering_end(self, **kwargs):
- self.status_container.success('Finish collecting information.')
+ self.status_container.success("Finish collecting information.")
def on_information_organization_start(self, **kwargs):
- self.status_container.info('Start organizing information into a hierarchical outline.')
+ self.status_container.info(
+ "Start organizing information into a hierarchical outline."
+ )
def on_direct_outline_generation_end(self, outline: str, **kwargs):
- self.status_container.success(f'Finish leveraging the internal knowledge of the large language model.')
+ self.status_container.success(
+ f"Finish leveraging the internal knowledge of the large language model."
+ )
def on_outline_refinement_end(self, outline: str, **kwargs):
- self.status_container.success(f'Finish leveraging the collected information.')
+ self.status_container.success(f"Finish leveraging the collected information.")
diff --git a/frontend/demo_light/pages_util/CreateNewArticle.py b/frontend/demo_light/pages_util/CreateNewArticle.py
index 2257044..fb30aae 100644
--- a/frontend/demo_light/pages_util/CreateNewArticle.py
+++ b/frontend/demo_light/pages_util/CreateNewArticle.py
@@ -3,39 +3,60 @@ import time
import demo_util
import streamlit as st
-from demo_util import DemoFileIOHelper, DemoTextProcessingHelper, DemoUIHelper, truncate_filename
+from demo_util import (
+ DemoFileIOHelper,
+ DemoTextProcessingHelper,
+ DemoUIHelper,
+ truncate_filename,
+)
+
def handle_not_started():
if st.session_state["page3_write_article_state"] == "not started":
-
_, search_form_column, _ = st.columns([2, 5, 2])
with search_form_column:
- with st.form(key='search_form'):
+ with st.form(key="search_form"):
# Text input for the search topic
- DemoUIHelper.st_markdown_adjust_size(content="Enter the topic you want to learn in depth:",
- font_size=18)
- st.session_state["page3_topic"] = st.text_input(label='page3_topic', label_visibility="collapsed")
+ DemoUIHelper.st_markdown_adjust_size(
+ content="Enter the topic you want to learn in depth:", font_size=18
+ )
+ st.session_state["page3_topic"] = st.text_input(
+ label="page3_topic", label_visibility="collapsed"
+ )
pass_appropriateness_check = True
# Submit button for the form
- submit_button = st.form_submit_button(label='Research')
+ submit_button = st.form_submit_button(label="Research")
# only start new search when button is clicked, not started, or already finished previous one
- if submit_button and st.session_state["page3_write_article_state"] in ["not started", "show results"]:
+ if submit_button and st.session_state["page3_write_article_state"] in [
+ "not started",
+ "show results",
+ ]:
if not st.session_state["page3_topic"].strip():
pass_appropriateness_check = False
- st.session_state["page3_warning_message"] = "topic could not be empty"
-
- st.session_state["page3_topic_name_cleaned"] = st.session_state["page3_topic"].replace(
- ' ', '_').replace('/', '_')
- st.session_state["page3_topic_name_truncated"] = truncate_filename(st.session_state["page3_topic_name_cleaned"])
+ st.session_state[
+ "page3_warning_message"
+ ] = "topic could not be empty"
+
+ st.session_state["page3_topic_name_cleaned"] = (
+ st.session_state["page3_topic"]
+ .replace(" ", "_")
+ .replace("/", "_")
+ )
+ st.session_state["page3_topic_name_truncated"] = truncate_filename(
+ st.session_state["page3_topic_name_cleaned"]
+ )
if not pass_appropriateness_check:
st.session_state["page3_write_article_state"] = "not started"
- alert = st.warning(st.session_state["page3_warning_message"], icon="⚠️")
+ alert = st.warning(
+ st.session_state["page3_warning_message"], icon="⚠️"
+ )
time.sleep(5)
alert.empty()
else:
st.session_state["page3_write_article_state"] = "initiated"
+
def handle_initiated():
if st.session_state["page3_write_article_state"] == "initiated":
current_working_dir = os.path.join(demo_util.get_demo_dir(), "DEMO_WORKING_DIR")
@@ -47,9 +68,12 @@ def handle_initiated():
st.session_state["page3_current_working_dir"] = current_working_dir
st.session_state["page3_write_article_state"] = "pre_writing"
+
def handle_pre_writing():
- if st.session_state["page3_write_article_state"] == "pre_writing":
- status = st.status("I am brain**STORM**ing now to research the topic. (This may take 2-3 minutes.)")
+ if st.session_state["page3_write_article_state"] == "pre_writing":
+ status = st.status(
+ "I am brain**STORM**ing now to research the topic. (This may take 2-3 minutes.)"
+ )
st_callback_handler = demo_util.StreamlitCallbackHandler(status)
with status:
# STORM main gen outline
@@ -59,23 +83,37 @@ def handle_pre_writing():
do_generate_outline=True,
do_generate_article=False,
do_polish_article=False,
- callback_handler=st_callback_handler
+ callback_handler=st_callback_handler,
+ )
+ conversation_log_path = os.path.join(
+ st.session_state["page3_current_working_dir"],
+ st.session_state["page3_topic_name_truncated"],
+ "conversation_log.json",
+ )
+ demo_util._display_persona_conversations(
+ DemoFileIOHelper.read_json_file(conversation_log_path)
)
- conversation_log_path = os.path.join(st.session_state["page3_current_working_dir"],
- st.session_state["page3_topic_name_truncated"], "conversation_log.json")
- demo_util._display_persona_conversations(DemoFileIOHelper.read_json_file(conversation_log_path))
st.session_state["page3_write_article_state"] = "final_writing"
status.update(label="brain**STORM**ing complete!", state="complete")
+
def handle_final_writing():
if st.session_state["page3_write_article_state"] == "final_writing":
# polish final article
with st.status(
- "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)") as status:
- st.info('Now I will connect the information I found for your reference. (This may take 4-5 minutes.)')
- st.session_state["runner"].run(topic=st.session_state["page3_topic"], do_research=False,
- do_generate_outline=False,
- do_generate_article=True, do_polish_article=True, remove_duplicate=False)
+ "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)"
+ ) as status:
+ st.info(
+ "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)"
+ )
+ st.session_state["runner"].run(
+ topic=st.session_state["page3_topic"],
+ do_research=False,
+ do_generate_outline=False,
+ do_generate_article=True,
+ do_polish_article=True,
+ remove_duplicate=False,
+ )
# finish the session
st.session_state["runner"].post_run()
@@ -83,6 +121,7 @@ def handle_final_writing():
st.session_state["page3_write_article_state"] = "prepare_to_show_result"
status.update(label="information snythesis complete!", state="complete")
+
def handle_prepare_to_show_result():
if st.session_state["page3_write_article_state"] == "prepare_to_show_result":
_, show_result_col, _ = st.columns([4, 3, 4])
@@ -91,16 +130,23 @@ def handle_prepare_to_show_result():
st.session_state["page3_write_article_state"] = "completed"
st.rerun()
+
def handle_completed():
-
if st.session_state["page3_write_article_state"] == "completed":
# display polished article
current_working_dir_paths = DemoFileIOHelper.read_structure_to_dict(
- st.session_state["page3_current_working_dir"])
- current_article_file_path_dict = current_working_dir_paths[st.session_state["page3_topic_name_truncated"]]
- demo_util.display_article_page(selected_article_name=st.session_state["page3_topic_name_cleaned"],
- selected_article_file_path_dict=current_article_file_path_dict,
- show_title=True, show_main_article=True)
+ st.session_state["page3_current_working_dir"]
+ )
+ current_article_file_path_dict = current_working_dir_paths[
+ st.session_state["page3_topic_name_truncated"]
+ ]
+ demo_util.display_article_page(
+ selected_article_name=st.session_state["page3_topic_name_cleaned"],
+ selected_article_file_path_dict=current_article_file_path_dict,
+ show_title=True,
+ show_main_article=True,
+ )
+
def create_new_article_page():
demo_util.clear_other_page_session_state(page_index=3)
diff --git a/frontend/demo_light/pages_util/MyArticles.py b/frontend/demo_light/pages_util/MyArticles.py
index e4e3bd1..3d1ee6f 100644
--- a/frontend/demo_light/pages_util/MyArticles.py
+++ b/frontend/demo_light/pages_util/MyArticles.py
@@ -11,7 +11,10 @@ def my_articles_page():
with st.sidebar:
_, return_button_col = st.columns([2, 5])
with return_button_col:
- if st.button("Select another article", disabled="page2_selected_my_article" not in st.session_state):
+ if st.button(
+ "Select another article",
+ disabled="page2_selected_my_article" not in st.session_state,
+ ):
if "page2_selected_my_article" in st.session_state:
del st.session_state["page2_selected_my_article"]
st.rerun()
@@ -20,17 +23,22 @@ def my_articles_page():
if "page2_user_articles_file_path_dict" not in st.session_state:
local_dir = os.path.join(demo_util.get_demo_dir(), "DEMO_WORKING_DIR")
os.makedirs(local_dir, exist_ok=True)
- st.session_state["page2_user_articles_file_path_dict"] = DemoFileIOHelper.read_structure_to_dict(local_dir)
+ st.session_state[
+ "page2_user_articles_file_path_dict"
+ ] = DemoFileIOHelper.read_structure_to_dict(local_dir)
# if no feature demo selected, display all featured articles as info cards
def article_card_setup(column_to_add, card_title, article_name):
with column_to_add:
cleaned_article_title = article_name.replace("_", " ")
- hasClicked = card(title=" / ".join(card_title),
- text=article_name.replace("_", " "),
- image=DemoFileIOHelper.read_image_as_base64(
- os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")),
- styles=DemoUIHelper.get_article_card_UI_style(boarder_color="#9AD8E1"))
+ hasClicked = card(
+ title=" / ".join(card_title),
+ text=article_name.replace("_", " "),
+ image=DemoFileIOHelper.read_image_as_base64(
+ os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")
+ ),
+ styles=DemoUIHelper.get_article_card_UI_style(boarder_color="#9AD8E1"),
+ )
if hasClicked:
st.session_state["page2_selected_my_article"] = article_name
st.rerun()
@@ -40,7 +48,9 @@ def my_articles_page():
my_article_columns = st.columns(3)
if len(st.session_state["page2_user_articles_file_path_dict"]) > 0:
# get article names
- article_names = sorted(list(st.session_state["page2_user_articles_file_path_dict"].keys()))
+ article_names = sorted(
+ list(st.session_state["page2_user_articles_file_path_dict"].keys())
+ )
# configure pagination
pagination = st.container()
bottom_menu = st.columns((1, 4, 1, 1, 1))[1:-1]
@@ -48,7 +58,9 @@ def my_articles_page():
batch_size = st.selectbox("Page Size", options=[24, 48, 72])
with bottom_menu[1]:
total_pages = (
- int(len(article_names) / batch_size) if int(len(article_names) / batch_size) > 0 else 1
+ int(len(article_names) / batch_size)
+ if int(len(article_names) / batch_size) > 0
+ else 1
)
current_page = st.number_input(
"Page", min_value=1, max_value=total_pages, step=1
@@ -60,19 +72,24 @@ def my_articles_page():
my_article_count = 0
start_index = (current_page - 1) * batch_size
end_index = min(current_page * batch_size, len(article_names))
- for article_name in article_names[start_index: end_index]:
+ for article_name in article_names[start_index:end_index]:
column_to_add = my_article_columns[my_article_count % 3]
my_article_count += 1
- article_card_setup(column_to_add=column_to_add,
- card_title=["My Article"],
- article_name=article_name)
+ article_card_setup(
+ column_to_add=column_to_add,
+ card_title=["My Article"],
+ article_name=article_name,
+ )
else:
with my_article_columns[0]:
- hasClicked = card(title="Get started",
- text="Start your first research!",
- image=DemoFileIOHelper.read_image_as_base64(
- os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")),
- styles=DemoUIHelper.get_article_card_UI_style())
+ hasClicked = card(
+ title="Get started",
+ text="Start your first research!",
+ image=DemoFileIOHelper.read_image_as_base64(
+ os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")
+ ),
+ styles=DemoUIHelper.get_article_card_UI_style(),
+ )
if hasClicked:
st.session_state.selected_page = 1
st.session_state["manual_selection_override"] = True
@@ -80,8 +97,13 @@ def my_articles_page():
st.rerun()
else:
selected_article_name = st.session_state["page2_selected_my_article"]
- selected_article_file_path_dict = st.session_state["page2_user_articles_file_path_dict"][selected_article_name]
+ selected_article_file_path_dict = st.session_state[
+ "page2_user_articles_file_path_dict"
+ ][selected_article_name]
- demo_util.display_article_page(selected_article_name=selected_article_name,
- selected_article_file_path_dict=selected_article_file_path_dict,
- show_title=True, show_main_article=True)
+ demo_util.display_article_page(
+ selected_article_name=selected_article_name,
+ selected_article_file_path_dict=selected_article_file_path_dict,
+ show_title=True,
+ show_main_article=True,
+ )
diff --git a/frontend/demo_light/stoc.py b/frontend/demo_light/stoc.py
index 7bd4402..2abd3c4 100644
--- a/frontend/demo_light/stoc.py
+++ b/frontend/demo_light/stoc.py
@@ -44,9 +44,9 @@ class stoc:
for title_size, title in self.toc_items:
h = int(title_size.replace("h", ""))
markdown_toc += (
- " " * 2 * h
- + "- "
- + f'<a href="#{normalize(title)}" class="toc"> {title}</a> \n'
+ " " * 2 * h
+ + "- "
+ + f'<a href="#{normalize(title)}" class="toc"> {title}</a> \n'
)
# st.sidebar.write(markdown_toc, unsafe_allow_html=True)
st.write(markdown_toc, unsafe_allow_html=True)
@@ -56,27 +56,35 @@ class stoc:
def increase_heading_depth_and_add_top_heading(markdown_text, new_top_heading):
lines = markdown_text.splitlines()
# Increase the depth of each heading by adding an extra '#'
- increased_depth_lines = ['#' + line if line.startswith('#') else line for line in lines]
+ increased_depth_lines = [
+ "#" + line if line.startswith("#") else line for line in lines
+ ]
# Add the new top-level heading at the beginning
increased_depth_lines.insert(0, f"# {new_top_heading}")
# Re-join the modified lines back into a single string
- modified_text = '\n'.join(increased_depth_lines)
+ modified_text = "\n".join(increased_depth_lines)
return modified_text
if topic:
- markdown_text = increase_heading_depth_and_add_top_heading(markdown_text, topic)
+ markdown_text = increase_heading_depth_and_add_top_heading(
+ markdown_text, topic
+ )
toc = []
for line in markdown_text.splitlines():
- if line.startswith('#'):
+ if line.startswith("#"):
# Remove the '#' characters and strip leading/trailing spaces
- heading_text = line.lstrip('#').strip()
+ heading_text = line.lstrip("#").strip()
# Create slug (lowercase, spaces to hyphens, remove non-alphanumeric characters)
- slug = re.sub(r'[^a-zA-Z0-9\s-]', '', heading_text).lower().replace(' ', '-')
+ slug = (
+ re.sub(r"[^a-zA-Z0-9\s-]", "", heading_text)
+ .lower()
+ .replace(" ", "-")
+ )
# Determine heading level for indentation
- level = line.count('#') - 1
+ level = line.count("#") - 1
# Add to the table of contents
- toc.append(' ' * level + f'- [{heading_text}](#{slug})')
- return '\n'.join(toc)
+ toc.append(" " * level + f"- [{heading_text}](#{slug})")
+ return "\n".join(toc)
@classmethod
def from_markdown(cls, text: str, expander=None):
diff --git a/frontend/demo_light/storm.py b/frontend/demo_light/storm.py
index 1d095c3..5aca259 100644
--- a/frontend/demo_light/storm.py
+++ b/frontend/demo_light/storm.py
@@ -11,13 +11,13 @@ from streamlit_option_menu import option_menu
def main():
global database
- st.set_page_config(layout='wide')
+ st.set_page_config(layout="wide")
if "first_run" not in st.session_state:
- st.session_state['first_run'] = True
+ st.session_state["first_run"] = True
# set api keys from secrets
- if st.session_state['first_run']:
+ if st.session_state["first_run"]:
for key, value in st.secrets.items():
if type(value) == str:
os.environ[key] = value
@@ -31,20 +31,26 @@ def main():
st.session_state["rerun_requested"] = False
st.rerun()
- st.write('<style>div.block-container{padding-top:2rem;}</style>', unsafe_allow_html=True)
+ st.write(
+ "<style>div.block-container{padding-top:2rem;}</style>", unsafe_allow_html=True
+ )
menu_container = st.container()
with menu_container:
pages = ["My Articles", "Create New Article"]
- styles={
- "container": {"padding": "0.2rem 0",
- "background-color": "#22222200"},
- }
- menu_selection = option_menu(None, pages,
- icons=['house', 'search'],
- menu_icon="cast", default_index=0, orientation="horizontal",
- manual_select=st.session_state.selected_page,
- styles=styles,
- key='menu_selection')
+ styles = {
+ "container": {"padding": "0.2rem 0", "background-color": "#22222200"},
+ }
+ menu_selection = option_menu(
+ None,
+ pages,
+ icons=["house", "search"],
+ menu_icon="cast",
+ default_index=0,
+ orientation="horizontal",
+ manual_select=st.session_state.selected_page,
+ styles=styles,
+ key="menu_selection",
+ )
if st.session_state.get("manual_selection_override", False):
menu_selection = pages[st.session_state["selected_page"]]
st.session_state["manual_selection_override"] = False
diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.py
index 8da7433..193d684 100644
--- a/knowledge_storm/collaborative_storm/engine.py
+++ b/knowledge_storm/collaborative_storm/engine.py
@@ -600,11 +600,13 @@ class CoStormRunner:
callback_handler=self.callback_handler,
)
- warmstart_conv, warmstart_revised_conv, warmstart_experts = (
- warm_start_module.initiate_warm_start(
- topic=self.runner_argument.topic,
- knowledge_base=self.knowledge_base,
- )
+ (
+ warmstart_conv,
+ warmstart_revised_conv,
+ warmstart_experts,
+ ) = warm_start_module.initiate_warm_start(
+ topic=self.runner_argument.topic,
+ knowledge_base=self.knowledge_base,
)
self.discourse_manager.experts = (
self.discourse_manager._parse_expert_names_to_agent(
@@ -618,7 +620,6 @@ class CoStormRunner:
self.warmstart_conv_archive = warmstart_conv
self.knowledge_base.reogranize()
else:
-
if self.knowledge_base is None:
self.knowledge_base = KnowledgeBase(
topic=self.runner_argument.topic,
diff --git a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
index 47daafb..05cbde3 100644
--- a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
+++ b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
@@ -270,9 +270,10 @@ class InsertInformationModule(dspy.Module):
root=insert_root,
)
- encoded_outlines, outlines = (
- knowledge_base.get_knowledge_base_structure_embedding(root=insert_root)
- )
+ (
+ encoded_outlines,
+ outlines,
+ ) = knowledge_base.get_knowledge_base_structure_embedding(root=insert_root)
to_return = []
if not allow_create_new_node:
# use multi thread as knowledge base structure does not change
@@ -295,10 +296,11 @@ class InsertInformationModule(dspy.Module):
else:
# use sequential insert as knowledge base structure might change
for question, query in intent_to_placement_dict:
- encoded_outlines, outlines = (
- knowledge_base.get_knowledge_base_structure_embedding(
- root=insert_root
- )
+ (
+ encoded_outlines,
+ outlines,
+ ) = knowledge_base.get_knowledge_base_structure_embedding(
+ root=insert_root
)
_, placement_prediction = process_intent(question=question, query=query)
intent_to_placement_dict[(question, query)] = placement_prediction
diff --git a/knowledge_storm/dataclass.py b/knowledge_storm/dataclass.py
index 53bd7e4..86e89ad 100644
--- a/knowledge_storm/dataclass.py
+++ b/knowledge_storm/dataclass.py
@@ -773,9 +773,9 @@ class KnowledgeBase:
def update_all_info_path(self):
def _helper(node):
for citation_idx in node.content:
- self.info_uuid_to_info_dict[citation_idx].meta["placement"] = (
- " -> ".join(node.get_path_from_root())
- )
+ self.info_uuid_to_info_dict[citation_idx].meta[
+ "placement"
+ ] = " -> ".join(node.get_path_from_root())
for child in node.children:
_helper(child)
diff --git a/knowledge_storm/interface.py b/knowledge_storm/interface.py
index 5922602..db02896 100644
--- a/knowledge_storm/interface.py
+++ b/knowledge_storm/interface.py
@@ -473,7 +473,6 @@ class LMConfigs(ABC):
return model_name_to_usage
def log(self):
-
return OrderedDict(
{
attr_name: getattr(self, attr_name).kwargs
@@ -503,9 +502,9 @@ class Engine(ABC):
logger.info(f"{func.__name__} executed in {execution_time:.4f} seconds")
self.lm_cost[func.__name__] = self.lm_configs.collect_and_reset_lm_usage()
if hasattr(self, "retriever"):
- self.rm_cost[func.__name__] = (
- self.retriever.collect_and_reset_rm_usage()
- )
+ self.rm_cost[
+ func.__name__
+ ] = self.retriever.collect_and_reset_rm_usage()
return result
return wrapper
diff --git a/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.py
index f0b3f55..c698a9a 100644
--- a/knowledge_storm/storm_wiki/engine.py
+++ b/knowledge_storm/storm_wiki/engine.py
@@ -213,16 +213,16 @@ class STORMWikiRunner(Engine):
ground_truth_url: str = "None",
callback_handler: BaseCallbackHandler = None,
) -> StormInformationTable:
-
- information_table, conversation_log = (
- self.storm_knowledge_curation_module.research(
- topic=self.topic,
- ground_truth_url=ground_truth_url,
- callback_handler=callback_handler,
- max_perspective=self.args.max_perspective,
- disable_perspective=False,
- return_conversation_log=True,
- )
+ (
+ information_table,
+ conversation_log,
+ ) = self.storm_knowledge_curation_module.research(
+ topic=self.topic,
+ ground_truth_url=ground_truth_url,
+ callback_handler=callback_handler,
+ max_perspective=self.args.max_perspective,
+ disable_perspective=False,
+ return_conversation_log=True,
)
FileIOHelper.dump_json(
diff --git a/knowledge_storm/storm_wiki/modules/article_generation.py b/knowledge_storm/storm_wiki/modules/article_generation.py
index a23b788..6289c7a 100644
--- a/knowledge_storm/storm_wiki/modules/article_generation.py
+++ b/knowledge_storm/storm_wiki/modules/article_generation.py
@@ -88,7 +88,6 @@ class StormArticleGenerationModule(ArticleGenerationModule):
)
section_output_dict_collection = [section_output_dict]
else:
-
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.max_thread_num
) as executor:
diff --git a/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.py
index 119869c..c43c409 100644
--- a/knowledge_storm/storm_wiki/modules/storm_dataclass.py
+++ b/knowledge_storm/storm_wiki/modules/storm_dataclass.py
@@ -58,9 +58,9 @@ class StormInformationTable(InformationTable):
def __init__(self, conversations=List[Tuple[str, List[DialogueTurn]]]):
super().__init__()
self.conversations = conversations
- self.url_to_info: Dict[str, Information] = (
- StormInformationTable.construct_url_to_info(self.conversations)
- )
+ self.url_to_info: Dict[
+ str, Information
+ ] = StormInformationTable.construct_url_to_info(self.conversations)
@staticmethod
def construct_url_to_info(