An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
fix code style with black
Yucheng-Jiang5fd4bc9bde2e
diff --git a/examples/costorm_examples/run_costorm_gpt.py b/examples/costorm_examples/run_costorm_gpt.pyindex 9046bd5..4013891 100644--- a/examples/costorm_examples/run_costorm_gpt.py+++ b/examples/costorm_examples/run_costorm_gpt.py@@ -16,51 +16,83 @@ args.output_dir/import osimport jsonfrom argparse import ArgumentParser-from knowledge_storm.collaborative_storm.engine import CollaborativeStormLMConfigs, RunnerArgument, CoStormRunner-from knowledge_storm.collaborative_storm.modules.callback import LocalConsolePrintCallBackHandler+from knowledge_storm.collaborative_storm.engine import (+ CollaborativeStormLMConfigs,+ RunnerArgument,+ CoStormRunner,+)+from knowledge_storm.collaborative_storm.modules.callback import (+ LocalConsolePrintCallBackHandler,+)from knowledge_storm.lm import OpenAIModel, AzureOpenAIModelfrom knowledge_storm.logging_wrapper import LoggingWrapper-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_config: CollaborativeStormLMConfigs = CollaborativeStormLMConfigs()- openai_kwargs = {- "api_key": os.getenv("OPENAI_API_KEY"),- "api_provider": "openai",- "temperature": 1.0,- "top_p": 0.9,- "api_base": None,- } if os.getenv('OPENAI_API_TYPE') == 'openai' else {- "api_key": os.getenv("AZURE_API_KEY"),- "temperature": 1.0,- "top_p": 0.9,- "api_base": os.getenv("AZURE_API_BASE"),- "api_version": os.getenv("AZURE_API_VERSION"),- }-- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel+ openai_kwargs = (+ {+ "api_key": os.getenv("OPENAI_API_KEY"),+ "api_provider": "openai",+ "temperature": 1.0,+ "top_p": 0.9,+ "api_base": None,+ }+ if os.getenv("OPENAI_API_TYPE") == "openai"+ else {+ "api_key": os.getenv("AZURE_API_KEY"),+ "temperature": 1.0,+ "top_p": 0.9,+ "api_base": os.getenv("AZURE_API_BASE"),+ "api_version": os.getenv("AZURE_API_VERSION"),+ }+ )++ ModelClass = (+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel+ )# If you are using Azure service, make sure the model name matches your own deployed model name.# The default name here is only used for demonstration and may not match your case.- gpt_4o_mini_model_name = 'gpt-4o-mini'- gpt_4o_model_name = 'gpt-4o'- if os.getenv('OPENAI_API_TYPE') == 'azure':- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')+ gpt_4o_mini_model_name = "gpt-4o-mini"+ gpt_4o_model_name = "gpt-4o"+ if os.getenv("OPENAI_API_TYPE") == "azure":+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")# STORM is a LM system so different components can be powered by different models.# For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.- question_answering_lm = ModelClass(model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs)- discourse_manage_lm = ModelClass(model=gpt_4o_model_name, max_tokens=500, **openai_kwargs)- utterance_polishing_lm = ModelClass(model=gpt_4o_model_name, max_tokens=2000, **openai_kwargs)- warmstart_outline_gen_lm = ModelClass(model=gpt_4o_model_name, max_tokens=500, **openai_kwargs)- question_asking_lm = ModelClass(model=gpt_4o_model_name, max_tokens=300, **openai_kwargs)- knowledge_base_lm = ModelClass(model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs)+ question_answering_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs+ )+ discourse_manage_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=500, **openai_kwargs+ )+ utterance_polishing_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=2000, **openai_kwargs+ )+ warmstart_outline_gen_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=500, **openai_kwargs+ )+ question_asking_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=300, **openai_kwargs+ )+ knowledge_base_lm = ModelClass(+ model=gpt_4o_model_name, max_tokens=1000, **openai_kwargs+ )lm_config.set_question_answering_lm(question_answering_lm)lm_config.set_discourse_manage_lm(discourse_manage_lm)@@ -69,7 +101,7 @@ def main(args):lm_config.set_question_asking_lm(question_asking_lm)lm_config.set_knowledge_base_lm(knowledge_base_lm)- topic = input('Topic: ')+ topic = input("Topic: ")runner_argument = RunnerArgument(topic=topic,retrieve_top_k=args.retrieve_top_k,@@ -83,49 +115,76 @@ def main(args):max_thread_num=args.max_thread_num,max_num_round_table_experts=args.max_num_round_table_experts,moderator_override_N_consecutive_answering_turn=args.moderator_override_N_consecutive_answering_turn,- node_expansion_trigger_count=args.node_expansion_trigger_count)+ node_expansion_trigger_count=args.node_expansion_trigger_count,+ )logging_wrapper = LoggingWrapper(lm_config)- callback_handler = LocalConsolePrintCallBackHandler() if args.enable_log_print else None+ callback_handler = (+ LocalConsolePrintCallBackHandler() if args.enable_log_print else None+ )# Co-STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=runner_argument.retrieve_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=runner_argument.retrieve_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=runner_argument.retrieve_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=runner_argument.retrieve_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=runner_argument.retrieve_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=runner_argument.retrieve_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=runner_argument.retrieve_top_k,+ )+ case "you":+ rm = YouRM(+ ydc_api_key=os.getenv("YDC_API_KEY"), k=runner_argument.retrieve_top_k+ )+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=runner_argument.retrieve_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=runner_argument.retrieve_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=runner_argument.retrieve_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"),+ k=runner_argument.retrieve_top_k,+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )- costorm_runner = CoStormRunner(lm_config=lm_config,- runner_argument=runner_argument,- logging_wrapper=logging_wrapper,- rm=rm,- callback_handler=callback_handler)+ costorm_runner = CoStormRunner(+ lm_config=lm_config,+ runner_argument=runner_argument,+ logging_wrapper=logging_wrapper,+ rm=rm,+ callback_handler=callback_handler,+ )# warm start the systemcostorm_runner.warm_start()# Below is an example of how users may interact with Co-STORM to seek information together# In actual deployment, we suggest allowing the user to decide whether to observe the agent utterance or inject a turn-+# observing Co-STORM LLM agent utterance for 5 turnsfor _ in range(1):conv_turn = costorm_runner.step()print(f"**{conv_turn.role}**: {conv_turn.utterance}\n")-+# active engaging by injecting your utterance- your_utterance = input('Your utterance: ')+ your_utterance = input("Your utterance: ")costorm_runner.step(user_utterance=your_utterance)# continue observing@@ -154,93 +213,105 @@ def main(args):json.dump(log_dump, f, indent=2)-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/co-storm',- help='Directory to store the outputs.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/co-storm",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )# hyperparameters for co-stormparser.add_argument(- '--retrieve_top_k',+ "--retrieve_top_k",type=int,default=10,- help='Retrieve top k results for each query in retriever.'+ help="Retrieve top k results for each query in retriever.",)parser.add_argument(- '--max_search_queries',+ "--max_search_queries",type=int,default=2,- help='Maximum number of search queries to consider for each question.'+ help="Maximum number of search queries to consider for each question.",)parser.add_argument(- '--total_conv_turn',+ "--total_conv_turn",type=int,default=20,- help='Maximum number of turns in conversation.'+ help="Maximum number of turns in conversation.",)parser.add_argument(- '--max_search_thread',+ "--max_search_thread",type=int,default=5,- help='Maximum number of parallel threads for retriever.'+ help="Maximum number of parallel threads for retriever.",)parser.add_argument(- '--max_search_queries_per_turn',+ "--max_search_queries_per_turn",type=int,default=3,- help='Maximum number of search queries to consider in each turn.'+ help="Maximum number of search queries to consider in each turn.",)parser.add_argument(- '--warmstart_max_num_experts',+ "--warmstart_max_num_experts",type=int,default=3,- help='Max number of experts in perspective-guided QA during warm start.'+ help="Max number of experts in perspective-guided QA during warm start.",)parser.add_argument(- '--warmstart_max_turn_per_experts',+ "--warmstart_max_turn_per_experts",type=int,default=2,- help='Max number of turns per perspective during warm start.'+ help="Max number of turns per perspective during warm start.",)parser.add_argument(- '--warmstart_max_thread',+ "--warmstart_max_thread",type=int,default=3,- help='Max number of threads for parallel perspective-guided QA during warm start.'+ help="Max number of threads for parallel perspective-guided QA during warm start.",)parser.add_argument(- '--max_thread_num',+ "--max_thread_num",type=int,default=10,- help=("Maximum number of threads to use. "- "Consider reducing it if you keep getting 'Exceed rate limit' errors when calling the LM API.")+ help=(+ "Maximum number of threads to use. "+ "Consider reducing it if you keep getting 'Exceed rate limit' errors when calling the LM API."+ ),)parser.add_argument(- '--max_num_round_table_experts',+ "--max_num_round_table_experts",type=int,default=2,- help='Max number of active experts in round table discussion.'+ help="Max number of active experts in round table discussion.",)parser.add_argument(- '--moderator_override_N_consecutive_answering_turn',+ "--moderator_override_N_consecutive_answering_turn",type=int,default=3,- help=('Number of consecutive expert answering turns before the moderator overrides the conversation.')+ help=(+ "Number of consecutive expert answering turns before the moderator overrides the conversation."+ ),)parser.add_argument(- '--node_expansion_trigger_count',+ "--node_expansion_trigger_count",type=int,default=10,- help='Trigger node expansion for nodes that contain more than N snippets.'+ help="Trigger node expansion for nodes that contain more than N snippets.",)# Boolean flagsparser.add_argument(- '--enable_log_print',- action='store_true',- help='If set, enable console log print.'+ "--enable_log_print",+ action="store_true",+ help="If set, enable console log print.",)main(parser.parse_args())diff --git a/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py b/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.pyindex 926192b..f3dfe51 100644--- a/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py+++ b/examples/storm_examples/helper/process_kaggle_arxiv_abstract_dataset.py@@ -9,21 +9,28 @@ import pandas as pdif __name__ == "__main__":parser = ArgumentParser()- parser.add_argument("--input-path", type=str, help="Path to arxiv_data_210930-054931.csv.")- parser.add_argument("--output-path", type=str,- help="Path to store the csv file that is compatible with VectorRM.")+ parser.add_argument(+ "--input-path", type=str, help="Path to arxiv_data_210930-054931.csv."+ )+ parser.add_argument(+ "--output-path",+ type=str,+ help="Path to store the csv file that is compatible with VectorRM.",+ )args = parser.parse_args()df = pd.read_csv(args.input_path)- print(f'The original dataset has {len(df)} samples.')+ print(f"The original dataset has {len(df)} samples.")# Downsample the dataset.- df = df[df['terms'] == "['cs.CV']"]+ df = df[df["terms"] == "['cs.CV']"]# Reformat the dataset to match the VectorRM input format.df.rename(columns={"abstracts": "content", "titles": "title"}, inplace=True)- df['url'] = ['uid_' + str(idx) for idx in range(len(df))] # Ensure the url is unique.- df['description'] = ''+ df["url"] = [+ "uid_" + str(idx) for idx in range(len(df))+ ] # Ensure the url is unique.+ df["description"] = ""- print(f'The downsampled dataset has {len(df)} samples.')- df.to_csv(args.output_path, index=False)\ No newline at end of file+ print(f"The downsampled dataset has {len(df)} samples.")+ df.to_csv(args.output_path, index=False)diff --git a/examples/storm_examples/run_storm_wiki_claude.py b/examples/storm_examples/run_storm_wiki_claude.pyindex 1da12c0..10f6e36 100644--- a/examples/storm_examples/run_storm_wiki_claude.py+++ b/examples/storm_examples/run_storm_wiki_claude.py@@ -19,19 +19,31 @@ args.output_dir/import osfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import ClaudeModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()claude_kwargs = {- 'api_key': os.getenv("ANTHROPIC_API_KEY"),- 'temperature': 1.0,- 'top_p': 0.9+ "api_key": os.getenv("ANTHROPIC_API_KEY"),+ "temperature": 1.0,+ "top_p": 0.9,}# STORM is a LM system so different components can be powered by different models.@@ -39,11 +51,21 @@ def main(args):# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.- conv_simulator_lm = ClaudeModel(model='claude-3-haiku-20240307', max_tokens=500, **claude_kwargs)- question_asker_lm = ClaudeModel(model='claude-3-sonnet-20240229', max_tokens=500, **claude_kwargs)- outline_gen_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=400, **claude_kwargs)- article_gen_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=700, **claude_kwargs)- article_polish_lm = ClaudeModel(model='claude-3-opus-20240229', max_tokens=4000, **claude_kwargs)+ conv_simulator_lm = ClaudeModel(+ model="claude-3-haiku-20240307", max_tokens=500, **claude_kwargs+ )+ question_asker_lm = ClaudeModel(+ model="claude-3-sonnet-20240229", max_tokens=500, **claude_kwargs+ )+ outline_gen_lm = ClaudeModel(+ model="claude-3-opus-20240229", max_tokens=400, **claude_kwargs+ )+ article_gen_lm = ClaudeModel(+ model="claude-3-opus-20240229", max_tokens=700, **claude_kwargs+ )+ article_polish_lm = ClaudeModel(+ model="claude-3-opus-20240229", max_tokens=4000, **claude_kwargs+ )lm_configs.set_conv_simulator_lm(conv_simulator_lm)lm_configs.set_question_asker_lm(question_asker_lm)@@ -62,26 +84,45 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')-+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )+runner = STORMWikiRunner(engine_args, lm_configs, rm)- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -93,38 +134,81 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/claude',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/claude",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_deepseek.py b/examples/storm_examples/run_storm_wiki_deepseek.pyindex e4cf5fc..e831797 100644--- a/examples/storm_examples/run_storm_wiki_deepseek.py+++ b/examples/storm_examples/run_storm_wiki_deepseek.py@@ -22,9 +22,21 @@ import reimport loggingfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import DeepSeekModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_key@@ -34,10 +46,10 @@ def sanitize_topic(topic):Remove or replace characters that are not allowed in file names."""# Replace spaces with underscores- topic = topic.replace(' ', '_')+ topic = topic.replace(" ", "_")# Remove any character that isn't alphanumeric, underscore, or hyphen- topic = re.sub(r'[^a-zA-Z0-9_-]', '', topic)+ topic = re.sub(r"[^a-zA-Z0-9_-]", "", topic)# Ensure the topic isn't empty after sanitizationif not topic:@@ -47,29 +59,37 @@ def sanitize_topic(topic):def main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()logger = logging.getLogger(__name__)# Ensure DEEPSEEK_API_KEY is setif not os.getenv("DEEPSEEK_API_KEY"):- raise ValueError("DEEPSEEK_API_KEY environment variable is not set. Please set it in your secrets.toml file.")+ raise ValueError(+ "DEEPSEEK_API_KEY environment variable is not set. Please set it in your secrets.toml file."+ )deepseek_kwargs = {- 'api_key': os.getenv("DEEPSEEK_API_KEY"),- 'api_base': os.getenv("DEEPSEEK_API_BASE", "https://api.deepseek.com"),- 'temperature': args.temperature,- 'top_p': args.top_p,+ "api_key": os.getenv("DEEPSEEK_API_KEY"),+ "api_base": os.getenv("DEEPSEEK_API_BASE", "https://api.deepseek.com"),+ "temperature": args.temperature,+ "top_p": args.top_p,}# DeepSeek offers two main models: 'deepseek-chat' for general tasks and 'deepseek-coder' for coding tasks# Users can choose the appropriate model based on their needs- conv_simulator_lm = DeepSeekModel(model=args.model, max_tokens=500, **deepseek_kwargs)- question_asker_lm = DeepSeekModel(model=args.model, max_tokens=500, **deepseek_kwargs)+ conv_simulator_lm = DeepSeekModel(+ model=args.model, max_tokens=500, **deepseek_kwargs+ )+ question_asker_lm = DeepSeekModel(+ model=args.model, max_tokens=500, **deepseek_kwargs+ )outline_gen_lm = DeepSeekModel(model=args.model, max_tokens=400, **deepseek_kwargs)article_gen_lm = DeepSeekModel(model=args.model, max_tokens=700, **deepseek_kwargs)- article_polish_lm = DeepSeekModel(model=args.model, max_tokens=4000, **deepseek_kwargs)+ article_polish_lm = DeepSeekModel(+ model=args.model, max_tokens=4000, **deepseek_kwargs+ )lm_configs.set_conv_simulator_lm(conv_simulator_lm)lm_configs.set_question_asker_lm(question_asker_lm)@@ -88,26 +108,45 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)- topic = input('Topic: ')+ topic = input("Topic: ")sanitized_topic = sanitize_topic(topic)try:@@ -126,44 +165,94 @@ def main(args):raise-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/deepseek',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')- parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',- help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')- parser.add_argument('--temperature', type=float, default=1.0,- help='Sampling temperature to use.')- parser.add_argument('--top_p', type=float, default=0.9,- help='Top-p sampling parameter.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/deepseek",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )+ parser.add_argument(+ "--model",+ type=str,+ choices=["deepseek-chat", "deepseek-coder"],+ default="deepseek-chat",+ help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.',+ )+ parser.add_argument(+ "--temperature", type=float, default=1.0, help="Sampling temperature to use."+ )+ parser.add_argument(+ "--top_p", type=float, default=0.9, help="Top-p sampling parameter."+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_gemini.py b/examples/storm_examples/run_storm_wiki_gemini.pyindex d574062..947832f 100644--- a/examples/storm_examples/run_storm_wiki_gemini.py+++ b/examples/storm_examples/run_storm_wiki_gemini.py@@ -19,18 +19,31 @@ args.output_dir/import osfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import GoogleModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_key+def main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()gemini_kwargs = {- 'api_key': os.getenv("GOOGLE_API_KEY"),- 'temperature': 1.0,- 'top_p': 0.9+ "api_key": os.getenv("GOOGLE_API_KEY"),+ "temperature": 1.0,+ "top_p": 0.9,}# STORM is a LM system so different components can be powered by different models.@@ -40,11 +53,21 @@ def main(args):# which is responsible for generating sections with citations.# To check out available Google models, see:# https://ai.google.dev/gemini-api/docs/get-started/tutorial?lang=python#list_models- conv_simulator_lm = GoogleModel(model='models/gemini-1.5-flash', max_tokens=500, **gemini_kwargs)- question_asker_lm = GoogleModel(model='models/gemini-1.5-flash', max_tokens=500, **gemini_kwargs)- outline_gen_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=400, **gemini_kwargs)- article_gen_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=700, **gemini_kwargs)- article_polish_lm = GoogleModel(model='models/gemini-1.5-pro-exp-0801', max_tokens=4000, **gemini_kwargs)+ conv_simulator_lm = GoogleModel(+ model="models/gemini-1.5-flash", max_tokens=500, **gemini_kwargs+ )+ question_asker_lm = GoogleModel(+ model="models/gemini-1.5-flash", max_tokens=500, **gemini_kwargs+ )+ outline_gen_lm = GoogleModel(+ model="models/gemini-1.5-pro-exp-0801", max_tokens=400, **gemini_kwargs+ )+ article_gen_lm = GoogleModel(+ model="models/gemini-1.5-pro-exp-0801", max_tokens=700, **gemini_kwargs+ )+ article_polish_lm = GoogleModel(+ model="models/gemini-1.5-pro-exp-0801", max_tokens=4000, **gemini_kwargs+ )lm_configs.set_conv_simulator_lm(conv_simulator_lm)lm_configs.set_question_asker_lm(question_asker_lm)@@ -63,26 +86,45 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -94,38 +136,81 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/gemini',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/gemini",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_gpt.py b/examples/storm_examples/run_storm_wiki_gpt.pyindex b1740a1..e96c680 100644--- a/examples/storm_examples/run_storm_wiki_gpt.py+++ b/examples/storm_examples/run_storm_wiki_gpt.py@@ -22,40 +22,63 @@ args.output_dir/import osfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG, AzureAISearch+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+ AzureAISearch,+)from knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()openai_kwargs = {- 'api_key': os.getenv("OPENAI_API_KEY"),- 'temperature': 1.0,- 'top_p': 0.9,+ "api_key": os.getenv("OPENAI_API_KEY"),+ "temperature": 1.0,+ "top_p": 0.9,}- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel+ ModelClass = (+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel+ )# If you are using Azure service, make sure the model name matches your own deployed model name.# The default name here is only used for demonstration and may not match your case.- gpt_35_model_name = 'gpt-3.5-turbo' if os.getenv('OPENAI_API_TYPE') == 'openai' else 'gpt-35-turbo'- gpt_4_model_name = 'gpt-4o'- if os.getenv('OPENAI_API_TYPE') == 'azure':- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')+ gpt_35_model_name = (+ "gpt-3.5-turbo" if os.getenv("OPENAI_API_TYPE") == "openai" else "gpt-35-turbo"+ )+ gpt_4_model_name = "gpt-4o"+ if os.getenv("OPENAI_API_TYPE") == "azure":+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")# STORM is a LM system so different components can be powered by different models.# For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.- conv_simulator_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)- question_asker_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)+ conv_simulator_lm = ModelClass(+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs+ )+ question_asker_lm = ModelClass(+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs+ )outline_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=400, **openai_kwargs)article_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=700, **openai_kwargs)- article_polish_lm = ModelClass(model=gpt_4_model_name, max_tokens=4000, **openai_kwargs)+ article_polish_lm = ModelClass(+ model=gpt_4_model_name, max_tokens=4000, **openai_kwargs+ )lm_configs.set_conv_simulator_lm(conv_simulator_lm)lm_configs.set_question_asker_lm(question_asker_lm)@@ -75,28 +98,50 @@ def main(args):# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)- case 'azure_ai_search':- rm = AzureAISearch(azure_ai_search_api_key=os.getenv('AZURE_AI_SEARCH_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )+ case "azure_ai_search":+ rm = AzureAISearch(+ azure_ai_search_api_key=os.getenv("AZURE_AI_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", "searxng", or "azure_ai_search"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", "searxng", or "azure_ai_search"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -108,38 +153,90 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/gpt',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng', 'azure_ai_search'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/gpt",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=[+ "bing",+ "you",+ "brave",+ "serper",+ "duckduckgo",+ "tavily",+ "searxng",+ "azure_ai_search",+ ],+ help="The search engine API to use for retrieving information.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py b/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.pyindex 8fa4d0d..77c9dbf 100644--- a/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py+++ b/examples/storm_examples/run_storm_wiki_gpt_with_VectorRM.py@@ -29,7 +29,11 @@ args.output_dir/import osfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.rm import VectorRMfrom knowledge_storm.lm import OpenAIModel, AzureOpenAIModelfrom knowledge_storm.utils import load_api_key, QdrantVectorStoreManager@@ -37,35 +41,45 @@ from knowledge_storm.utils import load_api_key, QdrantVectorStoreManagerdef main(args):# Load API key from the specified toml file path- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")# Initialize the language model configurationsengine_lm_configs = STORMWikiLMConfigs()openai_kwargs = {- 'api_key': os.getenv("OPENAI_API_KEY"),- 'temperature': 1.0,- 'top_p': 0.9,+ "api_key": os.getenv("OPENAI_API_KEY"),+ "temperature": 1.0,+ "top_p": 0.9,}- ModelClass = OpenAIModel if os.getenv('OPENAI_API_TYPE') == 'openai' else AzureOpenAIModel+ ModelClass = (+ OpenAIModel if os.getenv("OPENAI_API_TYPE") == "openai" else AzureOpenAIModel+ )# If you are using Azure service, make sure the model name matches your own deployed model name.# The default name here is only used for demonstration and may not match your case.- gpt_35_model_name = 'gpt-3.5-turbo' if os.getenv('OPENAI_API_TYPE') == 'openai' else 'gpt-35-turbo'- gpt_4_model_name = 'gpt-4o'- if os.getenv('OPENAI_API_TYPE') == 'azure':- openai_kwargs['api_base'] = os.getenv('AZURE_API_BASE')- openai_kwargs['api_version'] = os.getenv('AZURE_API_VERSION')+ gpt_35_model_name = (+ "gpt-3.5-turbo" if os.getenv("OPENAI_API_TYPE") == "openai" else "gpt-35-turbo"+ )+ gpt_4_model_name = "gpt-4o"+ if os.getenv("OPENAI_API_TYPE") == "azure":+ openai_kwargs["api_base"] = os.getenv("AZURE_API_BASE")+ openai_kwargs["api_version"] = os.getenv("AZURE_API_VERSION")# STORM is a LM system so different components can be powered by different models.- # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm+ # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.- conv_simulator_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)- question_asker_lm = ModelClass(model=gpt_35_model_name, max_tokens=500, **openai_kwargs)+ conv_simulator_lm = ModelClass(+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs+ )+ question_asker_lm = ModelClass(+ model=gpt_35_model_name, max_tokens=500, **openai_kwargs+ )outline_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=400, **openai_kwargs)article_gen_lm = ModelClass(model=gpt_4_model_name, max_tokens=700, **openai_kwargs)- article_polish_lm = ModelClass(model=gpt_4_model_name, max_tokens=4000, **openai_kwargs)+ article_polish_lm = ModelClass(+ model=gpt_4_model_name, max_tokens=4000, **openai_kwargs+ )engine_lm_configs.set_conv_simulator_lm(conv_simulator_lm)engine_lm_configs.set_question_asker_lm(question_asker_lm)@@ -85,43 +99,49 @@ def main(args):# Create / update the vector store with the documents in the csv fileif args.csv_file_path:kwargs = {- 'file_path': args.csv_file_path,- 'content_column': 'content',- 'title_column': 'title',- 'url_column': 'url',- 'desc_column': 'description',- 'batch_size': args.embed_batch_size,- 'vector_db_mode': args.vector_db_mode,- 'collection_name': args.collection_name,- 'embedding_model': args.embedding_model,- 'device': args.device,+ "file_path": args.csv_file_path,+ "content_column": "content",+ "title_column": "title",+ "url_column": "url",+ "desc_column": "description",+ "batch_size": args.embed_batch_size,+ "vector_db_mode": args.vector_db_mode,+ "collection_name": args.collection_name,+ "embedding_model": args.embedding_model,+ "device": args.device,}- if args.vector_db_mode == 'offline':+ if args.vector_db_mode == "offline":QdrantVectorStoreManager.create_or_update_vector_store(- vector_store_path=args.offline_vector_db_dir,- **kwargs+ vector_store_path=args.offline_vector_db_dir, **kwargs)- elif args.vector_db_mode == 'online':+ elif args.vector_db_mode == "online":QdrantVectorStoreManager.create_or_update_vector_store(url=args.online_vector_db_url,- api_key=os.getenv('QDRANT_API_KEY'),+ api_key=os.getenv("QDRANT_API_KEY"),**kwargs)# Setup VectorRM to retrieve information from your own data- rm = VectorRM(collection_name=args.collection_name, embedding_model=args.embedding_model, device=args.device, k=engine_args.search_top_k)+ rm = VectorRM(+ collection_name=args.collection_name,+ embedding_model=args.embedding_model,+ device=args.device,+ k=engine_args.search_top_k,+ )# initialize the vector store, either online (store the db on Qdrant server) or offline (store the db locally):- if args.vector_db_mode == 'offline':+ if args.vector_db_mode == "offline":rm.init_offline_vector_db(vector_store_path=args.offline_vector_db_dir)- elif args.vector_db_mode == 'online':- rm.init_online_vector_db(url=args.online_vector_db_url, api_key=os.getenv('QDRANT_API_KEY'))+ elif args.vector_db_mode == "online":+ rm.init_online_vector_db(+ url=args.online_vector_db_url, api_key=os.getenv("QDRANT_API_KEY")+ )# Initialize the STORM Wiki Runnerrunner = STORMWikiRunner(engine_args, engine_lm_configs, rm)# run the pipeline- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -136,50 +156,120 @@ def main(args):if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/gpt_retrieval',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/gpt_retrieval",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )# provide local corpus and set up vector db- parser.add_argument('--collection-name', type=str, default="my_documents",- help='The collection name for vector store.')- parser.add_argument('--embedding_model', type=str, default="BAAI/bge-m3",- help='The collection name for vector store.')- parser.add_argument('--device', type=str, default="mps",- help='The device used to run the retrieval model (mps, cuda, cpu, etc).')- parser.add_argument('--vector-db-mode', type=str, choices=['offline', 'online'],- help='The mode of the Qdrant vector store (offline or online).')- parser.add_argument('--offline-vector-db-dir', type=str, default='./vector_store',- help='If use offline mode, please provide the directory to store the vector store.')- parser.add_argument('--online-vector-db-url', type=str,- help='If use online mode, please provide the url of the Qdrant server.')- parser.add_argument('--csv-file-path', type=str, default=None,- help='The path of the custom document corpus in CSV format. The CSV file should include '- 'content, title, url, and description columns.')- parser.add_argument('--embed-batch-size', type=int, default=64,- help='Batch size for embedding the documents in the csv file.')+ parser.add_argument(+ "--collection-name",+ type=str,+ default="my_documents",+ help="The collection name for vector store.",+ )+ parser.add_argument(+ "--embedding_model",+ type=str,+ default="BAAI/bge-m3",+ help="The collection name for vector store.",+ )+ parser.add_argument(+ "--device",+ type=str,+ default="mps",+ help="The device used to run the retrieval model (mps, cuda, cpu, etc).",+ )+ parser.add_argument(+ "--vector-db-mode",+ type=str,+ choices=["offline", "online"],+ help="The mode of the Qdrant vector store (offline or online).",+ )+ parser.add_argument(+ "--offline-vector-db-dir",+ type=str,+ default="./vector_store",+ help="If use offline mode, please provide the directory to store the vector store.",+ )+ parser.add_argument(+ "--online-vector-db-url",+ type=str,+ help="If use online mode, please provide the url of the Qdrant server.",+ )+ parser.add_argument(+ "--csv-file-path",+ type=str,+ default=None,+ help="The path of the custom document corpus in CSV format. The CSV file should include "+ "content, title, url, and description columns.",+ )+ parser.add_argument(+ "--embed-batch-size",+ type=int,+ default=64,+ help="Batch size for embedding the documents in the csv file.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')- main(parser.parse_args())\ No newline at end of file+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_groq.py b/examples/storm_examples/run_storm_wiki_groq.pyindex 0dcaadb..de5f6ae 100644--- a/examples/storm_examples/run_storm_wiki_groq.py+++ b/examples/storm_examples/run_storm_wiki_groq.py@@ -21,12 +21,24 @@ import osimport refrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)# Now import lm directlyimport lmfrom lm import GroqModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_key@@ -36,10 +48,10 @@ def sanitize_topic(topic):Remove or replace characters that are not allowed in file names."""# Replace spaces with underscores- topic = topic.replace(' ', '_')+ topic = topic.replace(" ", "_")# Remove any character that isn't alphanumeric, underscore, or hyphen- topic = re.sub(r'[^a-zA-Z0-9_-]', '', topic)+ topic = re.sub(r"[^a-zA-Z0-9_-]", "", topic)# Ensure the topic isn't empty after sanitizationif not topic:@@ -49,26 +61,34 @@ def sanitize_topic(topic):def main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()# Ensure GROQ_API_KEY is setif not os.getenv("GROQ_API_KEY"):- raise ValueError("GROQ_API_KEY environment variable is not set. Please set it in your secrets.toml file.")+ raise ValueError(+ "GROQ_API_KEY environment variable is not set. Please set it in your secrets.toml file."+ )groq_kwargs = {- 'api_key': os.getenv("GROQ_API_KEY"),- 'api_base': "https://api.groq.com/openai/v1",- 'temperature': args.temperature,- 'top_p': args.top_p,+ "api_key": os.getenv("GROQ_API_KEY"),+ "api_base": "https://api.groq.com/openai/v1",+ "temperature": args.temperature,+ "top_p": args.top_p,}# Groq currently offers the "llama3-70b-8192" model with generous free API credits and the llama3.1 family of models as a preview for paying customers- conv_simulator_lm = GroqModel(model="llama3-70b-8192", max_tokens=500, **groq_kwargs)- question_asker_lm = GroqModel(model="llama3-70b-8192", max_tokens=500, **groq_kwargs)+ conv_simulator_lm = GroqModel(+ model="llama3-70b-8192", max_tokens=500, **groq_kwargs+ )+ question_asker_lm = GroqModel(+ model="llama3-70b-8192", max_tokens=500, **groq_kwargs+ )outline_gen_lm = GroqModel(model="llama3-70b-8192", max_tokens=400, **groq_kwargs)article_gen_lm = GroqModel(model="llama3-70b-8192", max_tokens=700, **groq_kwargs)- article_polish_lm = GroqModel(model="llama3-70b-8192", max_tokens=4000, **groq_kwargs)+ article_polish_lm = GroqModel(+ model="llama3-70b-8192", max_tokens=4000, **groq_kwargs+ )lm_configs.set_conv_simulator_lm(conv_simulator_lm)lm_configs.set_question_asker_lm(question_asker_lm)@@ -87,26 +107,45 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)- topic = input('Topic: ')+ topic = input("Topic: ")sanitized_topic = sanitize_topic(topic)try:@@ -125,42 +164,87 @@ def main(args):raise-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--output-dir', type=str, default='./results/groq',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')- parser.add_argument('--temperature', type=float, default=1.0,- help='Sampling temperature to use.')- parser.add_argument('--top_p', type=float, default=0.9,- help='Top-p sampling parameter.')+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/groq",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )+ parser.add_argument(+ "--temperature", type=float, default=1.0, help="Sampling temperature to use."+ )+ parser.add_argument(+ "--top_p", type=float, default=0.9, help="Top-p sampling parameter."+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_mistral.py b/examples/storm_examples/run_storm_wiki_mistral.pyindex ceabd47..1b52d7f 100644--- a/examples/storm_examples/run_storm_wiki_mistral.py+++ b/examples/storm_examples/run_storm_wiki_mistral.py@@ -20,21 +20,35 @@ from argparse import ArgumentParserfrom dspy import Example-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import VLLMClient-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)from knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()mistral_kwargs = {"model": "mistralai/Mistral-7B-Instruct-v0.2","port": args.port,"url": args.url,- "stop": ('\n\n---',) # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.+ "stop": (+ "\n\n---",+ ), # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.}conv_simulator_lm = VLLMClient(max_tokens=500, **mistral_kwargs)@@ -60,22 +74,41 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -87,26 +120,28 @@ def main(args):find_related_topic_example = Example(topic="Knowledge Curation",related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"- "https://en.wikipedia.org/wiki/Information_science\n"- "https://en.wikipedia.org/wiki/Library_science\n"+ "https://en.wikipedia.org/wiki/Information_science\n"+ "https://en.wikipedia.org/wiki/Library_science\n",)gen_persona_example = Example(topic="Knowledge Curation",examples="Title: Knowledge management\n"- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"- " Knowledge protection methods\n Formal methods\n Informal methods\n"- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"+ " Knowledge protection methods\n Formal methods\n Informal methods\n"+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",personas="1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge curation. They will provide context on how knowledge curation has changed over time and its impact on modern practices.\n"- "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"- "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"- "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"- "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm."+ "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"+ "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"+ "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"+ "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm.",)runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [- find_related_topic_example]+ find_related_topic_example+ ]runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [- gen_persona_example]+ gen_persona_example+ ]# A trade-off of adding one-shot example is that it will increase the input length of the prompt. Also, some# examples may be very long (e.g., an example for writing a section based on the given information), which may@@ -118,24 +153,28 @@ def main(args):topic="Example Topic",conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"- "# Section 2\n## Subsection 1\n## Subsection 2\n"- "# Section 3",+ "# Section 2\n## Subsection 1\n## Subsection 2\n"+ "# Section 3",outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"- "# New Section 2\n"- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"+ "# New Section 2\n"+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",)- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [+ write_page_outline_example+ ]write_section_example = Example(info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",topic="Example Topic",section="Example Section",output="# Example Topic\n## Subsection 1\n"- "This is an example sentence [1]. This is another example sentence [2][3].\n"- "## Subsection 2\nThis is one more example sentence [1]."+ "This is an example sentence [1]. This is another example sentence [2][3].\n"+ "## Subsection 2\nThis is one more example sentence [1].",)- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]+ runner.storm_article_generation.section_gen.write_section.demos = [+ write_section_example+ ]- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -147,42 +186,87 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--url', type=str, default='http://localhost',- help='URL of the VLLM server.')- parser.add_argument('--port', type=int, default=8000,- help='Port of the VLLM server.')- parser.add_argument('--output-dir', type=str, default='./results/mistral_7b',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--url", type=str, default="http://localhost", help="URL of the VLLM server."+ )+ parser.add_argument(+ "--port", type=int, default=8000, help="Port of the VLLM server."+ )+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/mistral_7b",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_ollama.py b/examples/storm_examples/run_storm_wiki_ollama.pyindex 465e065..11d6d93 100644--- a/examples/storm_examples/run_storm_wiki_ollama.py+++ b/examples/storm_examples/run_storm_wiki_ollama.py@@ -22,20 +22,34 @@ from argparse import ArgumentParserfrom dspy import Examplefrom knowledge_storm.lm import OllamaClient-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm.rm import (+ YouRM,+ BingSearch,+ BraveRM,+ SerperRM,+ DuckDuckGoSearchRM,+ TavilySearchRM,+ SearXNG,+)+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()ollama_kwargs = {"model": args.model,"port": args.port,"url": args.url,- "stop": ('\n\n---',) # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.+ "stop": (+ "\n\n---",+ ), # dspy uses "\n\n---" to separate examples. Open models sometimes generate this.}conv_simulator_lm = OllamaClient(max_tokens=500, **ollama_kwargs)@@ -61,22 +75,41 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.match args.retriever:- case 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- case 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- case 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- case 'duckduckgo':- rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')- case 'serper':- rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})- case 'tavily':- rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)- case 'searxng':- rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case "bing":+ rm = BingSearch(+ bing_search_api=os.getenv("BING_SEARCH_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "you":+ rm = YouRM(ydc_api_key=os.getenv("YDC_API_KEY"), k=engine_args.search_top_k)+ case "brave":+ rm = BraveRM(+ brave_search_api_key=os.getenv("BRAVE_API_KEY"),+ k=engine_args.search_top_k,+ )+ case "duckduckgo":+ rm = DuckDuckGoSearchRM(+ k=engine_args.search_top_k, safe_search="On", region="us-en"+ )+ case "serper":+ rm = SerperRM(+ serper_search_api_key=os.getenv("SERPER_API_KEY"),+ query_params={"autocorrect": True, "num": 10, "page": 1},+ )+ case "tavily":+ rm = TavilySearchRM(+ tavily_search_api_key=os.getenv("TAVILY_API_KEY"),+ k=engine_args.search_top_k,+ include_raw_content=True,+ )+ case "searxng":+ rm = SearXNG(+ searxng_api_key=os.getenv("SEARXNG_API_KEY"), k=engine_args.search_top_k+ )case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')+ raise ValueError(+ f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"'+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -88,26 +121,28 @@ def main(args):find_related_topic_example = Example(topic="Knowledge Curation",related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"- "https://en.wikipedia.org/wiki/Information_science\n"- "https://en.wikipedia.org/wiki/Library_science\n"+ "https://en.wikipedia.org/wiki/Information_science\n"+ "https://en.wikipedia.org/wiki/Library_science\n",)gen_persona_example = Example(topic="Knowledge Curation",examples="Title: Knowledge management\n"- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"- " Knowledge protection methods\n Formal methods\n Informal methods\n"- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"+ " Knowledge protection methods\n Formal methods\n Informal methods\n"+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",personas="1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge curation. They will provide context on how knowledge curation has changed over time and its impact on modern practices.\n"- "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"- "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"- "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"- "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm."+ "2. Information Science Professional: With insights from 'Information science', this editor will explore the foundational theories, definitions, and philosophy that underpin knowledge curation\n"+ "3. Digital Librarian: This editor will delve into the specifics of how digital libraries operate, including software, metadata, digital preservation.\n"+ "4. Technical expert: This editor will focus on the technical aspects of knowledge curation, such as common features of content management systems.\n"+ "5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and the transition of these practices into the digital realm.",)runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [- find_related_topic_example]+ find_related_topic_example+ ]runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [- gen_persona_example]+ gen_persona_example+ ]# A trade-off of adding one-shot example is that it will increase the input length of the prompt. Also, some# examples may be very long (e.g., an example for writing a section based on the given information), which may@@ -119,24 +154,28 @@ def main(args):topic="Example Topic",conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"- "# Section 2\n## Subsection 1\n## Subsection 2\n"- "# Section 3",+ "# Section 2\n## Subsection 1\n## Subsection 2\n"+ "# Section 3",outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"- "# New Section 2\n"- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"+ "# New Section 2\n"+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",)- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [+ write_page_outline_example+ ]write_section_example = Example(info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",topic="Example Topic",section="Example Section",output="# Example Topic\n## Subsection 1\n"- "This is an example sentence [1]. This is another example sentence [2][3].\n"- "## Subsection 2\nThis is one more example sentence [1]."+ "This is an example sentence [1]. This is another example sentence [2][3].\n"+ "## Subsection 2\nThis is one more example sentence [1].",)- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]+ runner.storm_article_generation.section_gen.write_section.demos = [+ write_section_example+ ]- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -148,44 +187,90 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--url', type=str, default='http://localhost',- help='URL of the Ollama server.')- parser.add_argument('--port', type=int, default=11434,- help='Port of the Ollama server.')- parser.add_argument('--model', type=str, default='llama3:latest',- help='Model of the Ollama server.')- parser.add_argument('--output-dir', type=str, default='./results/ollama',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],- help='The search engine API to use for retrieving information.')+ parser.add_argument(+ "--url", type=str, default="http://localhost", help="URL of the Ollama server."+ )+ parser.add_argument(+ "--port", type=int, default=11434, help="Port of the Ollama server."+ )+ parser.add_argument(+ "--model", type=str, default="llama3:latest", help="Model of the Ollama server."+ )+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/ollama",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["bing", "you", "brave", "serper", "duckduckgo", "tavily", "searxng"],+ help="The search engine API to use for retrieving information.",+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )- main(parser.parse_args())\ No newline at end of file+ main(parser.parse_args())diff --git a/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py b/examples/storm_examples/run_storm_wiki_ollama_with_searxng.pyindex a4be362..725a68b 100644--- a/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py+++ b/examples/storm_examples/run_storm_wiki_ollama_with_searxng.py@@ -3,21 +3,25 @@ from argparse import ArgumentParserfrom dspy import Example-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import OllamaClientfrom knowledge_storm.rm import SearXNGfrom knowledge_storm.utils import load_api_keydef main(args):- load_api_key(toml_file_path='secrets.toml')+ load_api_key(toml_file_path="secrets.toml")lm_configs = STORMWikiLMConfigs()ollama_kwargs = {"model": args.model,"port": args.port,"url": args.url,- "stop": ('\n\n---',)+ "stop": ("\n\n---",),}conv_simulator_lm = OllamaClient(max_tokens=500, **ollama_kwargs)@@ -40,23 +44,27 @@ def main(args):max_thread_num=args.max_thread_num,)- rm = SearXNG(searxng_api_url=args.searxng_api_url, searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ rm = SearXNG(+ searxng_api_url=args.searxng_api_url,+ searxng_api_key=os.getenv("SEARXNG_API_KEY"),+ k=engine_args.search_top_k,+ )runner = STORMWikiRunner(engine_args, lm_configs, rm)find_related_topic_example = Example(topic="Knowledge Curation",related_topics="https://en.wikipedia.org/wiki/Knowledge_management\n"- "https://en.wikipedia.org/wiki/Information_science\n"- "https://en.wikipedia.org/wiki/Library_science\n"+ "https://en.wikipedia.org/wiki/Information_science\n"+ "https://en.wikipedia.org/wiki/Library_science\n",)gen_persona_example = Example(topic="Knowledge Curation",examples="Title: Knowledge management\n"- "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"- "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"- " Knowledge protection methods\n Formal methods\n Informal methods\n"- " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",+ "Table of Contents: History\nResearch\n Dimensions\n Strategies\n Motivations\nKM technologies"+ "\nKnowledge barriers\nKnowledge retention\nKnowledge audit\nKnowledge protection\n"+ " Knowledge protection methods\n Formal methods\n Informal methods\n"+ " Balancing knowledge protection and knowledge sharing\n Knowledge protection risks",personas=("1. Historian of Knowledge Systems: This editor will focus on the history and evolution of knowledge ""curation. They will provide context on how knowledge curation has changed over time and its impact on "@@ -69,35 +77,41 @@ def main(args):"such as common features of content management systems.\n""5. Museum Curator: The museum curator will contribute expertise on the curation of physical items and ""the transition of these practices into the digital realm."- )+ ),)runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.find_related_topic.demos = [- find_related_topic_example]+ find_related_topic_example+ ]runner.storm_knowledge_curation_module.persona_generator.create_writer_with_persona.gen_persona.demos = [- gen_persona_example]+ gen_persona_example+ ]write_page_outline_example = Example(topic="Example Topic",conv="Wikipedia Writer: ...\nExpert: ...\nWikipedia Writer: ...\nExpert: ...",old_outline="# Section 1\n## Subsection 1\n## Subsection 2\n"- "# Section 2\n## Subsection 1\n## Subsection 2\n"- "# Section 3",+ "# Section 2\n## Subsection 1\n## Subsection 2\n"+ "# Section 3",outline="# New Section 1\n## New Subsection 1\n## New Subsection 2\n"- "# New Section 2\n"- "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3"+ "# New Section 2\n"+ "# New Section 3\n## New Subsection 1\n## New Subsection 2\n## New Subsection 3",)- runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [write_page_outline_example]+ runner.storm_outline_generation_module.write_outline.write_page_outline.demos = [+ write_page_outline_example+ ]write_section_example = Example(info="[1]\nInformation in document 1\n[2]\nInformation in document 2\n[3]\nInformation in document 3",topic="Example Topic",section="Example Section",output="# Example Topic\n## Subsection 1\n"- "This is an example sentence [1]. This is another example sentence [2][3].\n"- "## Subsection 2\nThis is one more example sentence [1]."+ "This is an example sentence [1]. This is another example sentence [2][3].\n"+ "## Subsection 2\nThis is one more example sentence [1].",)- runner.storm_article_generation.section_gen.write_section.demos = [write_section_example]+ runner.storm_article_generation.section_gen.write_section.demos = [+ write_section_example+ ]- topic = input('Topic: ')+ topic = input("Topic: ")runner.run(topic=topic,do_research=args.do_research,@@ -109,46 +123,93 @@ def main(args):runner.summary()-if __name__ == '__main__':+if __name__ == "__main__":parser = ArgumentParser()# global arguments- parser.add_argument('--url', type=str, default='http://localhost',- help='URL of the Ollama server.')- parser.add_argument('--port', type=int, default=11434,- help='Port of the Ollama server.')- parser.add_argument('--model', type=str, default='llama3:latest',- help='Model of the Ollama server.')- parser.add_argument('--output-dir', type=str, default='./results/ollama',- help='Directory to store the outputs.')- parser.add_argument('--max-thread-num', type=int, default=3,- help='Maximum number of threads to use. The information seeking part and the article generation'- 'part can speed up by using multiple threads. Consider reducing it if keep getting '- '"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['searxng'],- help='The search engine API to use for retrieving information.')- parser.add_argument('--searxng-api-url', type=str, required=True,- help='URL of the SearXNG API.')+ parser.add_argument(+ "--url", type=str, default="http://localhost", help="URL of the Ollama server."+ )+ parser.add_argument(+ "--port", type=int, default=11434, help="Port of the Ollama server."+ )+ parser.add_argument(+ "--model", type=str, default="llama3:latest", help="Model of the Ollama server."+ )+ parser.add_argument(+ "--output-dir",+ type=str,+ default="./results/ollama",+ help="Directory to store the outputs.",+ )+ parser.add_argument(+ "--max-thread-num",+ type=int,+ default=3,+ help="Maximum number of threads to use. The information seeking part and the article generation"+ "part can speed up by using multiple threads. Consider reducing it if keep getting "+ '"Exceed rate limit" error when calling LM API.',+ )+ parser.add_argument(+ "--retriever",+ type=str,+ choices=["searxng"],+ help="The search engine API to use for retrieving information.",+ )+ parser.add_argument(+ "--searxng-api-url", type=str, required=True, help="URL of the SearXNG API."+ )# stage of the pipeline- parser.add_argument('--do-research', action='store_true',- help='If True, simulate conversation to research the topic; otherwise, load the results.')- parser.add_argument('--do-generate-outline', action='store_true',- help='If True, generate an outline for the topic; otherwise, load the results.')- parser.add_argument('--do-generate-article', action='store_true',- help='If True, generate an article for the topic; otherwise, load the results.')- parser.add_argument('--do-polish-article', action='store_true',- help='If True, polish the article by adding a summarization section and (optionally) removing '- 'duplicate content.')+ parser.add_argument(+ "--do-research",+ action="store_true",+ help="If True, simulate conversation to research the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-outline",+ action="store_true",+ help="If True, generate an outline for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-generate-article",+ action="store_true",+ help="If True, generate an article for the topic; otherwise, load the results.",+ )+ parser.add_argument(+ "--do-polish-article",+ action="store_true",+ help="If True, polish the article by adding a summarization section and (optionally) removing "+ "duplicate content.",+ )# hyperparameters for the pre-writing stage- parser.add_argument('--max-conv-turn', type=int, default=3,- help='Maximum number of questions in conversational question asking.')- parser.add_argument('--max-perspective', type=int, default=3,- help='Maximum number of perspectives to consider in perspective-guided question asking.')- parser.add_argument('--search-top-k', type=int, default=3,- help='Top k search results to consider for each search query.')+ parser.add_argument(+ "--max-conv-turn",+ type=int,+ default=3,+ help="Maximum number of questions in conversational question asking.",+ )+ parser.add_argument(+ "--max-perspective",+ type=int,+ default=3,+ help="Maximum number of perspectives to consider in perspective-guided question asking.",+ )+ parser.add_argument(+ "--search-top-k",+ type=int,+ default=3,+ help="Top k search results to consider for each search query.",+ )# hyperparameters for the writing stage- parser.add_argument('--retrieve-top-k', type=int, default=3,- help='Top k collected references for each section title.')- parser.add_argument('--remove-duplicate', action='store_true',- help='If True, remove duplicate content from the article.')+ parser.add_argument(+ "--retrieve-top-k",+ type=int,+ default=3,+ help="Top k collected references for each section title.",+ )+ parser.add_argument(+ "--remove-duplicate",+ action="store_true",+ help="If True, remove duplicate content from the article.",+ )main(parser.parse_args())diff --git a/frontend/demo_light/demo_util.py b/frontend/demo_light/demo_util.pyindex 22258dd..cff1d9f 100644--- a/frontend/demo_light/demo_util.py+++ b/frontend/demo_light/demo_util.py@@ -13,7 +13,11 @@ import streamlit as st# Uncomment the following lines:# import sys# sys.path.append('../../')-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs+from knowledge_storm import (+ STORMWikiRunnerArguments,+ STORMWikiRunner,+ STORMWikiLMConfigs,+)from knowledge_storm.lm import OpenAIModelfrom knowledge_storm.rm import YouRMfrom knowledge_storm.storm_wiki.modules.callback import BaseCallbackHandler@@ -21,7 +25,7 @@ from knowledge_storm.utils import truncate_filenamefrom stoc import stoc-class DemoFileIOHelper():+class DemoFileIOHelper:@staticmethoddef read_structure_to_dict(articles_root_path):"""@@ -107,8 +111,10 @@ class DemoFileIOHelper():file_path (str): The path to the file.modification_time_string (str): The desired modification time in 'YYYY-MM-DD HH:MM:SS' format."""- california_tz = pytz.timezone('America/Los_Angeles')- modification_time = datetime.datetime.strptime(modification_time_string, '%Y-%m-%d %H:%M:%S')+ california_tz = pytz.timezone("America/Los_Angeles")+ modification_time = datetime.datetime.strptime(+ modification_time_string, "%Y-%m-%d %H:%M:%S"+ )modification_time = california_tz.localize(modification_time)modification_time_utc = modification_time.astimezone(datetime.timezone.utc)modification_timestamp = modification_time_utc.timestamp()@@ -125,7 +131,7 @@ class DemoFileIOHelper():Returns:str: The latest file's modification time in 'YYYY-MM-DD HH:MM:SS' format."""- california_tz = pytz.timezone('America/Los_Angeles')+ california_tz = pytz.timezone("America/Los_Angeles")latest_mod_time = Nonefile_paths = []@@ -138,17 +144,26 @@ class DemoFileIOHelper():for file_path in file_paths:modification_timestamp = os.path.getmtime(file_path)- modification_time_utc = datetime.datetime.utcfromtimestamp(modification_timestamp)- modification_time_utc = modification_time_utc.replace(tzinfo=datetime.timezone.utc)- modification_time_california = modification_time_utc.astimezone(california_tz)-- if latest_mod_time is None or modification_time_california > latest_mod_time:+ modification_time_utc = datetime.datetime.utcfromtimestamp(+ modification_timestamp+ )+ modification_time_utc = modification_time_utc.replace(+ tzinfo=datetime.timezone.utc+ )+ modification_time_california = modification_time_utc.astimezone(+ california_tz+ )++ if (+ latest_mod_time is None+ or modification_time_california > latest_mod_time+ ):latest_mod_time = modification_time_californiaif latest_mod_time is not None:- return latest_mod_time.strftime('%Y-%m-%d %H:%M:%S')+ return latest_mod_time.strftime("%Y-%m-%d %H:%M:%S")else:- return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')+ return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")@staticmethoddef assemble_article_data(article_file_path_dict):@@ -171,25 +186,44 @@ class DemoFileIOHelper():if neither the raw nor polished article text exists in theprovided file paths."""- if "storm_gen_article.txt" in article_file_path_dict or "storm_gen_article_polished.txt" in article_file_path_dict:- full_article_name = "storm_gen_article_polished.txt" if "storm_gen_article_polished.txt" in article_file_path_dict else "storm_gen_article.txt"- article_data = {"article": DemoTextProcessingHelper.parse(- DemoFileIOHelper.read_txt_file(article_file_path_dict[full_article_name]))}+ if (+ "storm_gen_article.txt" in article_file_path_dict+ or "storm_gen_article_polished.txt" in article_file_path_dict+ ):+ full_article_name = (+ "storm_gen_article_polished.txt"+ if "storm_gen_article_polished.txt" in article_file_path_dict+ else "storm_gen_article.txt"+ )+ article_data = {+ "article": DemoTextProcessingHelper.parse(+ DemoFileIOHelper.read_txt_file(+ article_file_path_dict[full_article_name]+ )+ )+ }if "url_to_info.json" in article_file_path_dict:article_data["citations"] = _construct_citation_dict_from_search_result(- DemoFileIOHelper.read_json_file(article_file_path_dict["url_to_info.json"]))+ DemoFileIOHelper.read_json_file(+ article_file_path_dict["url_to_info.json"]+ )+ )if "conversation_log.json" in article_file_path_dict:article_data["conversation_log"] = DemoFileIOHelper.read_json_file(- article_file_path_dict["conversation_log.json"])+ article_file_path_dict["conversation_log.json"]+ )return article_datareturn None-class DemoTextProcessingHelper():-+class DemoTextProcessingHelper:@staticmethoddef remove_citations(sent):- return re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent)).replace(" |", "").replace("]", "")+ return (+ re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent))+ .replace(" |", "")+ .replace("]", "")+ )@staticmethoddef parse_conversation_history(json_data):@@ -199,43 +233,54 @@ class DemoTextProcessingHelper():"""parsed_data = []for persona_conversation_data in json_data:- if ': ' in persona_conversation_data["perspective"]:- name, description = persona_conversation_data["perspective"].split(": ", 1)- elif '- ' in persona_conversation_data["perspective"]:- name, description = persona_conversation_data["perspective"].split("- ", 1)+ if ": " in persona_conversation_data["perspective"]:+ name, description = persona_conversation_data["perspective"].split(+ ": ", 1+ )+ elif "- " in persona_conversation_data["perspective"]:+ name, description = persona_conversation_data["perspective"].split(+ "- ", 1+ )else:name, description = "", persona_conversation_data["perspective"]cur_conversation = []for dialogue_turn in persona_conversation_data["dlg_turns"]:- cur_conversation.append({"role": "user", "content": dialogue_turn["user_utterance"]})cur_conversation.append(- {"role": "assistant",- "content": DemoTextProcessingHelper.remove_citations(dialogue_turn["agent_utterance"])})+ {"role": "user", "content": dialogue_turn["user_utterance"]}+ )+ cur_conversation.append(+ {+ "role": "assistant",+ "content": DemoTextProcessingHelper.remove_citations(+ dialogue_turn["agent_utterance"]+ ),+ }+ )parsed_data.append((name, description, cur_conversation))return parsed_data@staticmethoddef parse(text):regex = re.compile(r']:\s+"(.*?)"\s+http')- text = regex.sub(']: http', text)+ text = regex.sub("]: http", text)return text@staticmethoddef add_markdown_indentation(input_string):- lines = input_string.split('\n')+ lines = input_string.split("\n")processed_lines = [""]for line in lines:num_hashes = 0for char in line:- if char == '#':+ if char == "#":num_hashes += 1else:breaknum_hashes -= 1num_spaces = 4 * num_hashes- new_line = ' ' * num_spaces + line+ new_line = " " * num_spaces + lineprocessed_lines.append(new_line)- return '\n'.join(processed_lines)+ return "\n".join(processed_lines)@staticmethoddef get_current_time_string():@@ -245,13 +290,15 @@ class DemoTextProcessingHelper():Returns:str: The current California time in 'YYYY-MM-DD HH:MM:SS' format."""- california_tz = pytz.timezone('America/Los_Angeles')+ california_tz = pytz.timezone("America/Los_Angeles")utc_now = datetime.datetime.now(datetime.timezone.utc)california_now = utc_now.astimezone(california_tz)- return california_now.strftime('%Y-%m-%d %H:%M:%S')+ return california_now.strftime("%Y-%m-%d %H:%M:%S")@staticmethod- def compare_time_strings(time_string1, time_string2, time_format='%Y-%m-%d %H:%M:%S'):+ def compare_time_strings(+ time_string1, time_string2, time_format="%Y-%m-%d %H:%M:%S"+ ):"""Compares two time strings to determine if they represent the same point in time.@@ -273,13 +320,13 @@ class DemoTextProcessingHelper():@staticmethoddef add_inline_citation_link(article_text, citation_dict):# Regular expression to find citations like [i]- pattern = r'\[(\d+)\]'+ pattern = r"\[(\d+)\]"# Function to replace each citation with its Markdown linkdef replace_with_link(match):i = match.group(1)- url = citation_dict.get(int(i), {}).get('url', '#')- return f'[[{i}]]({url})'+ url = citation_dict.get(int(i), {}).get("url", "#")+ return f"[[{i}]]({url})"# Replace all citations in the text with Markdown linksreturn re.sub(pattern, replace_with_link, article_text)@@ -292,26 +339,34 @@ class DemoTextProcessingHelper():level = line.count("#")title = line.strip("# ").strip()anchor = title.lower().replace(" ", "-").replace(".", "")- toc.append(f"<li style='margin-left: {20 * (level - 1)}px;'><a href='#{anchor}'>{title}</a></li>")+ toc.append(+ f"<li style='margin-left: {20 * (level - 1)}px;'><a href='#{anchor}'>{title}</a></li>"+ )return "<ul>" + "".join(toc) + "</ul>"@staticmethoddef construct_bibliography_from_url_to_info(url_to_info):bibliography_list = []- sorted_url_to_unified_index = dict(sorted(url_to_info['url_to_unified_index'].items(),- key=lambda item: item[1]))+ sorted_url_to_unified_index = dict(+ sorted(+ url_to_info["url_to_unified_index"].items(), key=lambda item: item[1]+ )+ )for url, index in sorted_url_to_unified_index.items():- title = url_to_info['url_to_info'][url]['title']+ title = url_to_info["url_to_info"][url]["title"]bibliography_list.append(f"[{index}]: [{title}]({url})")bibliography_string = "\n\n".join(bibliography_list)return f"# References\n\n{bibliography_string}"-class DemoUIHelper():+class DemoUIHelper:def st_markdown_adjust_size(content, font_size=20):- st.markdown(f"""+ st.markdown(+ f"""<span style='font-size: {font_size}px;'>{content}</span>- """, unsafe_allow_html=True)+ """,+ unsafe_allow_html=True,+ )@staticmethoddef get_article_card_UI_style(boarder_color="#9AD8E1"):@@ -326,7 +381,7 @@ class DemoUIHelper():"border-radius": "5px","border-left": f"0.5rem solid {boarder_color}","box-shadow": "0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15)",- "margin": "0px"+ "margin": "0px",},"title": {"white-space": "nowrap",@@ -336,7 +391,7 @@ class DemoUIHelper():"color": "rgb(49, 51, 63)","text-align": "left","width": "95%",- "font-weight": "normal"+ "font-weight": "normal",},"text": {"white-space": "nowrap",@@ -345,11 +400,9 @@ class DemoUIHelper():"font-size": "25px","color": "rgb(49, 51, 63)","text-align": "left",- "width": "95%"+ "width": "95%",},- "filter": {- "background-color": "rgba(0, 0, 0, 0)"- }+ "filter": {"background-color": "rgba(0, 0, 0, 0)"},}@staticmethod@@ -373,7 +426,8 @@ class DemoUIHelper():line-height: 1.5; /* Adjust this value as needed */}</style>- """, unsafe_allow_html=True+ """,+ unsafe_allow_html=True,)@staticmethod@@ -405,10 +459,12 @@ def _construct_citation_dict_from_search_result(search_results):if search_results is None:return Nonecitation_dict = {}- for url, index in search_results['url_to_unified_index'].items():- citation_dict[index] = {'url': url,- 'title': search_results['url_to_info'][url]['title'],- 'snippets': search_results['url_to_info'][url]['snippets']}+ for url, index in search_results["url_to_unified_index"].items():+ citation_dict[index] = {+ "url": url,+ "title": search_results["url_to_info"][url]["title"],+ "snippets": search_results["url_to_info"][url]["snippets"],+ }return citation_dict@@ -416,10 +472,14 @@ def _display_main_article_text(article_text, citation_dict, table_content_sideba# Post-process the generated article for better display.if "Write the lead section:" in article_text:article_text = article_text[- article_text.find("Write the lead section:") + len("Write the lead section:"):]- if article_text[0] == '#':- article_text = '\n'.join(article_text.split('\n')[1:])- article_text = DemoTextProcessingHelper.add_inline_citation_link(article_text, citation_dict)+ article_text.find("Write the lead section:")+ + len("Write the lead section:") :+ ]+ if article_text[0] == "#":+ article_text = "\n".join(article_text.split("\n")[1:])+ article_text = DemoTextProcessingHelper.add_inline_citation_link(+ article_text, citation_dict+ )# '$' needs to be changed to '\$' to avoid being interpreted as LaTeX in st.markdown()article_text = article_text.replace("$", "\\$")stoc.from_markdown(article_text, table_content_sidebar)@@ -430,10 +490,10 @@ def _display_references(citation_dict):reference_list = [f"reference [{i}]" for i in range(1, len(citation_dict) + 1)]selected_key = st.selectbox("Select a reference", reference_list)citation_val = citation_dict[reference_list.index(selected_key) + 1]- citation_val['title'] = citation_val['title'].replace("$", "\\$")+ citation_val["title"] = citation_val["title"].replace("$", "\\$")st.markdown(f"**Title:** {citation_val['title']}")st.markdown(f"**Url:** {citation_val['url']}")- snippets = '\n\n'.join(citation_val['snippets']).replace("$", "\\$")+ snippets = "\n\n".join(citation_val["snippets"]).replace("$", "\\$")st.markdown(f"**Highlights:**\n\n {snippets}")else:st.markdown("**No references available**")@@ -444,7 +504,9 @@ def _display_persona_conversations(conversation_log):Display persona conversation in dialogue UI"""# get personas list as (persona_name, persona_description, dialogue turns list) tuple- parsed_conversation_history = DemoTextProcessingHelper.parse_conversation_history(conversation_log)+ parsed_conversation_history = DemoTextProcessingHelper.parse_conversation_history(+ conversation_log+ )# construct tabs for each persona conversationpersona_tabs = st.tabs([name for (name, _, _) in parsed_conversation_history])for idx, persona_tab in enumerate(persona_tabs):@@ -453,7 +515,7 @@ def _display_persona_conversations(conversation_log):st.info(parsed_conversation_history[idx][1])# show user / agent utterance in dialogue UIfor message in parsed_conversation_history[idx][2]:- message['content'] = message['content'].replace("$", "\\$")+ message["content"] = message["content"].replace("$", "\\$")with st.chat_message(message["role"]):if message["role"] == "user":st.markdown(f"**{message['content']}**")@@ -461,14 +523,22 @@ def _display_persona_conversations(conversation_log):st.markdown(message["content"])-def _display_main_article(selected_article_file_path_dict, show_reference=True, show_conversation=True):- article_data = DemoFileIOHelper.assemble_article_data(selected_article_file_path_dict)+def _display_main_article(+ selected_article_file_path_dict, show_reference=True, show_conversation=True+):+ article_data = DemoFileIOHelper.assemble_article_data(+ selected_article_file_path_dict+ )with st.container(height=1000, border=True):- table_content_sidebar = st.sidebar.expander("**Table of contents**", expanded=True)- _display_main_article_text(article_text=article_data.get("article", ""),- citation_dict=article_data.get("citations", {}),- table_content_sidebar=table_content_sidebar)+ table_content_sidebar = st.sidebar.expander(+ "**Table of contents**", expanded=True+ )+ _display_main_article_text(+ article_text=article_data.get("article", ""),+ citation_dict=article_data.get("citations", {}),+ table_content_sidebar=table_content_sidebar,+ )# display reference panelif show_reference and "citations" in article_data:@@ -479,9 +549,12 @@ def _display_main_article(selected_article_file_path_dict, show_reference=True,# display conversation historyif show_conversation and "conversation_log" in article_data:with st.expander(- "**STORM** is powered by a knowledge agent that proactively research a given topic by asking good questions coming from different perspectives.\n\n"- ":sunglasses: Click here to view the agent's brain**STORM**ing process!"):- _display_persona_conversations(conversation_log=article_data.get("conversation_log", {}))+ "**STORM** is powered by a knowledge agent that proactively research a given topic by asking good questions coming from different perspectives.\n\n"+ ":sunglasses: Click here to view the agent's brain**STORM**ing process!"+ ):+ _display_persona_conversations(+ conversation_log=article_data.get("conversation_log", {})+ )def get_demo_dir():@@ -492,7 +565,11 @@ def clear_other_page_session_state(page_index: Optional[int]):if page_index is None:keys_to_delete = [key for key in st.session_state if key.startswith("page")]else:- keys_to_delete = [key for key in st.session_state if key.startswith("page") and f"page{page_index}" not in key]+ keys_to_delete = [+ key+ for key in st.session_state+ if key.startswith("page") and f"page{page_index}" not in key+ ]for key in set(keys_to_delete):del st.session_state[key]@@ -504,29 +581,44 @@ def set_storm_runner():# configure STORM runnerllm_configs = STORMWikiLMConfigs()- llm_configs.init_openai_model(openai_api_key=st.secrets['OPENAI_API_KEY'], openai_type='openai')- llm_configs.set_question_asker_lm(OpenAIModel(model='gpt-4-1106-preview', api_key=st.secrets['OPENAI_API_KEY'],- api_provider='openai',- max_tokens=500, temperature=1.0, top_p=0.9))+ llm_configs.init_openai_model(+ openai_api_key=st.secrets["OPENAI_API_KEY"], openai_type="openai"+ )+ llm_configs.set_question_asker_lm(+ OpenAIModel(+ model="gpt-4-1106-preview",+ api_key=st.secrets["OPENAI_API_KEY"],+ api_provider="openai",+ max_tokens=500,+ temperature=1.0,+ top_p=0.9,+ )+ )engine_args = STORMWikiRunnerArguments(output_dir=current_working_dir,max_conv_turn=3,max_perspective=3,search_top_k=3,- retrieve_top_k=5+ retrieve_top_k=5,)- rm = YouRM(ydc_api_key=st.secrets['YDC_API_KEY'], k=engine_args.search_top_k)+ rm = YouRM(ydc_api_key=st.secrets["YDC_API_KEY"], k=engine_args.search_top_k)runner = STORMWikiRunner(engine_args, llm_configs, rm)st.session_state["runner"] = runner-def display_article_page(selected_article_name, selected_article_file_path_dict,- show_title=True, show_main_article=True):+def display_article_page(+ selected_article_name,+ selected_article_file_path_dict,+ show_title=True,+ show_main_article=True,+):if show_title:- st.markdown(f"<h2 style='text-align: center;'>{selected_article_name.replace('_', ' ')}</h2>",- unsafe_allow_html=True)+ st.markdown(+ f"<h2 style='text-align: center;'>{selected_article_name.replace('_', ' ')}</h2>",+ unsafe_allow_html=True,+ )if show_main_article:_display_main_article(selected_article_file_path_dict)@@ -537,20 +629,25 @@ class StreamlitCallbackHandler(BaseCallbackHandler):self.status_container = status_containerdef on_identify_perspective_start(self, **kwargs):- self.status_container.info('Start identifying different perspectives for researching the topic.')+ self.status_container.info(+ "Start identifying different perspectives for researching the topic."+ )def on_identify_perspective_end(self, perspectives: list[str], **kwargs):perspective_list = "\n- ".join(perspectives)- self.status_container.success(f'Finish identifying perspectives. Will now start gathering information'- f' from the following perspectives:\n- {perspective_list}')+ self.status_container.success(+ f"Finish identifying perspectives. Will now start gathering information"+ f" from the following perspectives:\n- {perspective_list}"+ )def on_information_gathering_start(self, **kwargs):- self.status_container.info('Start browsing the Internet.')+ self.status_container.info("Start browsing the Internet.")def on_dialogue_turn_end(self, dlg_turn, **kwargs):urls = list(set([r.url for r in dlg_turn.search_results]))for url in urls:- self.status_container.markdown(f"""+ self.status_container.markdown(+ f"""<style>.small-font {{font-size: 14px;@@ -559,16 +656,22 @@ class StreamlitCallbackHandler(BaseCallbackHandler):}}</style><div class="small-font">Finish browsing <a href="{url}" class="small-font" target="_blank">{url}</a>.</div>- """, unsafe_allow_html=True)+ """,+ unsafe_allow_html=True,+ )def on_information_gathering_end(self, **kwargs):- self.status_container.success('Finish collecting information.')+ self.status_container.success("Finish collecting information.")def on_information_organization_start(self, **kwargs):- self.status_container.info('Start organizing information into a hierarchical outline.')+ self.status_container.info(+ "Start organizing information into a hierarchical outline."+ )def on_direct_outline_generation_end(self, outline: str, **kwargs):- self.status_container.success(f'Finish leveraging the internal knowledge of the large language model.')+ self.status_container.success(+ f"Finish leveraging the internal knowledge of the large language model."+ )def on_outline_refinement_end(self, outline: str, **kwargs):- self.status_container.success(f'Finish leveraging the collected information.')+ self.status_container.success(f"Finish leveraging the collected information.")diff --git a/frontend/demo_light/pages_util/CreateNewArticle.py b/frontend/demo_light/pages_util/CreateNewArticle.pyindex 2257044..fb30aae 100644--- a/frontend/demo_light/pages_util/CreateNewArticle.py+++ b/frontend/demo_light/pages_util/CreateNewArticle.py@@ -3,39 +3,60 @@ import timeimport demo_utilimport streamlit as st-from demo_util import DemoFileIOHelper, DemoTextProcessingHelper, DemoUIHelper, truncate_filename+from demo_util import (+ DemoFileIOHelper,+ DemoTextProcessingHelper,+ DemoUIHelper,+ truncate_filename,+)+def handle_not_started():if st.session_state["page3_write_article_state"] == "not started":-_, search_form_column, _ = st.columns([2, 5, 2])with search_form_column:- with st.form(key='search_form'):+ with st.form(key="search_form"):# Text input for the search topic- DemoUIHelper.st_markdown_adjust_size(content="Enter the topic you want to learn in depth:",- font_size=18)- st.session_state["page3_topic"] = st.text_input(label='page3_topic', label_visibility="collapsed")+ DemoUIHelper.st_markdown_adjust_size(+ content="Enter the topic you want to learn in depth:", font_size=18+ )+ st.session_state["page3_topic"] = st.text_input(+ label="page3_topic", label_visibility="collapsed"+ )pass_appropriateness_check = True# Submit button for the form- submit_button = st.form_submit_button(label='Research')+ submit_button = st.form_submit_button(label="Research")# only start new search when button is clicked, not started, or already finished previous one- if submit_button and st.session_state["page3_write_article_state"] in ["not started", "show results"]:+ if submit_button and st.session_state["page3_write_article_state"] in [+ "not started",+ "show results",+ ]:if not st.session_state["page3_topic"].strip():pass_appropriateness_check = False- st.session_state["page3_warning_message"] = "topic could not be empty"-- st.session_state["page3_topic_name_cleaned"] = st.session_state["page3_topic"].replace(- ' ', '_').replace('/', '_')- st.session_state["page3_topic_name_truncated"] = truncate_filename(st.session_state["page3_topic_name_cleaned"])+ st.session_state[+ "page3_warning_message"+ ] = "topic could not be empty"++ st.session_state["page3_topic_name_cleaned"] = (+ st.session_state["page3_topic"]+ .replace(" ", "_")+ .replace("/", "_")+ )+ st.session_state["page3_topic_name_truncated"] = truncate_filename(+ st.session_state["page3_topic_name_cleaned"]+ )if not pass_appropriateness_check:st.session_state["page3_write_article_state"] = "not started"- alert = st.warning(st.session_state["page3_warning_message"], icon="⚠️")+ alert = st.warning(+ st.session_state["page3_warning_message"], icon="⚠️"+ )time.sleep(5)alert.empty()else:st.session_state["page3_write_article_state"] = "initiated"+def handle_initiated():if st.session_state["page3_write_article_state"] == "initiated":current_working_dir = os.path.join(demo_util.get_demo_dir(), "DEMO_WORKING_DIR")@@ -47,9 +68,12 @@ def handle_initiated():st.session_state["page3_current_working_dir"] = current_working_dirst.session_state["page3_write_article_state"] = "pre_writing"+def handle_pre_writing():- if st.session_state["page3_write_article_state"] == "pre_writing":- status = st.status("I am brain**STORM**ing now to research the topic. (This may take 2-3 minutes.)")+ if st.session_state["page3_write_article_state"] == "pre_writing":+ status = st.status(+ "I am brain**STORM**ing now to research the topic. (This may take 2-3 minutes.)"+ )st_callback_handler = demo_util.StreamlitCallbackHandler(status)with status:# STORM main gen outline@@ -59,23 +83,37 @@ def handle_pre_writing():do_generate_outline=True,do_generate_article=False,do_polish_article=False,- callback_handler=st_callback_handler+ callback_handler=st_callback_handler,+ )+ conversation_log_path = os.path.join(+ st.session_state["page3_current_working_dir"],+ st.session_state["page3_topic_name_truncated"],+ "conversation_log.json",+ )+ demo_util._display_persona_conversations(+ DemoFileIOHelper.read_json_file(conversation_log_path))- conversation_log_path = os.path.join(st.session_state["page3_current_working_dir"],- st.session_state["page3_topic_name_truncated"], "conversation_log.json")- demo_util._display_persona_conversations(DemoFileIOHelper.read_json_file(conversation_log_path))st.session_state["page3_write_article_state"] = "final_writing"status.update(label="brain**STORM**ing complete!", state="complete")+def handle_final_writing():if st.session_state["page3_write_article_state"] == "final_writing":# polish final articlewith st.status(- "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)") as status:- st.info('Now I will connect the information I found for your reference. (This may take 4-5 minutes.)')- st.session_state["runner"].run(topic=st.session_state["page3_topic"], do_research=False,- do_generate_outline=False,- do_generate_article=True, do_polish_article=True, remove_duplicate=False)+ "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)"+ ) as status:+ st.info(+ "Now I will connect the information I found for your reference. (This may take 4-5 minutes.)"+ )+ st.session_state["runner"].run(+ topic=st.session_state["page3_topic"],+ do_research=False,+ do_generate_outline=False,+ do_generate_article=True,+ do_polish_article=True,+ remove_duplicate=False,+ )# finish the sessionst.session_state["runner"].post_run()@@ -83,6 +121,7 @@ def handle_final_writing():st.session_state["page3_write_article_state"] = "prepare_to_show_result"status.update(label="information snythesis complete!", state="complete")+def handle_prepare_to_show_result():if st.session_state["page3_write_article_state"] == "prepare_to_show_result":_, show_result_col, _ = st.columns([4, 3, 4])@@ -91,16 +130,23 @@ def handle_prepare_to_show_result():st.session_state["page3_write_article_state"] = "completed"st.rerun()+def handle_completed():-if st.session_state["page3_write_article_state"] == "completed":# display polished articlecurrent_working_dir_paths = DemoFileIOHelper.read_structure_to_dict(- st.session_state["page3_current_working_dir"])- current_article_file_path_dict = current_working_dir_paths[st.session_state["page3_topic_name_truncated"]]- demo_util.display_article_page(selected_article_name=st.session_state["page3_topic_name_cleaned"],- selected_article_file_path_dict=current_article_file_path_dict,- show_title=True, show_main_article=True)+ st.session_state["page3_current_working_dir"]+ )+ current_article_file_path_dict = current_working_dir_paths[+ st.session_state["page3_topic_name_truncated"]+ ]+ demo_util.display_article_page(+ selected_article_name=st.session_state["page3_topic_name_cleaned"],+ selected_article_file_path_dict=current_article_file_path_dict,+ show_title=True,+ show_main_article=True,+ )+def create_new_article_page():demo_util.clear_other_page_session_state(page_index=3)diff --git a/frontend/demo_light/pages_util/MyArticles.py b/frontend/demo_light/pages_util/MyArticles.pyindex e4e3bd1..3d1ee6f 100644--- a/frontend/demo_light/pages_util/MyArticles.py+++ b/frontend/demo_light/pages_util/MyArticles.py@@ -11,7 +11,10 @@ def my_articles_page():with st.sidebar:_, return_button_col = st.columns([2, 5])with return_button_col:- if st.button("Select another article", disabled="page2_selected_my_article" not in st.session_state):+ if st.button(+ "Select another article",+ disabled="page2_selected_my_article" not in st.session_state,+ ):if "page2_selected_my_article" in st.session_state:del st.session_state["page2_selected_my_article"]st.rerun()@@ -20,17 +23,22 @@ def my_articles_page():if "page2_user_articles_file_path_dict" not in st.session_state:local_dir = os.path.join(demo_util.get_demo_dir(), "DEMO_WORKING_DIR")os.makedirs(local_dir, exist_ok=True)- st.session_state["page2_user_articles_file_path_dict"] = DemoFileIOHelper.read_structure_to_dict(local_dir)+ st.session_state[+ "page2_user_articles_file_path_dict"+ ] = DemoFileIOHelper.read_structure_to_dict(local_dir)# if no feature demo selected, display all featured articles as info cardsdef article_card_setup(column_to_add, card_title, article_name):with column_to_add:cleaned_article_title = article_name.replace("_", " ")- hasClicked = card(title=" / ".join(card_title),- text=article_name.replace("_", " "),- image=DemoFileIOHelper.read_image_as_base64(- os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")),- styles=DemoUIHelper.get_article_card_UI_style(boarder_color="#9AD8E1"))+ hasClicked = card(+ title=" / ".join(card_title),+ text=article_name.replace("_", " "),+ image=DemoFileIOHelper.read_image_as_base64(+ os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")+ ),+ styles=DemoUIHelper.get_article_card_UI_style(boarder_color="#9AD8E1"),+ )if hasClicked:st.session_state["page2_selected_my_article"] = article_namest.rerun()@@ -40,7 +48,9 @@ def my_articles_page():my_article_columns = st.columns(3)if len(st.session_state["page2_user_articles_file_path_dict"]) > 0:# get article names- article_names = sorted(list(st.session_state["page2_user_articles_file_path_dict"].keys()))+ article_names = sorted(+ list(st.session_state["page2_user_articles_file_path_dict"].keys())+ )# configure paginationpagination = st.container()bottom_menu = st.columns((1, 4, 1, 1, 1))[1:-1]@@ -48,7 +58,9 @@ def my_articles_page():batch_size = st.selectbox("Page Size", options=[24, 48, 72])with bottom_menu[1]:total_pages = (- int(len(article_names) / batch_size) if int(len(article_names) / batch_size) > 0 else 1+ int(len(article_names) / batch_size)+ if int(len(article_names) / batch_size) > 0+ else 1)current_page = st.number_input("Page", min_value=1, max_value=total_pages, step=1@@ -60,19 +72,24 @@ def my_articles_page():my_article_count = 0start_index = (current_page - 1) * batch_sizeend_index = min(current_page * batch_size, len(article_names))- for article_name in article_names[start_index: end_index]:+ for article_name in article_names[start_index:end_index]:column_to_add = my_article_columns[my_article_count % 3]my_article_count += 1- article_card_setup(column_to_add=column_to_add,- card_title=["My Article"],- article_name=article_name)+ article_card_setup(+ column_to_add=column_to_add,+ card_title=["My Article"],+ article_name=article_name,+ )else:with my_article_columns[0]:- hasClicked = card(title="Get started",- text="Start your first research!",- image=DemoFileIOHelper.read_image_as_base64(- os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")),- styles=DemoUIHelper.get_article_card_UI_style())+ hasClicked = card(+ title="Get started",+ text="Start your first research!",+ image=DemoFileIOHelper.read_image_as_base64(+ os.path.join(demo_util.get_demo_dir(), "assets", "void.jpg")+ ),+ styles=DemoUIHelper.get_article_card_UI_style(),+ )if hasClicked:st.session_state.selected_page = 1st.session_state["manual_selection_override"] = True@@ -80,8 +97,13 @@ def my_articles_page():st.rerun()else:selected_article_name = st.session_state["page2_selected_my_article"]- selected_article_file_path_dict = st.session_state["page2_user_articles_file_path_dict"][selected_article_name]+ selected_article_file_path_dict = st.session_state[+ "page2_user_articles_file_path_dict"+ ][selected_article_name]- demo_util.display_article_page(selected_article_name=selected_article_name,- selected_article_file_path_dict=selected_article_file_path_dict,- show_title=True, show_main_article=True)+ demo_util.display_article_page(+ selected_article_name=selected_article_name,+ selected_article_file_path_dict=selected_article_file_path_dict,+ show_title=True,+ show_main_article=True,+ )diff --git a/frontend/demo_light/stoc.py b/frontend/demo_light/stoc.pyindex 7bd4402..2abd3c4 100644--- a/frontend/demo_light/stoc.py+++ b/frontend/demo_light/stoc.py@@ -44,9 +44,9 @@ class stoc:for title_size, title in self.toc_items:h = int(title_size.replace("h", ""))markdown_toc += (- " " * 2 * h- + "- "- + f'<a href="#{normalize(title)}" class="toc"> {title}</a> \n'+ " " * 2 * h+ + "- "+ + f'<a href="#{normalize(title)}" class="toc"> {title}</a> \n')# st.sidebar.write(markdown_toc, unsafe_allow_html=True)st.write(markdown_toc, unsafe_allow_html=True)@@ -56,27 +56,35 @@ class stoc:def increase_heading_depth_and_add_top_heading(markdown_text, new_top_heading):lines = markdown_text.splitlines()# Increase the depth of each heading by adding an extra '#'- increased_depth_lines = ['#' + line if line.startswith('#') else line for line in lines]+ increased_depth_lines = [+ "#" + line if line.startswith("#") else line for line in lines+ ]# Add the new top-level heading at the beginningincreased_depth_lines.insert(0, f"# {new_top_heading}")# Re-join the modified lines back into a single string- modified_text = '\n'.join(increased_depth_lines)+ modified_text = "\n".join(increased_depth_lines)return modified_textif topic:- markdown_text = increase_heading_depth_and_add_top_heading(markdown_text, topic)+ markdown_text = increase_heading_depth_and_add_top_heading(+ markdown_text, topic+ )toc = []for line in markdown_text.splitlines():- if line.startswith('#'):+ if line.startswith("#"):# Remove the '#' characters and strip leading/trailing spaces- heading_text = line.lstrip('#').strip()+ heading_text = line.lstrip("#").strip()# Create slug (lowercase, spaces to hyphens, remove non-alphanumeric characters)- slug = re.sub(r'[^a-zA-Z0-9\s-]', '', heading_text).lower().replace(' ', '-')+ slug = (+ re.sub(r"[^a-zA-Z0-9\s-]", "", heading_text)+ .lower()+ .replace(" ", "-")+ )# Determine heading level for indentation- level = line.count('#') - 1+ level = line.count("#") - 1# Add to the table of contents- toc.append(' ' * level + f'- [{heading_text}](#{slug})')- return '\n'.join(toc)+ toc.append(" " * level + f"- [{heading_text}](#{slug})")+ return "\n".join(toc)@classmethoddef from_markdown(cls, text: str, expander=None):diff --git a/frontend/demo_light/storm.py b/frontend/demo_light/storm.pyindex 1d095c3..5aca259 100644--- a/frontend/demo_light/storm.py+++ b/frontend/demo_light/storm.py@@ -11,13 +11,13 @@ from streamlit_option_menu import option_menudef main():global database- st.set_page_config(layout='wide')+ st.set_page_config(layout="wide")if "first_run" not in st.session_state:- st.session_state['first_run'] = True+ st.session_state["first_run"] = True# set api keys from secrets- if st.session_state['first_run']:+ if st.session_state["first_run"]:for key, value in st.secrets.items():if type(value) == str:os.environ[key] = value@@ -31,20 +31,26 @@ def main():st.session_state["rerun_requested"] = Falsest.rerun()- st.write('<style>div.block-container{padding-top:2rem;}</style>', unsafe_allow_html=True)+ st.write(+ "<style>div.block-container{padding-top:2rem;}</style>", unsafe_allow_html=True+ )menu_container = st.container()with menu_container:pages = ["My Articles", "Create New Article"]- styles={- "container": {"padding": "0.2rem 0",- "background-color": "#22222200"},- }- menu_selection = option_menu(None, pages,- icons=['house', 'search'],- menu_icon="cast", default_index=0, orientation="horizontal",- manual_select=st.session_state.selected_page,- styles=styles,- key='menu_selection')+ styles = {+ "container": {"padding": "0.2rem 0", "background-color": "#22222200"},+ }+ menu_selection = option_menu(+ None,+ pages,+ icons=["house", "search"],+ menu_icon="cast",+ default_index=0,+ orientation="horizontal",+ manual_select=st.session_state.selected_page,+ styles=styles,+ key="menu_selection",+ )if st.session_state.get("manual_selection_override", False):menu_selection = pages[st.session_state["selected_page"]]st.session_state["manual_selection_override"] = Falsediff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.pyindex 8da7433..193d684 100644--- a/knowledge_storm/collaborative_storm/engine.py+++ b/knowledge_storm/collaborative_storm/engine.py@@ -600,11 +600,13 @@ class CoStormRunner:callback_handler=self.callback_handler,)- warmstart_conv, warmstart_revised_conv, warmstart_experts = (- warm_start_module.initiate_warm_start(- topic=self.runner_argument.topic,- knowledge_base=self.knowledge_base,- )+ (+ warmstart_conv,+ warmstart_revised_conv,+ warmstart_experts,+ ) = warm_start_module.initiate_warm_start(+ topic=self.runner_argument.topic,+ knowledge_base=self.knowledge_base,)self.discourse_manager.experts = (self.discourse_manager._parse_expert_names_to_agent(@@ -618,7 +620,6 @@ class CoStormRunner:self.warmstart_conv_archive = warmstart_convself.knowledge_base.reogranize()else:-if self.knowledge_base is None:self.knowledge_base = KnowledgeBase(topic=self.runner_argument.topic,diff --git a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py b/knowledge_storm/collaborative_storm/modules/information_insertion_module.pyindex 47daafb..05cbde3 100644--- a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py+++ b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py@@ -270,9 +270,10 @@ class InsertInformationModule(dspy.Module):root=insert_root,)- encoded_outlines, outlines = (- knowledge_base.get_knowledge_base_structure_embedding(root=insert_root)- )+ (+ encoded_outlines,+ outlines,+ ) = knowledge_base.get_knowledge_base_structure_embedding(root=insert_root)to_return = []if not allow_create_new_node:# use multi thread as knowledge base structure does not change@@ -295,10 +296,11 @@ class InsertInformationModule(dspy.Module):else:# use sequential insert as knowledge base structure might changefor question, query in intent_to_placement_dict:- encoded_outlines, outlines = (- knowledge_base.get_knowledge_base_structure_embedding(- root=insert_root- )+ (+ encoded_outlines,+ outlines,+ ) = knowledge_base.get_knowledge_base_structure_embedding(+ root=insert_root)_, placement_prediction = process_intent(question=question, query=query)intent_to_placement_dict[(question, query)] = placement_predictiondiff --git a/knowledge_storm/dataclass.py b/knowledge_storm/dataclass.pyindex 53bd7e4..86e89ad 100644--- a/knowledge_storm/dataclass.py+++ b/knowledge_storm/dataclass.py@@ -773,9 +773,9 @@ class KnowledgeBase:def update_all_info_path(self):def _helper(node):for citation_idx in node.content:- self.info_uuid_to_info_dict[citation_idx].meta["placement"] = (- " -> ".join(node.get_path_from_root())- )+ self.info_uuid_to_info_dict[citation_idx].meta[+ "placement"+ ] = " -> ".join(node.get_path_from_root())for child in node.children:_helper(child)diff --git a/knowledge_storm/interface.py b/knowledge_storm/interface.pyindex 5922602..db02896 100644--- a/knowledge_storm/interface.py+++ b/knowledge_storm/interface.py@@ -473,7 +473,6 @@ class LMConfigs(ABC):return model_name_to_usagedef log(self):-return OrderedDict({attr_name: getattr(self, attr_name).kwargs@@ -503,9 +502,9 @@ class Engine(ABC):logger.info(f"{func.__name__} executed in {execution_time:.4f} seconds")self.lm_cost[func.__name__] = self.lm_configs.collect_and_reset_lm_usage()if hasattr(self, "retriever"):- self.rm_cost[func.__name__] = (- self.retriever.collect_and_reset_rm_usage()- )+ self.rm_cost[+ func.__name__+ ] = self.retriever.collect_and_reset_rm_usage()return resultreturn wrapperdiff --git a/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.pyindex f0b3f55..c698a9a 100644--- a/knowledge_storm/storm_wiki/engine.py+++ b/knowledge_storm/storm_wiki/engine.py@@ -213,16 +213,16 @@ class STORMWikiRunner(Engine):ground_truth_url: str = "None",callback_handler: BaseCallbackHandler = None,) -> StormInformationTable:-- information_table, conversation_log = (- self.storm_knowledge_curation_module.research(- topic=self.topic,- ground_truth_url=ground_truth_url,- callback_handler=callback_handler,- max_perspective=self.args.max_perspective,- disable_perspective=False,- return_conversation_log=True,- )+ (+ information_table,+ conversation_log,+ ) = self.storm_knowledge_curation_module.research(+ topic=self.topic,+ ground_truth_url=ground_truth_url,+ callback_handler=callback_handler,+ max_perspective=self.args.max_perspective,+ disable_perspective=False,+ return_conversation_log=True,)FileIOHelper.dump_json(diff --git a/knowledge_storm/storm_wiki/modules/article_generation.py b/knowledge_storm/storm_wiki/modules/article_generation.pyindex a23b788..6289c7a 100644--- a/knowledge_storm/storm_wiki/modules/article_generation.py+++ b/knowledge_storm/storm_wiki/modules/article_generation.py@@ -88,7 +88,6 @@ class StormArticleGenerationModule(ArticleGenerationModule):)section_output_dict_collection = [section_output_dict]else:-with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_thread_num) as executor:diff --git a/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.pyindex 119869c..c43c409 100644--- a/knowledge_storm/storm_wiki/modules/storm_dataclass.py+++ b/knowledge_storm/storm_wiki/modules/storm_dataclass.py@@ -58,9 +58,9 @@ class StormInformationTable(InformationTable):def __init__(self, conversations=List[Tuple[str, List[DialogueTurn]]]):super().__init__()self.conversations = conversations- self.url_to_info: Dict[str, Information] = (- StormInformationTable.construct_url_to_info(self.conversations)- )+ self.url_to_info: Dict[+ str, Information+ ] = StormInformationTable.construct_url_to_info(self.conversations)@staticmethoddef construct_url_to_info(