An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Update example scripts to support BingSearch.
shaoyijia1f8ac2ddb4dd
diff --git a/examples/run_storm_wiki_claude.py b/examples/run_storm_wiki_claude.pyindex 86afded..f29ba1c 100644--- a/examples/run_storm_wiki_claude.py+++ b/examples/run_storm_wiki_claude.py@@ -2,7 +2,7 @@STORM Wiki pipeline powered by Claude family models and You.com search engine.You need to set up the following environment variables to run this script:- ANTHROPIC_API_KEY: Anthropic API key- - YDC_API_KEY: You.com API key+ - YDC_API_KEY: You.com API key; or, BING_SEARCH_API_KEY: Bing Search API keyOutput will be structured as belowargs.output_dir/@@ -22,6 +22,7 @@ from argparse import ArgumentParsersys.path.append('./src')from lm import ClaudeModel+from rm import YouRM, BingSearchfrom storm_wiki.engine import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom utils import load_api_key@@ -36,7 +37,7 @@ def main(args):}# STORM is a LM system so different components can be powered by different models.- # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm+ # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.@@ -59,7 +60,15 @@ def main(args):search_top_k=args.search_top_k,max_thread_num=args.max_thread_num,)- runner = STORMWikiRunner(engine_args, lm_configs)++ # STORM is a knowledge curation system which consumes information from the retrieval module.+ # Currently, the information source is the Internet and we use search engine API as the retrieval module.+ if args.retriever == 'bing':+ rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)+ elif args.retriever == 'you':+ rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)++ runner = STORMWikiRunner(engine_args, lm_configs, rm)topic = input('Topic: ')runner.run(@@ -82,6 +91,8 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')+ parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',help='If True, simulate conversation to research the topic; otherwise, load the results.')@@ -105,4 +116,4 @@ if __name__ == '__main__':parser.add_argument('--remove-duplicate', action='store_true',help='If True, remove duplicate content from the article.')- main(parser.parse_args())+ main(parser.parse_args())\ No newline at end of filediff --git a/examples/run_storm_wiki_gpt.py b/examples/run_storm_wiki_gpt.pyindex 6b07226..f7639d6 100644--- a/examples/run_storm_wiki_gpt.py+++ b/examples/run_storm_wiki_gpt.py@@ -5,7 +5,7 @@ You need to set up the following environment variables to run this script:- OPENAI_API_TYPE: OpenAI API type (e.g., 'openai' or 'azure')- AZURE_API_BASE: Azure API base URL if using Azure API- AZURE_API_VERSION: Azure API version if using Azure API- - YDC_API_KEY: You.com API key+ - YDC_API_KEY: You.com API key; or, BING_SEARCH_API_KEY: Bing Search API keyOutput will be structured as belowargs.output_dir/@@ -25,6 +25,7 @@ from argparse import ArgumentParsersys.path.append('./src')from lm import OpenAIModel+from rm import YouRM, BingSearchfrom storm_wiki.engine import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom utils import load_api_key@@ -42,7 +43,7 @@ def main(args):}# STORM is a LM system so different components can be powered by different models.- # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm+ # For a good balance between cost and quality, you can choose a cheaper/faster model for conv_simulator_lm# which is used to split queries, synthesize answers in the conversation. We recommend using stronger models# for outline_gen_lm which is responsible for organizing the collected information, and article_gen_lm# which is responsible for generating sections with citations.@@ -65,7 +66,15 @@ def main(args):search_top_k=args.search_top_k,max_thread_num=args.max_thread_num,)- runner = STORMWikiRunner(engine_args, lm_configs)++ # STORM is a knowledge curation system which consumes information from the retrieval module.+ # Currently, the information source is the Internet and we use search engine API as the retrieval module.+ if args.retriever == 'bing':+ rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)+ elif args.retriever == 'you':+ rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)++ runner = STORMWikiRunner(engine_args, lm_configs, rm)topic = input('Topic: ')runner.run(@@ -88,6 +97,8 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')+ parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',help='If True, simulate conversation to research the topic; otherwise, load the results.')@@ -111,4 +122,4 @@ if __name__ == '__main__':parser.add_argument('--remove-duplicate', action='store_true',help='If True, remove duplicate content from the article.')- main(parser.parse_args())+ main(parser.parse_args())\ No newline at end of filediff --git a/examples/run_storm_wiki_mistral.py b/examples/run_storm_wiki_mistral.pyindex e819136..f7bc22d 100644--- a/examples/run_storm_wiki_mistral.py+++ b/examples/run_storm_wiki_mistral.py@@ -1,7 +1,7 @@"""STORM Wiki pipeline powered by Mistral-7B-Instruct-v0.2 hosted by VLLM server and You.com search engine.You need to set up the following environment variables to run this script:- - YDC_API_KEY: You.com API key+ - YDC_API_KEY: You.com API key; or, BING_SEARCH_API_KEY: Bing Search API keyYou also need to have a VLLM server running with the Mistral-7B-Instruct-v0.2 model. Specify `--url` and `--port` accordingly.Output will be structured as below@@ -15,7 +15,7 @@ args.output_dir/storm_gen_article.txt # Final article generatedstorm_gen_article_polished.txt # Polished final article (if args.do_polish_article is True)"""-+import osimport sysfrom argparse import ArgumentParser@@ -23,6 +23,7 @@ from dspy import Examplesys.path.append('./src')from lm import VLLMClient+from rm import YouRM, BingSearchfrom storm_wiki.engine import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom utils import load_api_key@@ -57,7 +58,15 @@ def main(args):search_top_k=args.search_top_k,max_thread_num=args.max_thread_num,)- runner = STORMWikiRunner(engine_args, lm_configs)++ # STORM is a knowledge curation system which consumes information from the retrieval module.+ # Currently, the information source is the Internet and we use search engine API as the retrieval module.+ if args.retriever == 'bing':+ rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)+ elif args.retriever == 'you':+ rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)++ runner = STORMWikiRunner(engine_args, lm_configs, rm)# Open LMs are generally weaker in following output format.# One way for mitigation is to add one-shot example to the prompt to exemplify the desired output format.@@ -140,6 +149,8 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')+ parser.add_argument('--retriever', type=str, choices=['bing', 'you'],+ help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',help='If True, simulate conversation to research the topic; otherwise, load the results.')@@ -163,4 +174,4 @@ if __name__ == '__main__':parser.add_argument('--remove-duplicate', action='store_true',help='If True, remove duplicate content from the article.')- main(parser.parse_args())+ main(parser.parse_args())\ No newline at end of file