Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

remove legacy scripts

Yucheng-Jiang4c0e83e6ed01
diff --git a/src/scripts/run_prewriting.py b/src/scripts/run_prewriting.py
deleted file mode 100644
index 6ebade1..0000000
--- a/src/scripts/run_prewriting.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""
-STORM pipeline pre-writing stage:
-Collect information grounded on search engine and organize information into a detailed outline.
-
-Output will be structured as below
-args.output_dir/
- topic_name/ # topic_name will follow convention of underscore-connected topic name w/o space and slash
- conversation_log.json # Log of information-seeking conversation
- raw_search_results.json # Raw search results from search engine
- direct_gen_outline.txt # Outline directly generated with LLM's parametric knowledge
- storm_gen_outline.txt # Outline refined with collected information
-"""
-
-import os
-from argparse import ArgumentParser
-
-import pandas as pd
-from engine import DeepSearchRunnerArguments, DeepSearchRunner
-from modules.utils import MyOpenAIModel, load_api_key, LLMConfigs
-from tqdm import tqdm
-
-
-def main(args):
- load_api_key()
- llm_configs = LLMConfigs()
- llm_configs.init_openai_model(openai_api_key=os.getenv("OPENAI_API_KEY"), openai_type=os.getenv('OPENAI_API_TYPE'),
- api_base=os.getenv('AZURE_API_BASE'), api_version=os.getenv('AZURE_API_VERSION'))
-
- if args.engine == 'gpt-35-turbo': # If args.engine == 'gpt4', use the default config.
- if os.getenv("OPENAI_API_TYPE") and os.getenv("OPENAI_API_TYPE") == 'azure':
- model_name = 'gpt-35-turbo-16k'
- else:
- model_name = 'gpt-3.5-turbo-16k'
- llm_configs.set_outline_gen_lm(MyOpenAIModel(model=model_name, api_key=os.getenv("OPENAI_API_KEY"),
- api_provider=os.getenv("OPENAI_API_TYPE"),
- engine='gpt-35-turbo-16k',
- api_base=os.getenv("AZURE_API_BASE"),
- api_version=os.getenv("AZURE_API_VERSION"),
- max_tokens=400, temperature=1.0, top_p=0.9))
-
- engine_args = DeepSearchRunnerArguments(
- output_dir=args.output_dir,
- max_conv_turn=args.max_conv_turn,
- max_perspective=args.max_perspective,
- search_top_k=args.search_top_k,
- disable_perspective=args.disable_perspective,
-
- )
- runner = DeepSearchRunner(engine_args, llm_configs)
-
- if args.input_source == 'console':
- topic = input('Topic: ')
- ground_truth_url = input('Ground truth url (will be excluded from source): ')
- runner.run(topic=topic,
- ground_truth_url=ground_truth_url,
- do_research=args.do_research,
- do_generate_outline=True,
- do_generate_article=False,
- do_polish_article=False)
- runner.post_run()
- else:
- data = pd.read_csv(args.input_path)
-
- for _, row in tqdm(data.iterrows(), total=len(data)):
- topic = row['topic']
- ground_truth_url = row['url']
- runner.run(topic=topic,
- ground_truth_url=ground_truth_url,
- do_research=args.do_research,
- do_generate_outline=True,
- do_generate_article=False,
- do_polish_article=False)
- runner.post_run()
-
-
-if __name__ == '__main__':
- parser = ArgumentParser()
- parser.add_argument('--max-conv-turn', type=int, default=5,
- help='Maximum number of questions in conversational question asking.')
- parser.add_argument('--max-perspective', type=int, default=5,
- help='Maximum number of perspectives to consider in perspective-guided question asking.')
- parser.add_argument('--search-top-k', type=int, default=3,
- help='Top k search results to consider for each search query.')
- parser.add_argument('--disable-perspective', action='store_true',
- help='If True, disable perspective-guided question asking; set True only for ablation study.')
- parser.add_argument('--do-research', action='store_true',
- help='If True, simulate conversation to research the topic; otherwise, load the results.')
- parser.add_argument('--input-source', type=str, choices=['console', 'file'],
- help='Where does the input come from.')
- parser.add_argument('--input-path', type=str,
- help='Using csv file to store topic and ground truth url at present.')
- parser.add_argument('--output-dir', type=str, default='../results',
- help='Directory to store the outputs.')
- parser.add_argument('--engine', type=str, required=True, choices=['gpt-4', 'gpt-35-turbo'])
-
- main(parser.parse_args())
diff --git a/src/scripts/run_writing.py b/src/scripts/run_writing.py
deleted file mode 100644
index d9d19a1..0000000
--- a/src/scripts/run_writing.py
+++ /dev/null
@@ -1,94 +0,0 @@
-"""
-STORM pipeline final-writing stage:
-Generate use the outline and collected information to generate a full article.
-
-Expect the following files to exist:
-args.output_dir/
- topic_name/ # topic_name will follow convention of underscore-connected topic name w/o space and slash
- raw_search_results.json # Raw search results from search engine
- storm_gen_outline.txt # Outline refined with collected information
-
-Output will be structured as below
-args.output_dir/
- topic_name/
- url_to_info.json # Sources that are used in the final article
- storm_gen_article.txt # Final article generated
- storm_gen_article_polished.txt # Polished final article (if args.do_polish_article is True)
-"""
-
-import os
-from argparse import ArgumentParser
-
-import pandas as pd
-from tqdm import tqdm
-
-from engine import DeepSearchRunnerArguments, DeepSearchRunner
-from modules.utils import MyOpenAIModel, load_api_key, LLMConfigs
-
-
-def main(args):
- load_api_key()
- llm_configs = LLMConfigs()
- llm_configs.init_openai_model(openai_api_key=os.getenv("OPENAI_API_KEY"), openai_type=os.getenv('OPENAI_API_TYPE'),
- api_base=os.getenv('AZURE_API_BASE'), api_version=os.getenv('AZURE_API_VERSION'))
-
- if args.engine == 'gpt-35-turbo': # if args.engine == 'gpt-4', use the default.
- if os.getenv("OPENAI_API_TYPE") and os.getenv("OPENAI_API_TYPE") == 'azure':
- model_name = 'gpt-35-turbo'
- else:
- model_name = 'gpt-3.5-turbo'
- llm_configs.set_article_gen_lm(MyOpenAIModel(model=model_name, api_key=os.getenv("OPENAI_API_KEY"),
- api_provider=os.getenv("OPENAI_API_TYPE"),
- api_base=os.getenv("AZURE_API_BASE"),
- api_version=os.getenv("AZURE_API_VERSION"),
- max_tokens=700, temperature=1.0, top_p=0.9))
-
- engine_args = DeepSearchRunnerArguments(
- output_dir=args.output_dir,
- retrieve_top_k=args.retrieve_top_k
- )
- runner = DeepSearchRunner(engine_args, llm_configs)
-
- if args.input_source == 'console':
- topic = input('Topic: ')
- runner.run(topic=topic,
- ground_truth_url=None,
- do_research=False,
- do_generate_outline=False,
- do_generate_article=True,
- do_polish_article=args.do_polish_article,
- remove_duplicate=args.remove_duplicate)
- runner.post_run()
- else:
- data = pd.read_csv(args.input_path)
-
- for _, row in tqdm(data.iterrows(), total=len(data)):
- topic = row['topic']
- runner.run(topic=topic,
- ground_truth_url=None,
- do_research=False,
- do_generate_outline=False,
- do_generate_article=True,
- do_polish_article=args.do_polish_article,
- remove_duplicate=args.remove_duplicate)
- runner.post_run()
-
-
-if __name__ == '__main__':
- parser = ArgumentParser()
- parser.add_argument('--retrieve-top-k', type=int, default=5,
- help='Top k collected references for each section title.')
- parser.add_argument('--do-polish-article', action='store_true',
- help='If True, polish the article by adding a summarization section and (optionally) removing '
- 'duplicate content.')
- parser.add_argument('--remove-duplicate', action='store_true',
- help='If True, remove duplicate content from the article.')
- parser.add_argument('--input-source', type=str, choices=['console', 'file'],
- help='Where does the input come from.')
- parser.add_argument('--input-path', type=str,
- help='Using csv file to store topic and ground truth url at present.')
- parser.add_argument('--output-dir', type=str, default='../results',
- help='Directory to store the outputs.')
- parser.add_argument('--engine', type=str, required=True, choices=['gpt-4', 'gpt-35-turbo'])
-
- main(parser.parse_args())