An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Added tavily code
zenith1101902903f5e6b
diff --git a/README.md b/README.mdindex 88435eb..865cac9 100644--- a/README.md+++ b/README.md@@ -101,7 +101,7 @@ runner = STORMWikiRunner(engine_args, lm_configs, rm)Currently, our package support:- `OpenAIModel`, `AzureOpenAIModel`, `ClaudeModel`, `VLLMClient`, `TGIClient`, `TogetherClient`, `OllamaClient`, `GoogleModel`, `DeepSeekModel`, `GroqModel` as language model components-- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, and `DuckDuckGoSearchRM` as retrieval module components+- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, `DuckDuckGoSearchRM`, and `TavilySearchRM` as retrieval module components:star2: **PRs for integrating more language models into [knowledge_storm/lm.py](knowledge_storm/lm.py) and search engines/retrievers into [knowledge_storm/rm.py](knowledge_storm/rm.py) are highly appreciated!**diff --git a/examples/run_storm_wiki_claude.py b/examples/run_storm_wiki_claude.pyindex d80d793..1da12c0 100644--- a/examples/run_storm_wiki_claude.py+++ b/examples/run_storm_wiki_claude.py@@ -2,7 +2,7 @@STORM Wiki pipeline powered by Claude family models and You.com search engine.You need to set up the following environment variables to run this script:- ANTHROPIC_API_KEY: Anthropic API key- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyOutput will be structured as belowargs.output_dir/@@ -21,7 +21,7 @@ from argparse import ArgumentParserfrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import ClaudeModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key@@ -72,8 +72,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -98,7 +102,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_deepseek.py b/examples/run_storm_wiki_deepseek.pyindex b20b185..e4cf5fc 100644--- a/examples/run_storm_wiki_deepseek.py+++ b/examples/run_storm_wiki_deepseek.py@@ -3,7 +3,7 @@ STORM Wiki pipeline powered by DeepSeek models and You.com or Bing search engineYou need to set up the following environment variables to run this script:- DEEPSEEK_API_KEY: DeepSeek API key- DEEPSEEK_API_BASE: DeepSeek API base URL (default is https://api.deepseek.com)- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyOutput will be structured as belowargs.output_dir/@@ -18,14 +18,13 @@ args.output_dir/"""import os-import sysimport reimport loggingfrom argparse import ArgumentParserfrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import DeepSeekModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key@@ -99,8 +98,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -132,7 +135,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'], required=True,+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')diff --git a/examples/run_storm_wiki_gemini.py b/examples/run_storm_wiki_gemini.pyindex 137ff62..d574062 100644--- a/examples/run_storm_wiki_gemini.py+++ b/examples/run_storm_wiki_gemini.py@@ -2,7 +2,7 @@STORM Wiki pipeline powered by Google Gemini models and search engine.You need to set up the following environment variables to run this script:- GOOGLE_API_KEY: Google API key (Can be obtained from https://ai.google.dev/gemini-api/docs/api-key)- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyOutput will be structured as belowargs.output_dir/@@ -21,10 +21,9 @@ from argparse import ArgumentParserfrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import GoogleModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key-def main(args):load_api_key(toml_file_path='secrets.toml')lm_configs = STORMWikiLMConfigs()@@ -74,8 +73,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -100,7 +103,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_gpt.py b/examples/run_storm_wiki_gpt.pyindex 7292cdc..ac07968 100644--- a/examples/run_storm_wiki_gpt.py+++ b/examples/run_storm_wiki_gpt.py@@ -5,7 +5,7 @@ You need to set up the following environment variables to run this script:- OPENAI_API_TYPE: OpenAI API type (e.g., 'openai' or 'azure')- AZURE_API_BASE: Azure API base URL if using Azure API- AZURE_API_VERSION: Azure API version if using Azure API- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyOutput will be structured as belowargs.output_dir/@@ -21,10 +21,9 @@ args.output_dir/import osfrom argparse import ArgumentParser-from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import OpenAIModel, AzureOpenAIModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key@@ -84,8 +83,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -110,7 +113,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_groq.py b/examples/run_storm_wiki_groq.pyindex 788fe42..55419c2 100644--- a/examples/run_storm_wiki_groq.py+++ b/examples/run_storm_wiki_groq.py@@ -2,7 +2,7 @@STORM Wiki pipeline powered by llama3-70b-8192 hosted by Groq server and You.com search engine.You need to set up the following environment variables to run this script:- GROQ_API_KEY: You can get your Groq API Key at https://console.groq.com/keys- - YDC_API_KEY: You.com API key; or, BING_SEARCH_API_KEY: Bing Search API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyYou also need to have a VLLM server running with the Mistral-7B-Instruct-v0.2 model. Specify `--url` and `--port` accordingly.Output will be structured as below@@ -33,7 +33,7 @@ sys.path.insert(0, lm_path)# Now import lm directlyimport lmfrom lm import GroqModel-from knowledge_storm.rm import YouRM, BingSearch, BraveRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key@@ -93,14 +93,23 @@ def main(args):# STORM is a knowledge curation system which consumes information from the retrieval module.# Currently, the information source is the Internet and we use search engine API as the retrieval module.- if args.retriever == 'bing':- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)- elif args.retriever == 'you':- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)- elif args.retriever == 'brave':- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)- else:- raise ValueError(f"Invalid retriever: {args.retriever}. Choose either 'bing', 'you', or 'brave'.")+ match args.retriever:+ case 'bing':+ rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)+ case 'you':+ rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)+ case 'brave':+ rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)+ case 'duckduckgo':+ rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')+ case 'serper':+ rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)+ case _:+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -132,7 +141,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'], required=True,+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')parser.add_argument('--temperature', type=float, default=1.0,help='Sampling temperature to use.')diff --git a/examples/run_storm_wiki_mistral.py b/examples/run_storm_wiki_mistral.pyindex 1b7d589..ceabd47 100644--- a/examples/run_storm_wiki_mistral.py+++ b/examples/run_storm_wiki_mistral.py@@ -1,7 +1,7 @@"""STORM Wiki pipeline powered by Mistral-7B-Instruct-v0.2 hosted by VLLM server and You.com search engine.You need to set up the following environment variables to run this script:- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyYou also need to have a VLLM server running with the Mistral-7B-Instruct-v0.2 model. Specify `--url` and `--port` accordingly.Output will be structured as below@@ -22,7 +22,7 @@ from dspy import Examplefrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.lm import VLLMClient-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm.utils import load_api_key@@ -70,8 +70,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -156,7 +160,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/examples/run_storm_wiki_ollama.py b/examples/run_storm_wiki_ollama.pyindex 62e4e7b..519467b 100644--- a/examples/run_storm_wiki_ollama.py+++ b/examples/run_storm_wiki_ollama.py@@ -1,7 +1,7 @@"""STORM Wiki pipeline powered by local model hosted by Ollama server and You.com or Bing search engine.You need to set up the following environment variables to run this script:- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API keyYou also need to have a Ollama server running with the llama3 model or other. Specify `--url`, `--port` and `--model` accordingly.Output will be structured as below@@ -23,7 +23,7 @@ from dspy import Examplesys.path.append('./src')from knowledge_storm.lm import OllamaClient-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNGfrom knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.utils import load_api_key@@ -72,8 +72,12 @@ def main(args):rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')case 'serper':rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})+ case 'tavily':+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)+ case 'searxng':+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)case _:- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')runner = STORMWikiRunner(engine_args, lm_configs, rm)@@ -160,7 +164,7 @@ if __name__ == '__main__':help='Maximum number of threads to use. The information seeking part and the article generation''part can speed up by using multiple threads. Consider reducing it if keep getting ''"Exceed rate limit" error when calling LM API.')- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],help='The search engine API to use for retrieving information.')# stage of the pipelineparser.add_argument('--do-research', action='store_true',diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 0fc9d7f..85d1250 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -691,3 +691,121 @@ class DuckDuckGoSearchRM(dspy.Retrieve):return collected_results+class TavilySearchRM(dspy.Retrieve):+ """Retrieve information from custom queries using Tavily."""+ def __init__(+ self,+ tavily_search_api_key=None,+ k: int = 3,+ is_valid_source: Callable = None,+ min_char_count: int = 150,+ snippet_chunk_size: int = 1000,+ webpage_helper_max_threads=10,+ include_raw_content = False+ ):+ """+ Params:+ min_char_count: Minimum character count for the article to be considered valid.+ snippet_chunk_size: Maximum character count for each snippet.+ webpage_helper_max_threads: Maximum number of threads to use for webpage helper.+ **kwargs: Additional parameters for the OpenAI API.+ """+ super().__init__(k=k)+ try:+ from tavily import TavilyClient+ except ImportError as err:+ raise ImportError("Tavily requires `pip install tavily-python`.") from err++ if not tavily_search_api_key and not os.environ.get("TAVILY_API_KEY"):+ raise RuntimeError("You must supply tavily_search_api_key or set environment variable TAVILY_API_KEY")+ elif tavily_search_api_key:+ self.tavily_search_api_key = tavily_search_api_key+ else:+ self.tavily_search_api_key = os.environ["TAVILY_API_KEY"]++ self.k = k+ self.webpage_helper = WebPageHelper(+ min_char_count=min_char_count,+ snippet_chunk_size=snippet_chunk_size,+ max_thread_num=webpage_helper_max_threads,+ )+ self.usage = 0++ self.tavily_client = TavilyClient(api_key=self.tavily_search_api_key)+ self.include_raw_content = include_raw_content+ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.+ if is_valid_source:+ self.is_valid_source = is_valid_source+ else:+ self.is_valid_source = lambda x: True+++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0+ return {'TavilySearchRM': usage}++ def forward(+ self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []+ ):+ """Search with TavilySearch for self.k top passages for query or queries+ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.+ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)++ collected_results = []++ for query in queries:+ args = {+ "max_results": self.k,+ "include_raw_contents": self.include_raw_content+ }+ # list of dicts that will be parsed to return+ responseData = self.tavily_client.search(query)+ results = responseData.get("results")+ for d in results:+ # assert d is dict+ if not isinstance(d, dict):+ print(f'Invalid result: {d}\n')+ continue++ try:+ # ensure keys are present+ url = d.get('url', None)+ title = d.get('title', None)+ description = d.get('content', None)+ snippets = []+ if(d.get('raw_body_content')):+ snippets.append(d.get('raw_body_content'))+ else:+ snippets.append(d.get('content'))++ # raise exception of missing key(s)+ if not all([url, title, description, snippets]):+ raise ValueError(f'Missing key(s) in result: {d}')+ if self.is_valid_source(url) and url not in exclude_urls:+ result = {+ 'url': url,+ 'title': title,+ 'description': description,+ 'snippets': snippets,+ }+ collected_results.append(result)+ else:+ print(f'invalid source {url} or url in exclude_urls')+ except Exception as e:+ print(f'Error occurs when processing {result=}: {e}\n')+ print(f'Error occurs when searching query {query}: {e}')++ return collected_results++