Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

Added tavily code

zenith1101902903f5e6b
diff --git a/README.md b/README.md
index 88435eb..865cac9 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,7 @@ runner = STORMWikiRunner(engine_args, lm_configs, rm)
Currently, our package support:
- `OpenAIModel`, `AzureOpenAIModel`, `ClaudeModel`, `VLLMClient`, `TGIClient`, `TogetherClient`, `OllamaClient`, `GoogleModel`, `DeepSeekModel`, `GroqModel` as language model components
-- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, and `DuckDuckGoSearchRM` as retrieval module components
+- `YouRM`, `BingSearch`, `VectorRM`, `SerperRM`, `BraveRM`, `SearXNG`, `DuckDuckGoSearchRM`, and `TavilySearchRM` as retrieval module components
:star2: **PRs for integrating more language models into [knowledge_storm/lm.py](knowledge_storm/lm.py) and search engines/retrievers into [knowledge_storm/rm.py](knowledge_storm/rm.py) are highly appreciated!**
diff --git a/examples/run_storm_wiki_claude.py b/examples/run_storm_wiki_claude.py
index d80d793..1da12c0 100644
--- a/examples/run_storm_wiki_claude.py
+++ b/examples/run_storm_wiki_claude.py
@@ -2,7 +2,7 @@
STORM Wiki pipeline powered by Claude family models and You.com search engine.
You need to set up the following environment variables to run this script:
- ANTHROPIC_API_KEY: Anthropic API key
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
Output will be structured as below
args.output_dir/
@@ -21,7 +21,7 @@ from argparse import ArgumentParser
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.lm import ClaudeModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
@@ -72,8 +72,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -98,7 +102,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_deepseek.py b/examples/run_storm_wiki_deepseek.py
index b20b185..e4cf5fc 100644
--- a/examples/run_storm_wiki_deepseek.py
+++ b/examples/run_storm_wiki_deepseek.py
@@ -3,7 +3,7 @@ STORM Wiki pipeline powered by DeepSeek models and You.com or Bing search engine
You need to set up the following environment variables to run this script:
- DEEPSEEK_API_KEY: DeepSeek API key
- DEEPSEEK_API_BASE: DeepSeek API base URL (default is https://api.deepseek.com)
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
Output will be structured as below
args.output_dir/
@@ -18,14 +18,13 @@ args.output_dir/
"""
import os
-import sys
import re
import logging
from argparse import ArgumentParser
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.lm import DeepSeekModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
@@ -99,8 +98,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -132,7 +135,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'], required=True,
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
parser.add_argument('--model', type=str, choices=['deepseek-chat', 'deepseek-coder'], default='deepseek-chat',
help='DeepSeek model to use. "deepseek-chat" for general tasks, "deepseek-coder" for coding tasks.')
diff --git a/examples/run_storm_wiki_gemini.py b/examples/run_storm_wiki_gemini.py
index 137ff62..d574062 100644
--- a/examples/run_storm_wiki_gemini.py
+++ b/examples/run_storm_wiki_gemini.py
@@ -2,7 +2,7 @@
STORM Wiki pipeline powered by Google Gemini models and search engine.
You need to set up the following environment variables to run this script:
- GOOGLE_API_KEY: Google API key (Can be obtained from https://ai.google.dev/gemini-api/docs/api-key)
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
Output will be structured as below
args.output_dir/
@@ -21,10 +21,9 @@ from argparse import ArgumentParser
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.lm import GoogleModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
-
def main(args):
load_api_key(toml_file_path='secrets.toml')
lm_configs = STORMWikiLMConfigs()
@@ -74,8 +73,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -100,7 +103,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_gpt.py b/examples/run_storm_wiki_gpt.py
index 7292cdc..ac07968 100644
--- a/examples/run_storm_wiki_gpt.py
+++ b/examples/run_storm_wiki_gpt.py
@@ -5,7 +5,7 @@ You need to set up the following environment variables to run this script:
- OPENAI_API_TYPE: OpenAI API type (e.g., 'openai' or 'azure')
- AZURE_API_BASE: Azure API base URL if using Azure API
- AZURE_API_VERSION: Azure API version if using Azure API
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
Output will be structured as below
args.output_dir/
@@ -21,10 +21,9 @@ args.output_dir/
import os
from argparse import ArgumentParser
-
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
@@ -84,8 +83,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -110,7 +113,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_groq.py b/examples/run_storm_wiki_groq.py
index 788fe42..55419c2 100644
--- a/examples/run_storm_wiki_groq.py
+++ b/examples/run_storm_wiki_groq.py
@@ -2,7 +2,7 @@
STORM Wiki pipeline powered by llama3-70b-8192 hosted by Groq server and You.com search engine.
You need to set up the following environment variables to run this script:
- GROQ_API_KEY: You can get your Groq API Key at https://console.groq.com/keys
- - YDC_API_KEY: You.com API key; or, BING_SEARCH_API_KEY: Bing Search API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
You also need to have a VLLM server running with the Mistral-7B-Instruct-v0.2 model. Specify `--url` and `--port` accordingly.
Output will be structured as below
@@ -33,7 +33,7 @@ sys.path.insert(0, lm_path)
# Now import lm directly
import lm
from lm import GroqModel
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
@@ -93,14 +93,23 @@ def main(args):
# STORM is a knowledge curation system which consumes information from the retrieval module.
# Currently, the information source is the Internet and we use search engine API as the retrieval module.
- if args.retriever == 'bing':
- rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
- elif args.retriever == 'you':
- rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
- elif args.retriever == 'brave':
- rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
- else:
- raise ValueError(f"Invalid retriever: {args.retriever}. Choose either 'bing', 'you', or 'brave'.")
+ match args.retriever:
+ case 'bing':
+ rm = BingSearch(bing_search_api=os.getenv('BING_SEARCH_API_KEY'), k=engine_args.search_top_k)
+ case 'you':
+ rm = YouRM(ydc_api_key=os.getenv('YDC_API_KEY'), k=engine_args.search_top_k)
+ case 'brave':
+ rm = BraveRM(brave_search_api_key=os.getenv('BRAVE_API_KEY'), k=engine_args.search_top_k)
+ case 'duckduckgo':
+ rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
+ case 'serper':
+ rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
+ case _:
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -132,7 +141,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave'], required=True,
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
parser.add_argument('--temperature', type=float, default=1.0,
help='Sampling temperature to use.')
diff --git a/examples/run_storm_wiki_mistral.py b/examples/run_storm_wiki_mistral.py
index 1b7d589..ceabd47 100644
--- a/examples/run_storm_wiki_mistral.py
+++ b/examples/run_storm_wiki_mistral.py
@@ -1,7 +1,7 @@
"""
STORM Wiki pipeline powered by Mistral-7B-Instruct-v0.2 hosted by VLLM server and You.com search engine.
You need to set up the following environment variables to run this script:
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
You also need to have a VLLM server running with the Mistral-7B-Instruct-v0.2 model. Specify `--url` and `--port` accordingly.
Output will be structured as below
@@ -22,7 +22,7 @@ from dspy import Example
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.lm import VLLMClient
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm.utils import load_api_key
@@ -70,8 +70,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -156,7 +160,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/examples/run_storm_wiki_ollama.py b/examples/run_storm_wiki_ollama.py
index 62e4e7b..519467b 100644
--- a/examples/run_storm_wiki_ollama.py
+++ b/examples/run_storm_wiki_ollama.py
@@ -1,7 +1,7 @@
"""
STORM Wiki pipeline powered by local model hosted by Ollama server and You.com or Bing search engine.
You need to set up the following environment variables to run this script:
- - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, or BRAVE_API_KEY: Brave API key
+ - YDC_API_KEY: You.com API key; BING_SEARCH_API_KEY: Bing Search API key, SERPER_API_KEY: Serper API key, BRAVE_API_KEY: Brave API key, or TAVILY_API_KEY: Tavily API key
You also need to have a Ollama server running with the llama3 model or other. Specify `--url`, `--port` and `--model` accordingly.
Output will be structured as below
@@ -23,7 +23,7 @@ from dspy import Example
sys.path.append('./src')
from knowledge_storm.lm import OllamaClient
-from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM
+from knowledge_storm.rm import YouRM, BingSearch, BraveRM, SerperRM, DuckDuckGoSearchRM, TavilySearchRM, SearXNG
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.utils import load_api_key
@@ -72,8 +72,12 @@ def main(args):
rm = DuckDuckGoSearchRM(k=engine_args.search_top_k, safe_search='On', region='us-en')
case 'serper':
rm = SerperRM(serper_search_api_key=os.getenv('SERPER_API_KEY'), query_params={'autocorrect': True, 'num': 10, 'page': 1})
+ case 'tavily':
+ rm = TavilySearchRM(tavily_search_api_key=os.getenv('TAVILY_API_KEY'), k=engine_args.search_top_k, include_raw_content=True)
+ case 'searxng':
+ rm = SearXNG(searxng_api_key=os.getenv('SEARXNG_API_KEY'), k=engine_args.search_top_k)
case _:
- raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", or "serper"')
+ raise ValueError(f'Invalid retriever: {args.retriever}. Choose either "bing", "you", "brave", "duckduckgo", "serper", "tavily", or "searxng"')
runner = STORMWikiRunner(engine_args, lm_configs, rm)
@@ -160,7 +164,7 @@ if __name__ == '__main__':
help='Maximum number of threads to use. The information seeking part and the article generation'
'part can speed up by using multiple threads. Consider reducing it if keep getting '
'"Exceed rate limit" error when calling LM API.')
- parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo'],
+ parser.add_argument('--retriever', type=str, choices=['bing', 'you', 'brave', 'serper', 'duckduckgo', 'tavily', 'searxng'],
help='The search engine API to use for retrieving information.')
# stage of the pipeline
parser.add_argument('--do-research', action='store_true',
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.py
index 0fc9d7f..85d1250 100644
--- a/knowledge_storm/rm.py
+++ b/knowledge_storm/rm.py
@@ -691,3 +691,121 @@ class DuckDuckGoSearchRM(dspy.Retrieve):
return collected_results
+class TavilySearchRM(dspy.Retrieve):
+ """Retrieve information from custom queries using Tavily."""
+ def __init__(
+ self,
+ tavily_search_api_key=None,
+ k: int = 3,
+ is_valid_source: Callable = None,
+ min_char_count: int = 150,
+ snippet_chunk_size: int = 1000,
+ webpage_helper_max_threads=10,
+ include_raw_content = False
+ ):
+ """
+ Params:
+ min_char_count: Minimum character count for the article to be considered valid.
+ snippet_chunk_size: Maximum character count for each snippet.
+ webpage_helper_max_threads: Maximum number of threads to use for webpage helper.
+ **kwargs: Additional parameters for the OpenAI API.
+ """
+ super().__init__(k=k)
+ try:
+ from tavily import TavilyClient
+ except ImportError as err:
+ raise ImportError("Tavily requires `pip install tavily-python`.") from err
+
+ if not tavily_search_api_key and not os.environ.get("TAVILY_API_KEY"):
+ raise RuntimeError("You must supply tavily_search_api_key or set environment variable TAVILY_API_KEY")
+ elif tavily_search_api_key:
+ self.tavily_search_api_key = tavily_search_api_key
+ else:
+ self.tavily_search_api_key = os.environ["TAVILY_API_KEY"]
+
+ self.k = k
+ self.webpage_helper = WebPageHelper(
+ min_char_count=min_char_count,
+ snippet_chunk_size=snippet_chunk_size,
+ max_thread_num=webpage_helper_max_threads,
+ )
+ self.usage = 0
+
+ self.tavily_client = TavilyClient(api_key=self.tavily_search_api_key)
+ self.include_raw_content = include_raw_content
+ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.
+ if is_valid_source:
+ self.is_valid_source = is_valid_source
+ else:
+ self.is_valid_source = lambda x: True
+
+
+ def get_usage_and_reset(self):
+ usage = self.usage
+ self.usage = 0
+ return {'TavilySearchRM': usage}
+
+ def forward(
+ self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []
+ ):
+ """Search with TavilySearch for self.k top passages for query or queries
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ exclude_urls (List[str]): A list of urls to exclude from the search results.
+ Returns:
+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ self.usage += len(queries)
+
+ collected_results = []
+
+ for query in queries:
+ args = {
+ "max_results": self.k,
+ "include_raw_contents": self.include_raw_content
+ }
+ # list of dicts that will be parsed to return
+ responseData = self.tavily_client.search(query)
+ results = responseData.get("results")
+ for d in results:
+ # assert d is dict
+ if not isinstance(d, dict):
+ print(f'Invalid result: {d}\n')
+ continue
+
+ try:
+ # ensure keys are present
+ url = d.get('url', None)
+ title = d.get('title', None)
+ description = d.get('content', None)
+ snippets = []
+ if(d.get('raw_body_content')):
+ snippets.append(d.get('raw_body_content'))
+ else:
+ snippets.append(d.get('content'))
+
+ # raise exception of missing key(s)
+ if not all([url, title, description, snippets]):
+ raise ValueError(f'Missing key(s) in result: {d}')
+ if self.is_valid_source(url) and url not in exclude_urls:
+ result = {
+ 'url': url,
+ 'title': title,
+ 'description': description,
+ 'snippets': snippets,
+ }
+ collected_results.append(result)
+ else:
+ print(f'invalid source {url} or url in exclude_urls')
+ except Exception as e:
+ print(f'Error occurs when processing {result=}: {e}\n')
+ print(f'Error occurs when searching query {query}: {e}')
+
+ return collected_results
+
+