Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

feat: add Tavily web search provider alongside Exa (from #29) (#43)

Jordan Gibbsedf80b6b3b13
diff --git a/.gitignore b/.gitignore
index de8e6ad..ebd2b90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ GEMINI.md
# Per-vault research output
research/
+.knack/
diff --git a/pyproject.toml b/pyproject.toml
index eea22b9..6358200 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,8 +37,9 @@ dependencies = [
crawl4ai = ["Crawl4AI>=0.4"]
mcp = ["mcp>=1.6"]
exa = ["exa-py>=2.0.0"]
+tavily = ["tavily-python>=0.3"]
watch = ["watchdog>=4.0"]
-all = ["hyperresearch[crawl4ai,mcp,exa,watch]"]
+all = ["hyperresearch[crawl4ai,mcp,exa,tavily,watch]"]
dev = [
"pytest>=7.4",
"pytest-cov>=4.1",
diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.py
index a518043..cf95a4f 100644
--- a/src/hyperresearch/web/base.py
+++ b/src/hyperresearch/web/base.py
@@ -191,4 +191,12 @@ def get_provider(
return ExaProvider()
- raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa")
+ if name == "tavily":
+ try:
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ return TavilyProvider()
+ except ImportError:
+ raise ImportError("tavily provider requires: pip install \"hyperresearch[tavily]\"")
+
+ raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily")
diff --git a/src/hyperresearch/web/tavily_provider.py b/src/hyperresearch/web/tavily_provider.py
new file mode 100644
index 0000000..9e00d18
--- /dev/null
+++ b/src/hyperresearch/web/tavily_provider.py
@@ -0,0 +1,104 @@
+"""Tavily web provider — search API designed for LLMs and AI agents.
+
+Tavily (https://tavily.com) provides web search and content extraction optimized
+for retrieval-augmented generation workflows.
+
+Configuration:
+ export TAVILY_API_KEY="tvly-your-api-key" # https://app.tavily.com
+
+ # in .hyperresearch/config.toml
+ [web]
+ provider = "tavily"
+
+Optional install:
+ pip install "hyperresearch[tavily]"
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime
+from typing import Any
+
+from hyperresearch.web.base import WebResult
+
+
+class TavilyProvider:
+ """Web provider backed by the Tavily search API.
+
+ Supports both `search` (web search) and `fetch` (URL content extraction).
+ """
+
+ name = "tavily"
+
+ def __init__(
+ self,
+ api_key: str | None = None,
+ search_depth: str = "advanced",
+ topic: str = "general",
+ include_domains: list[str] | None = None,
+ exclude_domains: list[str] | None = None,
+ ):
+ try:
+ from tavily import TavilyClient
+ except ImportError as exc:
+ raise ImportError(
+ 'tavily provider requires: pip install "hyperresearch[tavily]"'
+ ) from exc
+
+ key = api_key or os.environ.get("TAVILY_API_KEY", "").strip()
+ if not key:
+ raise RuntimeError(
+ "TAVILY_API_KEY is not set. Get a free key at "
+ "https://app.tavily.com and export it."
+ )
+
+ self._client = TavilyClient(api_key=key)
+ self._search_depth = search_depth
+ self._topic = topic
+ self._include_domains = include_domains
+ self._exclude_domains = exclude_domains
+
+ def search(self, query: str, max_results: int = 5) -> list[WebResult]:
+ """Search the web via Tavily and return ranked results with content."""
+ kwargs: dict[str, Any] = {
+ "max_results": max_results,
+ "search_depth": self._search_depth,
+ "topic": self._topic,
+ }
+ if self._include_domains:
+ kwargs["include_domains"] = self._include_domains
+ if self._exclude_domains:
+ kwargs["exclude_domains"] = self._exclude_domains
+
+ response = self._client.search(query, **kwargs)
+ return [_to_web_result(r) for r in response.get("results", [])]
+
+ def fetch(self, url: str) -> WebResult:
+ """Fetch a single URL via Tavily extract and return clean text."""
+ response = self._client.extract(urls=[url])
+ results = response.get("results", [])
+ if not results:
+ raise RuntimeError(f"Tavily returned no contents for {url}")
+ return _to_web_result(results[0])
+
+
+def _to_web_result(item: dict[str, Any]) -> WebResult:
+ """Convert a Tavily result dict into a hyperresearch WebResult."""
+ content = item.get("content", "") or item.get("raw_content", "") or ""
+
+ metadata: dict[str, Any] = {}
+ score = item.get("score")
+ if score is not None:
+ metadata["score"] = score
+ published_date = item.get("published_date")
+ if published_date:
+ metadata["published_date"] = published_date
+
+ return WebResult(
+ url=item.get("url", ""),
+ title=item.get("title", "") or "",
+ content=content,
+ fetched_at=datetime.now(UTC),
+ metadata=metadata,
+ )
diff --git a/tests/test_web/test_tavily_provider.py b/tests/test_web/test_tavily_provider.py
new file mode 100644
index 0000000..3993122
--- /dev/null
+++ b/tests/test_web/test_tavily_provider.py
@@ -0,0 +1,135 @@
+"""Tests for the Tavily web provider — stubs the tavily-python SDK.
+
+Unlike the exa tests (which patch the real `exa_py` module and therefore need
+it installed), these inject a fake `tavily` module into sys.modules, so the
+suite passes without `tavily-python` in the dev extra.
+"""
+
+from __future__ import annotations
+
+import sys
+from types import ModuleType
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+
+
+def _install_fake_tavily(monkeypatch: pytest.MonkeyPatch, client: MagicMock) -> None:
+ fake = ModuleType("tavily")
+ fake.TavilyClient = MagicMock(return_value=client) # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "tavily", fake)
+ # The provider module may have been imported by an earlier test; force a
+ # fresh import so its `from tavily import TavilyClient` binds the fake.
+ sys.modules.pop("hyperresearch.web.tavily_provider", None)
+
+
+def _search_item(**overrides: Any) -> dict:
+ item = {
+ "url": "https://example.com/article",
+ "title": "Example Article",
+ "content": "Snippet of the article body.",
+ "score": 0.91,
+ "published_date": "2026-06-01",
+ }
+ item.update(overrides)
+ return item
+
+
+def test_provider_registered_via_factory(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
+ _install_fake_tavily(monkeypatch, MagicMock())
+
+ from hyperresearch.web.base import get_provider
+
+ prov = get_provider("tavily")
+ assert prov.name == "tavily"
+
+
+def test_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("TAVILY_API_KEY", raising=False)
+ _install_fake_tavily(monkeypatch, MagicMock())
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ with pytest.raises(RuntimeError, match="TAVILY_API_KEY"):
+ TavilyProvider()
+
+
+def test_search_maps_results_to_webresult(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = MagicMock()
+ client.search.return_value = {"results": [_search_item()]}
+ _install_fake_tavily(monkeypatch, client)
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ results = TavilyProvider(api_key="tvly-test").search("solid state batteries")
+ assert len(results) == 1
+ r = results[0]
+ assert r.url == "https://example.com/article"
+ assert r.title == "Example Article"
+ assert r.content == "Snippet of the article body."
+ assert r.metadata["score"] == 0.91
+ assert r.metadata["published_date"] == "2026-06-01"
+
+
+def test_search_passes_configured_options(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = MagicMock()
+ client.search.return_value = {"results": []}
+ _install_fake_tavily(monkeypatch, client)
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ prov = TavilyProvider(
+ api_key="tvly-test",
+ search_depth="basic",
+ include_domains=["nature.com"],
+ exclude_domains=["pinterest.com"],
+ )
+ prov.search("q", max_results=7)
+
+ kwargs = client.search.call_args.kwargs
+ assert kwargs["max_results"] == 7
+ assert kwargs["search_depth"] == "basic"
+ assert kwargs["include_domains"] == ["nature.com"]
+ assert kwargs["exclude_domains"] == ["pinterest.com"]
+
+
+def test_fetch_uses_extract_and_raw_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = MagicMock()
+ client.extract.return_value = {
+ "results": [
+ {"url": "https://example.com/p", "raw_content": "Full extracted page text."}
+ ]
+ }
+ _install_fake_tavily(monkeypatch, client)
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ r = TavilyProvider(api_key="tvly-test").fetch("https://example.com/p")
+ assert r.content == "Full extracted page text."
+ assert r.title == ""
+
+
+def test_fetch_no_results_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = MagicMock()
+ client.extract.return_value = {"results": []}
+ _install_fake_tavily(monkeypatch, client)
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ with pytest.raises(RuntimeError, match="no contents"):
+ TavilyProvider(api_key="tvly-test").fetch("https://example.com/p")
+
+
+def test_content_falls_back_to_raw_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = MagicMock()
+ client.search.return_value = {
+ "results": [_search_item(content="", raw_content="Raw body wins.")]
+ }
+ _install_fake_tavily(monkeypatch, client)
+
+ from hyperresearch.web.tavily_provider import TavilyProvider
+
+ results = TavilyProvider(api_key="tvly-test").search("q")
+ assert results[0].content == "Raw body wins."