Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
feat: add Tavily web search provider alongside Exa (from #29) (#43)
Jordan Gibbsedf80b6b3b13
diff --git a/.gitignore b/.gitignoreindex de8e6ad..ebd2b90 100644--- a/.gitignore+++ b/.gitignore@@ -29,3 +29,4 @@ GEMINI.md# Per-vault research outputresearch/+.knack/diff --git a/pyproject.toml b/pyproject.tomlindex eea22b9..6358200 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -37,8 +37,9 @@ dependencies = [crawl4ai = ["Crawl4AI>=0.4"]mcp = ["mcp>=1.6"]exa = ["exa-py>=2.0.0"]+tavily = ["tavily-python>=0.3"]watch = ["watchdog>=4.0"]-all = ["hyperresearch[crawl4ai,mcp,exa,watch]"]+all = ["hyperresearch[crawl4ai,mcp,exa,tavily,watch]"]dev = ["pytest>=7.4","pytest-cov>=4.1",diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.pyindex a518043..cf95a4f 100644--- a/src/hyperresearch/web/base.py+++ b/src/hyperresearch/web/base.py@@ -191,4 +191,12 @@ def get_provider(return ExaProvider()- raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa")+ if name == "tavily":+ try:+ from hyperresearch.web.tavily_provider import TavilyProvider++ return TavilyProvider()+ except ImportError:+ raise ImportError("tavily provider requires: pip install \"hyperresearch[tavily]\"")++ raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily")diff --git a/src/hyperresearch/web/tavily_provider.py b/src/hyperresearch/web/tavily_provider.pynew file mode 100644index 0000000..9e00d18--- /dev/null+++ b/src/hyperresearch/web/tavily_provider.py@@ -0,0 +1,104 @@+"""Tavily web provider — search API designed for LLMs and AI agents.++Tavily (https://tavily.com) provides web search and content extraction optimized+for retrieval-augmented generation workflows.++Configuration:+ export TAVILY_API_KEY="tvly-your-api-key" # https://app.tavily.com++ # in .hyperresearch/config.toml+ [web]+ provider = "tavily"++Optional install:+ pip install "hyperresearch[tavily]"+"""++from __future__ import annotations++import os+from datetime import UTC, datetime+from typing import Any++from hyperresearch.web.base import WebResult+++class TavilyProvider:+ """Web provider backed by the Tavily search API.++ Supports both `search` (web search) and `fetch` (URL content extraction).+ """++ name = "tavily"++ def __init__(+ self,+ api_key: str | None = None,+ search_depth: str = "advanced",+ topic: str = "general",+ include_domains: list[str] | None = None,+ exclude_domains: list[str] | None = None,+ ):+ try:+ from tavily import TavilyClient+ except ImportError as exc:+ raise ImportError(+ 'tavily provider requires: pip install "hyperresearch[tavily]"'+ ) from exc++ key = api_key or os.environ.get("TAVILY_API_KEY", "").strip()+ if not key:+ raise RuntimeError(+ "TAVILY_API_KEY is not set. Get a free key at "+ "https://app.tavily.com and export it."+ )++ self._client = TavilyClient(api_key=key)+ self._search_depth = search_depth+ self._topic = topic+ self._include_domains = include_domains+ self._exclude_domains = exclude_domains++ def search(self, query: str, max_results: int = 5) -> list[WebResult]:+ """Search the web via Tavily and return ranked results with content."""+ kwargs: dict[str, Any] = {+ "max_results": max_results,+ "search_depth": self._search_depth,+ "topic": self._topic,+ }+ if self._include_domains:+ kwargs["include_domains"] = self._include_domains+ if self._exclude_domains:+ kwargs["exclude_domains"] = self._exclude_domains++ response = self._client.search(query, **kwargs)+ return [_to_web_result(r) for r in response.get("results", [])]++ def fetch(self, url: str) -> WebResult:+ """Fetch a single URL via Tavily extract and return clean text."""+ response = self._client.extract(urls=[url])+ results = response.get("results", [])+ if not results:+ raise RuntimeError(f"Tavily returned no contents for {url}")+ return _to_web_result(results[0])+++def _to_web_result(item: dict[str, Any]) -> WebResult:+ """Convert a Tavily result dict into a hyperresearch WebResult."""+ content = item.get("content", "") or item.get("raw_content", "") or ""++ metadata: dict[str, Any] = {}+ score = item.get("score")+ if score is not None:+ metadata["score"] = score+ published_date = item.get("published_date")+ if published_date:+ metadata["published_date"] = published_date++ return WebResult(+ url=item.get("url", ""),+ title=item.get("title", "") or "",+ content=content,+ fetched_at=datetime.now(UTC),+ metadata=metadata,+ )diff --git a/tests/test_web/test_tavily_provider.py b/tests/test_web/test_tavily_provider.pynew file mode 100644index 0000000..3993122--- /dev/null+++ b/tests/test_web/test_tavily_provider.py@@ -0,0 +1,135 @@+"""Tests for the Tavily web provider — stubs the tavily-python SDK.++Unlike the exa tests (which patch the real `exa_py` module and therefore need+it installed), these inject a fake `tavily` module into sys.modules, so the+suite passes without `tavily-python` in the dev extra.+"""++from __future__ import annotations++import sys+from types import ModuleType+from typing import Any+from unittest.mock import MagicMock++import pytest+++def _install_fake_tavily(monkeypatch: pytest.MonkeyPatch, client: MagicMock) -> None:+ fake = ModuleType("tavily")+ fake.TavilyClient = MagicMock(return_value=client) # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, "tavily", fake)+ # The provider module may have been imported by an earlier test; force a+ # fresh import so its `from tavily import TavilyClient` binds the fake.+ sys.modules.pop("hyperresearch.web.tavily_provider", None)+++def _search_item(**overrides: Any) -> dict:+ item = {+ "url": "https://example.com/article",+ "title": "Example Article",+ "content": "Snippet of the article body.",+ "score": 0.91,+ "published_date": "2026-06-01",+ }+ item.update(overrides)+ return item+++def test_provider_registered_via_factory(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")+ _install_fake_tavily(monkeypatch, MagicMock())++ from hyperresearch.web.base import get_provider++ prov = get_provider("tavily")+ assert prov.name == "tavily"+++def test_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("TAVILY_API_KEY", raising=False)+ _install_fake_tavily(monkeypatch, MagicMock())++ from hyperresearch.web.tavily_provider import TavilyProvider++ with pytest.raises(RuntimeError, match="TAVILY_API_KEY"):+ TavilyProvider()+++def test_search_maps_results_to_webresult(monkeypatch: pytest.MonkeyPatch) -> None:+ client = MagicMock()+ client.search.return_value = {"results": [_search_item()]}+ _install_fake_tavily(monkeypatch, client)++ from hyperresearch.web.tavily_provider import TavilyProvider++ results = TavilyProvider(api_key="tvly-test").search("solid state batteries")+ assert len(results) == 1+ r = results[0]+ assert r.url == "https://example.com/article"+ assert r.title == "Example Article"+ assert r.content == "Snippet of the article body."+ assert r.metadata["score"] == 0.91+ assert r.metadata["published_date"] == "2026-06-01"+++def test_search_passes_configured_options(monkeypatch: pytest.MonkeyPatch) -> None:+ client = MagicMock()+ client.search.return_value = {"results": []}+ _install_fake_tavily(monkeypatch, client)++ from hyperresearch.web.tavily_provider import TavilyProvider++ prov = TavilyProvider(+ api_key="tvly-test",+ search_depth="basic",+ include_domains=["nature.com"],+ exclude_domains=["pinterest.com"],+ )+ prov.search("q", max_results=7)++ kwargs = client.search.call_args.kwargs+ assert kwargs["max_results"] == 7+ assert kwargs["search_depth"] == "basic"+ assert kwargs["include_domains"] == ["nature.com"]+ assert kwargs["exclude_domains"] == ["pinterest.com"]+++def test_fetch_uses_extract_and_raw_content(monkeypatch: pytest.MonkeyPatch) -> None:+ client = MagicMock()+ client.extract.return_value = {+ "results": [+ {"url": "https://example.com/p", "raw_content": "Full extracted page text."}+ ]+ }+ _install_fake_tavily(monkeypatch, client)++ from hyperresearch.web.tavily_provider import TavilyProvider++ r = TavilyProvider(api_key="tvly-test").fetch("https://example.com/p")+ assert r.content == "Full extracted page text."+ assert r.title == ""+++def test_fetch_no_results_raises(monkeypatch: pytest.MonkeyPatch) -> None:+ client = MagicMock()+ client.extract.return_value = {"results": []}+ _install_fake_tavily(monkeypatch, client)++ from hyperresearch.web.tavily_provider import TavilyProvider++ with pytest.raises(RuntimeError, match="no contents"):+ TavilyProvider(api_key="tvly-test").fetch("https://example.com/p")+++def test_content_falls_back_to_raw_content(monkeypatch: pytest.MonkeyPatch) -> None:+ client = MagicMock()+ client.search.return_value = {+ "results": [_search_item(content="", raw_content="Raw body wins.")]+ }+ _install_fake_tavily(monkeypatch, client)++ from hyperresearch.web.tavily_provider import TavilyProvider++ results = TavilyProvider(api_key="tvly-test").search("q")+ assert results[0].content == "Raw body wins."