Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
feat: add Serply web provider (#121)
Serplyb38613739e2b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bfa1ec7..ffe35ba 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,8 @@## [Unreleased]+- **`serply` web provider.** Google organic results with page fetch through the same `SERPLY_API_KEY`; opt-in via `[web] provider = "serply"`, no new dependency.+## [0.11.1] - 2026-09-11### Seven fixes from the backlog sweepdiff --git a/README.md b/README.mdindex b67f69f..6def787 100644--- a/README.md+++ b/README.md@@ -262,6 +262,7 @@ The `[web] provider` setting in `.hyperresearch/config.toml` picks how pages are- **`exa`** — neural web search and page extraction. Needs an API key. `pip install "hyperresearch[exa]"`.- **`tavily`** — search and extraction built for agents. Needs an API key. `pip install "hyperresearch[tavily]"`.- **`parallel`** — [Parallel](https://parallel.ai/)'s Search MCP endpoint, which needs no account or key. Search only — bulk fetch waves degrade to per-URL, so it is a good search provider rather than a replacement for the crawl4ai fetch path. Every request from one process carries a random session ID that Parallel uses for correlation and rate limiting on its side. `pip install "hyperresearch[parallel]"`.+- **`serply`** — [Serply](https://serply.io)'s Google search API, with page fetch through the same key ([API docs](https://serply.io/docs)). Search results are fetched to full page text, falling back to the snippet when a page cannot be fetched. Needs an API key; no extra install.```toml# .hyperresearch/config.tomldiff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.pyindex 9e00fa9..efae7d5 100644--- a/src/hyperresearch/web/base.py+++ b/src/hyperresearch/web/base.py@@ -231,6 +231,12 @@ def get_provider(except ImportError:raise ImportError('parallel provider requires: pip install "hyperresearch[parallel]"')+ if name == "serply":+ from hyperresearch.web.serply_provider import SerplyProvider++ return SerplyProvider()+raise ValueError(- f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily, parallel"+ f"Unknown web provider: {name!r}. "+ "Available: builtin, crawl4ai, exa, tavily, parallel, serply")diff --git a/src/hyperresearch/web/serply_provider.py b/src/hyperresearch/web/serply_provider.pynew file mode 100644index 0000000..3f4629c--- /dev/null+++ b/src/hyperresearch/web/serply_provider.py@@ -0,0 +1,110 @@+"""Serply web provider: Google search results plus page fetch behind one key.++Serply (https://serply.io) returns Google organic results as JSON and fetches+pages as Markdown. Search results carry only the SERP snippet, so `search`+fetches each page through the same API and falls back to the snippet when a+page cannot be fetched or comes back as a bot wall.++Configuration:+ export SERPLY_API_KEY="your-api-key" # https://serply.io++ # in .hyperresearch/config.toml+ [web]+ provider = "serply"++No extra install: uses the httpx dependency the core already carries.+"""++from __future__ import annotations++import os+from datetime import UTC, datetime+from typing import Any++import httpx++from hyperresearch.web.base import WebResult++_SEARCH_URL = "https://api.serply.io/v1/search/"+_FETCH_URL = "https://api.serply.io/v1/request"+_TIMEOUT = 30.0+++class SerplyProvider:+ """Web provider backed by the Serply API (https://serply.io/docs).++ Supports both `search` (Google organic results) and `fetch` (URL to Markdown).+ """++ name = "serply"++ def __init__(+ self,+ api_key: str | None = None,+ fetch_content: bool = True,+ max_characters: int = 8000,+ transport: httpx.BaseTransport | None = None,+ ):+ key = api_key or os.environ.get("SERPLY_API_KEY", "").strip()+ if not key:+ raise RuntimeError(+ "SERPLY_API_KEY is not set. Get a key at https://serply.io and export it."+ )++ # Serply sits behind Cloudflare, which rejects the default httpx User-Agent.+ self._client = httpx.Client(+ headers={"X-Api-Key": key, "User-Agent": "hyperresearch"},+ timeout=_TIMEOUT,+ transport=transport,+ )+ self._fetch_content = fetch_content+ self._max_characters = max_characters++ def search(self, query: str, max_results: int = 5) -> list[WebResult]:+ """Search Google via Serply and return ranked results with content."""+ resp = self._client.get(_SEARCH_URL, params={"q": query, "num": max_results})+ resp.raise_for_status()+ items = resp.json().get("results", [])[:max_results]++ results: list[WebResult] = []+ for item in items:+ result = _to_web_result(item)+ if not result.url:+ continue+ if self._fetch_content:+ try:+ fetched = self.fetch(result.url)+ except (httpx.HTTPError, RuntimeError):+ fetched = None+ if fetched is not None and fetched.looks_like_junk() is None:+ result.content = fetched.content+ results.append(result)+ return results++ def fetch(self, url: str) -> WebResult:+ """Fetch a single URL via Serply and return it as Markdown."""+ resp = self._client.post(_FETCH_URL, json={"url": url, "response_type": "markdown"})+ resp.raise_for_status()+ content = resp.text.strip()[: self._max_characters]+ if not content:+ raise RuntimeError(f"Serply returned no content for {url}")+ return WebResult(url=url, title="", content=content, fetched_at=datetime.now(UTC))+++def _to_web_result(item: dict[str, Any]) -> WebResult:+ """Convert a Serply organic result dict into a hyperresearch WebResult."""+ metadata: dict[str, Any] = {}+ position = item.get("position")+ if position is not None:+ metadata["position"] = position+ published_time = (item.get("metadata") or {}).get("published_time")+ if published_time:+ metadata["published_time"] = published_time++ return WebResult(+ url=item.get("link", "") or "",+ title=item.get("title", "") or "",+ content=item.get("description", "") or "",+ fetched_at=datetime.now(UTC),+ metadata=metadata,+ )diff --git a/tests/test_web/test_serply_provider.py b/tests/test_web/test_serply_provider.pynew file mode 100644index 0000000..3eb1a16--- /dev/null+++ b/tests/test_web/test_serply_provider.py@@ -0,0 +1,129 @@+"""Tests for the Serply web provider: offline via the httpx transport seam."""++from __future__ import annotations++import json+from typing import Any++import httpx+import pytest++from hyperresearch.web.serply_provider import SerplyProvider++SEARCH_PATH = "/v1/search/"+FETCH_PATH = "/v1/request"+++def _search_item(**overrides: Any) -> dict[str, Any]:+ item: dict[str, Any] = {+ "title": "Example Article",+ "description": "Snippet of the article body.",+ "position": 1,+ "result_type": "organic",+ "metadata": {"published_time": "May 15, 2025"},+ "link": "https://example.com/article",+ }+ item.update(overrides)+ return item+++def _transport(+ search_items: list[dict[str, Any]],+ fetch_response: httpx.Response,+ seen: list[httpx.Request],+) -> httpx.MockTransport:+ def handler(request: httpx.Request) -> httpx.Response:+ seen.append(request)+ if request.url.path == SEARCH_PATH:+ return httpx.Response(200, json={"results": search_items})+ if request.url.path == FETCH_PATH:+ return fetch_response+ raise AssertionError(f"unexpected request: {request.url}")++ return httpx.MockTransport(handler)+++def test_provider_registered_via_factory(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("SERPLY_API_KEY", "serply-test")++ from hyperresearch.web.base import get_provider++ prov = get_provider("serply")+ assert prov.name == "serply"+++def test_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("SERPLY_API_KEY", raising=False)++ with pytest.raises(RuntimeError, match="SERPLY_API_KEY"):+ SerplyProvider()+++def test_search_maps_results_and_fetches_content() -> None:+ seen: list[httpx.Request] = []+ page = "# Full page\n\n" + "Body text. " * 40+ transport = _transport([_search_item()], httpx.Response(200, text=page), seen)++ results = SerplyProvider(api_key="serply-test", transport=transport).search("q", max_results=3)++ assert len(results) == 1+ r = results[0]+ assert r.url == "https://example.com/article"+ assert r.title == "Example Article"+ assert r.content == page.strip()+ assert r.metadata == {"position": 1, "published_time": "May 15, 2025"}++ search, fetch = seen+ assert search.headers["X-Api-Key"] == "serply-test"+ assert search.url.params["q"] == "q"+ assert search.url.params["num"] == "3"+ assert json.loads(fetch.content) == {+ "url": "https://example.com/article",+ "response_type": "markdown",+ }+++def test_search_falls_back_to_snippet_when_fetch_fails() -> None:+ transport = _transport([_search_item()], httpx.Response(500), [])++ results = SerplyProvider(api_key="serply-test", transport=transport).search("q")++ assert results[0].content == "Snippet of the article body."+++def test_search_falls_back_to_snippet_when_fetch_is_a_bot_wall() -> None:+ wall = httpx.Response(200, text="Just a moment... Enable JavaScript and cookies to continue")+ transport = _transport([_search_item()], wall, [])++ results = SerplyProvider(api_key="serply-test", transport=transport).search("q")++ assert results[0].content == "Snippet of the article body."+++def test_fetch_clips_content_to_max_characters() -> None:+ transport = _transport([], httpx.Response(200, text="x" * 50), [])++ r = SerplyProvider(api_key="serply-test", max_characters=20, transport=transport).fetch(+ "https://example.com/p"+ )++ assert r.content == "x" * 20+++def test_search_without_fetch_content_makes_one_request() -> None:+ seen: list[httpx.Request] = []+ transport = _transport([_search_item(), _search_item(link="")], httpx.Response(200), seen)++ results = SerplyProvider(+ api_key="serply-test", fetch_content=False, transport=transport+ ).search("q")++ assert [r.content for r in results] == ["Snippet of the article body."]+ assert len(seen) == 1+++def test_fetch_no_content_raises() -> None:+ transport = _transport([], httpx.Response(200, text=" \n"), [])++ with pytest.raises(RuntimeError, match="no content"):+ SerplyProvider(api_key="serply-test", transport=transport).fetch("https://example.com/p")