Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

feat: add Serply web provider (#121)

Serplyb38613739e2b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bfa1ec7..ffe35ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
## [Unreleased]
+- **`serply` web provider.** Google organic results with page fetch through the same `SERPLY_API_KEY`; opt-in via `[web] provider = "serply"`, no new dependency.
+
## [0.11.1] - 2026-09-11
### Seven fixes from the backlog sweep
diff --git a/README.md b/README.md
index b67f69f..6def787 100644
--- a/README.md
+++ b/README.md
@@ -262,6 +262,7 @@ The `[web] provider` setting in `.hyperresearch/config.toml` picks how pages are
- **`exa`** — neural web search and page extraction. Needs an API key. `pip install "hyperresearch[exa]"`.
- **`tavily`** — search and extraction built for agents. Needs an API key. `pip install "hyperresearch[tavily]"`.
- **`parallel`** — [Parallel](https://parallel.ai/)'s Search MCP endpoint, which needs no account or key. Search only — bulk fetch waves degrade to per-URL, so it is a good search provider rather than a replacement for the crawl4ai fetch path. Every request from one process carries a random session ID that Parallel uses for correlation and rate limiting on its side. `pip install "hyperresearch[parallel]"`.
+- **`serply`** — [Serply](https://serply.io)'s Google search API, with page fetch through the same key ([API docs](https://serply.io/docs)). Search results are fetched to full page text, falling back to the snippet when a page cannot be fetched. Needs an API key; no extra install.
```toml
# .hyperresearch/config.toml
diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.py
index 9e00fa9..efae7d5 100644
--- a/src/hyperresearch/web/base.py
+++ b/src/hyperresearch/web/base.py
@@ -231,6 +231,12 @@ def get_provider(
except ImportError:
raise ImportError('parallel provider requires: pip install "hyperresearch[parallel]"')
+ if name == "serply":
+ from hyperresearch.web.serply_provider import SerplyProvider
+
+ return SerplyProvider()
+
raise ValueError(
- f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily, parallel"
+ f"Unknown web provider: {name!r}. "
+ "Available: builtin, crawl4ai, exa, tavily, parallel, serply"
)
diff --git a/src/hyperresearch/web/serply_provider.py b/src/hyperresearch/web/serply_provider.py
new file mode 100644
index 0000000..3f4629c
--- /dev/null
+++ b/src/hyperresearch/web/serply_provider.py
@@ -0,0 +1,110 @@
+"""Serply web provider: Google search results plus page fetch behind one key.
+
+Serply (https://serply.io) returns Google organic results as JSON and fetches
+pages as Markdown. Search results carry only the SERP snippet, so `search`
+fetches each page through the same API and falls back to the snippet when a
+page cannot be fetched or comes back as a bot wall.
+
+Configuration:
+ export SERPLY_API_KEY="your-api-key" # https://serply.io
+
+ # in .hyperresearch/config.toml
+ [web]
+ provider = "serply"
+
+No extra install: uses the httpx dependency the core already carries.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+
+from hyperresearch.web.base import WebResult
+
+_SEARCH_URL = "https://api.serply.io/v1/search/"
+_FETCH_URL = "https://api.serply.io/v1/request"
+_TIMEOUT = 30.0
+
+
+class SerplyProvider:
+ """Web provider backed by the Serply API (https://serply.io/docs).
+
+ Supports both `search` (Google organic results) and `fetch` (URL to Markdown).
+ """
+
+ name = "serply"
+
+ def __init__(
+ self,
+ api_key: str | None = None,
+ fetch_content: bool = True,
+ max_characters: int = 8000,
+ transport: httpx.BaseTransport | None = None,
+ ):
+ key = api_key or os.environ.get("SERPLY_API_KEY", "").strip()
+ if not key:
+ raise RuntimeError(
+ "SERPLY_API_KEY is not set. Get a key at https://serply.io and export it."
+ )
+
+ # Serply sits behind Cloudflare, which rejects the default httpx User-Agent.
+ self._client = httpx.Client(
+ headers={"X-Api-Key": key, "User-Agent": "hyperresearch"},
+ timeout=_TIMEOUT,
+ transport=transport,
+ )
+ self._fetch_content = fetch_content
+ self._max_characters = max_characters
+
+ def search(self, query: str, max_results: int = 5) -> list[WebResult]:
+ """Search Google via Serply and return ranked results with content."""
+ resp = self._client.get(_SEARCH_URL, params={"q": query, "num": max_results})
+ resp.raise_for_status()
+ items = resp.json().get("results", [])[:max_results]
+
+ results: list[WebResult] = []
+ for item in items:
+ result = _to_web_result(item)
+ if not result.url:
+ continue
+ if self._fetch_content:
+ try:
+ fetched = self.fetch(result.url)
+ except (httpx.HTTPError, RuntimeError):
+ fetched = None
+ if fetched is not None and fetched.looks_like_junk() is None:
+ result.content = fetched.content
+ results.append(result)
+ return results
+
+ def fetch(self, url: str) -> WebResult:
+ """Fetch a single URL via Serply and return it as Markdown."""
+ resp = self._client.post(_FETCH_URL, json={"url": url, "response_type": "markdown"})
+ resp.raise_for_status()
+ content = resp.text.strip()[: self._max_characters]
+ if not content:
+ raise RuntimeError(f"Serply returned no content for {url}")
+ return WebResult(url=url, title="", content=content, fetched_at=datetime.now(UTC))
+
+
+def _to_web_result(item: dict[str, Any]) -> WebResult:
+ """Convert a Serply organic result dict into a hyperresearch WebResult."""
+ metadata: dict[str, Any] = {}
+ position = item.get("position")
+ if position is not None:
+ metadata["position"] = position
+ published_time = (item.get("metadata") or {}).get("published_time")
+ if published_time:
+ metadata["published_time"] = published_time
+
+ return WebResult(
+ url=item.get("link", "") or "",
+ title=item.get("title", "") or "",
+ content=item.get("description", "") or "",
+ fetched_at=datetime.now(UTC),
+ metadata=metadata,
+ )
diff --git a/tests/test_web/test_serply_provider.py b/tests/test_web/test_serply_provider.py
new file mode 100644
index 0000000..3eb1a16
--- /dev/null
+++ b/tests/test_web/test_serply_provider.py
@@ -0,0 +1,129 @@
+"""Tests for the Serply web provider: offline via the httpx transport seam."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import httpx
+import pytest
+
+from hyperresearch.web.serply_provider import SerplyProvider
+
+SEARCH_PATH = "/v1/search/"
+FETCH_PATH = "/v1/request"
+
+
+def _search_item(**overrides: Any) -> dict[str, Any]:
+ item: dict[str, Any] = {
+ "title": "Example Article",
+ "description": "Snippet of the article body.",
+ "position": 1,
+ "result_type": "organic",
+ "metadata": {"published_time": "May 15, 2025"},
+ "link": "https://example.com/article",
+ }
+ item.update(overrides)
+ return item
+
+
+def _transport(
+ search_items: list[dict[str, Any]],
+ fetch_response: httpx.Response,
+ seen: list[httpx.Request],
+) -> httpx.MockTransport:
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request)
+ if request.url.path == SEARCH_PATH:
+ return httpx.Response(200, json={"results": search_items})
+ if request.url.path == FETCH_PATH:
+ return fetch_response
+ raise AssertionError(f"unexpected request: {request.url}")
+
+ return httpx.MockTransport(handler)
+
+
+def test_provider_registered_via_factory(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SERPLY_API_KEY", "serply-test")
+
+ from hyperresearch.web.base import get_provider
+
+ prov = get_provider("serply")
+ assert prov.name == "serply"
+
+
+def test_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("SERPLY_API_KEY", raising=False)
+
+ with pytest.raises(RuntimeError, match="SERPLY_API_KEY"):
+ SerplyProvider()
+
+
+def test_search_maps_results_and_fetches_content() -> None:
+ seen: list[httpx.Request] = []
+ page = "# Full page\n\n" + "Body text. " * 40
+ transport = _transport([_search_item()], httpx.Response(200, text=page), seen)
+
+ results = SerplyProvider(api_key="serply-test", transport=transport).search("q", max_results=3)
+
+ assert len(results) == 1
+ r = results[0]
+ assert r.url == "https://example.com/article"
+ assert r.title == "Example Article"
+ assert r.content == page.strip()
+ assert r.metadata == {"position": 1, "published_time": "May 15, 2025"}
+
+ search, fetch = seen
+ assert search.headers["X-Api-Key"] == "serply-test"
+ assert search.url.params["q"] == "q"
+ assert search.url.params["num"] == "3"
+ assert json.loads(fetch.content) == {
+ "url": "https://example.com/article",
+ "response_type": "markdown",
+ }
+
+
+def test_search_falls_back_to_snippet_when_fetch_fails() -> None:
+ transport = _transport([_search_item()], httpx.Response(500), [])
+
+ results = SerplyProvider(api_key="serply-test", transport=transport).search("q")
+
+ assert results[0].content == "Snippet of the article body."
+
+
+def test_search_falls_back_to_snippet_when_fetch_is_a_bot_wall() -> None:
+ wall = httpx.Response(200, text="Just a moment... Enable JavaScript and cookies to continue")
+ transport = _transport([_search_item()], wall, [])
+
+ results = SerplyProvider(api_key="serply-test", transport=transport).search("q")
+
+ assert results[0].content == "Snippet of the article body."
+
+
+def test_fetch_clips_content_to_max_characters() -> None:
+ transport = _transport([], httpx.Response(200, text="x" * 50), [])
+
+ r = SerplyProvider(api_key="serply-test", max_characters=20, transport=transport).fetch(
+ "https://example.com/p"
+ )
+
+ assert r.content == "x" * 20
+
+
+def test_search_without_fetch_content_makes_one_request() -> None:
+ seen: list[httpx.Request] = []
+ transport = _transport([_search_item(), _search_item(link="")], httpx.Response(200), seen)
+
+ results = SerplyProvider(
+ api_key="serply-test", fetch_content=False, transport=transport
+ ).search("q")
+
+ assert [r.content for r in results] == ["Snippet of the article body."]
+ assert len(seen) == 1
+
+
+def test_fetch_no_content_raises() -> None:
+ transport = _transport([], httpx.Response(200, text=" \n"), [])
+
+ with pytest.raises(RuntimeError, match="no content"):
+ SerplyProvider(api_key="serply-test", transport=transport).fetch("https://example.com/p")