Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

feat: add Parallel Search MCP provider (#57)

George Pickettdf641899f34e
diff --git a/README.md b/README.md
index 81dc90e..b67f69f 100644
--- a/README.md
+++ b/README.md
@@ -253,6 +253,24 @@ Resolved URLs from third-party APIs get the same treatment. An open-access locat
---
+## Web providers
+
+The `[web] provider` setting in `.hyperresearch/config.toml` picks how pages are fetched and, for the providers that support it, how the web is searched. All are optional extras except the default.
+
+- **`builtin`** (default) — plain HTTP fetch with no search. Zero extra dependencies; the starting point everything else improves on.
+- **`crawl4ai`** — headless browser fetch with stealth, PDF extraction and the browser-escalation lane. The one the pipeline is tuned for. `pip install "hyperresearch[crawl4ai]"`.
+- **`exa`** — neural web search and page extraction. Needs an API key. `pip install "hyperresearch[exa]"`.
+- **`tavily`** — search and extraction built for agents. Needs an API key. `pip install "hyperresearch[tavily]"`.
+- **`parallel`** — [Parallel](https://parallel.ai/)'s Search MCP endpoint, which needs no account or key. Search only — bulk fetch waves degrade to per-URL, so it is a good search provider rather than a replacement for the crawl4ai fetch path. Every request from one process carries a random session ID that Parallel uses for correlation and rate limiting on its side. `pip install "hyperresearch[parallel]"`.
+
+```toml
+# .hyperresearch/config.toml
+[web]
+provider = "crawl4ai"
+```
+
+---
+
## Authenticated crawling + the browser lane
Fetch from LinkedIn, Twitter, paywalled sites or anything you can log into:
diff --git a/pyproject.toml b/pyproject.toml
index 062b6ce..1b5dcc1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,10 +40,14 @@ crawl4ai = ["Crawl4AI>=0.7.3"]
# fresh `pip install hyperresearch[mcp]` resolves 2.x and `hyperresearch mcp`
# dies on import, reporting the extra as missing when it is in fact installed.
mcp = ["mcp>=1.6,<2"]
+# The Parallel provider needs the Streamable HTTP client introduced in mcp 1.8.
+# Keep this separate so installing the Hyperresearch MCP server does not force
+# an unrelated SDK upgrade.
+parallel = ["mcp>=1.8,<2"]
exa = ["exa-py>=2.0.0"]
tavily = ["tavily-python>=0.3"]
watch = ["watchdog>=4.0"]
-all = ["hyperresearch[crawl4ai,mcp,exa,tavily,watch]"]
+all = ["hyperresearch[crawl4ai,mcp,parallel,exa,tavily,watch]"]
dev = [
"pytest>=7.4",
"pytest-cov>=4.1",
diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.py
index ebf5570..9e00fa9 100644
--- a/src/hyperresearch/web/base.py
+++ b/src/hyperresearch/web/base.py
@@ -223,4 +223,14 @@ def get_provider(
except ImportError:
raise ImportError("tavily provider requires: pip install \"hyperresearch[tavily]\"")
- raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily")
+ if name == "parallel":
+ try:
+ from hyperresearch.web.parallel_provider import ParallelProvider
+
+ return ParallelProvider(settings=settings)
+ except ImportError:
+ raise ImportError('parallel provider requires: pip install "hyperresearch[parallel]"')
+
+ raise ValueError(
+ f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily, parallel"
+ )
diff --git a/src/hyperresearch/web/parallel_provider.py b/src/hyperresearch/web/parallel_provider.py
new file mode 100644
index 0000000..1669669
--- /dev/null
+++ b/src/hyperresearch/web/parallel_provider.py
@@ -0,0 +1,233 @@
+"""Parallel Search MCP provider — no-account web search and page extraction.
+
+Configuration:
+ # in .hyperresearch/config.toml
+ [web]
+ provider = "parallel"
+
+Optional install:
+ pip install "hyperresearch[parallel]"
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import Mapping
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
+
+from mcp import ClientSession
+from mcp.client.streamable_http import streamablehttp_client
+
+from hyperresearch.core.config import FetchSettings
+from hyperresearch.web.base import WebResult
+
+
+def _run_coro(coro: Any) -> Any:
+ """Run a coroutine to completion from synchronous code, loop or no loop.
+
+ `asyncio.run` refuses to start when the calling thread already has a
+ running loop — which is exactly the situation inside the MCP server,
+ where FastMCP dispatches sync tool functions inline on its own loop, and
+ inside any test harness that wraps the call in one. In that case the
+ coroutine runs on a fresh thread with its own loop and the result is
+ joined back. Outside a loop this is plain `asyncio.run`.
+ """
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(coro)
+
+ import threading
+
+ box: dict[str, Any] = {}
+
+ def _target() -> None:
+ try:
+ box["value"] = asyncio.run(coro)
+ except BaseException as exc: # re-raised on the caller's thread
+ box["error"] = exc
+
+ thread = threading.Thread(target=_target, name="parallel-mcp-call", daemon=True)
+ thread.start()
+ thread.join()
+ if "error" in box:
+ raise box["error"]
+ return box["value"]
+
+if TYPE_CHECKING:
+ from mcp.types import CallToolResult
+
+_ENDPOINT = "https://search.parallel.ai/mcp"
+_MAX_OBJECTIVE_CHARS = 5000
+_MAX_SEARCH_QUERY_CHARS = 200
+_SESSION_ID = uuid4().hex
+
+
+class ParallelProvider:
+ """Web provider backed by Parallel's keyless Search MCP endpoint.
+
+ Each operation uses a short-lived MCP connection. All provider instances in
+ a process send the same random ``session_id``; Parallel uses it to correlate
+ requests in its logs and for rate limiting. Transport, tool
+ arguments, and result validation stay behind the synchronous ``WebProvider``
+ interface used by Hyperresearch's CLI.
+ """
+
+ name = "parallel"
+
+ def __init__(self, settings: FetchSettings | None = None) -> None:
+ settings = settings or FetchSettings()
+ self._timeout = timedelta(milliseconds=settings.page_timeout_ms)
+
+ def search(self, query: str, max_results: int = 5) -> list[WebResult]:
+ """Search the web and return ranked results with Markdown excerpts."""
+ objective, search_query = _normalize_query(query)
+ payload = self._call_tool(
+ "web_search",
+ {
+ "objective": objective,
+ "search_queries": [search_query],
+ "session_id": _SESSION_ID,
+ },
+ )
+ results = _result_items(payload, "web_search")
+ return [_to_web_result(item) for item in results[: max(0, max_results)]]
+
+ def fetch(self, url: str) -> WebResult:
+ """Fetch one URL as clean Markdown."""
+ payload = self._call_tool(
+ "web_fetch",
+ {
+ "urls": [url],
+ "full_content": True,
+ "session_id": _SESSION_ID,
+ },
+ )
+ results = _result_items(payload, "web_fetch")
+ if not results:
+ raise RuntimeError(_fetch_error(payload, url))
+ return _to_web_result(results[0])
+
+ def _call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ payload: dict[str, Any] = _run_coro(self._call_tool_async(name, arguments))
+ return payload
+
+ async def _call_tool_async(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ async with (
+ streamablehttp_client(
+ _ENDPOINT,
+ timeout=self._timeout,
+ sse_read_timeout=self._timeout,
+ ) as (read_stream, write_stream, _),
+ ClientSession(
+ read_stream,
+ write_stream,
+ read_timeout_seconds=self._timeout,
+ ) as session,
+ ):
+ await session.initialize()
+ result = await session.call_tool(
+ name,
+ arguments=arguments,
+ read_timeout_seconds=self._timeout,
+ )
+ return _decode_tool_result(result, name)
+
+
+def _normalize_query(query: str) -> tuple[str, str]:
+ value = query.strip()
+ if not value:
+ raise ValueError("search query must not be empty")
+ return (
+ _truncate_at_word_boundary(value, _MAX_OBJECTIVE_CHARS),
+ _truncate_at_word_boundary(value, _MAX_SEARCH_QUERY_CHARS),
+ )
+
+
+def _truncate_at_word_boundary(value: str, limit: int) -> str:
+ if len(value) <= limit:
+ return value
+ truncated = value[:limit]
+ head, separator, _ = truncated.rpartition(" ")
+ return head.rstrip() if separator and head.strip() else truncated
+
+
+def _decode_tool_result(result: CallToolResult, tool_name: str) -> dict[str, Any]:
+ text_blocks = [block.text for block in result.content if block.type == "text"]
+ text = "\n".join(text_blocks).strip()
+
+ if result.isError:
+ detail = text or "unknown MCP error"
+ raise RuntimeError(f"Parallel {tool_name} failed: {detail}")
+
+ structured_content = getattr(result, "structuredContent", None)
+ if structured_content is not None:
+ return dict(structured_content)
+
+ if not text:
+ raise RuntimeError(f"Parallel {tool_name} returned no structured or text content")
+
+ try:
+ payload = json.loads(text)
+ except json.JSONDecodeError as exc:
+ raise RuntimeError(f"Parallel {tool_name} returned invalid JSON") from exc
+ if not isinstance(payload, dict):
+ raise RuntimeError(f"Parallel {tool_name} returned a non-object payload")
+ return payload
+
+
+def _result_items(payload: Mapping[str, Any], tool_name: str) -> list[Mapping[str, Any]]:
+ results = payload.get("results")
+ if not isinstance(results, list):
+ raise RuntimeError(f"Parallel {tool_name} response is missing a results list")
+ if not all(isinstance(item, dict) for item in results):
+ raise RuntimeError(f"Parallel {tool_name} returned a malformed result")
+ return results
+
+
+def _fetch_error(payload: Mapping[str, Any], url: str) -> str:
+ errors = payload.get("errors")
+ if isinstance(errors, list) and errors and isinstance(errors[0], dict):
+ error_type = errors[0].get("error_type")
+ if isinstance(error_type, str) and error_type:
+ return f"Parallel could not fetch {url}: {error_type}"
+ return f"Parallel returned no contents for {url}"
+
+
+def _to_web_result(item: Mapping[str, Any]) -> WebResult:
+ url = item.get("url")
+ if not isinstance(url, str) or not url:
+ raise RuntimeError("Parallel returned a result without a URL")
+
+ title = item.get("title")
+ if title is None:
+ title = ""
+ if not isinstance(title, str):
+ raise RuntimeError(f"Parallel returned an invalid title for {url}")
+
+ full_content = item.get("full_content")
+ excerpts = item.get("excerpts")
+ if isinstance(full_content, str) and full_content.strip():
+ content = full_content
+ elif isinstance(excerpts, list) and all(isinstance(excerpt, str) for excerpt in excerpts):
+ content = "\n\n".join(excerpts)
+ else:
+ raise RuntimeError(f"Parallel returned invalid content for {url}")
+ if not content.strip():
+ raise RuntimeError(f"Parallel returned no content for {url}")
+
+ metadata: dict[str, Any] = {}
+ publish_date = item.get("publish_date")
+ if isinstance(publish_date, str) and publish_date:
+ metadata["published_date"] = publish_date
+
+ return WebResult(
+ url=url,
+ title=title,
+ content=content,
+ fetched_at=datetime.now(UTC),
+ metadata=metadata,
+ )
diff --git a/tests/test_core/test_packaging.py b/tests/test_core/test_packaging.py
index 3024ecb..5972fef 100644
--- a/tests/test_core/test_packaging.py
+++ b/tests/test_core/test_packaging.py
@@ -26,6 +26,9 @@ def test_dev_extra_covers_tested_optional_providers():
dev_deps = set(optional["dev"])
assert "exa-py>=2.0.0" in dev_deps
+ assert "mcp>=1.6,<2" in dev_deps
+ assert optional["mcp"] == ["mcp>=1.6,<2"]
+ assert optional["parallel"] == ["mcp>=1.8,<2"]
assert "exa" in optional
assert "crawl4ai" in optional
assert "watch" in optional
diff --git a/tests/test_web/test_parallel_provider.py b/tests/test_web/test_parallel_provider.py
new file mode 100644
index 0000000..a694ef9
--- /dev/null
+++ b/tests/test_web/test_parallel_provider.py
@@ -0,0 +1,315 @@
+"""Tests for the Parallel provider — stubs the optional MCP SDK.
+
+The test suite must collect without ``hyperresearch[parallel]`` installed.
+These tests inject the small MCP surface the provider imports into
+``sys.modules``, following the same isolation pattern as the Tavily tests.
+"""
+
+from __future__ import annotations
+
+import builtins
+import importlib
+import json
+import sys
+from collections.abc import Iterator
+from datetime import timedelta
+from types import ModuleType, SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+
+from hyperresearch.core.config import FetchSettings
+
+_MISSING = object()
+
+
+def parallel_module(monkeypatch: pytest.MonkeyPatch) -> Iterator[ModuleType]:
+ fake_mcp = ModuleType("mcp")
+ fake_mcp.__path__ = [] # type: ignore[attr-defined]
+ fake_client = ModuleType("mcp.client")
+ fake_client.__path__ = [] # type: ignore[attr-defined]
+ fake_transport = ModuleType("mcp.client.streamable_http")
+
+ fake_mcp.ClientSession = MagicMock() # type: ignore[attr-defined]
+ fake_transport.streamablehttp_client = MagicMock() # type: ignore[attr-defined]
+ fake_mcp.client = fake_client # type: ignore[attr-defined]
+ fake_client.streamable_http = fake_transport # type: ignore[attr-defined]
+
+ monkeypatch.setitem(sys.modules, "mcp", fake_mcp)
+ monkeypatch.setitem(sys.modules, "mcp.client", fake_client)
+ monkeypatch.setitem(sys.modules, "mcp.client.streamable_http", fake_transport)
+ sys.modules.pop("hyperresearch.web.parallel_provider", None)
+
+ yield importlib.import_module("hyperresearch.web.parallel_provider")
+
+ sys.modules.pop("hyperresearch.web.parallel_provider", None)
+
+
+def _search_item(**overrides: Any) -> dict[str, Any]:
+ item: dict[str, Any] = {
+ "url": "https://example.com/article",
+ "title": "Example Article",
+ "publish_date": "2026-07-22",
+ "excerpts": ["First excerpt.", "Second excerpt."],
+ }
+ item.update(overrides)
+ return item
+
+
+def _tool_result(
+ text: str = "",
+ *,
+ structured_content: object = _MISSING,
+ is_error: bool = False,
+) -> SimpleNamespace:
+ content = [SimpleNamespace(type="text", text=text)] if text else []
+ result = SimpleNamespace(content=content, isError=is_error)
+ if structured_content is not _MISSING:
+ result.structuredContent = structured_content
+ return result
+
+
+def test_provider_registered_via_factory(parallel_module: ModuleType) -> None:
+ from hyperresearch.web.base import get_provider
+
+ provider = get_provider("parallel", settings=FetchSettings(page_timeout_ms=4321))
+
+ assert isinstance(provider, parallel_module.ParallelProvider)
+ assert provider._timeout == timedelta(milliseconds=4321)
+
+
+def test_missing_sdk_names_parallel_extra(monkeypatch: pytest.MonkeyPatch) -> None:
+ from hyperresearch.web.base import get_provider
+
+ real_import = builtins.__import__
+
+ def import_without_mcp(name: str, *args: Any, **kwargs: Any) -> Any:
+ if name == "mcp" or name.startswith("mcp."):
+ raise ImportError("No module named 'mcp'")
+ return real_import(name, *args, **kwargs)
+
+ sys.modules.pop("hyperresearch.web.parallel_provider", None)
+ monkeypatch.setattr(builtins, "__import__", import_without_mcp)
+
+ with pytest.raises(ImportError, match=r"hyperresearch\[parallel\]"):
+ get_provider("parallel")
+
+
+def test_search_maps_results_limits_output_and_reuses_process_session(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ calls: list[tuple[str, dict[str, Any]]] = []
+
+ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ calls.append((name, arguments))
+ return {"results": [_search_item(), _search_item(url="https://example.com/second")]}
+
+ monkeypatch.setattr(provider_type, "_call_tool", fake_call)
+ results = provider_type().search("solid state batteries", max_results=1)
+ provider_type().fetch("https://example.com/article")
+
+ assert len(results) == 1
+ assert results[0].url == "https://example.com/article"
+ assert results[0].content == "First excerpt.\n\nSecond excerpt."
+ assert results[0].metadata["published_date"] == "2026-07-22"
+ assert calls[0][0] == "web_search"
+ assert calls[0][1]["objective"] == "solid state batteries"
+ assert calls[0][1]["search_queries"] == ["solid state batteries"]
+ assert calls[0][1]["session_id"] == calls[1][1]["session_id"]
+
+
+ ("length", "objective_length", "search_query_length"),
+ [(200, 200, 200), (201, 201, 200), (5000, 5000, 200), (5001, 5000, 200)],
+)
+def test_search_bounds_tool_arguments(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ length: int,
+ objective_length: int,
+ search_query_length: int,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ captured: dict[str, Any] = {}
+
+ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ captured.update(arguments)
+ return {"results": []}
+
+ monkeypatch.setattr(provider_type, "_call_tool", fake_call)
+ provider_type().search("x" * length)
+
+ assert len(captured["objective"]) == objective_length
+ assert len(captured["search_queries"][0]) == search_query_length
+
+
+def test_search_rejects_empty_query(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+
+ def fail_call(self: Any, name: str, arguments: dict[str, Any]) -> None:
+ pytest.fail("MCP should not be called")
+
+ monkeypatch.setattr(provider_type, "_call_tool", fail_call)
+
+ with pytest.raises(ValueError, match="must not be empty"):
+ provider_type().search(" ")
+
+
+def test_fetch_requests_full_content_and_prefers_it(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ captured: dict[str, Any] = {}
+
+ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ captured.update({"name": name, "arguments": arguments})
+ return {
+ "results": [
+ _search_item(full_content="Full page Markdown.", excerpts=["Short excerpt."])
+ ],
+ "errors": [],
+ }
+
+ monkeypatch.setattr(provider_type, "_call_tool", fake_call)
+ result = provider_type().fetch("https://example.com/article")
+
+ assert captured["name"] == "web_fetch"
+ assert captured["arguments"]["urls"] == ["https://example.com/article"]
+ assert captured["arguments"]["full_content"] is True
+ assert result.content == "Full page Markdown."
+
+
+def test_fetch_falls_back_to_excerpts(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ monkeypatch.setattr(
+ provider_type,
+ "_call_tool",
+ lambda self, name, arguments: {
+ "results": [_search_item(full_content=None, excerpts=["One.", "Two."])],
+ "errors": [],
+ },
+ )
+
+ result = provider_type().fetch("https://example.com/article")
+ assert result.content == "One.\n\nTwo."
+
+
+def test_fetch_surfaces_extract_error(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ monkeypatch.setattr(
+ provider_type,
+ "_call_tool",
+ lambda self, name, arguments: {
+ "results": [],
+ "errors": [{"url": "https://example.com/missing", "error_type": "not_found"}],
+ },
+ )
+
+ with pytest.raises(RuntimeError, match="not_found"):
+ provider_type().fetch("https://example.com/missing")
+
+
+def test_malformed_results_raise(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider_type = parallel_module.ParallelProvider
+ monkeypatch.setattr(
+ provider_type,
+ "_call_tool",
+ lambda self, name, arguments: {"results": "not-a-list"},
+ )
+
+ with pytest.raises(RuntimeError, match="results list"):
+ provider_type().search("query")
+
+
+def test_timeout_applies_to_transport_session_and_tool_call(
+ parallel_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ observed: dict[str, Any] = {}
+
+ class AsyncContext:
+ def __init__(self, value: Any) -> None:
+ self.value = value
+
+ async def __aenter__(self) -> Any:
+ return self.value
+
+ async def __aexit__(self, *args: Any) -> None:
+ return None
+
+ class FakeSession:
+ def __init__(self, read: object, write: object, **kwargs: Any) -> None:
+ observed["session_timeout"] = kwargs["read_timeout_seconds"]
+
+ async def __aenter__(self) -> FakeSession:
+ return self
+
+ async def __aexit__(self, *args: Any) -> None:
+ return None
+
+ async def initialize(self) -> None:
+ observed["initialized"] = True
+
+ async def call_tool(self, name: str, **kwargs: Any) -> SimpleNamespace:
+ observed["tool_timeout"] = kwargs["read_timeout_seconds"]
+ return _tool_result(json.dumps({"results": []}))
+
+ def fake_transport(endpoint: str, **kwargs: Any) -> AsyncContext:
+ observed["endpoint"] = endpoint
+ observed["transport_timeout"] = kwargs["timeout"]
+ observed["sse_timeout"] = kwargs["sse_read_timeout"]
+ return AsyncContext((object(), object(), None))
+
+ monkeypatch.setattr(parallel_module, "streamablehttp_client", fake_transport)
+ monkeypatch.setattr(parallel_module, "ClientSession", FakeSession)
+
+ timeout = timedelta(milliseconds=1234)
+ result = parallel_module.ParallelProvider(FetchSettings(page_timeout_ms=1234))._call_tool(
+ "web_search", {}
+ )
+
+ assert result == {"results": []}
+ assert observed == {
+ "endpoint": "https://search.parallel.ai/mcp",
+ "transport_timeout": timeout,
+ "sse_timeout": timeout,
+ "session_timeout": timeout,
+ "initialized": True,
+ "tool_timeout": timeout,
+ }
+
+
+def test_tool_result_uses_structured_content(parallel_module: ModuleType) -> None:
+ result = _tool_result("ignored", structured_content={"results": []})
+ assert parallel_module._decode_tool_result(result, "web_search") == {"results": []}
+
+
+def test_old_tool_result_without_structured_content_uses_json_text(
+ parallel_module: ModuleType,
+) -> None:
+ result = _tool_result(json.dumps({"results": []}))
+ assert not hasattr(result, "structuredContent")
+ assert parallel_module._decode_tool_result(result, "web_search") == {"results": []}
+
+
+def test_tool_error_raises_with_detail(parallel_module: ModuleType) -> None:
+ result = _tool_result("rate limited", is_error=True)
+ with pytest.raises(RuntimeError, match="rate limited"):
+ parallel_module._decode_tool_result(result, "web_search")