Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
feat: add Parallel Search MCP provider (#57)
George Pickettdf641899f34e
diff --git a/README.md b/README.mdindex 81dc90e..b67f69f 100644--- a/README.md+++ b/README.md@@ -253,6 +253,24 @@ Resolved URLs from third-party APIs get the same treatment. An open-access locat---+## Web providers++The `[web] provider` setting in `.hyperresearch/config.toml` picks how pages are fetched and, for the providers that support it, how the web is searched. All are optional extras except the default.++- **`builtin`** (default) — plain HTTP fetch with no search. Zero extra dependencies; the starting point everything else improves on.+- **`crawl4ai`** — headless browser fetch with stealth, PDF extraction and the browser-escalation lane. The one the pipeline is tuned for. `pip install "hyperresearch[crawl4ai]"`.+- **`exa`** — neural web search and page extraction. Needs an API key. `pip install "hyperresearch[exa]"`.+- **`tavily`** — search and extraction built for agents. Needs an API key. `pip install "hyperresearch[tavily]"`.+- **`parallel`** — [Parallel](https://parallel.ai/)'s Search MCP endpoint, which needs no account or key. Search only — bulk fetch waves degrade to per-URL, so it is a good search provider rather than a replacement for the crawl4ai fetch path. Every request from one process carries a random session ID that Parallel uses for correlation and rate limiting on its side. `pip install "hyperresearch[parallel]"`.++```toml+# .hyperresearch/config.toml+[web]+provider = "crawl4ai"+```++---+## Authenticated crawling + the browser laneFetch from LinkedIn, Twitter, paywalled sites or anything you can log into:diff --git a/pyproject.toml b/pyproject.tomlindex 062b6ce..1b5dcc1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,10 +40,14 @@ crawl4ai = ["Crawl4AI>=0.7.3"]# fresh `pip install hyperresearch[mcp]` resolves 2.x and `hyperresearch mcp`# dies on import, reporting the extra as missing when it is in fact installed.mcp = ["mcp>=1.6,<2"]+# The Parallel provider needs the Streamable HTTP client introduced in mcp 1.8.+# Keep this separate so installing the Hyperresearch MCP server does not force+# an unrelated SDK upgrade.+parallel = ["mcp>=1.8,<2"]exa = ["exa-py>=2.0.0"]tavily = ["tavily-python>=0.3"]watch = ["watchdog>=4.0"]-all = ["hyperresearch[crawl4ai,mcp,exa,tavily,watch]"]+all = ["hyperresearch[crawl4ai,mcp,parallel,exa,tavily,watch]"]dev = ["pytest>=7.4","pytest-cov>=4.1",diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.pyindex ebf5570..9e00fa9 100644--- a/src/hyperresearch/web/base.py+++ b/src/hyperresearch/web/base.py@@ -223,4 +223,14 @@ def get_provider(except ImportError:raise ImportError("tavily provider requires: pip install \"hyperresearch[tavily]\"")- raise ValueError(f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily")+ if name == "parallel":+ try:+ from hyperresearch.web.parallel_provider import ParallelProvider++ return ParallelProvider(settings=settings)+ except ImportError:+ raise ImportError('parallel provider requires: pip install "hyperresearch[parallel]"')++ raise ValueError(+ f"Unknown web provider: {name!r}. Available: builtin, crawl4ai, exa, tavily, parallel"+ )diff --git a/src/hyperresearch/web/parallel_provider.py b/src/hyperresearch/web/parallel_provider.pynew file mode 100644index 0000000..1669669--- /dev/null+++ b/src/hyperresearch/web/parallel_provider.py@@ -0,0 +1,233 @@+"""Parallel Search MCP provider — no-account web search and page extraction.++Configuration:+ # in .hyperresearch/config.toml+ [web]+ provider = "parallel"++Optional install:+ pip install "hyperresearch[parallel]"+"""++from __future__ import annotations++import asyncio+import json+from collections.abc import Mapping+from datetime import UTC, datetime, timedelta+from typing import TYPE_CHECKING, Any+from uuid import uuid4++from mcp import ClientSession+from mcp.client.streamable_http import streamablehttp_client++from hyperresearch.core.config import FetchSettings+from hyperresearch.web.base import WebResult+++def _run_coro(coro: Any) -> Any:+ """Run a coroutine to completion from synchronous code, loop or no loop.++ `asyncio.run` refuses to start when the calling thread already has a+ running loop — which is exactly the situation inside the MCP server,+ where FastMCP dispatches sync tool functions inline on its own loop, and+ inside any test harness that wraps the call in one. In that case the+ coroutine runs on a fresh thread with its own loop and the result is+ joined back. Outside a loop this is plain `asyncio.run`.+ """+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ return asyncio.run(coro)++ import threading++ box: dict[str, Any] = {}++ def _target() -> None:+ try:+ box["value"] = asyncio.run(coro)+ except BaseException as exc: # re-raised on the caller's thread+ box["error"] = exc++ thread = threading.Thread(target=_target, name="parallel-mcp-call", daemon=True)+ thread.start()+ thread.join()+ if "error" in box:+ raise box["error"]+ return box["value"]++if TYPE_CHECKING:+ from mcp.types import CallToolResult++_ENDPOINT = "https://search.parallel.ai/mcp"+_MAX_OBJECTIVE_CHARS = 5000+_MAX_SEARCH_QUERY_CHARS = 200+_SESSION_ID = uuid4().hex+++class ParallelProvider:+ """Web provider backed by Parallel's keyless Search MCP endpoint.++ Each operation uses a short-lived MCP connection. All provider instances in+ a process send the same random ``session_id``; Parallel uses it to correlate+ requests in its logs and for rate limiting. Transport, tool+ arguments, and result validation stay behind the synchronous ``WebProvider``+ interface used by Hyperresearch's CLI.+ """++ name = "parallel"++ def __init__(self, settings: FetchSettings | None = None) -> None:+ settings = settings or FetchSettings()+ self._timeout = timedelta(milliseconds=settings.page_timeout_ms)++ def search(self, query: str, max_results: int = 5) -> list[WebResult]:+ """Search the web and return ranked results with Markdown excerpts."""+ objective, search_query = _normalize_query(query)+ payload = self._call_tool(+ "web_search",+ {+ "objective": objective,+ "search_queries": [search_query],+ "session_id": _SESSION_ID,+ },+ )+ results = _result_items(payload, "web_search")+ return [_to_web_result(item) for item in results[: max(0, max_results)]]++ def fetch(self, url: str) -> WebResult:+ """Fetch one URL as clean Markdown."""+ payload = self._call_tool(+ "web_fetch",+ {+ "urls": [url],+ "full_content": True,+ "session_id": _SESSION_ID,+ },+ )+ results = _result_items(payload, "web_fetch")+ if not results:+ raise RuntimeError(_fetch_error(payload, url))+ return _to_web_result(results[0])++ def _call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ payload: dict[str, Any] = _run_coro(self._call_tool_async(name, arguments))+ return payload++ async def _call_tool_async(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ async with (+ streamablehttp_client(+ _ENDPOINT,+ timeout=self._timeout,+ sse_read_timeout=self._timeout,+ ) as (read_stream, write_stream, _),+ ClientSession(+ read_stream,+ write_stream,+ read_timeout_seconds=self._timeout,+ ) as session,+ ):+ await session.initialize()+ result = await session.call_tool(+ name,+ arguments=arguments,+ read_timeout_seconds=self._timeout,+ )+ return _decode_tool_result(result, name)+++def _normalize_query(query: str) -> tuple[str, str]:+ value = query.strip()+ if not value:+ raise ValueError("search query must not be empty")+ return (+ _truncate_at_word_boundary(value, _MAX_OBJECTIVE_CHARS),+ _truncate_at_word_boundary(value, _MAX_SEARCH_QUERY_CHARS),+ )+++def _truncate_at_word_boundary(value: str, limit: int) -> str:+ if len(value) <= limit:+ return value+ truncated = value[:limit]+ head, separator, _ = truncated.rpartition(" ")+ return head.rstrip() if separator and head.strip() else truncated+++def _decode_tool_result(result: CallToolResult, tool_name: str) -> dict[str, Any]:+ text_blocks = [block.text for block in result.content if block.type == "text"]+ text = "\n".join(text_blocks).strip()++ if result.isError:+ detail = text or "unknown MCP error"+ raise RuntimeError(f"Parallel {tool_name} failed: {detail}")++ structured_content = getattr(result, "structuredContent", None)+ if structured_content is not None:+ return dict(structured_content)++ if not text:+ raise RuntimeError(f"Parallel {tool_name} returned no structured or text content")++ try:+ payload = json.loads(text)+ except json.JSONDecodeError as exc:+ raise RuntimeError(f"Parallel {tool_name} returned invalid JSON") from exc+ if not isinstance(payload, dict):+ raise RuntimeError(f"Parallel {tool_name} returned a non-object payload")+ return payload+++def _result_items(payload: Mapping[str, Any], tool_name: str) -> list[Mapping[str, Any]]:+ results = payload.get("results")+ if not isinstance(results, list):+ raise RuntimeError(f"Parallel {tool_name} response is missing a results list")+ if not all(isinstance(item, dict) for item in results):+ raise RuntimeError(f"Parallel {tool_name} returned a malformed result")+ return results+++def _fetch_error(payload: Mapping[str, Any], url: str) -> str:+ errors = payload.get("errors")+ if isinstance(errors, list) and errors and isinstance(errors[0], dict):+ error_type = errors[0].get("error_type")+ if isinstance(error_type, str) and error_type:+ return f"Parallel could not fetch {url}: {error_type}"+ return f"Parallel returned no contents for {url}"+++def _to_web_result(item: Mapping[str, Any]) -> WebResult:+ url = item.get("url")+ if not isinstance(url, str) or not url:+ raise RuntimeError("Parallel returned a result without a URL")++ title = item.get("title")+ if title is None:+ title = ""+ if not isinstance(title, str):+ raise RuntimeError(f"Parallel returned an invalid title for {url}")++ full_content = item.get("full_content")+ excerpts = item.get("excerpts")+ if isinstance(full_content, str) and full_content.strip():+ content = full_content+ elif isinstance(excerpts, list) and all(isinstance(excerpt, str) for excerpt in excerpts):+ content = "\n\n".join(excerpts)+ else:+ raise RuntimeError(f"Parallel returned invalid content for {url}")+ if not content.strip():+ raise RuntimeError(f"Parallel returned no content for {url}")++ metadata: dict[str, Any] = {}+ publish_date = item.get("publish_date")+ if isinstance(publish_date, str) and publish_date:+ metadata["published_date"] = publish_date++ return WebResult(+ url=url,+ title=title,+ content=content,+ fetched_at=datetime.now(UTC),+ metadata=metadata,+ )diff --git a/tests/test_core/test_packaging.py b/tests/test_core/test_packaging.pyindex 3024ecb..5972fef 100644--- a/tests/test_core/test_packaging.py+++ b/tests/test_core/test_packaging.py@@ -26,6 +26,9 @@ def test_dev_extra_covers_tested_optional_providers():dev_deps = set(optional["dev"])assert "exa-py>=2.0.0" in dev_deps+ assert "mcp>=1.6,<2" in dev_deps+ assert optional["mcp"] == ["mcp>=1.6,<2"]+ assert optional["parallel"] == ["mcp>=1.8,<2"]assert "exa" in optionalassert "crawl4ai" in optionalassert "watch" in optionaldiff --git a/tests/test_web/test_parallel_provider.py b/tests/test_web/test_parallel_provider.pynew file mode 100644index 0000000..a694ef9--- /dev/null+++ b/tests/test_web/test_parallel_provider.py@@ -0,0 +1,315 @@+"""Tests for the Parallel provider — stubs the optional MCP SDK.++The test suite must collect without ``hyperresearch[parallel]`` installed.+These tests inject the small MCP surface the provider imports into+``sys.modules``, following the same isolation pattern as the Tavily tests.+"""++from __future__ import annotations++import builtins+import importlib+import json+import sys+from collections.abc import Iterator+from datetime import timedelta+from types import ModuleType, SimpleNamespace+from typing import Any+from unittest.mock import MagicMock++import pytest++from hyperresearch.core.config import FetchSettings++_MISSING = object()+++def parallel_module(monkeypatch: pytest.MonkeyPatch) -> Iterator[ModuleType]:+ fake_mcp = ModuleType("mcp")+ fake_mcp.__path__ = [] # type: ignore[attr-defined]+ fake_client = ModuleType("mcp.client")+ fake_client.__path__ = [] # type: ignore[attr-defined]+ fake_transport = ModuleType("mcp.client.streamable_http")++ fake_mcp.ClientSession = MagicMock() # type: ignore[attr-defined]+ fake_transport.streamablehttp_client = MagicMock() # type: ignore[attr-defined]+ fake_mcp.client = fake_client # type: ignore[attr-defined]+ fake_client.streamable_http = fake_transport # type: ignore[attr-defined]++ monkeypatch.setitem(sys.modules, "mcp", fake_mcp)+ monkeypatch.setitem(sys.modules, "mcp.client", fake_client)+ monkeypatch.setitem(sys.modules, "mcp.client.streamable_http", fake_transport)+ sys.modules.pop("hyperresearch.web.parallel_provider", None)++ yield importlib.import_module("hyperresearch.web.parallel_provider")++ sys.modules.pop("hyperresearch.web.parallel_provider", None)+++def _search_item(**overrides: Any) -> dict[str, Any]:+ item: dict[str, Any] = {+ "url": "https://example.com/article",+ "title": "Example Article",+ "publish_date": "2026-07-22",+ "excerpts": ["First excerpt.", "Second excerpt."],+ }+ item.update(overrides)+ return item+++def _tool_result(+ text: str = "",+ *,+ structured_content: object = _MISSING,+ is_error: bool = False,+) -> SimpleNamespace:+ content = [SimpleNamespace(type="text", text=text)] if text else []+ result = SimpleNamespace(content=content, isError=is_error)+ if structured_content is not _MISSING:+ result.structuredContent = structured_content+ return result+++def test_provider_registered_via_factory(parallel_module: ModuleType) -> None:+ from hyperresearch.web.base import get_provider++ provider = get_provider("parallel", settings=FetchSettings(page_timeout_ms=4321))++ assert isinstance(provider, parallel_module.ParallelProvider)+ assert provider._timeout == timedelta(milliseconds=4321)+++def test_missing_sdk_names_parallel_extra(monkeypatch: pytest.MonkeyPatch) -> None:+ from hyperresearch.web.base import get_provider++ real_import = builtins.__import__++ def import_without_mcp(name: str, *args: Any, **kwargs: Any) -> Any:+ if name == "mcp" or name.startswith("mcp."):+ raise ImportError("No module named 'mcp'")+ return real_import(name, *args, **kwargs)++ sys.modules.pop("hyperresearch.web.parallel_provider", None)+ monkeypatch.setattr(builtins, "__import__", import_without_mcp)++ with pytest.raises(ImportError, match=r"hyperresearch\[parallel\]"):+ get_provider("parallel")+++def test_search_maps_results_limits_output_and_reuses_process_session(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider+ calls: list[tuple[str, dict[str, Any]]] = []++ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ calls.append((name, arguments))+ return {"results": [_search_item(), _search_item(url="https://example.com/second")]}++ monkeypatch.setattr(provider_type, "_call_tool", fake_call)+ results = provider_type().search("solid state batteries", max_results=1)+ provider_type().fetch("https://example.com/article")++ assert len(results) == 1+ assert results[0].url == "https://example.com/article"+ assert results[0].content == "First excerpt.\n\nSecond excerpt."+ assert results[0].metadata["published_date"] == "2026-07-22"+ assert calls[0][0] == "web_search"+ assert calls[0][1]["objective"] == "solid state batteries"+ assert calls[0][1]["search_queries"] == ["solid state batteries"]+ assert calls[0][1]["session_id"] == calls[1][1]["session_id"]+++ ("length", "objective_length", "search_query_length"),+ [(200, 200, 200), (201, 201, 200), (5000, 5000, 200), (5001, 5000, 200)],+)+def test_search_bounds_tool_arguments(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+ length: int,+ objective_length: int,+ search_query_length: int,+) -> None:+ provider_type = parallel_module.ParallelProvider+ captured: dict[str, Any] = {}++ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ captured.update(arguments)+ return {"results": []}++ monkeypatch.setattr(provider_type, "_call_tool", fake_call)+ provider_type().search("x" * length)++ assert len(captured["objective"]) == objective_length+ assert len(captured["search_queries"][0]) == search_query_length+++def test_search_rejects_empty_query(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider++ def fail_call(self: Any, name: str, arguments: dict[str, Any]) -> None:+ pytest.fail("MCP should not be called")++ monkeypatch.setattr(provider_type, "_call_tool", fail_call)++ with pytest.raises(ValueError, match="must not be empty"):+ provider_type().search(" ")+++def test_fetch_requests_full_content_and_prefers_it(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider+ captured: dict[str, Any] = {}++ def fake_call(self: Any, name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ captured.update({"name": name, "arguments": arguments})+ return {+ "results": [+ _search_item(full_content="Full page Markdown.", excerpts=["Short excerpt."])+ ],+ "errors": [],+ }++ monkeypatch.setattr(provider_type, "_call_tool", fake_call)+ result = provider_type().fetch("https://example.com/article")++ assert captured["name"] == "web_fetch"+ assert captured["arguments"]["urls"] == ["https://example.com/article"]+ assert captured["arguments"]["full_content"] is True+ assert result.content == "Full page Markdown."+++def test_fetch_falls_back_to_excerpts(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider+ monkeypatch.setattr(+ provider_type,+ "_call_tool",+ lambda self, name, arguments: {+ "results": [_search_item(full_content=None, excerpts=["One.", "Two."])],+ "errors": [],+ },+ )++ result = provider_type().fetch("https://example.com/article")+ assert result.content == "One.\n\nTwo."+++def test_fetch_surfaces_extract_error(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider+ monkeypatch.setattr(+ provider_type,+ "_call_tool",+ lambda self, name, arguments: {+ "results": [],+ "errors": [{"url": "https://example.com/missing", "error_type": "not_found"}],+ },+ )++ with pytest.raises(RuntimeError, match="not_found"):+ provider_type().fetch("https://example.com/missing")+++def test_malformed_results_raise(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ provider_type = parallel_module.ParallelProvider+ monkeypatch.setattr(+ provider_type,+ "_call_tool",+ lambda self, name, arguments: {"results": "not-a-list"},+ )++ with pytest.raises(RuntimeError, match="results list"):+ provider_type().search("query")+++def test_timeout_applies_to_transport_session_and_tool_call(+ parallel_module: ModuleType,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ observed: dict[str, Any] = {}++ class AsyncContext:+ def __init__(self, value: Any) -> None:+ self.value = value++ async def __aenter__(self) -> Any:+ return self.value++ async def __aexit__(self, *args: Any) -> None:+ return None++ class FakeSession:+ def __init__(self, read: object, write: object, **kwargs: Any) -> None:+ observed["session_timeout"] = kwargs["read_timeout_seconds"]++ async def __aenter__(self) -> FakeSession:+ return self++ async def __aexit__(self, *args: Any) -> None:+ return None++ async def initialize(self) -> None:+ observed["initialized"] = True++ async def call_tool(self, name: str, **kwargs: Any) -> SimpleNamespace:+ observed["tool_timeout"] = kwargs["read_timeout_seconds"]+ return _tool_result(json.dumps({"results": []}))++ def fake_transport(endpoint: str, **kwargs: Any) -> AsyncContext:+ observed["endpoint"] = endpoint+ observed["transport_timeout"] = kwargs["timeout"]+ observed["sse_timeout"] = kwargs["sse_read_timeout"]+ return AsyncContext((object(), object(), None))++ monkeypatch.setattr(parallel_module, "streamablehttp_client", fake_transport)+ monkeypatch.setattr(parallel_module, "ClientSession", FakeSession)++ timeout = timedelta(milliseconds=1234)+ result = parallel_module.ParallelProvider(FetchSettings(page_timeout_ms=1234))._call_tool(+ "web_search", {}+ )++ assert result == {"results": []}+ assert observed == {+ "endpoint": "https://search.parallel.ai/mcp",+ "transport_timeout": timeout,+ "sse_timeout": timeout,+ "session_timeout": timeout,+ "initialized": True,+ "tool_timeout": timeout,+ }+++def test_tool_result_uses_structured_content(parallel_module: ModuleType) -> None:+ result = _tool_result("ignored", structured_content={"results": []})+ assert parallel_module._decode_tool_result(result, "web_search") == {"results": []}+++def test_old_tool_result_without_structured_content_uses_json_text(+ parallel_module: ModuleType,+) -> None:+ result = _tool_result(json.dumps({"results": []}))+ assert not hasattr(result, "structuredContent")+ assert parallel_module._decode_tool_result(result, "web_search") == {"results": []}+++def test_tool_error_raises_with_detail(parallel_module: ModuleType) -> None:+ result = _tool_result("rate limited", is_error=True)+ with pytest.raises(RuntimeError, match="rate limited"):+ parallel_module._decode_tool_result(result, "web_search")