Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: run fetch_url off the running event loop, and drop the false DUPLICATE_URL on a re-fetched, deleted note (#86)

Amir Fathi20188cadfcbe
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39fef4b..8d95b34 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ The "Academic APIs before web search" section of the injected agent instructions
### Contributed fixes
- **`run finish` no longer blocks every light-classified run started on the installed gear (@maximilliangrand in #95).** `verify_run()` took its required-artifact step set from the manifest's `profile_steps`, so a run initialized with `--profile full` whose step-1 decomposition classified it `light` was asked for `critic-findings-*.json` and `patch-log.json` — artifacts the light tier correctly never writes, because steps 12 and 14 are skipped by the tier gate. The run did all its light-tier work and then sat at `blocked (verify)` with no legitimate way to pass. The gate now resolves the tier declared in `prompt-decomposition.json` when it disagrees with the manifest profile, which is what the router already documents ("the manifest's profile field is informational — the decomposition's tier rules"). A missing, unreadable or unknown tier still falls back to `profile_steps`.
+- **The MCP `fetch_url` tool works again, and a deleted note's URL can be fetched again (#84, fixed by @AmirF194 in #86).** Two bugs on the same path. `Crawl4AIProvider.fetch()` and `fetch_many()` called `asyncio.run()`, which raises before the coroutine runs when the caller already has a loop — and the MCP server dispatches sync tools on its own loop thread, so the tool failed on every call while the CLI never noticed. Both now go through a wrapper that runs the coroutine on a dedicated thread when a loop is present. Separately, `sources.note_id` is `ON DELETE SET NULL`, so deleting a note leaves its row behind with a NULL id; every duplicate-URL check tested row truthiness, so that URL was `DUPLICATE_URL` ("already fetched as note 'None'") forever. All four fetch paths (`fetch_and_save`, `hpr fetch`, `hpr fetch-batch`, `hpr research`) now share one orphan-aware check. The three CLI paths also record the source with `INSERT OR IGNORE`, which was a no-op on the orphaned row too — so the new note was left with no source record, and `hpr fetch`'s duplicate-race detector then deleted the note it had just written and reported `note_id: null`. Those paths now reclaim the orphaned row after the insert; the guard on `note_id IS NULL` leaves a genuine race winner untouched.
## [0.10.1] - 2026-09-11
diff --git a/src/hyperresearch/cli/fetch.py b/src/hyperresearch/cli/fetch.py
index 7258152..6e946d9 100644
--- a/src/hyperresearch/cli/fetch.py
+++ b/src/hyperresearch/cli/fetch.py
@@ -12,6 +12,7 @@ import typer
from hyperresearch.cli._output import console, err_console, output
from hyperresearch.core.config import AssetSettings, FetchSettings
+from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
from hyperresearch.models.output import error, success
app = typer.Typer()
@@ -242,8 +243,9 @@ def fetch(
vault.auto_sync()
conn = vault.db
- # Check if URL already fetched
- existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ # Check if URL already fetched (an orphaned row — note deleted — is not a duplicate,
+ # so --suggested-by never tries to append a breadcrumb to a note_id of None)
+ existing = existing_live_note_for_url(conn, url)
if existing:
note_id = existing["note_id"]
# Graceful duplicate handling for the guided reading loop:
@@ -540,6 +542,12 @@ def fetch(
VALUES (?, ?, ?, ?, ?, ?)""",
(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),
)
+ # An orphaned row (note deleted → ON DELETE SET NULL) also makes that INSERT
+ # a no-op, but nobody owns the url, so claim it. Guarded on note_id IS NULL,
+ # so a genuine race winner is left alone and the check below still fires.
+ reclaim_orphaned_source_row(
+ conn, url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash
+ )
conn.commit()
# Detect the race: if the committed row's note_id != ours, another
diff --git a/src/hyperresearch/cli/fetch_batch.py b/src/hyperresearch/cli/fetch_batch.py
index 2c8a4bc..a1c9f58 100644
--- a/src/hyperresearch/cli/fetch_batch.py
+++ b/src/hyperresearch/cli/fetch_batch.py
@@ -9,6 +9,7 @@ from urllib.parse import urlparse
import typer
from hyperresearch.cli._output import console, output
+from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
from hyperresearch.models.output import error, success
@@ -79,7 +80,8 @@ def fetch_batch(
# Filter out already-fetched URLs
new_urls = []
for url in all_urls:
- existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ # An orphaned row (note deleted) is not a duplicate; only a live note skips
+ existing = existing_live_note_for_url(conn, url)
if existing:
if not json_output:
console.print(f" [dim]Skip:[/] {url} (already fetched as {existing['note_id']})")
@@ -275,6 +277,10 @@ def fetch_batch(
VALUES (?, ?, ?, ?, ?, ?)""",
(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),
)
+ # The INSERT is also a no-op on an orphaned row (note deleted); claim it
+ reclaim_orphaned_source_row(
+ conn, url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash
+ )
# Save assets if requested
if save_assets:
diff --git a/src/hyperresearch/cli/research.py b/src/hyperresearch/cli/research.py
index 55aff1f..e3b874d 100644
--- a/src/hyperresearch/cli/research.py
+++ b/src/hyperresearch/cli/research.py
@@ -10,6 +10,7 @@ from urllib.parse import urlparse
import typer
from hyperresearch.cli._output import console, output
+from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
from hyperresearch.models.output import error, success
@@ -109,11 +110,8 @@ def research(
break
if link_url in fetched_urls:
continue
- # Check if already in DB
- existing = conn.execute(
- "SELECT note_id FROM sources WHERE url = ?", (link_url,)
- ).fetchone()
- if existing:
+ # Check if already in DB (an orphaned row — note deleted — is not a duplicate)
+ if existing_live_note_for_url(conn, link_url):
fetched_urls.add(link_url)
continue
@@ -238,9 +236,8 @@ def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
if result.looks_like_login_wall(url):
return None
- # Skip if already fetched
- existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
- if existing:
+ # Skip if already fetched (an orphaned row — note deleted — is not a duplicate)
+ if existing_live_note_for_url(conn, url):
return None
title = result.title or urlparse(url).path.split("/")[-1] or "Untitled"
@@ -283,6 +280,10 @@ def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
VALUES (?, ?, ?, ?, ?, ?)""",
(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),
)
+ # The INSERT is also a no-op on an orphaned row (note deleted); claim it
+ reclaim_orphaned_source_row(
+ conn, url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash
+ )
conn.commit()
return {
diff --git a/src/hyperresearch/core/fetcher.py b/src/hyperresearch/core/fetcher.py
index 0b9406e..b8f5312 100644
--- a/src/hyperresearch/core/fetcher.py
+++ b/src/hyperresearch/core/fetcher.py
@@ -3,9 +3,53 @@
from __future__ import annotations
import hashlib
+import sqlite3
from urllib.parse import urlparse
+def existing_live_note_for_url(conn: sqlite3.Connection, url: str) -> sqlite3.Row | None:
+ """Return the ``sources`` row for ``url`` only if it still points at a live note.
+
+ ``sources.note_id`` is ``ON DELETE SET NULL``, so deleting a note leaves its
+ row behind with ``note_id = NULL``. That orphan is not a duplicate — the url
+ may be fetched again — so this returns None both when the row is absent and
+ when it is orphaned. Every duplicate-url check must go through here rather
+ than testing row truthiness.
+ """
+ row = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ if row is None or row["note_id"] is None:
+ return None
+ return row
+
+
+def reclaim_orphaned_source_row(
+ conn: sqlite3.Connection,
+ url: str,
+ note_id: str,
+ domain: str,
+ fetched_at: str,
+ provider: str,
+ content_hash: str,
+) -> None:
+ """Point an orphaned ``sources`` row (``note_id IS NULL``) for ``url`` at ``note_id``.
+
+ The CLI fetch paths record a source with ``INSERT OR IGNORE`` so that a
+ duplicate-url race is a silent no-op instead of an IntegrityError. An
+ orphaned row makes that INSERT a no-op too, which would leave the freshly
+ written note with no source record (and, in ``fetch``, trip the race
+ detector into deleting it). Call this right after the INSERT: the
+ ``note_id IS NULL`` guard means it claims only an orphan and stays a no-op
+ when another fetch already owns the row, so the race semantics are unchanged.
+ """
+ conn.execute(
+ """UPDATE sources
+ SET note_id = ?, domain = ?, fetched_at = ?, provider = ?,
+ content_hash = ?, status = 'active'
+ WHERE url = ? AND note_id IS NULL""",
+ (note_id, domain, fetched_at, provider, content_hash, url),
+ )
+
+
def fetch_and_save(
vault,
url: str,
@@ -29,8 +73,8 @@ def fetch_and_save(
tags = tags or []
conn = vault.db
- # Check if URL already fetched
- existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ # Check if URL already fetched (an orphaned row — note deleted — is not a duplicate)
+ existing = existing_live_note_for_url(conn, url)
if existing:
raise ValueError(f"URL already fetched as note '{existing['note_id']}'")
@@ -157,11 +201,18 @@ def fetch_and_save(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
- # Record source
+ # Record source (upsert: an orphaned row for this url may already exist)
content_hash = hashlib.sha256(result.content.encode("utf-8")).hexdigest()[:16]
conn.execute(
"""INSERT INTO sources (url, note_id, domain, fetched_at, provider, content_hash)
- VALUES (?, ?, ?, ?, ?, ?)""",
+ VALUES (?, ?, ?, ?, ?, ?)
+ ON CONFLICT(url) DO UPDATE SET
+ note_id = excluded.note_id,
+ domain = excluded.domain,
+ fetched_at = excluded.fetched_at,
+ provider = excluded.provider,
+ content_hash = excluded.content_hash,
+ status = 'active'""",
(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),
)
conn.commit()
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index e2f33fd..1a3a13b 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -13,6 +13,7 @@ import logging
import os
import pathlib
import sys
+import threading
from datetime import UTC, datetime
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator
@@ -35,6 +36,33 @@ if sys.platform == "win32":
pass
+def _run_coro(coro):
+ """Run ``coro`` to completion, whether or not this thread already has a running
+ event loop (the MCP server dispatches sync tools on its own loop's thread; the
+ CLI does not). A plain ``asyncio.run(coro)`` only works in the second case, so
+ fall back to a dedicated thread, which has no running loop of its own.
+ """
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(coro)
+
+ box: dict = {}
+
+ def _target() -> None:
+ try:
+ box["result"] = asyncio.run(coro)
+ except BaseException as exc: # re-raised on the caller's thread below
+ box["error"] = exc
+
+ thread = threading.Thread(target=_target)
+ thread.start()
+ thread.join()
+ if "error" in box:
+ raise box["error"]
+ return box["result"]
+
+
def _is_pdf_url(url: str) -> bool:
"""Check if URL likely points to a PDF."""
from urllib.parse import urlparse
@@ -393,9 +421,9 @@ class Crawl4AIProvider:
# When visible + profile: use Playwright directly (crawl4ai managed browser ignores headless=False)
if not self._headless and self._data_dir:
- return asyncio.run(self._fetch_visible(url))
+ return _run_coro(self._fetch_visible(url))
- result = asyncio.run(self._fetch_async(url))
+ result = _run_coro(self._fetch_async(url))
# Post-fetch PDF detection: if the browser got binary garbage (PDF served
# inline without proper content-type handling), re-fetch as a direct PDF download.
@@ -520,7 +548,7 @@ class Crawl4AIProvider:
def fetch_many(self, urls: list[str]) -> list[WebResult]:
"""Fetch multiple URLs concurrently using crawl4ai's arun_many."""
- return asyncio.run(self._fetch_many_async(urls))
+ return _run_coro(self._fetch_many_async(urls))
async def _fetch_many_async(self, urls: list[str]) -> list[WebResult]:
# SSRF gate: same entry-point check as fetch(). Refused URLs are
diff --git a/tests/test_cli/test_orphan_dedup_call_sites.py b/tests/test_cli/test_orphan_dedup_call_sites.py
new file mode 100644
index 0000000..bb78328
--- /dev/null
+++ b/tests/test_cli/test_orphan_dedup_call_sites.py
@@ -0,0 +1,169 @@
+"""The three CLI-side duplicate-url call sites: `fetch`, `fetch-batch`, and
+research's `_save_result`. All three read `sources.note_id`, which is NULL
+(not row-absent) once the note is deleted (ON DELETE SET NULL), so a bare
+`if existing:` treated that as a live duplicate forever.
+
+All three also record the source with `INSERT OR IGNORE`, which is a no-op on
+the orphaned row too — so passing the read-side check alone left the new note
+with no source record, and in `fetch` tripped the duplicate-url race detector
+into deleting the note it had just written. Every test here drives the real
+deletion path (the note is actually removed and the vault re-synced) rather
+than faking a NULL, and asserts that the row ends up pointing at the new note.
+Mirrors tests/test_core/test_fetcher_orphan_dedup.py for fetch_and_save.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from hyperresearch.cli import app
+from hyperresearch.web.base import WebResult
+
+runner = CliRunner()
+
+
+class _FakeProvider:
+ name = "fake"
+
+ def fetch(self, url: str) -> WebResult:
+ return WebResult(url=url, title="Fake Title", content="enough content to clear the junk gate. " * 10)
+
+
+def vault_dir(tmp_path: Path) -> Path:
+ result = runner.invoke(app, ["init", str(tmp_path / "kb"), "--name", "Orphan Dedup Test"])
+ assert result.exit_code == 0
+ return tmp_path / "kb"
+
+
+def _source_row(vault_dir: Path, url: str):
+ from hyperresearch.core.vault import Vault
+
+ return Vault.discover(vault_dir).db.execute(
+ "SELECT note_id FROM sources WHERE url = ?", (url,)
+ ).fetchone()
+
+
+def _delete_note_for_real(vault_dir: Path, note_id: str, url: str) -> None:
+ """`note rm` — the real path: the notes row goes, and ON DELETE SET NULL
+ leaves the sources row behind with note_id = NULL."""
+ result = runner.invoke(app, ["note", "rm", note_id, "--force", "--json"])
+ assert result.exit_code == 0, result.output
+ row = _source_row(vault_dir, url)
+ assert row is not None and row["note_id"] is None # orphaned, not deleted
+
+
+def test_fetch_refetches_after_note_is_deleted(vault_dir: Path, monkeypatch):
+ os.chdir(vault_dir)
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ url = "https://example.com/cli-fetch-refetch"
+
+ first = runner.invoke(app, ["fetch", url, "--json"])
+ assert first.exit_code == 0
+ note_id = json.loads(first.output)["data"]["note_id"]
+
+ _delete_note_for_real(vault_dir, note_id, url)
+
+ second = runner.invoke(app, ["fetch", url, "--json"])
+ assert second.exit_code == 0, second.output # must not report DUPLICATE_URL
+ data = json.loads(second.output)["data"]
+ # A None here means the race detector mistook the orphan for a lost race
+ # and deleted the note it had just written.
+ assert data["note_id"] is not None
+ assert (vault_dir / data["path"]).exists()
+ assert _source_row(vault_dir, url)["note_id"] == data["note_id"]
+
+
+def test_fetch_with_suggested_by_refetches_rather_than_breadcrumbing_none(
+ vault_dir: Path, monkeypatch
+):
+ """--suggested-by on a live duplicate appends a breadcrumb to the existing
+ note; on an orphan there is no note to breadcrumb, so it must re-fetch."""
+ os.chdir(vault_dir)
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ url = "https://example.com/cli-fetch-suggested"
+
+ first = runner.invoke(app, ["fetch", url, "--json"])
+ assert first.exit_code == 0
+ note_id = json.loads(first.output)["data"]["note_id"]
+
+ live = runner.invoke(app, ["fetch", url, "--suggested-by", "some-note", "--json"])
+ assert live.exit_code == 0, live.output
+ assert json.loads(live.output)["data"]["duplicate"] is True
+
+ _delete_note_for_real(vault_dir, note_id, url)
+
+ second = runner.invoke(app, ["fetch", url, "--suggested-by", "some-note", "--json"])
+ assert second.exit_code == 0, second.output
+ data = json.loads(second.output)["data"]
+ assert "duplicate" not in data
+ assert data["note_id"] is not None
+ assert _source_row(vault_dir, url)["note_id"] == data["note_id"]
+
+
+def test_fetch_batch_does_not_skip_a_deleted_note_url(vault_dir: Path, monkeypatch):
+ os.chdir(vault_dir)
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ url = "https://example.com/cli-batch-refetch"
+
+ first = runner.invoke(app, ["fetch", url, "--json"])
+ assert first.exit_code == 0
+ note_id = json.loads(first.output)["data"]["note_id"]
+ _delete_note_for_real(vault_dir, note_id, url)
+
+ batch = runner.invoke(app, ["fetch-batch", url, "--json"])
+ assert batch.exit_code == 0, batch.output
+ data = json.loads(batch.output)["data"]
+ assert data["skipped"] == 0 # a skip here means the orphan read as a duplicate
+ assert len(data["notes_created"]) == 1
+ new_id = data["notes_created"][0]["note_id"]
+ assert _source_row(vault_dir, url)["note_id"] == new_id # row re-linked, not left orphaned
+
+
+def test_fetch_batch_still_skips_a_live_duplicate(vault_dir: Path, monkeypatch):
+ os.chdir(vault_dir)
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ url = "https://example.com/cli-batch-live"
+
+ assert runner.invoke(app, ["fetch", url, "--json"]).exit_code == 0
+
+ batch = runner.invoke(app, ["fetch-batch", url, "--json"])
+ assert batch.exit_code == 0, batch.output
+ data = json.loads(batch.output)["data"]
+ assert data["skipped"] == 1
+ assert data["notes_created"] == []
+
+
+def test_save_result_skips_a_live_duplicate_but_not_a_deleted_note(tmp_vault, monkeypatch):
+ from hyperresearch.cli.research import _save_result
+ from hyperresearch.core.sync import compute_sync_plan, execute_sync
+
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ conn = tmp_vault.db
+ prov = _FakeProvider()
+ url = "https://example.com/research-save-result"
+
+ first = _save_result(tmp_vault, conn, prov, prov.fetch(url), [], None)
+ assert first is not None
+
+ # live duplicate: the row still carries a real note_id
+ again = _save_result(tmp_vault, conn, prov, prov.fetch(url), [], None)
+ assert again is None
+
+ # delete the note for real: unlink + sync drops the notes row, and
+ # ON DELETE SET NULL orphans the sources row
+ (tmp_vault.root / first["path"]).unlink()
+ execute_sync(tmp_vault, compute_sync_plan(tmp_vault))
+ row = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ assert row is not None and row["note_id"] is None
+
+ after_delete = _save_result(tmp_vault, conn, prov, prov.fetch(url), [], None)
+ assert after_delete is not None # must not skip an orphaned row
+ assert (tmp_vault.root / after_delete["path"]).exists()
+ row = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ assert row["note_id"] == after_delete["note_id"] # row re-linked, not left orphaned
diff --git a/tests/test_core/test_fetcher_orphan_dedup.py b/tests/test_core/test_fetcher_orphan_dedup.py
new file mode 100644
index 0000000..acb588b
--- /dev/null
+++ b/tests/test_core/test_fetcher_orphan_dedup.py
@@ -0,0 +1,58 @@
+"""A sources row survives note deletion (ON DELETE SET NULL); fetch_and_save's
+duplicate-URL check and its own INSERT must both handle that orphaned row rather
+than treat it as a live duplicate forever."""
+
+from __future__ import annotations
+
+from hyperresearch.core.fetcher import fetch_and_save
+from hyperresearch.core.sync import compute_sync_plan, execute_sync
+from hyperresearch.web.base import WebResult
+
+
+class _FakeProvider:
+ name = "fake"
+
+ def fetch(self, url: str) -> WebResult:
+ return WebResult(url=url, title="Fake Title", content="enough content to clear the junk gate. " * 10)
+
+
+def test_refetch_succeeds_after_the_note_is_deleted(tmp_vault, monkeypatch):
+ monkeypatch.setattr(
+ "hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider()
+ )
+ url = "https://example.com/refetch-me"
+
+ first = fetch_and_save(tmp_vault, url)
+ note_path = tmp_vault.root / first["path"]
+ assert note_path.exists()
+
+ note_path.unlink()
+ plan = compute_sync_plan(tmp_vault)
+ execute_sync(tmp_vault, plan)
+
+ row = tmp_vault.db.execute(
+ "SELECT note_id FROM sources WHERE url = ?", (url,)
+ ).fetchone()
+ assert row is not None and row["note_id"] is None # orphaned, not deleted
+
+ second = fetch_and_save(tmp_vault, url) # must not raise ValueError
+ assert (tmp_vault.root / second["path"]).exists()
+ row = tmp_vault.db.execute(
+ "SELECT note_id FROM sources WHERE url = ?", (url,)
+ ).fetchone()
+ assert row["note_id"] == second["note_id"] # the upsert re-linked the row
+
+
+def test_live_duplicate_is_still_rejected(tmp_vault, monkeypatch):
+ monkeypatch.setattr(
+ "hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider()
+ )
+ url = "https://example.com/still-a-duplicate"
+
+ fetch_and_save(tmp_vault, url)
+
+ try:
+ fetch_and_save(tmp_vault, url)
+ raise AssertionError("expected ValueError for a URL with a live note")
+ except ValueError as exc:
+ assert "already fetched" in str(exc)
diff --git a/tests/test_web/test_crawl4ai_running_loop.py b/tests/test_web/test_crawl4ai_running_loop.py
new file mode 100644
index 0000000..5d12db7
--- /dev/null
+++ b/tests/test_web/test_crawl4ai_running_loop.py
@@ -0,0 +1,97 @@
+"""fetch() must work when called on a thread that already has a running event loop.
+
+The MCP server's fastmcp tool dispatch calls synchronous tool functions directly on
+its own running event-loop thread (it does not offload them to a worker thread), so
+fetch_url's call chain into Crawl4AIProvider.fetch() lands there too. A plain
+asyncio.run() inside fetch() would raise "cannot be called from a running event
+loop" in that case; the CLI's plain synchronous invocation has no such loop and must
+keep working exactly as before.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+
+import pytest
+
+pytest.importorskip("crawl4ai.browser_adapter")
+
+from hyperresearch.web.base import WebResult
+from hyperresearch.web.crawl4ai_provider import Crawl4AIProvider
+
+
+def _run_on_a_fresh_thread(coro):
+ """Run ``coro`` to completion on a brand-new thread, guaranteeing a real,
+ isolated running event loop for the duration of the call.
+
+ See tests/test_web/test_fetch_many_fallback.py's ``_run`` for why: a prior
+ test in the full suite can leave the main thread's asyncio state marked
+ "running" (observed once crawl4ai's sync Playwright bridge has been
+ exercised), which makes a plain ``asyncio.run()`` here raise regardless of
+ what this test is trying to isolate. A fresh thread has no such leftover
+ state.
+ """
+ box: dict = {}
+
+ def target() -> None:
+ try:
+ box["result"] = asyncio.run(coro)
+ except BaseException as exc:
+ box["error"] = exc
+
+ thread = threading.Thread(target=target)
+ thread.start()
+ thread.join()
+ if "error" in box:
+ raise box["error"]
+ return box["result"]
+
+
+def test_fetch_succeeds_when_called_from_a_running_event_loop(monkeypatch) -> None:
+ async def fake_fetch_async(self, url):
+ return WebResult(url=url, title="ok", content="fetched content")
+
+ monkeypatch.setattr(Crawl4AIProvider, "_fetch_async", fake_fetch_async)
+ provider = Crawl4AIProvider(headless=True)
+
+ async def call_fetch_on_the_running_loop():
+ return provider.fetch("https://example.com")
+
+ result = _run_on_a_fresh_thread(call_fetch_on_the_running_loop())
+
+ assert result.title == "ok"
+ assert result.content == "fetched content"
+
+
+def test_fetch_still_succeeds_with_no_running_loop(monkeypatch) -> None:
+ """The CLI's plain synchronous call path (no event loop on the thread)."""
+ async def fake_fetch_async(self, url):
+ return WebResult(url=url, title="ok", content="fetched content")
+
+ monkeypatch.setattr(Crawl4AIProvider, "_fetch_async", fake_fetch_async)
+ provider = Crawl4AIProvider(headless=True)
+
+ result = provider.fetch("https://example.com")
+
+ assert result.title == "ok"
+ assert result.content == "fetched content"
+
+
+def test_fetch_many_succeeds_when_called_from_a_running_event_loop(monkeypatch) -> None:
+ """fetch_many() shares fetch()'s wrapper; a bare asyncio.run() here would
+ fail the same way on the MCP server's loop thread."""
+ async def fake_fetch_many_async(self, urls):
+ return [WebResult(url=u, title="ok", content="fetched content") for u in urls]
+
+ monkeypatch.setattr(Crawl4AIProvider, "_fetch_many_async", fake_fetch_many_async)
+ provider = Crawl4AIProvider(headless=True)
+ urls = ["https://example.com/a", "https://example.com/b"]
+
+ async def call_fetch_many_on_the_running_loop():
+ return provider.fetch_many(urls)
+
+ results = _run_on_a_fresh_thread(call_fetch_many_on_the_running_loop())
+
+ assert [r.url for r in results] == urls
+ assert all(r.content == "fetched content" for r in results)