The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(web): web_extract reports a target's HTTP 4xx/5xx as an error on every extract provider
Tekniumf7dde7f56da8
diff --git a/plugins/web/_common.py b/plugins/web/_common.pyindex 7508a3f36c4..091e58efd6d 100644--- a/plugins/web/_common.py+++ b/plugins/web/_common.py@@ -67,6 +67,44 @@ def extract_fail(urls: list[str], error: str) -> list[dict[str, Any]]:return [page_error(u, error) for u in urls]+# --- Origin HTTP status (the target page's answer, not the vendor API's) ---+# Scraping vendors report the target's own status inside a successful API reply (Firecrawl+# ``metadata.statusCode``, Parallel ``errors[].http_status_code``, Exa ``statuses[].error``).+# Unchecked, a 403/404 page reads as a successful (often empty) extraction: the agent can't tell+# "blocked" from "blank", the keyless rescue never runs, and the error page is cached as content.+def origin_http_error(status: Any, reason: Any = None) -> Optional[str]:+ """``HTTP <status> from origin[: reason]`` when the target answered >= 400, else None."""+ if isinstance(status, bool):+ return None+ try:+ code = int(status)+ except (TypeError, ValueError):+ return None+ if code < 400:+ return None+ detail = str(reason or "").strip()+ return f"HTTP {code} from origin: {detail}" if detail else f"HTTP {code} from origin"+++def with_origin_status(entry: dict[str, Any], status: Any, reason: Any = None) -> dict[str, Any]:+ """Set ``error`` on an extract entry whose target answered >= 400; the error page body+ stays in ``content`` (it can still explain the refusal)."""+ if error := origin_http_error(status, reason):+ entry["error"] = error+ return entry+++def vendor_error_entry(url: str, *, status: Any = None, reason: Any = None, body: Any = None) -> dict[str, Any]:+ """Per-URL failure from a vendor's own error list. Names the origin status when the vendor+ reports one (body kept as content; ``reason`` is a vendor tag like ``http_error`` that adds+ nothing to the code); otherwise the vendor's body/reason is the error text."""+ if origin := origin_http_error(status):+ entry = {**page_error(url, origin), "content": str(body or "")}+ else:+ entry = page_error(url, str(body or reason or "extraction failed"))+ return {**entry, "metadata": {"sourceURL": url}}++# --- Keyless ring hand-off (shared by exa / parallel / keenable) ---------------def keyless_search(display: str, name: str, query: str, limit: int, logger: logging.Logger) -> dict[str, Any]:from plugins.web.keyless_mcp import search_with_failoverdiff --git a/plugins/web/exa/provider.py b/plugins/web/exa/provider.pyindex 90caafadae1..865c29ea995 100644--- a/plugins/web/exa/provider.py+++ b/plugins/web/exa/provider.py@@ -11,7 +11,7 @@ from typing import Any, Dict, Listfrom plugins.web._common import (BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search, keyless_variant_schema,- provider_env, run_extract, run_search, search_ok, use_keyless, web_hit,+ provider_env, run_extract, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,)logger = logging.getLogger(__name__)@@ -57,7 +57,12 @@ class ExaWebSearchProvider(BaseWebSearchProvider):return keyless_extract("Exa", "exa", urls, logger)logger.info("Exa extract: %d URL(s)", len(urls))response = _get_exa_client().get_contents(urls, text=True)- return [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]+ results = [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]+ # A URL Exa could not fetch is absent from ``results``; its reason is in ``statuses``.+ return results + [+ vendor_error_entry(s.id or "", status=getattr(s.error, "http_status_code", None), reason=getattr(s.error, "tag", None))+ for s in getattr(response, "statuses", None) or [] if s.status == "error"+ ]return run_extract("Exa", logger, urls, _body, sdk=True)diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.pyindex 9b9753ce635..0b7168139fe 100644--- a/plugins/web/firecrawl/provider.py+++ b/plugins/web/firecrawl/provider.py@@ -12,7 +12,7 @@ from typing import Any, Dict, List, Optionalimport httpx-from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema+from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema, with_origin_statusfrom tools import managed_tool_gateway as _gatewayfrom tools import tool_backend_helpers as _backend_helpersfrom tools.url_safety import is_safe_url@@ -263,6 +263,14 @@ def _extract_scrape_payload(scrape_result: Any) -> dict[str, Any]:return plain["data"] if isinstance(plain.get("data"), dict) else plain+def _with_scrape_status(entry: dict[str, Any], metadata: Any) -> dict[str, Any]:+ """Flag a scrape whose target answered >= 400. The REST API reports ``statusCode``; the v4 SDK's+ ``model_dump`` (direct key and Tool Gateway) reports ``status_code``."""+ if not isinstance(metadata, dict):+ return entry+ return with_origin_status(entry, metadata.get("statusCode", metadata.get("status_code")), metadata.get("error"))++def _error_entry(url: str, error: str, *, title: str = "", raw: bool = False, blocked: Optional[dict[str, Any]] = None) -> dict[str, Any]:"""Per-URL extract failure. ``raw`` adds ``raw_content`` (post-scrape failures carryit, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``."""@@ -313,7 +321,8 @@ async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> direturn _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked)markdown, html = payload.get("markdown"), payload.get("html")content = markdown if format == "markdown" or (format is None and markdown) else html or markdown or ""- return {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}+ entry = {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}+ return _with_scrape_status(entry, metadata)except Exception as scrape_err:logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)return _error_entry(url, str(scrape_err), raw=True)diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.pyindex a29d9261b3e..8ae85a3e7a9 100644--- a/plugins/web/keyless_mcp.py+++ b/plugins/web/keyless_mcp.py@@ -14,7 +14,7 @@ import threadingimport uuidfrom typing import Any, Callable, Dict, List, Optional-from plugins.web._common import document as _page, page_error as _page_error, search_fail, search_ok, web_hit as _row+from plugins.web._common import document as _page, page_error as _page_error, search_fail, search_ok, vendor_error_entry, web_hit as _rowlogger = logging.getLogger(__name__)@@ -228,9 +228,10 @@ def parallel_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:_page(r.get("url") or "", r.get("title") or "", r.get("full_content") or r.get("content") or "\n\n".join(r.get("excerpts") or []))for r in data.get("results") or []]- for error in data.get("errors") or []:- url = error.get("url") or ""- results.append({**_page_error(url, str(error.get("content") or error.get("error_type") or "extraction failed")), "metadata": {"sourceURL": url}})+ results += [+ vendor_error_entry(e.get("url") or "", status=e.get("http_status_code"), reason=e.get("error_type"), body=e.get("content"))+ for e in data.get("errors") or []+ ]# URLs the endpoint silently dropped still get an error entry (per-URL contract).seen = {r["url"] for r in results}results.extend(_page_error(u, "no content returned") for u in urls if u not in seen)@@ -292,14 +293,14 @@ def firecrawl_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:def firecrawl_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:- from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload+ from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload, _with_scrape_statusclient = _KeylessFirecrawlClient()def _fetch(url: str) -> dict[str, Any]:payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}metadata = payload.get("metadata") or {}title = metadata.get("title") if isinstance(metadata, dict) else None- return _page(url, title or "", payload.get("markdown") or payload.get("html") or "")+ return _with_scrape_status(_page(url, title or "", payload.get("markdown") or payload.get("html") or ""), metadata)return _per_url(urls, _fetch, "firecrawl")diff --git a/plugins/web/parallel/provider.py b/plugins/web/parallel/provider.pyindex da0a299b2b6..472e17b57c1 100644--- a/plugins/web/parallel/provider.py+++ b/plugins/web/parallel/provider.py@@ -13,7 +13,7 @@ from typing import Any, Dict, Listfrom plugins.web._common import (SEARCH_LIMIT_CAP, BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search,- keyless_variant_schema, page_error, provider_env, run_extract_async, run_search, search_ok, use_keyless, web_hit,+ keyless_variant_schema, provider_env, run_extract_async, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,)logger = logging.getLogger(__name__)@@ -76,7 +76,7 @@ class ParallelWebSearchProvider(BaseWebSearchProvider):response = await _get_async_client().beta.extract(urls=urls, full_content=True)results = [document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or [])) for r in response.results or []]return results + [- {**page_error(e.url or "", e.content or e.error_type or "extraction failed"), "metadata": {"sourceURL": e.url or ""}}+ vendor_error_entry(e.url or "", status=getattr(e, "http_status_code", None), reason=e.error_type, body=e.content)for e in response.errors or []]diff --git a/tests/plugins/web/test_extract_origin_status.py b/tests/plugins/web/test_extract_origin_status.pynew file mode 100644index 00000000000..3e376ec3522--- /dev/null+++ b/tests/plugins/web/test_extract_origin_status.py@@ -0,0 +1,94 @@+"""web_extract must report a target page's HTTP >= 400 as an error, on every extract provider.++Scraping vendors answer 200 and tuck the target's own status into the payload; reading only the+content made a 403/404 look like a successful (often blank) page.+"""++import asyncio+import json+from types import SimpleNamespace+from unittest.mock import patch++import pytest++from plugins.web import keyless_mcp+from plugins.web.exa import provider as exa_provider+from plugins.web.firecrawl import provider as firecrawl_provider+from plugins.web.parallel import provider as parallel_provider++STUB_HTML = "<!DOCTYPE html><html><body></body></html>"+++def _scrape(payload: dict) -> dict:+ client = SimpleNamespace(scrape=lambda **_kw: payload)+ with patch.object(firecrawl_provider, "_get_firecrawl_client", return_value=client), \+ patch.object(firecrawl_provider, "check_website_access", return_value=None), \+ patch.object(firecrawl_provider, "is_safe_url", return_value=True):+ return asyncio.run(firecrawl_provider._scrape_one("https://t.test/x", ["markdown", "html"], None))++[email protected]("metadata, expected", [+ # Firecrawl SDK v4 model_dump (direct key and Nous Tool Gateway): snake_case, stub body.+ ({"status_code": 403, "error": "Forbidden", "source_url": "https://t.test/x"}, "HTTP 403 from origin: Forbidden"),+ # REST payload (keyless / self-hosted): camelCase.+ ({"statusCode": 404, "sourceURL": "https://t.test/x"}, "HTTP 404 from origin"),+])+def test_firecrawl_scrape_flags_origin_error_status(metadata, expected):+ entry = _scrape({"markdown": "", "html": STUB_HTML, "metadata": metadata})+ assert entry["error"] == expected+ assert entry["content"] == STUB_HTML # the error page body is kept, not discarded+++def test_firecrawl_scrape_success_has_no_error():+ entry = _scrape({"markdown": "# Hi", "metadata": {"status_code": 200, "title": "Hi"}})+ assert entry["content"] == "# Hi"+ assert "error" not in entry+++def test_keyless_firecrawl_flags_origin_error_status():+ class _Client:+ def scrape(self, **_kw):+ return {"success": True, "data": {"markdown": "", "metadata": {"statusCode": 403, "error": "Forbidden"}}}++ with patch.object(firecrawl_provider, "_KeylessFirecrawlClient", _Client):+ [entry] = keyless_mcp.firecrawl_extract_keyless(["https://t.test/x"])+ assert entry["error"] == "HTTP 403 from origin: Forbidden"+++def test_keyless_parallel_names_origin_status():+ payload = json.dumps({"results": [], "errors": [+ {"url": "https://t.test/x", "error_type": "http_error", "http_status_code": 403, "content": None},+ ]})+ with patch.object(keyless_mcp, "mcp_call", return_value=payload):+ [entry] = keyless_mcp.parallel_extract_keyless(["https://t.test/x"])+ assert entry["error"] == "HTTP 403 from origin"+++def test_parallel_sdk_names_origin_status():+ error = SimpleNamespace(url="https://t.test/x", error_type="http_error", http_status_code=404, content="nope")+ response = SimpleNamespace(results=[], errors=[error])++ async def _extract(**_kw):+ return response++ client = SimpleNamespace(beta=SimpleNamespace(extract=_extract))+ with patch.object(parallel_provider, "use_keyless", return_value=False), \+ patch.object(parallel_provider, "_get_async_client", return_value=client):+ [entry] = asyncio.run(parallel_provider.ParallelWebSearchProvider().extract(["https://t.test/x"]))+ assert entry["error"] == "HTTP 404 from origin"+ assert entry["content"] == "nope"+++def test_exa_sdk_reports_failed_urls_from_statuses():+ ok = SimpleNamespace(url="https://t.test/ok", title="OK", text="body")+ statuses = [+ SimpleNamespace(id="https://t.test/ok", status="success", error=None),+ SimpleNamespace(id="https://t.test/x", status="error",+ error=SimpleNamespace(tag="CRAWL_HTTP_403", http_status_code=403)),+ ]+ client = SimpleNamespace(get_contents=lambda *_a, **_kw: SimpleNamespace(results=[ok], statuses=statuses))+ with patch.object(exa_provider, "use_keyless", return_value=False), \+ patch.object(exa_provider, "_get_exa_client", return_value=client):+ out = exa_provider.ExaWebSearchProvider().extract(["https://t.test/ok", "https://t.test/x"])+ assert [r.get("error") for r in out] == [None, "HTTP 403 from origin"]+ assert out[1]["url"] == "https://t.test/x"