Skip to content

The agent that grows with you

fix(web): web_extract reports a target's HTTP 4xx/5xx as an error on every extract provider

Tekniumf7dde7f56da8
diff --git a/plugins/web/_common.py b/plugins/web/_common.py
index 7508a3f36c4..091e58efd6d 100644
--- a/plugins/web/_common.py
+++ b/plugins/web/_common.py
@@ -67,6 +67,44 @@ def extract_fail(urls: list[str], error: str) -> list[dict[str, Any]]:
return [page_error(u, error) for u in urls]
+# --- Origin HTTP status (the target page's answer, not the vendor API's) ---
+# Scraping vendors report the target's own status inside a successful API reply (Firecrawl
+# ``metadata.statusCode``, Parallel ``errors[].http_status_code``, Exa ``statuses[].error``).
+# Unchecked, a 403/404 page reads as a successful (often empty) extraction: the agent can't tell
+# "blocked" from "blank", the keyless rescue never runs, and the error page is cached as content.
+def origin_http_error(status: Any, reason: Any = None) -> Optional[str]:
+ """``HTTP <status> from origin[: reason]`` when the target answered >= 400, else None."""
+ if isinstance(status, bool):
+ return None
+ try:
+ code = int(status)
+ except (TypeError, ValueError):
+ return None
+ if code < 400:
+ return None
+ detail = str(reason or "").strip()
+ return f"HTTP {code} from origin: {detail}" if detail else f"HTTP {code} from origin"
+
+
+def with_origin_status(entry: dict[str, Any], status: Any, reason: Any = None) -> dict[str, Any]:
+ """Set ``error`` on an extract entry whose target answered >= 400; the error page body
+ stays in ``content`` (it can still explain the refusal)."""
+ if error := origin_http_error(status, reason):
+ entry["error"] = error
+ return entry
+
+
+def vendor_error_entry(url: str, *, status: Any = None, reason: Any = None, body: Any = None) -> dict[str, Any]:
+ """Per-URL failure from a vendor's own error list. Names the origin status when the vendor
+ reports one (body kept as content; ``reason`` is a vendor tag like ``http_error`` that adds
+ nothing to the code); otherwise the vendor's body/reason is the error text."""
+ if origin := origin_http_error(status):
+ entry = {**page_error(url, origin), "content": str(body or "")}
+ else:
+ entry = page_error(url, str(body or reason or "extraction failed"))
+ return {**entry, "metadata": {"sourceURL": url}}
+
+
# --- Keyless ring hand-off (shared by exa / parallel / keenable) ---------------
def keyless_search(display: str, name: str, query: str, limit: int, logger: logging.Logger) -> dict[str, Any]:
from plugins.web.keyless_mcp import search_with_failover
diff --git a/plugins/web/exa/provider.py b/plugins/web/exa/provider.py
index 90caafadae1..865c29ea995 100644
--- a/plugins/web/exa/provider.py
+++ b/plugins/web/exa/provider.py
@@ -11,7 +11,7 @@ from typing import Any, Dict, List
from plugins.web._common import (
BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search, keyless_variant_schema,
- provider_env, run_extract, run_search, search_ok, use_keyless, web_hit,
+ provider_env, run_extract, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,
)
logger = logging.getLogger(__name__)
@@ -57,7 +57,12 @@ class ExaWebSearchProvider(BaseWebSearchProvider):
return keyless_extract("Exa", "exa", urls, logger)
logger.info("Exa extract: %d URL(s)", len(urls))
response = _get_exa_client().get_contents(urls, text=True)
- return [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]
+ results = [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]
+ # A URL Exa could not fetch is absent from ``results``; its reason is in ``statuses``.
+ return results + [
+ vendor_error_entry(s.id or "", status=getattr(s.error, "http_status_code", None), reason=getattr(s.error, "tag", None))
+ for s in getattr(response, "statuses", None) or [] if s.status == "error"
+ ]
return run_extract("Exa", logger, urls, _body, sdk=True)
diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py
index 9b9753ce635..0b7168139fe 100644
--- a/plugins/web/firecrawl/provider.py
+++ b/plugins/web/firecrawl/provider.py
@@ -12,7 +12,7 @@ from typing import Any, Dict, List, Optional
import httpx
-from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema
+from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema, with_origin_status
from tools import managed_tool_gateway as _gateway
from tools import tool_backend_helpers as _backend_helpers
from tools.url_safety import is_safe_url
@@ -263,6 +263,14 @@ def _extract_scrape_payload(scrape_result: Any) -> dict[str, Any]:
return plain["data"] if isinstance(plain.get("data"), dict) else plain
+def _with_scrape_status(entry: dict[str, Any], metadata: Any) -> dict[str, Any]:
+ """Flag a scrape whose target answered >= 400. The REST API reports ``statusCode``; the v4 SDK's
+ ``model_dump`` (direct key and Tool Gateway) reports ``status_code``."""
+ if not isinstance(metadata, dict):
+ return entry
+ return with_origin_status(entry, metadata.get("statusCode", metadata.get("status_code")), metadata.get("error"))
+
+
def _error_entry(url: str, error: str, *, title: str = "", raw: bool = False, blocked: Optional[dict[str, Any]] = None) -> dict[str, Any]:
"""Per-URL extract failure. ``raw`` adds ``raw_content`` (post-scrape failures carry
it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``."""
@@ -313,7 +321,8 @@ async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> di
return _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked)
markdown, html = payload.get("markdown"), payload.get("html")
content = markdown if format == "markdown" or (format is None and markdown) else html or markdown or ""
- return {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}
+ entry = {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}
+ return _with_scrape_status(entry, metadata)
except Exception as scrape_err:
logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
return _error_entry(url, str(scrape_err), raw=True)
diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py
index a29d9261b3e..8ae85a3e7a9 100644
--- a/plugins/web/keyless_mcp.py
+++ b/plugins/web/keyless_mcp.py
@@ -14,7 +14,7 @@ import threading
import uuid
from typing import Any, Callable, Dict, List, Optional
-from plugins.web._common import document as _page, page_error as _page_error, search_fail, search_ok, web_hit as _row
+from plugins.web._common import document as _page, page_error as _page_error, search_fail, search_ok, vendor_error_entry, web_hit as _row
logger = logging.getLogger(__name__)
@@ -228,9 +228,10 @@ def parallel_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
_page(r.get("url") or "", r.get("title") or "", r.get("full_content") or r.get("content") or "\n\n".join(r.get("excerpts") or []))
for r in data.get("results") or []
]
- for error in data.get("errors") or []:
- url = error.get("url") or ""
- results.append({**_page_error(url, str(error.get("content") or error.get("error_type") or "extraction failed")), "metadata": {"sourceURL": url}})
+ results += [
+ vendor_error_entry(e.get("url") or "", status=e.get("http_status_code"), reason=e.get("error_type"), body=e.get("content"))
+ for e in data.get("errors") or []
+ ]
# URLs the endpoint silently dropped still get an error entry (per-URL contract).
seen = {r["url"] for r in results}
results.extend(_page_error(u, "no content returned") for u in urls if u not in seen)
@@ -292,14 +293,14 @@ def firecrawl_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
def firecrawl_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
- from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload
+ from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload, _with_scrape_status
client = _KeylessFirecrawlClient()
def _fetch(url: str) -> dict[str, Any]:
payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}
metadata = payload.get("metadata") or {}
title = metadata.get("title") if isinstance(metadata, dict) else None
- return _page(url, title or "", payload.get("markdown") or payload.get("html") or "")
+ return _with_scrape_status(_page(url, title or "", payload.get("markdown") or payload.get("html") or ""), metadata)
return _per_url(urls, _fetch, "firecrawl")
diff --git a/plugins/web/parallel/provider.py b/plugins/web/parallel/provider.py
index da0a299b2b6..472e17b57c1 100644
--- a/plugins/web/parallel/provider.py
+++ b/plugins/web/parallel/provider.py
@@ -13,7 +13,7 @@ from typing import Any, Dict, List
from plugins.web._common import (
SEARCH_LIMIT_CAP, BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search,
- keyless_variant_schema, page_error, provider_env, run_extract_async, run_search, search_ok, use_keyless, web_hit,
+ keyless_variant_schema, provider_env, run_extract_async, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,
)
logger = logging.getLogger(__name__)
@@ -76,7 +76,7 @@ class ParallelWebSearchProvider(BaseWebSearchProvider):
response = await _get_async_client().beta.extract(urls=urls, full_content=True)
results = [document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or [])) for r in response.results or []]
return results + [
- {**page_error(e.url or "", e.content or e.error_type or "extraction failed"), "metadata": {"sourceURL": e.url or ""}}
+ vendor_error_entry(e.url or "", status=getattr(e, "http_status_code", None), reason=e.error_type, body=e.content)
for e in response.errors or []
]
diff --git a/tests/plugins/web/test_extract_origin_status.py b/tests/plugins/web/test_extract_origin_status.py
new file mode 100644
index 00000000000..3e376ec3522
--- /dev/null
+++ b/tests/plugins/web/test_extract_origin_status.py
@@ -0,0 +1,94 @@
+"""web_extract must report a target page's HTTP >= 400 as an error, on every extract provider.
+
+Scraping vendors answer 200 and tuck the target's own status into the payload; reading only the
+content made a 403/404 look like a successful (often blank) page.
+"""
+
+import asyncio
+import json
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from plugins.web import keyless_mcp
+from plugins.web.exa import provider as exa_provider
+from plugins.web.firecrawl import provider as firecrawl_provider
+from plugins.web.parallel import provider as parallel_provider
+
+STUB_HTML = "<!DOCTYPE html><html><body></body></html>"
+
+
+def _scrape(payload: dict) -> dict:
+ client = SimpleNamespace(scrape=lambda **_kw: payload)
+ with patch.object(firecrawl_provider, "_get_firecrawl_client", return_value=client), \
+ patch.object(firecrawl_provider, "check_website_access", return_value=None), \
+ patch.object(firecrawl_provider, "is_safe_url", return_value=True):
+ return asyncio.run(firecrawl_provider._scrape_one("https://t.test/x", ["markdown", "html"], None))
+
+
[email protected]("metadata, expected", [
+ # Firecrawl SDK v4 model_dump (direct key and Nous Tool Gateway): snake_case, stub body.
+ ({"status_code": 403, "error": "Forbidden", "source_url": "https://t.test/x"}, "HTTP 403 from origin: Forbidden"),
+ # REST payload (keyless / self-hosted): camelCase.
+ ({"statusCode": 404, "sourceURL": "https://t.test/x"}, "HTTP 404 from origin"),
+])
+def test_firecrawl_scrape_flags_origin_error_status(metadata, expected):
+ entry = _scrape({"markdown": "", "html": STUB_HTML, "metadata": metadata})
+ assert entry["error"] == expected
+ assert entry["content"] == STUB_HTML # the error page body is kept, not discarded
+
+
+def test_firecrawl_scrape_success_has_no_error():
+ entry = _scrape({"markdown": "# Hi", "metadata": {"status_code": 200, "title": "Hi"}})
+ assert entry["content"] == "# Hi"
+ assert "error" not in entry
+
+
+def test_keyless_firecrawl_flags_origin_error_status():
+ class _Client:
+ def scrape(self, **_kw):
+ return {"success": True, "data": {"markdown": "", "metadata": {"statusCode": 403, "error": "Forbidden"}}}
+
+ with patch.object(firecrawl_provider, "_KeylessFirecrawlClient", _Client):
+ [entry] = keyless_mcp.firecrawl_extract_keyless(["https://t.test/x"])
+ assert entry["error"] == "HTTP 403 from origin: Forbidden"
+
+
+def test_keyless_parallel_names_origin_status():
+ payload = json.dumps({"results": [], "errors": [
+ {"url": "https://t.test/x", "error_type": "http_error", "http_status_code": 403, "content": None},
+ ]})
+ with patch.object(keyless_mcp, "mcp_call", return_value=payload):
+ [entry] = keyless_mcp.parallel_extract_keyless(["https://t.test/x"])
+ assert entry["error"] == "HTTP 403 from origin"
+
+
+def test_parallel_sdk_names_origin_status():
+ error = SimpleNamespace(url="https://t.test/x", error_type="http_error", http_status_code=404, content="nope")
+ response = SimpleNamespace(results=[], errors=[error])
+
+ async def _extract(**_kw):
+ return response
+
+ client = SimpleNamespace(beta=SimpleNamespace(extract=_extract))
+ with patch.object(parallel_provider, "use_keyless", return_value=False), \
+ patch.object(parallel_provider, "_get_async_client", return_value=client):
+ [entry] = asyncio.run(parallel_provider.ParallelWebSearchProvider().extract(["https://t.test/x"]))
+ assert entry["error"] == "HTTP 404 from origin"
+ assert entry["content"] == "nope"
+
+
+def test_exa_sdk_reports_failed_urls_from_statuses():
+ ok = SimpleNamespace(url="https://t.test/ok", title="OK", text="body")
+ statuses = [
+ SimpleNamespace(id="https://t.test/ok", status="success", error=None),
+ SimpleNamespace(id="https://t.test/x", status="error",
+ error=SimpleNamespace(tag="CRAWL_HTTP_403", http_status_code=403)),
+ ]
+ client = SimpleNamespace(get_contents=lambda *_a, **_kw: SimpleNamespace(results=[ok], statuses=statuses))
+ with patch.object(exa_provider, "use_keyless", return_value=False), \
+ patch.object(exa_provider, "_get_exa_client", return_value=client):
+ out = exa_provider.ExaWebSearchProvider().extract(["https://t.test/ok", "https://t.test/x"])
+ assert [r.get("error") for r in out] == [None, "HTTP 403 from origin"]
+ assert out[1]["url"] == "https://t.test/x"