The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(web): Firecrawl re-checks the website policy against the URL it was redirected to
Teknium6fc7c7dbf28e
diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.pyindex 0b7168139fe..e3b7a5105c1 100644--- a/plugins/web/firecrawl/provider.py+++ b/plugins/web/firecrawl/provider.py@@ -282,6 +282,12 @@ _SCRAPE_TIMEOUT_MSG = "Scrape timed out after 60s — page may be too large or u_UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address"+def _final_url(metadata: dict[str, Any], requested: str) -> str:+ """The URL Firecrawl actually fetched. ``url`` is the scrape engine's post-redirect URL on every shape+ (REST, v4 SDK ``model_dump``, Tool Gateway); ``sourceURL`` / ``source_url`` echo the REQUESTED url."""+ return metadata.get("url") or requested++async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> dict[str, Any]:"""Scrape one URL (60s timeout) and re-check SSRF + website policy against thepost-redirect URL. Never raises for scrape errors; returns an error entry instead."""@@ -312,7 +318,9 @@ async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> di# SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object).if not isinstance(metadata, dict):metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else getattr(metadata, "__dict__", {})- title, final_url = metadata.get("title", ""), metadata.get("sourceURL", url)+ # The v4 SDK names it ``source_url``; the extract cache keys a redirected page by ``sourceURL``.+ metadata.setdefault("sourceURL", metadata.get("source_url") or url)+ title, final_url = metadata.get("title", ""), _final_url(metadata, url)if not is_safe_url(final_url):logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True)diff --git a/tests/plugins/web/test_firecrawl_redirect_recheck.py b/tests/plugins/web/test_firecrawl_redirect_recheck.pynew file mode 100644index 00000000000..931c0863c94--- /dev/null+++ b/tests/plugins/web/test_firecrawl_redirect_recheck.py@@ -0,0 +1,74 @@+"""Firecrawl extract re-checks the website policy against the URL it was redirected to.++Firecrawl reports the requested URL as ``sourceURL`` (REST) / ``source_url`` (v4 SDK ``model_dump``, used+by the direct-key and Tool Gateway paths) and the post-redirect URL as ``url``. Reading the requested URL+made the re-check a no-op, so a public redirector returned a blocklisted page.+"""++import asyncio+import json++import pytest++from plugins.web.firecrawl import provider as firecrawl_provider+from tools import website_policy++REQUESTED = "https://redirector.example/go"+++class _SdkDocument:+ """Stand-in for firecrawl v4 ``Document``: ``_to_plain_object`` reads it via ``model_dump``."""++ def __init__(self, final_url: str):+ self._final_url = final_url++ def model_dump(self):+ return {+ "markdown": "# landed",+ "metadata": {"title": "Landed", "url": self._final_url, "source_url": REQUESTED, "status_code": 200},+ }+++def _rest_payload(final_url: str) -> dict:+ return {"success": True, "data": {"markdown": "# landed", "metadata": {"title": "Landed", "url": final_url, "sourceURL": REQUESTED, "statusCode": 200}}}+++SHAPES = {"sdk": _SdkDocument, "rest": _rest_payload}+++def scrape(monkeypatch, tmp_path):+ config = tmp_path / "config.yaml"+ config.write_text(json.dumps({"security": {"website_blocklist": {"enabled": True, "domains": ["blocked.example"]}}}))+ monkeypatch.setattr(firecrawl_provider, "check_website_access", lambda url: website_policy.check_website_access(url, config_path=config))+ monkeypatch.setattr(firecrawl_provider, "is_safe_url", lambda url: True)++ def _run(shape: str, final_url: str) -> dict:+ class _Client:+ def scrape(self, **_kwargs):+ return SHAPES[shape](final_url)++ monkeypatch.setattr(firecrawl_provider, "_get_firecrawl_client", lambda _capability=None: _Client())+ return asyncio.run(firecrawl_provider._scrape_one(REQUESTED, ["markdown"], "markdown"))++ return _run++[email protected]("shape", sorted(SHAPES))+def test_redirect_to_blocklisted_host_is_refused(scrape, shape):+ entry = scrape(shape, "https://blocked.example/page")++ assert entry["url"] == "https://blocked.example/page"+ assert entry["content"] == ""+ assert entry["blocked_by_policy"]["host"] == "blocked.example"++[email protected]("shape", sorted(SHAPES))+def test_allowed_redirect_keeps_content_and_requested_url_for_the_cache(scrape, shape):+ entry = scrape(shape, "https://allowed.example/page")++ assert not entry.get("error")+ assert entry["url"] == "https://allowed.example/page"+ assert entry["content"] == "# landed"+ # web_tools_extract files a redirected page under metadata.sourceURL (the requested URL).+ assert entry["metadata"]["sourceURL"] == REQUESTED