Skip to content

The agent that grows with you

fix(web): Firecrawl re-checks the website policy against the URL it was redirected to

Teknium6fc7c7dbf28e
diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py
index 0b7168139fe..e3b7a5105c1 100644
--- a/plugins/web/firecrawl/provider.py
+++ b/plugins/web/firecrawl/provider.py
@@ -282,6 +282,12 @@ _SCRAPE_TIMEOUT_MSG = "Scrape timed out after 60s — page may be too large or u
_UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address"
+def _final_url(metadata: dict[str, Any], requested: str) -> str:
+ """The URL Firecrawl actually fetched. ``url`` is the scrape engine's post-redirect URL on every shape
+ (REST, v4 SDK ``model_dump``, Tool Gateway); ``sourceURL`` / ``source_url`` echo the REQUESTED url."""
+ return metadata.get("url") or requested
+
+
async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> dict[str, Any]:
"""Scrape one URL (60s timeout) and re-check SSRF + website policy against the
post-redirect URL. Never raises for scrape errors; returns an error entry instead."""
@@ -312,7 +318,9 @@ async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> di
# SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object).
if not isinstance(metadata, dict):
metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else getattr(metadata, "__dict__", {})
- title, final_url = metadata.get("title", ""), metadata.get("sourceURL", url)
+ # The v4 SDK names it ``source_url``; the extract cache keys a redirected page by ``sourceURL``.
+ metadata.setdefault("sourceURL", metadata.get("source_url") or url)
+ title, final_url = metadata.get("title", ""), _final_url(metadata, url)
if not is_safe_url(final_url):
logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True)
diff --git a/tests/plugins/web/test_firecrawl_redirect_recheck.py b/tests/plugins/web/test_firecrawl_redirect_recheck.py
new file mode 100644
index 00000000000..931c0863c94
--- /dev/null
+++ b/tests/plugins/web/test_firecrawl_redirect_recheck.py
@@ -0,0 +1,74 @@
+"""Firecrawl extract re-checks the website policy against the URL it was redirected to.
+
+Firecrawl reports the requested URL as ``sourceURL`` (REST) / ``source_url`` (v4 SDK ``model_dump``, used
+by the direct-key and Tool Gateway paths) and the post-redirect URL as ``url``. Reading the requested URL
+made the re-check a no-op, so a public redirector returned a blocklisted page.
+"""
+
+import asyncio
+import json
+
+import pytest
+
+from plugins.web.firecrawl import provider as firecrawl_provider
+from tools import website_policy
+
+REQUESTED = "https://redirector.example/go"
+
+
+class _SdkDocument:
+ """Stand-in for firecrawl v4 ``Document``: ``_to_plain_object`` reads it via ``model_dump``."""
+
+ def __init__(self, final_url: str):
+ self._final_url = final_url
+
+ def model_dump(self):
+ return {
+ "markdown": "# landed",
+ "metadata": {"title": "Landed", "url": self._final_url, "source_url": REQUESTED, "status_code": 200},
+ }
+
+
+def _rest_payload(final_url: str) -> dict:
+ return {"success": True, "data": {"markdown": "# landed", "metadata": {"title": "Landed", "url": final_url, "sourceURL": REQUESTED, "statusCode": 200}}}
+
+
+SHAPES = {"sdk": _SdkDocument, "rest": _rest_payload}
+
+
+def scrape(monkeypatch, tmp_path):
+ config = tmp_path / "config.yaml"
+ config.write_text(json.dumps({"security": {"website_blocklist": {"enabled": True, "domains": ["blocked.example"]}}}))
+ monkeypatch.setattr(firecrawl_provider, "check_website_access", lambda url: website_policy.check_website_access(url, config_path=config))
+ monkeypatch.setattr(firecrawl_provider, "is_safe_url", lambda url: True)
+
+ def _run(shape: str, final_url: str) -> dict:
+ class _Client:
+ def scrape(self, **_kwargs):
+ return SHAPES[shape](final_url)
+
+ monkeypatch.setattr(firecrawl_provider, "_get_firecrawl_client", lambda _capability=None: _Client())
+ return asyncio.run(firecrawl_provider._scrape_one(REQUESTED, ["markdown"], "markdown"))
+
+ return _run
+
+
[email protected]("shape", sorted(SHAPES))
+def test_redirect_to_blocklisted_host_is_refused(scrape, shape):
+ entry = scrape(shape, "https://blocked.example/page")
+
+ assert entry["url"] == "https://blocked.example/page"
+ assert entry["content"] == ""
+ assert entry["blocked_by_policy"]["host"] == "blocked.example"
+
+
[email protected]("shape", sorted(SHAPES))
+def test_allowed_redirect_keeps_content_and_requested_url_for_the_cache(scrape, shape):
+ entry = scrape(shape, "https://allowed.example/page")
+
+ assert not entry.get("error")
+ assert entry["url"] == "https://allowed.example/page"
+ assert entry["content"] == "# landed"
+ # web_tools_extract files a redirected page under metadata.sourceURL (the requested URL).
+ assert entry["metadata"]["sourceURL"] == REQUESTED