Skip to content

The agent that grows with you

fix(web): pair a cached body with its own final URL; keyless Firecrawl reports its redirect target

Tekniumf7641ace7888
diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py
index f97f77a1158..d92155f98c0 100644
--- a/plugins/web/keyless_mcp.py
+++ b/plugins/web/keyless_mcp.py
@@ -302,14 +302,19 @@ def firecrawl_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
def firecrawl_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
- from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload, _with_scrape_status
+ from plugins.web.firecrawl.provider import (
+ _KeylessFirecrawlClient, _extract_scrape_payload, _final_url, _with_scrape_status,
+ )
client = _KeylessFirecrawlClient()
def _fetch(url: str) -> dict[str, Any]:
payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}
- metadata = payload.get("metadata") or {}
- title = metadata.get("title") if isinstance(metadata, dict) else None
- return _with_scrape_status(_page(url, title or "", payload.get("markdown") or payload.get("html") or ""), metadata)
+ metadata = payload.get("metadata")
+ metadata = metadata if isinstance(metadata, dict) else {}
+ content = payload.get("markdown") or payload.get("html") or ""
+ # Report the post-redirect page as ``url`` so the shared dispatch re-gates it.
+ page = _page(_final_url(metadata, url), metadata.get("title") or "", content, source_url=url)
+ return _with_scrape_status(page, metadata)
return _per_url(urls, _fetch, "firecrawl")
diff --git a/tests/plugins/web/test_firecrawl_redirect_recheck.py b/tests/plugins/web/test_firecrawl_redirect_recheck.py
index 931c0863c94..6526d2ac4a7 100644
--- a/tests/plugins/web/test_firecrawl_redirect_recheck.py
+++ b/tests/plugins/web/test_firecrawl_redirect_recheck.py
@@ -72,3 +72,23 @@ def test_allowed_redirect_keeps_content_and_requested_url_for_the_cache(scrape,
assert entry["content"] == "# landed"
# web_tools_extract files a redirected page under metadata.sourceURL (the requested URL).
assert entry["metadata"]["sourceURL"] == REQUESTED
+
+
+def test_keyless_extract_reports_the_redirect_target_for_the_shared_gate(monkeypatch, tmp_path):
+ """The keyless REST path must surface ``metadata.url`` too: reporting the requested URL hid the
+ redirect target from the dispatch-level re-gate, so a blocklisted landing page was returned."""
+ from plugins.web import keyless_mcp
+ from tools import web_tools_extract
+
+ monkeypatch.setattr(firecrawl_provider._KeylessFirecrawlClient, "scrape",
+ lambda self, **_kw: _rest_payload("https://blocked.example/page"))
+ entry = keyless_mcp.firecrawl_extract_keyless([REQUESTED])[0]
+ assert entry["url"] == "https://blocked.example/page"
+ assert entry["metadata"]["sourceURL"] == REQUESTED # the cache still keys it by the requested URL
+
+ config = tmp_path / "config.yaml"
+ config.write_text(json.dumps({"security": {"website_blocklist": {"enabled": True, "domains": ["blocked.example"]}}}))
+ real_check = website_policy.check_website_access
+ monkeypatch.setattr(website_policy, "check_website_access", lambda url: real_check(url, config_path=config))
+ gated = asyncio.run(web_tools_extract._gate_final_urls([entry], {REQUESTED}))[0]
+ assert gated["content"] == "" and gated["blocked_by_policy"]["host"] == "blocked.example"
diff --git a/tests/tools/test_web_extract_cache_final_url.py b/tests/tools/test_web_extract_cache_final_url.py
index 7cd9349efb4..26ded7a8f75 100644
--- a/tests/tools/test_web_extract_cache_final_url.py
+++ b/tests/tools/test_web_extract_cache_final_url.py
@@ -98,3 +98,37 @@ def test_an_uncached_refetch_never_retargets_the_old_entry(monkeypatch):
later = _RedirectingProvider()
result = _extract(later)[0]
assert later.calls == 1 and not result.get("cached"), "the old blocked page was served from cache"
+
+
+def test_interleaved_writers_never_pair_one_body_with_anothers_final_url(monkeypatch):
+ """Two fetches of one key: A writes its body, B writes its body and final URL, then A's late
+ final-URL write lands. B's body must stay paired with B's destination, so blocking B's host
+ makes the hit a miss instead of serving it under A's still-allowed destination."""
+ check, safe = _policy()
+ monkeypatch.setattr("tools.website_policy.check_website_access", check)
+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)
+
+ def landing(host, body):
+ class _Provider(_RedirectingProvider):
+ async def extract(self, urls, format=None):
+ self.calls += 1
+ return [{"url": f"https://{host}/", "content": body, "raw_content": body,
+ "metadata": {"sourceURL": urls[0]}}]
+ return _Provider()
+
+ real_record, deferred = wrc.extract_cache_record_final_url, []
+ monkeypatch.setattr(wrc, "extract_cache_record_final_url", lambda *a, **k: deferred.append((a, k)))
+ def fetch(provider): # straight to the fetch+cache-write path, as two concurrent misses would
+ asyncio.run(wte._dispatch_extract(provider, [REDIRECTOR], None))
+
+ fetch(landing("allowed-a.example", "BODY A")) # A-put; A-final held back
+ monkeypatch.setattr(wrc, "extract_cache_record_final_url", real_record)
+ fetch(landing("origin-b.example", "BODY B")) # B-put, B-final
+ for args, kwargs in deferred: # A-final lands last
+ real_record(*args, **kwargs)
+
+ check, safe = _policy(blocked_host="origin-b.example")
+ monkeypatch.setattr("tools.website_policy.check_website_access", check)
+ later = _RedirectingProvider()
+ result = _extract(later)[0]
+ assert later.calls == 1 and not result.get("cached"), "B's blocked body was served under A's final URL"
diff --git a/tools/web_result_cache.py b/tools/web_result_cache.py
index 2f445053586..7453d1be3c6 100644
--- a/tools/web_result_cache.py
+++ b/tools/web_result_cache.py
@@ -274,20 +274,30 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = ""
content = file_path.read_text(encoding="utf-8-sig")
except Exception:
return None
+ # The body file and its index entry are separate writes; serve only the body the entry (and its
+ # ``final_url``) was recorded for. Entries from before the digest carry no ``final_url`` either.
+ if ("content_sha256" in entry or "final_url" in entry) and entry.get("content_sha256") != _content_sha(content):
+ return None
logger.info("web_extract cache hit: %s", url)
return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True,
"final_url": entry.get("final_url") or url}
-def extract_cache_record_final_url(url: str, final_url: str, format: Optional[str] = None, provider: str = "") -> None:
+def _content_sha(content: str) -> str:
+ return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def extract_cache_record_final_url(
+ url: str, final_url: str, content: str, format: Optional[str] = None, provider: str = ""
+) -> None:
"""Remember the post-redirect URL behind a cached page, so a hit can re-run the website policy and
SSRF gates against the page it actually serves (a blocklist added after caching must still bite).
- Call only after ``extract_cache_put`` wrote this content: a skipped write must not retarget the
- older entry still on disk."""
+ Compare-and-set on *content*: when another fetch of the same key wrote its body in between (or this
+ write was skipped), the entry is left alone rather than pairing that body with this final URL."""
with _index_lock:
index = _load_index()
entry = index.get(_url_digest(url, format, provider))
- if entry is None:
+ if entry is None or entry.get("content_sha256") != _content_sha(content):
return
entry["final_url"] = final_url
_save_index(index)
@@ -312,6 +322,7 @@ def extract_cache_put(
index = _load_index()
index[_url_digest(url, format, provider)] = {
"url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),
+ "content_sha256": _content_sha(content),
}
_save_index(index)
return True
diff --git a/tools/web_tools_extract.py b/tools/web_tools_extract.py
index 4b63be934eb..c07a75a6671 100644
--- a/tools/web_tools_extract.py
+++ b/tools/web_tools_extract.py
@@ -250,7 +250,7 @@ async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[st
if url and _content and not fetched.get("error"):
stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
if stored and fetched.get("url") and fetched["url"] != url:
- extract_cache_record_final_url(url, fetched["url"], format=format, provider=provider.name)
+ extract_cache_record_final_url(url, fetched["url"], _content, format=format, provider=provider.name)
return results