The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(web): pair a cached body with its own final URL; keyless Firecrawl reports its redirect target
Tekniumf7641ace7888
diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.pyindex f97f77a1158..d92155f98c0 100644--- a/plugins/web/keyless_mcp.py+++ b/plugins/web/keyless_mcp.py@@ -302,14 +302,19 @@ def firecrawl_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:def firecrawl_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:- from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload, _with_scrape_status+ from plugins.web.firecrawl.provider import (+ _KeylessFirecrawlClient, _extract_scrape_payload, _final_url, _with_scrape_status,+ )client = _KeylessFirecrawlClient()def _fetch(url: str) -> dict[str, Any]:payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}- metadata = payload.get("metadata") or {}- title = metadata.get("title") if isinstance(metadata, dict) else None- return _with_scrape_status(_page(url, title or "", payload.get("markdown") or payload.get("html") or ""), metadata)+ metadata = payload.get("metadata")+ metadata = metadata if isinstance(metadata, dict) else {}+ content = payload.get("markdown") or payload.get("html") or ""+ # Report the post-redirect page as ``url`` so the shared dispatch re-gates it.+ page = _page(_final_url(metadata, url), metadata.get("title") or "", content, source_url=url)+ return _with_scrape_status(page, metadata)return _per_url(urls, _fetch, "firecrawl")diff --git a/tests/plugins/web/test_firecrawl_redirect_recheck.py b/tests/plugins/web/test_firecrawl_redirect_recheck.pyindex 931c0863c94..6526d2ac4a7 100644--- a/tests/plugins/web/test_firecrawl_redirect_recheck.py+++ b/tests/plugins/web/test_firecrawl_redirect_recheck.py@@ -72,3 +72,23 @@ def test_allowed_redirect_keeps_content_and_requested_url_for_the_cache(scrape,assert entry["content"] == "# landed"# web_tools_extract files a redirected page under metadata.sourceURL (the requested URL).assert entry["metadata"]["sourceURL"] == REQUESTED+++def test_keyless_extract_reports_the_redirect_target_for_the_shared_gate(monkeypatch, tmp_path):+ """The keyless REST path must surface ``metadata.url`` too: reporting the requested URL hid the+ redirect target from the dispatch-level re-gate, so a blocklisted landing page was returned."""+ from plugins.web import keyless_mcp+ from tools import web_tools_extract++ monkeypatch.setattr(firecrawl_provider._KeylessFirecrawlClient, "scrape",+ lambda self, **_kw: _rest_payload("https://blocked.example/page"))+ entry = keyless_mcp.firecrawl_extract_keyless([REQUESTED])[0]+ assert entry["url"] == "https://blocked.example/page"+ assert entry["metadata"]["sourceURL"] == REQUESTED # the cache still keys it by the requested URL++ config = tmp_path / "config.yaml"+ config.write_text(json.dumps({"security": {"website_blocklist": {"enabled": True, "domains": ["blocked.example"]}}}))+ real_check = website_policy.check_website_access+ monkeypatch.setattr(website_policy, "check_website_access", lambda url: real_check(url, config_path=config))+ gated = asyncio.run(web_tools_extract._gate_final_urls([entry], {REQUESTED}))[0]+ assert gated["content"] == "" and gated["blocked_by_policy"]["host"] == "blocked.example"diff --git a/tests/tools/test_web_extract_cache_final_url.py b/tests/tools/test_web_extract_cache_final_url.pyindex 7cd9349efb4..26ded7a8f75 100644--- a/tests/tools/test_web_extract_cache_final_url.py+++ b/tests/tools/test_web_extract_cache_final_url.py@@ -98,3 +98,37 @@ def test_an_uncached_refetch_never_retargets_the_old_entry(monkeypatch):later = _RedirectingProvider()result = _extract(later)[0]assert later.calls == 1 and not result.get("cached"), "the old blocked page was served from cache"+++def test_interleaved_writers_never_pair_one_body_with_anothers_final_url(monkeypatch):+ """Two fetches of one key: A writes its body, B writes its body and final URL, then A's late+ final-URL write lands. B's body must stay paired with B's destination, so blocking B's host+ makes the hit a miss instead of serving it under A's still-allowed destination."""+ check, safe = _policy()+ monkeypatch.setattr("tools.website_policy.check_website_access", check)+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)++ def landing(host, body):+ class _Provider(_RedirectingProvider):+ async def extract(self, urls, format=None):+ self.calls += 1+ return [{"url": f"https://{host}/", "content": body, "raw_content": body,+ "metadata": {"sourceURL": urls[0]}}]+ return _Provider()++ real_record, deferred = wrc.extract_cache_record_final_url, []+ monkeypatch.setattr(wrc, "extract_cache_record_final_url", lambda *a, **k: deferred.append((a, k)))+ def fetch(provider): # straight to the fetch+cache-write path, as two concurrent misses would+ asyncio.run(wte._dispatch_extract(provider, [REDIRECTOR], None))++ fetch(landing("allowed-a.example", "BODY A")) # A-put; A-final held back+ monkeypatch.setattr(wrc, "extract_cache_record_final_url", real_record)+ fetch(landing("origin-b.example", "BODY B")) # B-put, B-final+ for args, kwargs in deferred: # A-final lands last+ real_record(*args, **kwargs)++ check, safe = _policy(blocked_host="origin-b.example")+ monkeypatch.setattr("tools.website_policy.check_website_access", check)+ later = _RedirectingProvider()+ result = _extract(later)[0]+ assert later.calls == 1 and not result.get("cached"), "B's blocked body was served under A's final URL"diff --git a/tools/web_result_cache.py b/tools/web_result_cache.pyindex 2f445053586..7453d1be3c6 100644--- a/tools/web_result_cache.py+++ b/tools/web_result_cache.py@@ -274,20 +274,30 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = ""content = file_path.read_text(encoding="utf-8-sig")except Exception:return None+ # The body file and its index entry are separate writes; serve only the body the entry (and its+ # ``final_url``) was recorded for. Entries from before the digest carry no ``final_url`` either.+ if ("content_sha256" in entry or "final_url" in entry) and entry.get("content_sha256") != _content_sha(content):+ return Nonelogger.info("web_extract cache hit: %s", url)return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True,"final_url": entry.get("final_url") or url}-def extract_cache_record_final_url(url: str, final_url: str, format: Optional[str] = None, provider: str = "") -> None:+def _content_sha(content: str) -> str:+ return hashlib.sha256(content.encode("utf-8")).hexdigest()+++def extract_cache_record_final_url(+ url: str, final_url: str, content: str, format: Optional[str] = None, provider: str = ""+) -> None:"""Remember the post-redirect URL behind a cached page, so a hit can re-run the website policy andSSRF gates against the page it actually serves (a blocklist added after caching must still bite).- Call only after ``extract_cache_put`` wrote this content: a skipped write must not retarget the- older entry still on disk."""+ Compare-and-set on *content*: when another fetch of the same key wrote its body in between (or this+ write was skipped), the entry is left alone rather than pairing that body with this final URL."""with _index_lock:index = _load_index()entry = index.get(_url_digest(url, format, provider))- if entry is None:+ if entry is None or entry.get("content_sha256") != _content_sha(content):returnentry["final_url"] = final_url_save_index(index)@@ -312,6 +322,7 @@ def extract_cache_put(index = _load_index()index[_url_digest(url, format, provider)] = {"url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),+ "content_sha256": _content_sha(content),}_save_index(index)return Truediff --git a/tools/web_tools_extract.py b/tools/web_tools_extract.pyindex 4b63be934eb..c07a75a6671 100644--- a/tools/web_tools_extract.py+++ b/tools/web_tools_extract.py@@ -250,7 +250,7 @@ async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[stif url and _content and not fetched.get("error"):stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)if stored and fetched.get("url") and fetched["url"] != url:- extract_cache_record_final_url(url, fetched["url"], format=format, provider=provider.name)+ extract_cache_record_final_url(url, fetched["url"], _content, format=format, provider=provider.name)return results