The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(web): a skipped cache write never retargets the older cached page
Tekniumb02099ef0ed2
diff --git a/tests/tools/test_web_extract_cache_final_url.py b/tests/tools/test_web_extract_cache_final_url.pyindex b49f4698e5a..7cd9349efb4 100644--- a/tests/tools/test_web_extract_cache_final_url.py+++ b/tests/tools/test_web_extract_cache_final_url.py@@ -71,3 +71,30 @@ def test_cached_redirect_is_regated_on_its_final_url(monkeypatch, gate):assert provider.calls == 2, "a hit whose final URL is now refused must be a miss (live fetch re-checks)"assert not result.get("cached")++++def test_an_uncached_refetch_never_retargets_the_old_entry(monkeypatch):+ """A refused hit re-fetches; if that fetch lands somewhere allowed but is too big to cache, the old+ entry keeps its own final URL, so its blocked content is never served under the new target."""+ check, safe = _policy()+ monkeypatch.setattr("tools.website_policy.check_website_access", check)+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)+ _extract(_RedirectingProvider())++ check, safe = _policy(blocked_host="target.example")+ monkeypatch.setattr("tools.website_policy.check_website_access", check)+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)+ from tools.web_tools_truncate import MAX_STORED_TEXT_CHARS+ huge = "x" * (MAX_STORED_TEXT_CHARS + 1)++ class _NowElsewhere(_RedirectingProvider):+ async def extract(self, urls, format=None):+ self.calls += 1+ return [{"url": "https://allowed.example/", "content": huge, "raw_content": huge,+ "metadata": {"sourceURL": urls[0]}}]++ _extract(_NowElsewhere())+ later = _RedirectingProvider()+ result = _extract(later)[0]+ assert later.calls == 1 and not result.get("cached"), "the old blocked page was served from cache"diff --git a/tools/web_result_cache.py b/tools/web_result_cache.pyindex f1492805d26..2f445053586 100644--- a/tools/web_result_cache.py+++ b/tools/web_result_cache.py@@ -281,7 +281,9 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = ""def extract_cache_record_final_url(url: str, final_url: str, format: Optional[str] = None, provider: str = "") -> None:"""Remember the post-redirect URL behind a cached page, so a hit can re-run the website policy and- SSRF gates against the page it actually serves (a blocklist added after caching must still bite)."""+ SSRF gates against the page it actually serves (a blocklist added after caching must still bite).+ Call only after ``extract_cache_put`` wrote this content: a skipped write must not retarget the+ older entry still on disk."""with _index_lock:index = _load_index()entry = index.get(_url_digest(url, format, provider))@@ -293,16 +295,17 @@ def extract_cache_record_final_url(url: str, final_url: str, format: Optional[stdef extract_cache_put(url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = ""-) -> None:+) -> bool:"""Store one successful extraction's full clean text for TTL reuse; pages over the truncate-store- ceiling are not cached (serving a capped copy back as if whole would silently lose the tail)."""+ ceiling are not cached (serving a capped copy back as if whole would silently lose the tail).+ True only when this call wrote the entry."""if not content or not _cacheable(url):- return+ return Falsetry:from tools.web_tools_truncate import MAX_STORED_TEXT_CHARSfile_path = _entry_file_path(url, format, provider)if len(content) > MAX_STORED_TEXT_CHARS or file_path is None:- return+ return Falsefrom tools.spill_safety import write_text_exclusivewrite_text_exclusive(file_path, content, private=False, overwrite=True)with _index_lock:@@ -311,5 +314,7 @@ def extract_cache_put("url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),}_save_index(index)+ return Trueexcept Exception as exc:logger.debug("Failed to cache web extract for %s: %s", url, exc)+ return Falsediff --git a/tools/web_tools_extract.py b/tools/web_tools_extract.pyindex dfc20709890..846fb9abcb5 100644--- a/tools/web_tools_extract.py+++ b/tools/web_tools_extract.py@@ -199,8 +199,8 @@ async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[sturl = next((u for u in (fetched.get("url"), source) if u in requested), None)_content = fetched.get("raw_content", "") or fetched.get("content", "")if url and _content and not fetched.get("error"):- extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)- if fetched.get("url") and fetched["url"] != url:+ stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)+ if stored and fetched.get("url") and fetched["url"] != url:extract_cache_record_final_url(url, fetched["url"], format=format, provider=provider.name)return results