Skip to content

The agent that grows with you

fix(web): a skipped cache write never retargets the older cached page

Tekniumb02099ef0ed2
diff --git a/tests/tools/test_web_extract_cache_final_url.py b/tests/tools/test_web_extract_cache_final_url.py
index b49f4698e5a..7cd9349efb4 100644
--- a/tests/tools/test_web_extract_cache_final_url.py
+++ b/tests/tools/test_web_extract_cache_final_url.py
@@ -71,3 +71,30 @@ def test_cached_redirect_is_regated_on_its_final_url(monkeypatch, gate):
assert provider.calls == 2, "a hit whose final URL is now refused must be a miss (live fetch re-checks)"
assert not result.get("cached")
+
+
+
+def test_an_uncached_refetch_never_retargets_the_old_entry(monkeypatch):
+ """A refused hit re-fetches; if that fetch lands somewhere allowed but is too big to cache, the old
+ entry keeps its own final URL, so its blocked content is never served under the new target."""
+ check, safe = _policy()
+ monkeypatch.setattr("tools.website_policy.check_website_access", check)
+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)
+ _extract(_RedirectingProvider())
+
+ check, safe = _policy(blocked_host="target.example")
+ monkeypatch.setattr("tools.website_policy.check_website_access", check)
+ monkeypatch.setattr("tools.url_safety.async_is_safe_url", safe)
+ from tools.web_tools_truncate import MAX_STORED_TEXT_CHARS
+ huge = "x" * (MAX_STORED_TEXT_CHARS + 1)
+
+ class _NowElsewhere(_RedirectingProvider):
+ async def extract(self, urls, format=None):
+ self.calls += 1
+ return [{"url": "https://allowed.example/", "content": huge, "raw_content": huge,
+ "metadata": {"sourceURL": urls[0]}}]
+
+ _extract(_NowElsewhere())
+ later = _RedirectingProvider()
+ result = _extract(later)[0]
+ assert later.calls == 1 and not result.get("cached"), "the old blocked page was served from cache"
diff --git a/tools/web_result_cache.py b/tools/web_result_cache.py
index f1492805d26..2f445053586 100644
--- a/tools/web_result_cache.py
+++ b/tools/web_result_cache.py
@@ -281,7 +281,9 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = ""
def extract_cache_record_final_url(url: str, final_url: str, format: Optional[str] = None, provider: str = "") -> None:
"""Remember the post-redirect URL behind a cached page, so a hit can re-run the website policy and
- SSRF gates against the page it actually serves (a blocklist added after caching must still bite)."""
+ SSRF gates against the page it actually serves (a blocklist added after caching must still bite).
+ Call only after ``extract_cache_put`` wrote this content: a skipped write must not retarget the
+ older entry still on disk."""
with _index_lock:
index = _load_index()
entry = index.get(_url_digest(url, format, provider))
@@ -293,16 +295,17 @@ def extract_cache_record_final_url(url: str, final_url: str, format: Optional[st
def extract_cache_put(
url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = ""
-) -> None:
+) -> bool:
"""Store one successful extraction's full clean text for TTL reuse; pages over the truncate-store
- ceiling are not cached (serving a capped copy back as if whole would silently lose the tail)."""
+ ceiling are not cached (serving a capped copy back as if whole would silently lose the tail).
+ True only when this call wrote the entry."""
if not content or not _cacheable(url):
- return
+ return False
try:
from tools.web_tools_truncate import MAX_STORED_TEXT_CHARS
file_path = _entry_file_path(url, format, provider)
if len(content) > MAX_STORED_TEXT_CHARS or file_path is None:
- return
+ return False
from tools.spill_safety import write_text_exclusive
write_text_exclusive(file_path, content, private=False, overwrite=True)
with _index_lock:
@@ -311,5 +314,7 @@ def extract_cache_put(
"url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),
}
_save_index(index)
+ return True
except Exception as exc:
logger.debug("Failed to cache web extract for %s: %s", url, exc)
+ return False
diff --git a/tools/web_tools_extract.py b/tools/web_tools_extract.py
index dfc20709890..846fb9abcb5 100644
--- a/tools/web_tools_extract.py
+++ b/tools/web_tools_extract.py
@@ -199,8 +199,8 @@ async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[st
url = next((u for u in (fetched.get("url"), source) if u in requested), None)
_content = fetched.get("raw_content", "") or fetched.get("content", "")
if url and _content and not fetched.get("error"):
- extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
- if fetched.get("url") and fetched["url"] != url:
+ stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
+ if stored and fetched.get("url") and fetched["url"] != url:
extract_cache_record_final_url(url, fetched["url"], format=format, provider=provider.name)
return results