~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for the single shared research content cache.
Every consumer that needs the full text of a webpage, a RAG chunk or a
library document — the ``web_search`` tool (engines / FullSearchResults),
the ``fetch_content`` tool (ContentFetcher + summary-prompt construction),
the library resolver, and the extraction pipeline — resolves through one
``ResearchContentCache`` per research run. These tests pin that contract:
* key normalisation (tracking params collapse, chunk vs document distinct),
* store/lookup semantics (longest fetch wins, failures never cached),
* concurrent coalescing (parallel subagents fetch once),
* registry binding via the thread-local search context (incl. worker threads),
* cross-consumer sharing (pipeline fills, fetcher/fetch-tool/engine serve),
* lead + subagent fetch tools sharing one instance,
* one cache owned by ``AdvancedSearchSystem`` for quick phase + subsections.
No real network, DB, or LLM is touched — downloaders and fetchers are mocked.
"""
from __future__ import annotations
import threading
import uuid
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.utilities.research_content_cache import (
LIBRARY_PROVENANCE,
ResearchContentCache,
bind_cache_to_research,
content_cache_key,
get_cache_for_research,
get_current_cache,
resolve_cache,
unbind_cache_from_research,
)
from local_deep_research.utilities.thread_context import (
clear_search_context,
get_search_context,
search_context,
set_search_context,
)
@pytest.fixture
def fresh_context():
"""Guarantee no search-context or registry leakage between tests."""
clear_search_context()
yield
clear_search_context()
@pytest.fixture
def bound_cache(fresh_context):
"""One cache bound to a unique research id, unbound on teardown."""
rid = f"test-{uuid.uuid4().hex}"
cache = ResearchContentCache()
bind_cache_to_research(rid, cache)
set_search_context({"research_id": rid})
yield rid, cache
clear_search_context()
unbind_cache_from_research(rid)
class TestContentCacheKey:
def test_tracking_params_collapse(self):
"""Tracking params collapse; accepts over-merge on sites varying content per param."""
assert content_cache_key(
"https://example.com/p?utm_source=x&foo=1"
) == content_cache_key("https://example.com/p?foo=1")
def test_fragment_stays_distinct_for_web(self):
"""Fragments select distinct views in client-routed SPAs (JS rendering)."""
assert content_cache_key(
"https://example.com/p#section"
) != content_cache_key("https://example.com/p")
assert content_cache_key(
"https://spa.example.com/app#/route-alpha"
) != content_cache_key("https://spa.example.com/app#/route-beta")
def test_different_paths_differ(self):
assert content_cache_key("https://example.com/a") != content_cache_key(
"https://example.com/b"
)
def test_trailing_slash_stays_distinct_for_web(self):
"""PR #6759: ``/article`` and ``/article/`` are different HTTP
resources and must never share a content entry (the bibliography
``canonical_url_key`` collapses them; the content key must not)."""
assert content_cache_key(
"https://example.com/article"
) != content_cache_key("https://example.com/article/")
assert content_cache_key(
"http://127.0.0.1:9/article"
) != content_cache_key("http://127.0.0.1:9/article/")
def test_userinfo_stays_distinct_for_web(self):
"""PR #6759: userinfo selects a different origin view — a
credentialed URL must never hit the bare-host entry."""
assert content_cache_key(
"https://user:[email protected]/p"
) != content_cache_key("https://example.com/p")
assert content_cache_key(
"https://[email protected]/p"
) != content_cache_key("https://[email protected]/p")
def test_query_params_stay_distinct_for_web(self):
assert content_cache_key(
"https://example.com/p?a=1"
) != content_cache_key("https://example.com/p?a=2")
assert content_cache_key(
"https://example.com/p?foo=1"
) != content_cache_key("https://example.com/p")
def test_library_chunk_distinct_from_document(self):
doc = "/library/document/12345678123456781234567812345678"
chunk = doc + "#chunk-3"
assert content_cache_key(doc) != content_cache_key(chunk)
def test_library_lib_spelling_matches(self):
assert content_cache_key(
"/lib/document/12345678123456781234567812345678"
) == content_cache_key(
"/library/document/12345678123456781234567812345678"
)
def test_non_string_safe(self):
assert content_cache_key(None) # must not raise
assert content_cache_key("") != content_cache_key(
"https://example.com/"
)
class TestStoreSemantics:
def test_put_get_roundtrip(self):
cache = ResearchContentCache()
cache.put("https://example.com/x", "body", title="T")
hit = cache.get("https://example.com/x")
assert hit is not None and hit["content"] == "body"
assert hit["title"] == "T"
def test_get_returns_copy(self):
cache = ResearchContentCache()
cache.put("https://example.com/x", "body")
hit = cache.get("https://example.com/x")
assert hit is not None
hit["content"] = "mutated"
assert cache.get("https://example.com/x")["content"] == "body"
def test_longest_fetch_wins(self):
cache = ResearchContentCache()
cache.put("https://example.com/x", "short")
cache.put("https://example.com/x", "a much longer content here")
assert (
cache.get("https://example.com/x")["content"]
== "a much longer content here"
)
cache.put("https://example.com/x", "tiny")
assert (
cache.get("https://example.com/x")["content"]
== "a much longer content here"
)
def test_miss_returns_none(self):
assert ResearchContentCache().get("https://nope.example/") is None
def test_stats_and_len(self):
cache = ResearchContentCache()
cache.put("https://example.com/x", "short")
cache.put("https://example.com/x", "a much longer content here")
cache.get("https://example.com/x")
cache.get("https://example.com/missing")
stats = cache.stats()
assert stats["entries"] == 1
assert stats["puts"] == 2
assert stats["hits"] >= 1
assert stats["misses"] >= 1
assert len(cache) == 1
def test_clear(self):
cache = ResearchContentCache()
cache.put("https://example.com/x", "body")
cache.clear()
assert len(cache) == 0
assert cache.get("https://example.com/x") is None
stats = cache.stats()
assert stats["hits"] == 0
assert stats["misses"] == 1
assert stats["puts"] == 0
assert stats["entries"] == 0
assert stats["evictions"] == 0
assert stats["total_bytes"] == 0
def test_get_or_fetch_caches_success(self):
cache = ResearchContentCache()
calls = {"n": 0}
def fetch():
calls["n"] += 1
return {
"content": "hello",
"title": "T",
"url": "https://example.com/c",
}
first = cache.get_or_fetch("https://example.com/c", fetch)
second = cache.get_or_fetch("https://example.com/c", fetch)
assert first is not None and first["content"] == "hello"
assert second is not None and second["content"] == "hello"
assert calls["n"] == 1
def test_get_or_fetch_never_caches_failure(self):
cache = ResearchContentCache()
assert cache.get_or_fetch("https://example.com/n", lambda: None) is None
assert len(cache) == 0
assert cache.get("https://example.com/n") is None
def test_get_or_fetch_cleans_inflight_on_exception(self):
"""Ordinary exceptions release the slot so a retry refetches."""
cache = ResearchContentCache()
url = "https://example.com/flaky"
def boom():
raise ValueError("boom")
with pytest.raises(ValueError, match="boom"):
cache.get_or_fetch(url, boom)
# Slot released and waiters woken: no dead event retained.
assert cache._inflight == {}
assert len(cache) == 0
calls = {"n": 0}
def ok():
calls["n"] += 1
return {
"content": "recovered",
"title": "T",
"url": url,
}
result = cache.get_or_fetch(url, ok)
assert result is not None and result["content"] == "recovered"
assert calls["n"] == 1
def test_get_or_fetch_cleans_inflight_on_cancellation(self):
"""BaseException (cancellation) releases the slot like Exception."""
class _Cancelled(BaseException):
pass
cache = ResearchContentCache()
url = "https://example.com/cancelled"
def boom():
raise _Cancelled("cancelled")
with pytest.raises(_Cancelled):
cache.get_or_fetch(url, boom)
# Old `except Exception` cleanup leaked here: the unset event stayed
# in _inflight and a later caller waited on it past wait_timeout.
assert cache._inflight == {}
assert len(cache) == 0
def ok():
return {
"content": "recovered",
"title": "T",
"url": url,
}
# Must become the fetcher immediately, not wait on a dead event.
result = cache.get_or_fetch(url, ok, wait_timeout=5.0)
assert result is not None and result["content"] == "recovered"
def test_get_or_fetch_cleans_inflight_on_put_failure(self):
"""A put failure still wakes waiters so they retry, not hang."""
cache = ResearchContentCache()
url = "https://example.com/put-boom"
def ok():
return {
"content": "hello",
"title": "T",
"url": url,
}
with patch.object(
ResearchContentCache, "put", side_effect=RuntimeError("put boom")
):
with pytest.raises(RuntimeError, match="put boom"):
cache.get_or_fetch(url, ok)
assert cache._inflight == {}
# Retry after the put failure stores normally.
result = cache.get_or_fetch(url, ok)
assert result is not None and result["content"] == "hello"
def test_get_or_fetch_waiter_recovers_after_cancellation(self):
"""A waiter on a cancelled fetch falls through to its own fetch."""
import time
class _Cancelled(BaseException):
pass
cache = ResearchContentCache()
url = "https://example.com/cancel-race"
def cancelling_fetch():
time.sleep(0.2) # allow: unmarked-sleep
raise _Cancelled("cancelled")
def waiter_fetch():
return {
"content": "waiter-recovered",
"title": "T",
"url": url,
}
results: dict[str, object] = {}
def run_fetcher():
with pytest.raises(_Cancelled):
cache.get_or_fetch(url, cancelling_fetch, wait_timeout=5.0)
def run_waiter():
# Starts while the fetcher holds the slot; must not hang on
# the dead event after cancellation.
time.sleep(0.05) # allow: unmarked-sleep
results["waiter"] = cache.get_or_fetch(
url, waiter_fetch, wait_timeout=5.0
)
fetcher_thread = threading.Thread(target=run_fetcher)
waiter_thread = threading.Thread(target=run_waiter)
fetcher_thread.start()
waiter_thread.start()
fetcher_thread.join(timeout=10)
waiter_thread.join(timeout=10)
assert not fetcher_thread.is_alive()
assert not waiter_thread.is_alive()
assert results.get("waiter") is not None
assert results["waiter"]["content"] == "waiter-recovered"
@pytest.mark.slow
def test_concurrent_fetchers_coalesce_to_one(self):
cache = ResearchContentCache()
calls = {"n": 0}
def slow_fetch():
import time
calls["n"] += 1
time.sleep(0.2) # allow: unmarked-sleep
return {
"content": "hello",
"title": "T",
"url": "https://example.com/c",
}
results: list = []
threads = [
threading.Thread(
target=lambda: results.append(
cache.get_or_fetch("https://example.com/c", slow_fetch)
)
)
for _ in range(8)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert calls["n"] == 1
assert len(results) == 8
assert all(r is not None and r["content"] == "hello" for r in results)
class TestRegistry:
def test_bind_get_unbind(self):
cache = ResearchContentCache()
bind_cache_to_research("rid-1", cache)
try:
assert get_cache_for_research("rid-1") is cache
finally:
unbind_cache_from_research("rid-1")
assert get_cache_for_research("rid-1") is None
def test_bind_ignores_empty(self):
cache = ResearchContentCache()
bind_cache_to_research("", cache)
bind_cache_to_research(None, cache) # type: ignore[arg-type]
assert get_cache_for_research("") is None
def test_current_cache_via_search_context(self, fresh_context):
cache = ResearchContentCache()
bind_cache_to_research("rid-ctx", cache)
try:
set_search_context({"research_id": "rid-ctx"})
assert get_current_cache() is cache
finally:
clear_search_context()
unbind_cache_from_research("rid-ctx")
def test_no_context_means_no_cache(self, fresh_context):
assert get_current_cache() is None
def test_resolve_prefers_explicit(self, bound_cache):
_, bound = bound_cache
other = ResearchContentCache()
assert resolve_cache(other) is other
assert resolve_cache(None) is bound
def test_worker_thread_with_captured_context_sees_same_cache(
self, fresh_context
):
cache = ResearchContentCache()
bind_cache_to_research("rid-thread", cache)
set_search_context({"research_id": "rid-thread"})
try:
captured = get_search_context()
seen: dict = {}
def run():
with search_context(captured):
seen["cache"] = get_current_cache()
thread = threading.Thread(target=run)
thread.start()
thread.join()
assert seen.get("cache") is cache
finally:
clear_search_context()
unbind_cache_from_research("rid-thread")
def test_memory_bounds_evict_oldest(self):
cache = ResearchContentCache(max_entries=3, max_bytes=10**9)
for i in range(5):
cache.put(f"https://example.com/p{i}", f"body-{i}")
assert len(cache) == 3
stats = cache.stats()
assert stats["evictions"] == 2
# Oldest evicted, newest retained.
assert cache.get("https://example.com/p0") is None
assert cache.get("https://example.com/p4") is not None
def test_memory_bounds_byte_cap(self):
cache = ResearchContentCache(max_entries=1000, max_bytes=100)
cache.put("https://example.com/a", "x" * 60)
cache.put("https://example.com/b", "y" * 60)
# 120 bytes > 100 cap with 2 entries: oldest evicted, newest kept.
assert len(cache) == 1
assert cache.get("https://example.com/a") is None
assert cache.get("https://example.com/b") is not None
assert cache.stats()["total_bytes"] == 60
def test_get_or_fetch_wakes_waiters_after_put(self):
"""Waiters must see the entry (not re-fetch) — put before set."""
import time
cache = ResearchContentCache()
calls: list[str] = []
def slow_fetcher():
calls.append("fetcher")
time.sleep(0.3) # allow: unmarked-sleep
return {
"content": "hello",
"title": "T",
"url": "https://example.com/c",
}
def waiter_fetch():
# Must never run: the waiter should observe the cached entry.
calls.append("waiter-retry")
return {
"content": "hello",
"title": "T",
"url": "https://example.com/c",
}
results: dict[str, object] = {}
def run_waiter():
# Start slightly after the fetcher holds the in-flight slot.
time.sleep(0.05) # allow: unmarked-sleep
results["waiter"] = cache.get_or_fetch(
"https://example.com/c", waiter_fetch
)
fetcher_thread = threading.Thread(
target=lambda: results.__setitem__(
"fetcher",
cache.get_or_fetch("https://example.com/c", slow_fetcher),
)
)
waiter_thread = threading.Thread(target=run_waiter)
fetcher_thread.start()
waiter_thread.start()
fetcher_thread.join()
waiter_thread.join()
assert calls == ["fetcher"]
assert results["fetcher"] is not None
assert results["waiter"] is not None
def _mock_downloader(content: bytes, title: str = "PT"):
downloader = MagicMock()
downloader.download_with_result.return_value = MagicMock(
is_success=True, content=content, skip_reason=None
)
downloader.get_metadata.return_value = {"title": title}
return downloader
class TestContentFetcherCache:
def test_second_fetch_served_without_downloader(self):
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
downloader = _mock_downloader(b"full page text here")
fetcher._get_downloader = MagicMock(return_value=downloader)
first = fetcher.fetch("https://example.com/page", max_length=10)
assert first["status"] == "success"
assert "[... content truncated ...]" in first["content"]
assert downloader.download_with_result.call_count == 1
second = fetcher.fetch(
"https://example.com/page", max_length=100000
)
assert downloader.download_with_result.call_count == 1
assert second["content"] == "full page text here"
def test_cache_holds_fullest_version(self):
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
fetcher._get_downloader = MagicMock(
return_value=_mock_downloader(b"full page text here")
)
fetcher.fetch("https://example.com/page", max_length=10)
assert (
cache.get("https://example.com/page")["content"]
== "full page text here"
)
def test_registry_fallback_without_explicit_cache(self, bound_cache):
_, bound = bound_cache
from local_deep_research.content_fetcher.fetcher import ContentFetcher
bound.put("https://example.com/reg", "registry text", title="RT")
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher()
downloader = MagicMock()
fetcher._get_downloader = MagicMock(return_value=downloader)
result = fetcher.fetch("https://example.com/reg")
assert result["status"] == "success"
assert result["content"] == "registry text"
assert downloader.download_with_result.call_count == 0
def test_errors_are_not_cached(self):
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
failing = MagicMock()
failing.download_with_result.return_value = MagicMock(
is_success=False, content=None, skip_reason="boom"
)
fetcher._get_downloader = MagicMock(return_value=failing)
result = fetcher.fetch("https://example.com/err")
assert result["status"] == "error"
assert len(cache) == 0
def test_fetcher_keeps_trailing_slash_variants_apart(self):
"""PR #6759: the fetcher must not serve ``/article/`` from the
``/article`` entry — each variant downloads and caches separately."""
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
downloader = MagicMock()
calls: list[str] = []
def _download(url, content_type):
calls.append(url)
body = (
b"slash body text"
if url.endswith("/article/")
else b"noslash body text"
)
return MagicMock(
is_success=True, content=body, skip_reason=None
)
downloader.download_with_result.side_effect = _download
downloader.get_metadata.return_value = {"title": "T"}
fetcher._get_downloader = MagicMock(return_value=downloader)
first = fetcher.fetch("https://example.com/article")
second = fetcher.fetch("https://example.com/article/")
assert first["status"] == "success"
assert second["status"] == "success"
assert first["content"] == "noslash body text"
assert second["content"] == "slash body text"
# Both variants reached the downloader (no cross-serving).
assert calls == [
"https://example.com/article",
"https://example.com/article/",
]
class TestTrailingSlashContentCorrectness:
"""PR #6759 end-to-end: distinct web resources stay distinct in the
shared cache through the real download/extraction code."""
@staticmethod
def _page_bytes(marker: str) -> bytes:
paragraph = "Lorem ipsum dolor sit amet. " * 60
return (
f"<html><head><title>T</title></head><body><article>"
f"<h1>{marker}</h1><p>{paragraph} {marker}</p>"
f"<p>{paragraph}</p></article></body></html>"
).encode()
def test_pipeline_real_http_slash_variants_distinct(self):
"""Different bodies at ``/article`` vs ``/article/``: both correct,
and the second URL is actually requested (no cache cross-serving)."""
import http.server
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
hits: dict[str, int] = {}
class _Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self): # noqa: N802
hits[self.path] = hits.get(self.path, 0) + 1
marker = (
"ARTICLE-SLASH-BODY"
if self.path == "/article/"
else "ARTICLE-NOSLASH-BODY"
)
body = TestTrailingSlashContentCorrectness._page_bytes(marker)
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
port = server.server_address[1]
cache = ResearchContentCache()
first = pl.fetch_and_extract(
f"http://127.0.0.1:{port}/article",
content_cache=cache,
allow_private_ips=True,
)
second = pl.fetch_and_extract(
f"http://127.0.0.1:{port}/article/",
content_cache=cache,
allow_private_ips=True,
)
assert first is not None and "NOSLASH" in first
assert second is not None and "SLASH-BODY" in second
assert "NOSLASH" not in second
# The second URL was really fetched — a collapsed key would
# have served it from the first entry with zero hits.
assert hits.get("/article") == 1
assert hits.get("/article/") == 1
finally:
server.shutdown()
class TestPipelineCache:
def test_batch_second_call_hits_cache(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = b"batch hello"
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
first = pl.batch_fetch_and_extract(
["https://example.com/b1"], content_cache=cache
)
assert first["https://example.com/b1"] == "batch hello"
fake.download.reset_mock()
second = pl.batch_fetch_and_extract(
["https://example.com/b1"], content_cache=cache
)
assert second["https://example.com/b1"] == "batch hello"
assert fake.download.call_count == 0
def test_fetch_and_extract_caches(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = b"single hello"
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
assert (
pl.fetch_and_extract(
"https://example.com/s1", content_cache=cache
)
== "single hello"
)
fake.download.reset_mock()
assert (
pl.fetch_and_extract(
"https://example.com/s1", content_cache=cache
)
== "single hello"
)
assert fake.download.call_count == 0
def test_specialized_result_cached(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content="arxiv text", fallback_allowed=True
),
):
out = pl.batch_fetch_and_extract(
["https://arxiv.org/abs/1234.5678"], content_cache=cache
)
assert out["https://arxiv.org/abs/1234.5678"] == "arxiv text"
assert (
cache.get("https://arxiv.org/abs/1234.5678")["content"]
== "arxiv text"
)
def _fetch_cm(content="shared body", title="T", url="https://example.com/s"):
fetcher = MagicMock()
fetcher.fetch.return_value = {
"status": "success",
"title": title,
"content": content,
"url": url,
}
cm = MagicMock()
cm.__enter__.return_value = fetcher
cm.__exit__.return_value = False
return cm
class TestFetchToolSharing:
def test_lead_and_subagent_share_one_cache(self):
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
build_fetch_tool,
)
shared = ResearchContentCache()
collector = SearchResultsCollector([])
lead = build_fetch_tool(
"full",
collector,
settings_snapshot={},
max_fetches=0,
subagent_id="lead",
content_cache=shared,
)
sub = build_fetch_tool(
"full",
collector,
settings_snapshot={},
max_fetches=0,
subagent_id="subagent",
content_cache=shared,
)
with patch(
"local_deep_research.content_fetcher.ContentFetcher",
return_value=_fetch_cm(),
):
first = lead.invoke({"url": "https://example.com/s"})
assert "shared body" in first
with patch(
"local_deep_research.content_fetcher.ContentFetcher._get_downloader"
) as downloader:
second = sub.invoke({"url": "https://example.com/s"})
downloader.assert_not_called()
assert "shared body" in second
assert shared.stats()["entries"] >= 1
def test_summary_mode_reuses_raw_but_reruns_extractor(self):
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
build_fetch_tool,
)
model = MagicMock()
message = MagicMock()
message.content = "quote1"
model.invoke.return_value = message
collector = SearchResultsCollector([])
shared = ResearchContentCache()
first_tool = build_fetch_tool(
"summary_focus",
collector,
model=model,
settings_snapshot={},
max_fetches=0,
subagent_id="lead",
content_cache=shared,
)
second_tool = build_fetch_tool(
"summary_focus",
collector,
model=model,
settings_snapshot={},
max_fetches=0,
subagent_id="subagent",
content_cache=shared,
)
with patch(
"local_deep_research.content_fetcher.ContentFetcher",
return_value=_fetch_cm(content="long page about cats and dogs"),
):
first_tool.invoke({"url": "https://example.com/m", "focus": "cats"})
assert model.invoke.call_count == 1
with patch(
"local_deep_research.content_fetcher.ContentFetcher._get_downloader"
) as downloader:
second_tool.invoke(
{"url": "https://example.com/m", "focus": "dogs"}
)
downloader.assert_not_called()
assert model.invoke.call_count == 2
def test_summary_prompt_built_from_cached_raw(self, bound_cache):
_, bound = bound_cache
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
build_fetch_tool,
)
bound.put("https://example.com/p", "cached page text", title="CT")
model = MagicMock()
message = MagicMock()
message.content = "quote"
model.invoke.return_value = message
tool = build_fetch_tool(
"summary_focus",
SearchResultsCollector([]),
model=model,
settings_snapshot={},
max_fetches=0,
)
with patch(
"local_deep_research.content_fetcher.ContentFetcher._get_downloader"
) as downloader:
tool.invoke({"url": "https://example.com/p", "focus": "f"})
downloader.assert_not_called()
prompt = model.invoke.call_args[0][0]
assert "cached page text" in prompt
class TestEngineCache:
def _fake_engine_class(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class FakeEngine(BaseSearchEngine):
def __init__(self, **kwargs):
super().__init__(search_snippets_only=False, **kwargs)
self.calls = 0
def _get_previews(self, query):
return [
{
"title": "T",
"link": "https://example.com/e1",
"snippet": "s",
}
]
def _get_full_content(self, items):
self.calls += 1
for item in items:
item["full_content"] = "engine full text"
item["full_content_is_complete"] = True
return items
return FakeEngine
def test_second_run_served_from_cache(self):
engine = self._fake_engine_class()(content_cache=ResearchContentCache())
first = engine.run("q")
assert engine.calls == 1
assert first[0]["full_content"] == "engine full text"
second = engine.run("q")
assert engine.calls == 1
assert second[0]["full_content"] == "engine full text"
def test_without_cache_fetches_every_time(self):
engine = self._fake_engine_class()()
engine.run("q")
engine.run("q")
assert engine.calls == 2
def test_cross_consumer_pipeline_then_engine(self, bound_cache):
_, bound = bound_cache
bound.put("https://example.com/e1", "pipeline text", title="PT")
engine = self._fake_engine_class()()
results = engine.run("anything")
assert results[0]["full_content"] == "pipeline text"
assert engine.calls == 0
def test_factory_threads_cache_into_engine(self):
cache = ResearchContentCache()
from local_deep_research.web_search_engines import (
search_engine_factory as factory,
)
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
created: dict = {}
class StubEngine(BaseSearchEngine):
def __init__(self, **kwargs):
super().__init__(**kwargs)
created.update(kwargs)
def _get_previews(self, query):
return []
fake_config = {
"whatever": {
"module_path": "fake.module",
"class_name": "StubEngine",
"default_params": {},
}
}
with (
patch.object(factory, "search_config", return_value=fake_config),
patch.object(
factory, "get_safe_module_class", return_value=StubEngine
),
patch(
"local_deep_research.security.egress.policy.evaluate_engine",
return_value=MagicMock(allowed=True),
),
patch(
"local_deep_research.security.egress.policy.context_from_snapshot",
return_value=MagicMock(),
),
patch(
"local_deep_research.security.egress.policy.resolve_run_primary_engine",
return_value="whatever",
),
):
engine = factory.create_search_engine(
"whatever",
settings_snapshot={"search.tool": "whatever"},
content_cache=cache,
)
assert engine is not None
assert engine.content_cache is cache
class TestFullSearchCache:
def test_run_threads_cache_into_batch_funnel(self):
from local_deep_research.web_search_engines.engines.full_search import (
FullSearchResults,
)
cache = ResearchContentCache()
web_search = MagicMock()
web_search.invoke.return_value = [
{"title": "T", "link": "https://example.com/f1"}
]
full = FullSearchResults(
llm=None, web_search=web_search, content_cache=cache
)
with patch(
"local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
return_value={"https://example.com/f1": "page one"},
) as batch:
first = full.run("q")
assert first[0]["full_content"] == "page one"
assert batch.call_count == 1
# The shared cache instance must reach the funnel so repeat
# URLs (quick phase vs subagents) hit one cache.
assert batch.call_args.kwargs.get("content_cache") is cache
def test_funnel_serves_repeat_urls_from_cache(self):
# The funnel-level cache (pinned in TestPipelineCache) is what
# serves FullSearchResults repeats end to end; here we prove the
# wiring with the real funnel and a mocked downloader.
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
from local_deep_research.web_search_engines.engines.full_search import (
FullSearchResults,
)
cache = ResearchContentCache()
web_search = MagicMock()
web_search.invoke.return_value = [
{"title": "T", "link": "https://example.com/f2"}
]
full = FullSearchResults(
llm=None, web_search=web_search, content_cache=cache
)
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = b"page two"
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
first = full.run("q")
assert first[0]["full_content"] == "page two"
fake.download.reset_mock()
second = full.run("q")
assert second[0]["full_content"] == "page two"
assert fake.download.call_count == 0
class TestLibraryResolverCache:
def test_prepopulated_entry_skips_db(self):
from local_deep_research.advanced_search_system.tools.fetch import (
library_resolver as lr,
)
cache = ResearchContentCache()
cache.put(
"/library/document/12345678123456781234567812345678",
"cached doc text",
title="DocT",
url_out="/library/document/12345678123456781234567812345678",
provenance=LIBRARY_PROVENANCE,
)
result = lr.resolve_library_document(
"/library/document/12345678123456781234567812345678",
username="someone",
content_cache=cache,
)
assert result is not None
assert result["content"] == "cached doc text"
assert result["title"] == "DocT"
def test_chunk_and_document_keys_are_distinct(self):
cache = ResearchContentCache()
cache.put(
"/library/document/12345678123456781234567812345678#chunk-3",
"chunk3 text",
title="D",
url_out="/library/document/12345678123456781234567812345678#chunk-3",
)
assert (
cache.get("/library/document/12345678123456781234567812345678")
is None
)
assert (
cache.get(
"/library/document/12345678123456781234567812345678#chunk-3"
)["content"]
== "chunk3 text"
)
def test_make_resolver_forwards_cache(self):
from local_deep_research.advanced_search_system.tools.fetch import (
library_resolver as lr,
)
cache = ResearchContentCache()
cache.put(
"/library/document/12345678123456781234567812345678",
"cached",
title="T",
url_out="/library/document/12345678123456781234567812345678",
provenance=LIBRARY_PROVENANCE,
)
resolver = lr.make_library_resolver("someone", content_cache=cache)
assert (
resolver("/library/document/12345678123456781234567812345678")[
"content"
]
== "cached"
)
class TestSingleSharedCachePerResearch:
def test_system_strategy_and_engine_share_one_instance(self):
from unittest.mock import Mock
from local_deep_research.search_system import AdvancedSearchSystem
from local_deep_research.utilities.research_content_cache import (
get_cache_for_research,
)
rid = f"sys-{uuid.uuid4().hex}"
system = AdvancedSearchSystem(
llm=Mock(),
search=Mock(),
strategy_name="source-based",
settings_snapshot={},
research_id=rid,
)
try:
assert system.content_cache is not None
assert system.strategy.content_cache is system.content_cache
assert system.search.content_cache is system.content_cache
assert get_cache_for_research(rid) is system.content_cache
finally:
system.close()
assert get_cache_for_research(rid) is None
def test_strategy_creates_cache_when_standalone(self):
from local_deep_research.advanced_search_system.strategies.base_strategy import (
BaseSearchStrategy,
)
class Dummy(BaseSearchStrategy):
def analyze_topic(self, query):
return {}
first = Dummy()
second = Dummy()
assert first.content_cache is not None
assert second.content_cache is not None
assert first.content_cache is not second.content_cache
# Explicit sharing works when requested.
shared = ResearchContentCache()
assert Dummy(content_cache=shared).content_cache is shared
class TestProductionCoalescing:
"""Production-shaped contract: concurrent real fetches collapse to one."""
def test_concurrent_content_fetcher_calls_fetch_once(self):
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
downloader = _mock_downloader(b"shared page body")
fetcher._get_downloader = MagicMock(return_value=downloader)
results: list = []
def run():
results.append(fetcher.fetch("https://example.com/coalesce"))
threads = [threading.Thread(target=run) for _ in range(8)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert len(results) == 8
assert all(
r["status"] == "success" and r["content"] == "shared page body"
for r in results
)
assert downloader.download_with_result.call_count == 1
def test_truncation_marker_never_overwrites_full(self):
"""10_001-char page + 10_000 limit: full must survive tool put."""
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
full = "x" * 10_001
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
fetcher._get_downloader = MagicMock(
return_value=_mock_downloader(full.encode())
)
first = fetcher.fetch("https://example.com/long", max_length=10_000)
assert first["status"] == "success"
assert "[... content truncated ...]" in first["content"]
assert (
len(cache.get("https://example.com/long")["content"]) == 10_001
)
# Simulate the tool layer storing its truncated display output:
# it must be ignored (contains the marker).
truncated_display = first["content"]
assert len(truncated_display) > 10_001
# Tool-layer guard: skip puts containing the marker.
assert "[... content truncated ...]" in truncated_display
second = fetcher.fetch(
"https://example.com/long", max_length=100_000
)
assert second["content"] == full
def test_failure_then_retry_refetches(self):
from local_deep_research.content_fetcher.fetcher import ContentFetcher
cache = ResearchContentCache()
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
fetcher = ContentFetcher(content_cache=cache)
failing = MagicMock()
failing.download_with_result.return_value = MagicMock(
is_success=False, content=None, skip_reason="boom"
)
fetcher._get_downloader = MagicMock(return_value=failing)
assert (
fetcher.fetch("https://example.com/flaky")["status"] == "error"
)
assert len(cache) == 0
fetcher._get_downloader = MagicMock(
return_value=_mock_downloader(b"recovered")
)
retry = fetcher.fetch("https://example.com/flaky")
assert retry["status"] == "success"
assert retry["content"] == "recovered"
assert (
cache.get("https://example.com/flaky")["content"] == "recovered"
)
class TestLibraryResolverIsolation:
def test_web_url_cache_hit_is_not_library_document(self):
from local_deep_research.advanced_search_system.tools.fetch import (
library_resolver as lr,
)
cache = ResearchContentCache()
cache.put("https://example.com/page", "web body", title="WT")
assert (
lr.resolve_library_document(
"https://example.com/page",
username="someone",
content_cache=cache,
)
is None
)
def test_missing_chunk_not_cached_as_source_text(self):
from local_deep_research.advanced_search_system.tools.fetch import (
library_resolver as lr,
)
from local_deep_research.database.models.library import (
Document,
DocumentChunk,
)
cache = ResearchContentCache()
doc_id = "12345678123456781234567812345678"
chunk_url = f"/library/document/{doc_id}/chunks#chunk-5"
doc = MagicMock()
doc.id = doc_id
doc.title = "Test Doc"
doc.text_content = "full doc"
chunk = MagicMock()
chunk.id = 1
chunk.chunk_index = 5
chunk.chunk_text = "real chunk text"
chunk.collection_name = "default"
chunk_exists = False
session = MagicMock()
def query_mock(model):
q = MagicMock()
if model == Document:
q.filter_by.return_value.first.return_value = doc
elif model == DocumentChunk:
q.filter.return_value.order_by.return_value.first.return_value = (
chunk if chunk_exists else None
)
return q
session.query.side_effect = query_mock
session_cm = MagicMock()
session_cm.__enter__.return_value = session
session_cm.__exit__.return_value = False
with patch(
"local_deep_research.database.session_context.get_user_db_session",
return_value=session_cm,
):
# Attempt 1: missing chunk returns not found, NOT cached
res1 = lr.resolve_library_document(
chunk_url, "someone", content_cache=cache
)
assert res1 is not None
assert "Chunk 5 not found" in res1["content"]
assert cache.get(chunk_url) is None
# Attempt 2: chunk becomes available -> retry hits DB and returns real text
chunk_exists = True
session.query.reset_mock()
res2 = lr.resolve_library_document(
chunk_url, "someone", content_cache=cache
)
assert res2 is not None
assert res2["content"] == "real chunk text"
assert session.query.call_count > 0
assert cache.get(chunk_url) is not None
# Control: repeat returns cached text without DB query
session.query.reset_mock()
res3 = lr.resolve_library_document(
chunk_url, "someone", content_cache=cache
)
assert res3 is not None
assert res3["content"] == "real chunk text"
assert session.query.call_count == 0
def test_quick_phase_to_subsection_shares_via_registry(self, bound_cache):
"""Quick phase puts via explicit cache; subsection reads via registry."""
rid, quick_cache = bound_cache
from local_deep_research.content_fetcher.fetcher import ContentFetcher
quick_cache.put(
"https://example.com/shared", "quick phase body", title="QT"
)
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
):
# Subsection fetcher has no explicit cache: resolves via
# thread-local research context bound above.
fetcher = ContentFetcher()
downloader = MagicMock()
fetcher._get_downloader = MagicMock(return_value=downloader)
result = fetcher.fetch("https://example.com/shared")
assert result["status"] == "success"
assert result["content"] == "quick phase body"
assert downloader.download_with_result.call_count == 0
class TestChunkCollectionScoping:
"""Same (document, chunk) in two collections must not share an entry."""
_DOC = "12345678123456781234567812345678"
def _chunk_url(self):
return f"/library/document/{self._DOC}/chunks#chunk-3"
def test_keys_differ_per_collection(self):
url = self._chunk_url()
assert content_cache_key(url, "coll-A") != content_cache_key(
url, "coll-B"
)
def test_collection_spellings_collapse(self):
url = self._chunk_url()
assert content_cache_key(url, "coll-B") == content_cache_key(
url, "collection_coll-B"
)
def test_chunk_spellings_merge_for_same_collection(self):
base = f"/library/document/{self._DOC}#chunk-3"
suffixed = f"/library/document/{self._DOC}/chunks#chunk-3"
assert content_cache_key(base, "coll-A") == content_cache_key(
suffixed, "coll-A"
)
def test_document_url_ignores_collection(self):
doc = f"/library/document/{self._DOC}"
assert content_cache_key(doc) == content_cache_key(doc, "coll-A")
assert content_cache_key(doc, "coll-A") == content_cache_key(
doc, "coll-B"
)
def _collection_engine(self, cache, calls):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class CollEngine(BaseSearchEngine):
# Stands in for the collection engine, which may write library keys.
writes_library_cache_keys = True
def __init__(self, **kwargs):
super().__init__(search_snippets_only=False, **kwargs)
def _get_previews(self, query):
return []
def _get_full_content(self, items):
for item in items:
cid = (item.get("metadata") or {}).get("collection_id")
calls[cid] = calls.get(cid, 0) + 1
item["full_content"] = f"text-for-{cid}"
item["full_content_is_complete"] = True
return items
return CollEngine(content_cache=cache)
def test_two_collections_same_chunk_do_not_collide(self):
"""A fills, B fetches its own text, B repeats from cache."""
cache = ResearchContentCache()
calls: dict = {}
engine = self._collection_engine(cache, calls)
url = self._chunk_url()
items_a = [
{
"link": url,
"title": "T",
"metadata": {"collection_id": "coll-A", "chunk_index": 3},
}
]
result_a = engine._get_full_content_cached(items_a)
assert result_a[0]["full_content"] == "text-for-coll-A"
assert calls == {"coll-A": 1}
items_b = [
{
"link": url,
"title": "T",
"metadata": {"collection_id": "coll-B", "chunk_index": 3},
}
]
result_b = engine._get_full_content_cached(items_b)
# Must be B's text (not A's) and B's DB lookup must have run.
assert result_b[0]["full_content"] == "text-for-coll-B"
assert calls == {"coll-A": 1, "coll-B": 1}
# Repeat B: served from B's namespaced entry, no new lookup.
items_b2 = [
{
"link": url,
"title": "T",
"metadata": {"collection_id": "coll-B", "chunk_index": 3},
}
]
result_b2 = engine._get_full_content_cached(items_b2)
assert result_b2[0]["full_content"] == "text-for-coll-B"
assert calls == {"coll-A": 1, "coll-B": 1}
# Repeat A: still A's text.
items_a2 = [
{
"link": url,
"title": "T",
"metadata": {"collection_id": "coll-A", "chunk_index": 3},
}
]
result_a2 = engine._get_full_content_cached(items_a2)
assert result_a2[0]["full_content"] == "text-for-coll-A"
assert calls == {"coll-A": 1, "coll-B": 1}
def test_chunk_without_collection_bypasses_shared_entry(self):
"""No collection → fresh lookup every time, never poisons cache."""
cache = ResearchContentCache()
calls: dict = {}
engine = self._collection_engine(cache, calls)
url = self._chunk_url()
# Seed a scoped entry first.
scoped = [
{
"link": url,
"title": "T",
"metadata": {"collection_id": "coll-A", "chunk_index": 3},
}
]
assert (
engine._get_full_content_cached(scoped)[0]["full_content"]
== "text-for-coll-A"
)
assert calls == {"coll-A": 1}
# Unscoped chunk must not hit the scoped entry.
unscoped = [{"link": url, "title": "T", "metadata": {}}]
first = engine._get_full_content_cached(unscoped)
assert first[0]["full_content"] == "text-for-None"
second = engine._get_full_content_cached(
[{"link": url, "title": "T", "metadata": {}}]
)
# Bypassed both times: two lookups, never cached.
assert calls.get(None) == 2
assert second[0]["full_content"] == "text-for-None"
# Scoped entry still intact.
assert (
cache.get(url, collection_id="coll-A")["content"]
== "text-for-coll-A"
)
class TestGrantAwareCache:
"""Grant x cache: private entries must not leak to non-granted callers."""
_LOOPBACK = "http://127.0.0.1:9/private-page"
def test_batch_skips_private_hit_without_grant(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "private secret", title="S")
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = None
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
out = pl.batch_fetch_and_extract(
[self._LOOPBACK],
content_cache=cache,
allow_private_ips=False,
block_link_local=True,
)
assert out[self._LOOPBACK] is None
# Cache bypassed: the downloader gate was consulted.
assert fake.download.call_count >= 1
def test_batch_serves_private_hit_with_grant(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "private secret", title="S")
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = b"should not be used"
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
out = pl.batch_fetch_and_extract(
[self._LOOPBACK],
content_cache=cache,
allow_private_ips=True,
block_link_local=True,
)
assert out[self._LOOPBACK] == "private secret"
assert fake.download.call_count == 0
def test_fetch_and_extract_singular_skips_private_without_grant(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "private secret", title="S")
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = None
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
assert (
pl.fetch_and_extract(
self._LOOPBACK,
content_cache=cache,
allow_private_ips=False,
block_link_local=True,
)
is None
)
assert fake.download.call_count >= 1
def test_engine_fill_skips_private_without_grant(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def __init__(self, **kwargs):
super().__init__(search_snippets_only=False, **kwargs)
self.calls = 0
def _get_previews(self, query):
return []
def _get_full_content(self, items):
self.calls += 1
for item in items:
item["full_content"] = "fresh"
return items
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "cached private", title="T")
plain = StubEngine(content_cache=cache)
assert plain.allow_private_result_fetch is False
items = [{"link": self._LOOPBACK, "title": "T"}]
cached, missing = plain._fill_full_content_from_cache(items)
assert cached == [] and len(missing) == 1
assert items[0].get("full_content") is None
granted = StubEngine(content_cache=cache)
granted.allow_private_result_fetch = True
items2 = [{"link": self._LOOPBACK, "title": "T"}]
cached2, _ = granted._fill_full_content_from_cache(items2)
assert len(cached2) == 1
assert items2[0]["full_content"] == "cached private"
def test_engine_fill_still_serves_public_and_library(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
cache = ResearchContentCache()
cache.put("https://example.com/pub", "public body", title="T")
doc = "/library/document/12345678123456781234567812345678"
cache.put(
doc,
"doc text",
title="D",
url_out=doc,
provenance=LIBRARY_PROVENANCE,
)
engine = StubEngine(content_cache=cache)
items = [
{"link": "https://example.com/pub", "title": "T"},
{"link": doc, "title": "D"},
]
cached, missing = engine._fill_full_content_from_cache(items)
assert len(cached) == 2 and missing == []
assert items[0]["full_content"] == "public body"
assert items[1]["full_content"] == "doc text"
class TestFetchToolGrantGate:
"""Item 1: fetch-tool fast path must apply the SSRF grant gate."""
_LOOPBACK = "http://127.0.0.1:9/private-page"
def _unprotected_ctx(self):
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
return EgressContext(
scope=EgressScope.UNPROTECTED,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
def _private_ctx(self):
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
return EgressContext(
scope=EgressScope.PRIVATE_ONLY,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
def test_skips_private_hit_under_unprotected(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_fetch_raw_content,
)
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "private secret body", title="S")
ctx = self._unprotected_ctx()
with patch(
"local_deep_research.content_fetcher.ContentFetcher"
) as mock_cls:
inst = mock_cls.return_value.__enter__.return_value
inst.fetch.return_value = {
"status": "error",
"error": "URL failed security validation",
"url": self._LOOPBACK,
}
result, _, _, _ = _fetch_raw_content(
self._LOOPBACK,
None,
MagicMock(),
ctx,
None,
"full",
max_fetches=None,
content_cache=cache,
)
# Cache must NOT be served: the fetcher gate is consulted.
assert inst.fetch.call_count == 1
assert result is not None
assert result.get("status") == "error"
def test_serves_private_hit_with_private_grant(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_fetch_raw_content,
)
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "private secret body", title="S")
ctx = self._private_ctx()
with patch(
"local_deep_research.content_fetcher.ContentFetcher._get_downloader"
) as downloader:
result, _, _, _ = _fetch_raw_content(
self._LOOPBACK,
None,
MagicMock(),
ctx,
None,
"full",
max_fetches=None,
content_cache=cache,
)
downloader.assert_not_called()
assert result is not None
assert result.get("status") == "success"
assert result.get("content") == "private secret body"
def test_does_not_store_private_without_grant(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_fetch_raw_content,
)
cache = ResearchContentCache()
ctx = self._unprotected_ctx()
with patch(
"local_deep_research.content_fetcher.ContentFetcher"
) as mock_cls:
inst = mock_cls.return_value.__enter__.return_value
# Mocked fetcher bypasses its own gate (as in unit tests);
# the tool layer must still refuse to persist the entry.
inst.fetch.return_value = {
"status": "success",
"content": "private body",
"url": self._LOOPBACK,
"title": "S",
}
result, _, _, _ = _fetch_raw_content(
self._LOOPBACK,
None,
MagicMock(),
ctx,
None,
"full",
max_fetches=None,
content_cache=cache,
)
assert result is not None
assert result.get("status") == "success"
assert cache.get(self._LOOPBACK) is None
class TestCacheHitUrlLogging:
"""#7159: the cache-hit INFO event must not log a raw URL.
The ``source=cache`` event is new in this branch, so it has to redact at
the call site like every other ``[FETCH]`` event in the module. A raw URL
there would put credentials, query tokens, private paths and the fragment
into normal stderr/container logs (INFO, no debug mode needed).
Two traps this test deliberately pins:
* the package disables its loguru namespace at import (until
``config_logger`` runs), so the capture must ``logger.enable(...)``
first — without it the sink captures nothing and the non-empty
``source=cache`` assertion fails loudly (the test cannot pass
silently);
* the fragment is the marker that actually leaked. The global record
scrubber already strips userinfo and query values, so a test asserting
only those two would pass against the unfixed line.
"""
# Loopback + PRIVATE_ONLY so the real grant check admits the hit and the
# real cache-hit branch is reached (see TestFetchToolGrantGate).
_SECRET_URL = (
"http://alice:[email protected]:9/private/FAKE_PATH"
"?token=FAKE_TOKEN"
"#FAKE_FRAGMENT"
)
_MARKERS = ("FAKE_PASSWORD", "FAKE_PATH", "FAKE_TOKEN", "FAKE_FRAGMENT")
def _private_ctx(self):
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
return EgressContext(
scope=EgressScope.PRIVATE_ONLY,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
def test_cache_hit_log_redacts_url_and_still_serves_content(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_fetch_raw_content,
)
from loguru import logger
cache = ResearchContentCache()
cache.put(self._SECRET_URL, "cached body text", title="S")
ctx = self._private_ctx()
# loguru's namespace is disabled at import time; without re-enabling
# it the added sink below captures nothing and the non-empty
# cache-hit assertion below fails loudly (no silent pass).
logger.enable("local_deep_research")
records: list[str] = []
sink_id = logger.add(
lambda message: records.append(str(message)),
level="INFO",
)
try:
with patch(
"local_deep_research.content_fetcher.ContentFetcher._get_downloader"
) as downloader:
result, _, _, _ = _fetch_raw_content(
self._SECRET_URL,
None,
MagicMock(),
ctx,
None,
"full",
max_fetches=None,
content_cache=cache,
)
finally:
logger.remove(sink_id)
logger.disable("local_deep_research")
# The real cache-hit branch ran (no network fetch) and served content.
downloader.assert_not_called()
assert result is not None
assert result.get("status") == "success"
assert result.get("content") == "cached body text"
# The event was actually captured, and the hit was logged.
cache_hit = [
r for r in records if "source=cache" in r and "mode=full" in r
]
assert cache_hit, f"no cache-hit INFO record captured: {records}"
assert len(cache_hit) == 1
# Control: the markers were present in the URL being logged before the
# fix, so this assertion is discriminating rather than incidental.
for marker in self._MARKERS:
assert marker in self._SECRET_URL
assert all(marker not in rec for rec in cache_hit), (
f"{marker} leaked into the cache-hit log record: {cache_hit}"
)
# The redacted host is still there, so the record is not just empty.
assert "127.0.0.1" in cache_hit[0]
class TestPipelineGetOrFetchExceptionRedaction:
"""Optional note from the 6511e834 APPROVED review: pin pipeline.py:767.
``fetch_and_extract`` wraps ``_cache.get_or_fetch`` in try/except and
logs ``fetch_and_extract failed for {url}``. The message must use
``redact_url_for_log`` (like the file's other two sites) *and* must not
attach the traceback: plain ``logger.exception`` renders frame locals
via loguru diagnose, so the ``url`` local (credentials, path, query,
fragment) leaks even when the message itself is redacted. Reverting just
that hunk still passes the whole suite today, so this test guards it.
Traps pinned deliberately (mirrors ``TestCacheHitUrlLogging``):
* ``logger.enable(...)`` first — the package disables its loguru
namespace at import, so without it the sink captures nothing and the
non-empty failure-event assertion fails loudly (no silent pass);
* ``diagnose=True`` + ``backtrace=True`` with ``{exception}`` in the
format — the sibling ``test_pipeline_logging_sinks.py`` fixture uses
``diagnose=False``, which hides the frame-locals channel this test
exists to pin;
* all four marker classes asserted, plus an explicit ``Traceback``
absence — a generic (secret-free) ``get_or_fetch`` failure means any
marker in the output must come from the ``url`` local/message.
"""
_SECRET_URL = (
"http://alice:[email protected]:9/private/FAKE_PATH"
"?token=FAKE_TOKEN"
"#FAKE_FRAGMENT"
)
_MARKERS = ("FAKE_PASSWORD", "FAKE_PATH", "FAKE_TOKEN", "FAKE_FRAGMENT")
def test_fetch_and_extract_get_or_fetch_failure_redacted_no_traceback(
self,
):
from io import StringIO
from loguru import logger
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
for marker in self._MARKERS:
assert marker in self._SECRET_URL
logger.enable("local_deep_research")
output = StringIO()
sink_id = logger.add(
output,
level="DEBUG",
format="{message}\n{exception}",
backtrace=True,
diagnose=True,
)
try:
with (
patch.object(pl, "_is_cache_hit_allowed", return_value=True),
patch.object(
cache,
"get_or_fetch",
side_effect=RuntimeError("boom"),
) as mock_fetch,
):
result = pl.fetch_and_extract(
self._SECRET_URL,
content_cache=cache,
)
finally:
logger.remove(sink_id)
logger.disable("local_deep_research")
# The failing-cache branch ran and surfaced as a miss.
assert result is None
assert mock_fetch.call_count == 1
# The failure event was actually captured (no silent pass).
rendered = output.getvalue()
lines = [
line
for line in rendered.splitlines()
if "fetch_and_extract failed for" in line
]
assert lines, f"no failure event captured: {rendered!r}"
assert len(lines) == 1
# Redacted host retained, so the record is not just empty.
assert "127.0.0.1" in lines[0]
# No raw secret reaches the sink via message, exception text, or
# diagnose frame locals; and no traceback is attached at all.
for marker in self._MARKERS:
assert marker not in rendered, (
f"{marker} leaked into pipeline failure log: {rendered!r}"
)
assert "Traceback" not in rendered
class TestLibraryAliasGrantGate:
"""Item 2: absolute alias must pass the egress check; path-only stays exempt."""
_DOC = "/library/document/12345678123456781234567812345678"
_ALIAS = "https://library.document/12345678123456781234567812345678"
def test_pipeline_path_exempt_alias_requires_check(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
# Path-only: always allowed, even when the SSRF gate would refuse.
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
):
assert pl._is_cache_hit_allowed(self._DOC, False, True) is True
# Alias: falls through to the egress check.
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
) as mock_v:
assert pl._is_cache_hit_allowed(self._ALIAS, True, True) is False
assert mock_v.call_count >= 1
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=True,
):
assert pl._is_cache_hit_allowed(self._ALIAS, True, True) is True
def test_engine_path_exempt_alias_requires_check(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
engine = StubEngine(content_cache=ResearchContentCache())
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
):
assert engine._is_cache_url_allowed(self._DOC) is True
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
) as mock_v:
assert engine._is_cache_url_allowed(self._ALIAS) is False
assert mock_v.call_count >= 1
def test_pipeline_batch_alias_bypasses_cache(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
cache.put(self._ALIAS, "alias secret", title="A")
with patch.object(
pl,
"_try_specialized_downloader",
return_value=pl._SpecializedResult(
content=None, fallback_allowed=True
),
):
fake = MagicMock()
fake.download.return_value = None
with patch(
"local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
return_value=fake,
):
out = pl.batch_fetch_and_extract(
[self._ALIAS],
content_cache=cache,
allow_private_ips=False,
block_link_local=False,
)
assert out[self._ALIAS] is None
assert fake.download.call_count >= 1
def test_engine_fill_alias_bypasses_cache(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
cache = ResearchContentCache()
cache.put(self._ALIAS, "alias secret", title="A")
engine = StubEngine(content_cache=cache)
items = [{"link": self._ALIAS, "title": "A"}]
cached, missing = engine._fill_full_content_from_cache(items)
assert cached == [] and len(missing) == 1
assert items[0].get("full_content") is None
class TestOptionalHardening:
"""Follow-up to APPROVED-OPTIONAL review at d58a9443.
- Per-run SSRF-verdict memo removes per-hit getaddrinfo.
- All three gates fail closed when the validator import fails.
- Engine cache gate is scope-aware (withholds grant + evaluate_url).
"""
_LOOPBACK = "http://127.0.0.1:9/private-page"
def test_pipeline_memoizes_ssrf_verdict(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=True,
) as mock_v:
assert (
pl._is_cache_hit_allowed(
"https://example.com/a", False, True, cache
)
is True
)
assert (
pl._is_cache_hit_allowed(
"https://example.com/a", False, True, cache
)
is True
)
# One DNS-backed validation for two identical hits.
assert mock_v.call_count == 1
assert cache.stats()["validation_memo"] >= 1
def test_pipeline_memo_distinguishes_grants(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
side_effect=lambda url, allow_private_ips=False, block_link_local=False: (
bool(allow_private_ips) if "127.0.0.1" in url else True
),
) as mock_v:
assert (
pl._is_cache_hit_allowed(self._LOOPBACK, True, True, cache)
is True
)
assert (
pl._is_cache_hit_allowed(self._LOOPBACK, False, True, cache)
is False
)
assert (
pl._is_cache_hit_allowed(self._LOOPBACK, True, True, cache)
is True
)
# Two distinct (url, grant) keys → two validations.
assert mock_v.call_count == 2
def test_memoized_success_survives_dns_outage(self):
import socket
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=True,
):
assert (
pl._is_cache_hit_allowed(
"https://example.com/offline", False, True, cache
)
is True
)
# DNS down: validator would raise gaierror, but the memoized True
# verdict serves without re-resolving.
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
side_effect=socket.gaierror("dns down"),
) as mock_v:
assert (
pl._is_cache_hit_allowed(
"https://example.com/offline", False, True, cache
)
is True
)
assert mock_v.call_count == 0
def test_pipeline_fails_closed_on_import_error(self):
import sys
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
with patch.dict(
sys.modules,
{"local_deep_research.security.ssrf_validator": None},
):
assert (
pl._is_cache_hit_allowed(
"https://example.com/pub", False, True, cache
)
is False
)
def test_engine_fails_closed_on_import_error(self):
import sys
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
engine = StubEngine(content_cache=ResearchContentCache())
with patch.dict(
sys.modules,
{"local_deep_research.security.ssrf_validator": None},
):
assert (
engine._is_cache_url_allowed("https://example.com/pub") is False
)
def test_fetch_tool_fails_closed_on_import_error(self):
import sys
from local_deep_research.advanced_search_system.tools.fetch import (
_is_fetch_cache_hit_allowed,
)
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
ctx = EgressContext(
scope=EgressScope.PRIVATE_ONLY,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
cache = ResearchContentCache()
with patch.dict(
sys.modules,
{
"local_deep_research.security.egress.fetch": None,
"local_deep_research.security.ssrf_validator": None,
},
):
assert (
_is_fetch_cache_hit_allowed(
"https://example.com/pub", ctx, cache
)
is False
)
def test_engine_scope_withholds_private_under_public_only(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
public_snapshot = {
"policy.egress_scope": {"value": "public_only"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "cached private", title="T")
engine = StubEngine(
content_cache=cache, settings_snapshot=public_snapshot
)
engine.allow_private_result_fetch = True
# Grant is withheld by scope (mirrors _downloader_allow_private_ips)
# and evaluate_url denies private under PUBLIC_ONLY.
assert engine._is_cache_url_allowed(self._LOOPBACK) is False
items = [{"link": self._LOOPBACK, "title": "T"}]
cached, missing = engine._fill_full_content_from_cache(items)
assert cached == [] and len(missing) == 1
def test_engine_scope_allows_private_under_private_only(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
private_snapshot = {
"policy.egress_scope": {"value": "private_only"},
"search.tool": {"value": "library"},
}
cache = ResearchContentCache()
cache.put(self._LOOPBACK, "cached private", title="T")
engine = StubEngine(
content_cache=cache, settings_snapshot=private_snapshot
)
engine.allow_private_result_fetch = True
assert engine._is_cache_url_allowed(self._LOOPBACK) is True
def test_fetcher_shares_memo_with_tool_gates(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_is_fetch_cache_hit_allowed,
)
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
ctx = EgressContext(
scope=EgressScope.PRIVATE_ONLY,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
cache = ResearchContentCache()
with patch(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
return_value=True,
) as mock_v:
assert (
_is_fetch_cache_hit_allowed(
"https://example.com/shared", ctx, cache
)
is True
)
assert (
_is_fetch_cache_hit_allowed(
"https://example.com/shared", ctx, cache
)
is True
)
assert mock_v.call_count == 1
class TestMemoRecoveryAndScopeParity:
"""Required follow-ups from the 18c65d45 review.
1. A transient DNS failure (validate_url maps gaierror to False)
must not pin the URL for the run — blip then heal recovers.
2. An unevaluable egress policy must refuse the engine cache hit,
matching the network path which disables fetching.
3. Memo keys mirror the computed SSRF booleans across gates.
"""
_LOOPBACK = "http://127.0.0.1:9/private-page"
def test_dns_blip_then_heal_recovers(self):
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
cache = ResearchContentCache()
calls = {"n": 0}
def _flaky(url, allow_private_ips=False, block_link_local=False):
calls["n"] += 1
# First call: DNS blip (gaierror maps to False in validate_url).
return calls["n"] > 1
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
side_effect=_flaky,
):
assert (
pl._is_cache_hit_allowed(
"https://example.com/blip", False, True, cache
)
is False
)
# Heal: recomputed (not pinned), now True and pinned.
assert (
pl._is_cache_hit_allowed(
"https://example.com/blip", False, True, cache
)
is True
)
assert (
pl._is_cache_hit_allowed(
"https://example.com/blip", False, True, cache
)
is True
)
assert calls["n"] == 2
assert (
cache.get_validation("https://example.com/blip", False, True)
is True
)
def test_tool_gate_blip_then_fetcher_heal(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_is_fetch_cache_hit_allowed,
)
from local_deep_research.content_fetcher.fetcher import ContentFetcher
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
ctx = EgressContext(
scope=EgressScope.UNPROTECTED,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
cache = ResearchContentCache()
url = "https://example.com/blip-heal"
with patch(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
return_value=False,
):
assert _is_fetch_cache_hit_allowed(url, ctx, cache) is False
# Blip False is not pinned.
assert cache.get_validation(url, False, False) is None
# NOTE: fetcher.py binds policy_aware_validate_url at import, so
# the fetcher-side symbol (not the source module) is patched.
with patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
) as mock_v:
fetcher = ContentFetcher(egress_context=ctx, content_cache=cache)
with patch.object(fetcher, "_get_downloader", return_value=None):
out = fetcher.fetch(url)
# Passes the SSRF gate (fails later at downloader),
# proving the blip did not pin.
assert out.get("error") != (
"URL failed security validation "
"(blocked by SSRF protection)"
)
assert mock_v.call_count >= 1
def test_import_failure_false_not_pinned(self):
import sys
from local_deep_research.advanced_search_system.tools.fetch import (
_is_fetch_cache_hit_allowed,
)
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
ctx = EgressContext(
scope=EgressScope.PRIVATE_ONLY,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
cache = ResearchContentCache()
url = "https://example.com/import-blip"
with patch.dict(
sys.modules,
{"local_deep_research.security.egress.fetch": None},
):
assert _is_fetch_cache_hit_allowed(url, ctx, cache) is False
with patch(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
return_value=True,
):
assert _is_fetch_cache_hit_allowed(url, ctx, cache) is True
def test_engine_unevaluable_policy_refuses_cache(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
bogus_snapshot = {
"policy.egress_scope": {"value": "bogus_scope"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
cache.put("https://example.com/pub", "public body", title="T")
engine = StubEngine(
content_cache=cache, settings_snapshot=bogus_snapshot
)
# Network path disables fetching on unevaluable policy; the cache
# must refuse too rather than serve SSRF-only.
assert engine._is_cache_url_allowed("https://example.com/pub") is False
items = [{"link": "https://example.com/pub", "title": "T"}]
cached, missing = engine._fill_full_content_from_cache(items)
assert cached == [] and len(missing) == 1
assert items[0].get("full_content") is None
def test_engine_no_snapshot_still_ssrf_only(self):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
cache = ResearchContentCache()
cache.put("https://example.com/pub", "public body", title="T")
engine = StubEngine(content_cache=cache)
assert engine._is_cache_url_allowed("https://example.com/pub") is True
def test_memo_key_mirrors_compute(self):
from local_deep_research.advanced_search_system.tools.fetch import (
_is_fetch_cache_hit_allowed,
)
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
)
ctx = EgressContext(
scope=EgressScope.UNPROTECTED,
primary_engine="test",
require_local_llm=False,
require_local_embeddings=False,
)
cache = ResearchContentCache()
with patch(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
return_value=True,
):
assert (
_is_fetch_cache_hit_allowed(
"https://example.com/keyed", ctx, cache
)
is True
)
# UNPROTECTED computes strict validate_url(url) defaults
# (False, False) — the cell must live under that key, not the
# pipeline-style (False, True) cell sharing a different compute.
assert (
cache.get_validation("https://example.com/keyed", False, False)
is True
)
assert (
cache.get_validation("https://example.com/keyed", False, True)
is None
)
class TestReviewOptionalFollowups:
"""Optional notes from the c65fde9f APPROVED review, plus the 557de142
APPROVED review.
1. Under an unevaluable policy the engine cache refused path-only
``/library/...`` hits while the fetch tool still serves library
documents from the local DB without any egress check — local-only
content must stay available under a corrupted scope setting.
2. Cache-probe scope checks must not burn the run-wide denial quota —
only real fetches count toward ``MAX_DENIED_FETCHES_PER_RUN``.
3. (557de142 #1) Library exemption grammar matches the fetch tool's
strict 32/64-hex-or-UUID: slug-shaped routes are not exempt.
4. (557de142 #2) Exhausted quota still refuses cache probes
(``denial_quota_exceeded``) — probes cannot evade the cliff.
"""
_DOC = "/library/document/12345678123456781234567812345678"
def _stub_engine(self, cache, snapshot=None):
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
class StubEngine(BaseSearchEngine):
def _get_previews(self, query):
return []
def _get_full_content(self, items):
return items
kwargs = {"content_cache": cache}
if snapshot is not None:
kwargs["settings_snapshot"] = snapshot
return StubEngine(**kwargs)
def test_unevaluable_policy_still_serves_library(self):
bogus_snapshot = {
"policy.egress_scope": {"value": "bogus_scope"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
cache.put(
self._DOC, "local body", title="T", provenance=LIBRARY_PROVENANCE
)
cache.put("https://example.com/pub", "public body", title="T")
engine = self._stub_engine(cache, bogus_snapshot)
# Web still fails closed (matches the network path disabling fetch).
assert engine._is_cache_url_allowed("https://example.com/pub") is False
# Library is a local DB read — same as the fetch tool's
# _try_resolve_url fast path which skips the egress gate.
assert engine._is_cache_url_allowed(self._DOC) is True
items = [
{"link": self._DOC, "title": "T"},
{"link": "https://example.com/pub", "title": "T"},
]
cached, missing = engine._fill_full_content_from_cache(items)
assert {i["link"] for i in cached} == {self._DOC}
assert {i["link"] for i in missing} == {"https://example.com/pub"}
assert items[0].get("full_content") == "local body"
def test_evaluate_url_no_record_matches_verdict_without_quota(self):
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
MAX_DENIED_FETCHES_PER_RUN,
evaluate_url,
)
ctx = EgressContext(
scope=EgressScope.PUBLIC_ONLY,
primary_engine="arxiv",
require_local_llm=False,
require_local_embeddings=False,
)
# Same verdict both ways for a scope-mismatched private host.
denied_recorded = evaluate_url("http://10.0.0.5/", ctx)
assert denied_recorded.allowed is False
assert denied_recorded.reason == "scope_mismatch_public_only"
assert ctx._fetch_denial_count["count"] == 1
denied_probe = evaluate_url(
"http://10.0.0.6/", ctx, record_denial=False
)
assert denied_probe.allowed is False
assert denied_probe.reason == "scope_mismatch_public_only"
assert ctx._fetch_denial_count["count"] == 1
# Allowed verdicts never touch the quota either way.
allowed = evaluate_url(
"http://93.184.216.34/", ctx, record_denial=False
)
assert allowed.allowed is True
assert ctx._fetch_denial_count["count"] == 1
# Flooding probes past the cap never trips denial_quota_exceeded.
for _ in range(MAX_DENIED_FETCHES_PER_RUN + 5):
d = evaluate_url("http://10.0.0.5/", ctx, record_denial=False)
assert d.reason == "scope_mismatch_public_only"
assert ctx._fetch_denial_count["count"] == 1
def test_engine_cache_probes_do_not_burn_quota(self):
public_snapshot = {
"policy.egress_scope": {"value": "public_only"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
loopback = "http://127.0.0.1:9/private-page"
cache.put(loopback, "cached private", title="T")
engine = self._stub_engine(cache, public_snapshot)
engine.allow_private_result_fetch = True
# 60 scope-mismatched cache candidates — the old recording probe
# would have hit the 50-denial cliff; the non-recording probe must
# leave the run counter at zero.
for _ in range(60):
assert engine._is_cache_url_allowed(loopback) is False
egress_ctx = engine._get_cache_egress_context()
assert egress_ctx is not None
assert egress_ctx._fetch_denial_count["count"] == 0
# The run can still fetch an allowed public URL (quota intact).
from local_deep_research.security.egress.policy import evaluate_url
decision = evaluate_url("http://93.184.216.34/", egress_ctx)
assert decision.allowed is True
def test_engine_exhausted_quota_refuses_cache_probes(self):
"""Exhausted denial quota fails closed on cache probes too.
Optional note #2 from the 557de142 APPROVED review: the
non-recording probe verdict flips to refusal past the
``MAX_DENIED_FETCHES_PER_RUN`` cliff, same as the fetch path —
so a probe cannot serve as a quota-evading oracle.
"""
from local_deep_research.security.egress.policy import (
MAX_DENIED_FETCHES_PER_RUN,
evaluate_url,
)
public_snapshot = {
"policy.egress_scope": {"value": "public_only"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
public_url = "http://93.184.216.34/"
cache.put(public_url, "public body", title="T")
engine = self._stub_engine(cache, public_snapshot)
# Sanity: allowed before the cliff (SSRF passes for a public
# IP literal; scope allows it under PUBLIC_ONLY).
assert engine._is_cache_url_allowed(public_url) is True
egress_ctx = engine._get_cache_egress_context()
assert egress_ctx is not None
# Exhaust the run quota via the recording fetch path.
for _ in range(MAX_DENIED_FETCHES_PER_RUN):
denied = evaluate_url("http://10.0.0.5/", egress_ctx)
assert denied.allowed is False
assert (
egress_ctx._fetch_denial_count["count"]
== MAX_DENIED_FETCHES_PER_RUN
)
# The non-recording probe now refuses with the quota reason.
probe = evaluate_url(public_url, egress_ctx, record_denial=False)
assert probe.allowed is False
assert probe.reason == "denial_quota_exceeded"
# And the engine cache gate follows it fail-closed.
assert engine._is_cache_url_allowed(public_url) is False
# The cliff never records, so the counter stays pinned at the cap.
assert (
egress_ctx._fetch_denial_count["count"]
== MAX_DENIED_FETCHES_PER_RUN
)
def test_library_still_served_past_quota_cliff(self):
"""Valid library routes stay servable past the quota cliff.
Optional note from the 488e1865 APPROVED review: library
servability past an exhausted quota depends on the exemption
running before the quota check inside the gate — that ordering
is what spares local DB reads. This pins it (the quota test
above pins only the web refusal).
"""
from local_deep_research.security.egress.policy import (
MAX_DENIED_FETCHES_PER_RUN,
evaluate_url,
)
public_snapshot = {
"policy.egress_scope": {"value": "public_only"},
"search.tool": {"value": "arxiv"},
}
cache = ResearchContentCache()
public_url = "http://93.184.216.34/"
cache.put(public_url, "public body", title="T")
cache.put(
self._DOC, "local body", title="T", provenance=LIBRARY_PROVENANCE
)
engine = self._stub_engine(cache, public_snapshot)
# Sanity: both servable before the cliff.
assert engine._is_cache_url_allowed(public_url) is True
assert engine._is_cache_url_allowed(self._DOC) is True
egress_ctx = engine._get_cache_egress_context()
assert egress_ctx is not None
# Exhaust the run quota via the recording fetch path.
for _ in range(MAX_DENIED_FETCHES_PER_RUN):
denied = evaluate_url("http://10.0.0.5/", egress_ctx)
assert denied.allowed is False
assert (
egress_ctx._fetch_denial_count["count"]
== MAX_DENIED_FETCHES_PER_RUN
)
# Web now fails closed on the quota cliff ...
assert engine._is_cache_url_allowed(public_url) is False
# ... but the path-only library exemption runs before the quota
# check, so local DB reads stay servable (same as the fetch
# tool's _try_resolve_url fast path and the unevaluable-policy
# exemption).
assert engine._is_cache_url_allowed(self._DOC) is True
items = [
{"link": self._DOC, "title": "T"},
{"link": public_url, "title": "T"},
]
cached, missing = engine._fill_full_content_from_cache(items)
assert {i["link"] for i in cached} == {self._DOC}
assert {i["link"] for i in missing} == {public_url}
assert items[0].get("full_content") == "local body"
# Library probes must not burn quota either — counter stays
# pinned at the cap.
assert (
egress_ctx._fetch_denial_count["count"]
== MAX_DENIED_FETCHES_PER_RUN
)
def test_library_exemption_matches_fetch_tool_grammar(self):
"""Engine/pipeline exemption uses the fetch tool's strict grammar.
Optional note #1 from the 557de142 APPROVED review: the old
exemption used ``library_display_url`` (any ``[0-9A-Za-z_-]+``
slug), while the fetch tool's resolver accepts only
32/64-hex or UUID ids. A slug-shaped route can never resolve via
the fetch tool, so exempting it here would let the two surfaces
drift. Both gates now share ``parse_library_url``.
"""
from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (
parse_library_url,
)
strict = "/library/document/12345678123456781234567812345678"
strict_chunk = (
"/library/document/12345678123456781234567812345678/chunks#chunk-3"
)
slug = "/library/document/report"
# Fetch-tool ground truth.
assert parse_library_url(strict) is not None
assert parse_library_url(strict_chunk) is not None
assert parse_library_url(slug) is None
# Engine mirrors it (SSRF mocked to refuse, so only the
# exemption can return True).
engine = self._stub_engine(ResearchContentCache())
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
):
assert engine._is_cache_url_allowed(strict) is True
assert engine._is_cache_url_allowed(strict_chunk) is True
assert engine._is_cache_url_allowed(slug) is False
# Pipeline mirrors it too.
from local_deep_research.research_library.downloaders.extraction import (
pipeline as pl,
)
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=False,
):
assert pl._is_cache_hit_allowed(strict, False, True) is True
assert pl._is_cache_hit_allowed(strict_chunk, False, True) is True
assert pl._is_cache_hit_allowed(slug, False, True) is False