~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Single shared content cache for an entire research run.
Every consumer that needs the full text of a webpage, a RAG chunk, a
library document — whether it is the ``web_search`` tool (via
``FullSearchResults`` / ``BaseSearchEngine._get_full_content``), the
``fetch_content`` tool (via ``ContentFetcher`` + the summary-LLM prompt
construction), or the library / collection RAG engines — resolves through
this one cache. The initial quick-report phase, the detailed-report
subsections and all parallel subagents therefore share previously fetched
content instead of re-fetching the same URLs on the spot.
Lifetime: one instance per research run, owned by ``AdvancedSearchSystem``
(``system.content_cache``) and threaded explicitly into strategies, tools
and engines. Deep funnels that cannot receive an explicit reference
(``batch_fetch_and_extract``, ``ContentFetcher.fetch``,
``resolve_library_document``) fall back to the research-scoped registry
below, keyed by the ``research_id`` carried in the thread-local search
context — so freshly created per-call engines still hit the same cache.
When no cache is bound (no explicit instance and no research context, e.g.
legacy unit tests) all helpers return ``None`` and callers fall back to
direct fetching, preserving historical behaviour.
"""
from __future__ import annotations
import threading
from typing import Any, Callable
def normalize_collection_scope(collection_id: Any) -> str | None:
"""Normalise a collection identifier to its ``collection_<id>`` form.
Returns ``None`` for missing/empty values. A value already in
``collection_<id>`` form is returned unchanged so ``"coll-B"`` and
``"collection_coll-B"`` collapse to one cache namespace (matching the
``DocumentChunk.collection_name`` normalisation in
``LibraryRAGSearchEngine._get_full_content``). Human-readable names
that do not look like ``collection_*`` still get a distinct namespace
(under-merge, never cross-contamination).
"""
if collection_id is None or isinstance(collection_id, bool):
return None
if not isinstance(collection_id, str):
collection_id = str(collection_id)
stripped = collection_id.strip()
if not stripped:
return None
if stripped.startswith("collection_"):
return stripped
return f"collection_{stripped}"
def _normalize_chunk_display(display: str) -> str:
"""Merge the two chunk spellings to one cache identity.
``/library/document/<id>/chunks#chunk-N`` (emitted by
``build_chunk_anchor_url``) and ``/library/document/<id>#chunk-N``
(accepted by the resolver/tests) denote the same chunk text. Strip a
trailing ``/chunks`` view when a chunk fragment is present so both
share one entry (under-merge only, no corruption).
"""
if "#chunk-" not in display:
return display
path, _, fragment = display.partition("#")
if path.endswith("/chunks"):
path = path[: -len("/chunks")]
return f"{path}#{fragment}"
return display
def content_cache_key(url: Any, collection_id: Any = None) -> str:
"""Normalise *url* into a stable cache key.
Web URLs collapse tracking params / default ports via
``content_web_cache_key`` (NOT the bibliography ``canonical_url_key``:
the path keeps its trailing slash, userinfo is preserved, and
fragments are preserved for client-routed SPAs) while
``https://example.com/p?utm_x=1`` and ``https://example.com/p`` share
one entry (accepting potential over-merge on sites that vary content per
tracking param). Library routes keep their chunk anchor (``#chunk-<n>``)
via ``library_display_url`` because a chunk and its parent document
are *different* content and must not collide. Anything unparseable
falls back to the stripped raw string.
Library *chunk* URLs are additionally scoped by *collection_id*:
the same ``(document, chunk_index)`` can hold different text in two
collections (different chunk settings), so ``collection_A#chunk-3``
and ``collection_B#chunk-3`` must not share an entry. Document-level
library URLs (``Document.text_content`` is collection-independent)
and web URLs ignore *collection_id*.
"""
if not isinstance(url, str):
return f"raw:{url!r}"
stripped = url.strip()
if not stripped:
return "raw:"
try:
from .url_utils import content_web_cache_key, library_display_url
except Exception: # pragma: no cover - defensive
return f"raw:{stripped}"
try:
display = library_display_url(stripped)
except Exception: # pragma: no cover - defensive
display = None
if display is not None:
if "#chunk-" in display:
normalised = _normalize_chunk_display(display)
scope = normalize_collection_scope(collection_id)
if scope is not None:
return f"lib:{normalised}|collection:{scope}"
return f"lib:{normalised}|collection:__unscoped__"
return f"lib:{display}"
try:
canon = content_web_cache_key(stripped)
except Exception: # pragma: no cover - defensive
canon = ""
if canon:
return f"web:{canon}"
return f"raw:{stripped}"
#: Provenance tag for entries written by the code paths that read the
#: user's own library database (the library resolver and the library /
#: collection search engines). Library-keyed entries without this tag are
#: never served as the user's library document.
LIBRARY_PROVENANCE = "library"
def is_library_cache_key(url: Any) -> bool:
"""Whether *url* maps to a library (``lib:``) cache key."""
try:
return content_cache_key(url).startswith("lib:")
except Exception: # pragma: no cover - defensive
return False
def is_library_provenance_entry(entry: Any) -> bool:
"""Whether a cache *entry* was written by a library DB read path."""
return (
isinstance(entry, dict)
and entry.get("provenance") == LIBRARY_PROVENANCE
)
class ResearchContentCache:
"""Thread-safe in-memory store of fetched page / chunk text.
Entries hold the *fullest* fetched content (before per-caller
truncation). Callers slice to their own ``max_length`` on the way out,
so a ``fetch_content`` call capped at 10k chars and a ``web_search``
full-content call share one entry without data loss: the longer fetch
wins on ``put``.
Memory is bounded: at most ``max_entries`` entries and
``max_bytes`` of content bytes are retained (evict-oldest/FIFO).
A single oversized page is still stored (it evicts everything else
rather than being dropped) so callers never lose the entry they just
fetched.
"""
# Defaults keep a full research run (quick phase + subsections +
# subagents) in memory while capping multi-user server growth:
# 200 x ~100KB typical pages ~= 20MB; worst case 32MB per run.
DEFAULT_MAX_ENTRIES = 200
DEFAULT_MAX_BYTES = 32 * 1024 * 1024
# Cap for the SSRF-verdict memo (per-run, FIFO). Each entry is a
# small bool keyed by (url, allow_private, block_link_local); 1000
# covers a full run's distinct URLs without unbounded growth.
MAX_VALIDATION_MEMO_ENTRIES = 1000
def __init__(
self,
max_entries: int = DEFAULT_MAX_ENTRIES,
max_bytes: int = DEFAULT_MAX_BYTES,
) -> None:
self._lock = threading.RLock()
# key -> {"content": str, "title": str|None, "url": str}
self._entries: dict[str, dict[str, Any]] = {}
# key -> threading.Event for in-flight fetches (thundering-herd guard)
self._inflight: dict[str, threading.Event] = {}
self._max_entries = max(1, int(max_entries))
self._max_bytes = max(1, int(max_bytes))
self._total_bytes = 0
self._hits = 0
self._misses = 0
self._puts = 0
self._evictions = 0
# (url, allow_private, block_link_local) -> True. Per-run memo
# for the SSRF grant gates so repeat cache hits do not pay a
# getaddrinfo per read (and stay servable when DNS drops
# mid-run after a successful verdict). Only True verdicts are
# pinned: validate_url converts a transient DNS failure
# (socket.gaierror) into a False verdict rather than raising,
# so pinning False would block every later fetch of that URL
# for the rest of the run after one blip — a False verdict only
# skips the cache fast path and the fetcher/downloader
# re-validates on its own, so recomputing False never blocks
# recovery.
self._validation_memo: dict[tuple[str, bool, bool], bool] = {}
# -- basic ops ------------------------------------------------------
def get(self, url: str, collection_id: Any = None) -> dict[str, Any] | None:
key = content_cache_key(url, collection_id)
with self._lock:
entry = self._entries.get(key)
if entry is not None:
self._hits += 1
return dict(entry)
self._misses += 1
return None
def _evict_if_needed(self) -> None:
"""Evict oldest entries until within entry/byte caps (caller holds lock)."""
while (
len(self._entries) > self._max_entries
or self._total_bytes > self._max_bytes
) and len(self._entries) > 1:
oldest_key = next(iter(self._entries))
oldest = self._entries.pop(oldest_key)
self._total_bytes -= len(oldest.get("content", "") or "")
if self._total_bytes < 0:
self._total_bytes = 0
self._evictions += 1
def put(
self,
url: str,
content: str,
title: str | None = None,
url_out: str | None = None,
author: str | None = None,
published_date: Any | None = None,
collection_id: Any = None,
provenance: str | None = None,
) -> None:
"""Store *content* for *url*, keeping the longest version on conflict.
Only full pre-truncation text must be stored here — callers slice
to their own ``max_length`` (plus any truncation marker) on the way
out. Storing truncated display output would let a
``10_000 + marker`` slice overwrite the complete ``10_001``-char
page under longest-wins and lose the tail for later callers with
larger limits.
Library chunk URLs are namespaced by *collection_id* (see
:func:`content_cache_key`): the same chunk index can hold
different text per collection.
*provenance* records which path wrote the entry
(:data:`LIBRARY_PROVENANCE` for library DB reads). Provenance takes
precedence over length: a library-provenance entry is never
replaced by an untagged write, and a library-provenance write
always replaces an untagged entry under the same key.
"""
if not isinstance(content, str):
return
key = content_cache_key(url, collection_id)
new_is_library = provenance == LIBRARY_PROVENANCE
with self._lock:
existing = self._entries.get(key)
if existing is not None:
old_is_library = is_library_provenance_entry(existing)
if old_is_library and not new_is_library:
return
if old_is_library == new_is_library and len(
existing.get("content", "")
) >= len(content):
return
if existing is not None:
self._total_bytes -= len(existing.get("content", "") or "")
# Re-insert as newest so FIFO eviction drops truly old URLs.
self._entries.pop(key, None)
entry: dict[str, Any] = {
"content": content,
"title": title,
"url": url_out if url_out is not None else url,
}
if author is not None:
entry["author"] = author
if published_date is not None:
entry["published_date"] = published_date
if provenance is not None:
entry["provenance"] = provenance
self._entries[key] = entry
self._total_bytes += len(content)
self._puts += 1
self._evict_if_needed()
def get_or_fetch(
self,
url: str,
fetch_fn: Callable[[], dict[str, Any] | None],
*,
wait_timeout: float = 120.0,
collection_id: Any = None,
) -> dict[str, Any] | None:
"""Return the cached entry for *url* or fetch it exactly once.
Concurrent callers for the same URL coalesce: the first becomes the
fetcher, the rest wait on an event and then read the cache. If the
fetcher fails (exception or ``None``) waiters fall through and try
their own fetch instead of caching the failure. A waiter whose owner
does not finish within ``wait_timeout`` raises ``TimeoutError`` without
cancelling the owner; later callers can still reuse its result.
Results marked ``content_is_complete=False`` are returned without
caching, so a metadata fallback cannot prevent a later full fetch.
The result is :meth:`put` *before* waiters are woken — waking first
would let a waiter re-enter as a second fetcher and double-fetch.
Library chunk URLs are namespaced by *collection_id* (see
:func:`content_cache_key`).
"""
key = content_cache_key(url, collection_id)
while True:
with self._lock:
entry = self._entries.get(key)
if entry is not None:
self._hits += 1
return dict(entry)
self._misses += 1
event = self._inflight.get(key)
if event is None:
event = threading.Event()
self._inflight[key] = event
break
if not event.wait(timeout=wait_timeout):
# Keep the original owner's claim: a timed-out waiter must
# neither recurse forever nor release another thread's fetch.
raise TimeoutError("Timed out waiting for shared content")
# The owner completed without caching a result. Retry election
# iteratively so repeated failures never grow the call stack.
try:
result = fetch_fn()
if result is None:
return None
content = (
result.get("content") if isinstance(result, dict) else None
)
if (
isinstance(content, str)
and result.get("content_is_complete") is not False
):
# Cache BEFORE waking waiters so they observe the fresh
# entry. The finally below wakes waiters; put failures
# still wake via finally so waiters retry instead of
# hanging on a dead event.
self.put(
url,
content,
title=result.get("title")
if isinstance(result, dict)
else None,
url_out=result.get("url")
if isinstance(result, dict)
else None,
author=result.get("author")
if isinstance(result, dict)
else None,
published_date=result.get("published_date")
if isinstance(result, dict)
else None,
collection_id=collection_id,
)
with self._lock:
cached = self._entries.get(key)
if cached is not None:
return dict(cached)
return result
finally:
# Always release the in-flight slot and wake waiters — even
# for BaseException (cancellation/KeyboardInterrupt) and put
# failures — or a later caller would block until wait_timeout
# on a dead event. Store-before-wake holds because put runs
# inside try before this finally signals the event.
#
# Release only our own claim: after ``clear()`` another caller
# may already own a fresh event under the same key, and an
# unconditional pop would drop that claim and let a third
# caller start a duplicate fetch.
with self._lock:
if self._inflight.get(key) is event:
self._inflight.pop(key, None)
event.set()
# -- SSRF-verdict memo (per-run, defense-in-depth) ------------------
@staticmethod
def _validation_memo_key(
url: Any, allow_private_ips: bool, block_link_local: bool
) -> tuple[str, bool, bool] | None:
if not isinstance(url, str):
return None
stripped = url.strip()
if not stripped:
return None
return (stripped, bool(allow_private_ips), bool(block_link_local))
def get_validation(
self, url: str, allow_private_ips: bool, block_link_local: bool
) -> bool | None:
"""Return the memoized SSRF verdict, or None when not memoized."""
key = self._validation_memo_key(
url, allow_private_ips, block_link_local
)
if key is None:
return None
with self._lock:
return self._validation_memo.get(key)
def set_validation(
self,
url: str,
allow_private_ips: bool,
block_link_local: bool,
allowed: bool,
) -> None:
"""Memoize a successful SSRF verdict (FIFO-bounded, per-run).
Only True is stored — see :meth:`validated` for why False is
never pinned.
"""
if not allowed:
return
key = self._validation_memo_key(
url, allow_private_ips, block_link_local
)
if key is None:
return
with self._lock:
if key not in self._validation_memo:
while (
len(self._validation_memo)
>= self.MAX_VALIDATION_MEMO_ENTRIES
):
try:
self._validation_memo.pop(
next(iter(self._validation_memo))
)
except StopIteration:
break
self._validation_memo[key] = True
def validated(
self,
url: str,
allow_private_ips: bool,
block_link_local: bool,
compute: Callable[[], bool],
) -> bool:
"""Return memoized success or compute, pinning only True.
Thread-safe: concurrent callers for the same key may compute in
parallel (DNS outside the lock); the first True wins and stays
for the run like ``policy._dns_cache``. False is never stored:
``validate_url`` maps a transient DNS failure (socket.gaierror)
to False instead of raising, so pinning it would refuse every
later fetch of that URL until the run ends even after DNS heals.
Recomputing False costs one resolution per refused hit (bounded,
perf-only) and keeps the fetcher/downloader recovery path open.
"""
key = self._validation_memo_key(
url, allow_private_ips, block_link_local
)
if key is None:
try:
return bool(compute())
except Exception:
return False
with self._lock:
if key in self._validation_memo:
return True
try:
verdict = bool(compute())
except Exception:
return False
if not verdict:
return False
with self._lock:
if key not in self._validation_memo:
while (
len(self._validation_memo)
>= self.MAX_VALIDATION_MEMO_ENTRIES
):
try:
self._validation_memo.pop(
next(iter(self._validation_memo))
)
except StopIteration:
break
self._validation_memo[key] = True
return True
def clear(self) -> None:
with self._lock:
self._entries.clear()
self._inflight.clear()
self._validation_memo.clear()
self._total_bytes = 0
self._hits = 0
self._misses = 0
self._puts = 0
self._evictions = 0
def stats(self) -> dict[str, int]:
with self._lock:
return {
"hits": self._hits,
"misses": self._misses,
"puts": self._puts,
"entries": len(self._entries),
"evictions": self._evictions,
"total_bytes": self._total_bytes,
"validation_memo": len(self._validation_memo),
}
def __len__(self) -> int:
with self._lock:
return len(self._entries)
# -- research-scoped registry -------------------------------------------
# Maps research_id -> ResearchContentCache. Owned by AdvancedSearchSystem
# (one cache per research run); deep funnels resolve it via the
# thread-local search context so per-call engines share it without
# signature changes across dozens of files.
_registry_lock = threading.RLock()
_registry: dict[str, ResearchContentCache] = {}
def bind_cache_to_research(
research_id: str | None, cache: ResearchContentCache | None
) -> None:
"""Bind *cache* to *research_id* (no-op for empty ids / None cache)."""
if not research_id or cache is None:
return
with _registry_lock:
_registry[research_id] = cache
def unbind_cache_from_research(
research_id: str | None, *, cache: ResearchContentCache | None = None
) -> None:
"""Unbind a run, optionally only when *cache* still owns its binding."""
if not research_id:
return
with _registry_lock:
if cache is None or _registry.get(research_id) is cache:
_registry.pop(research_id, None)
def get_cache_for_research(
research_id: str | None,
) -> ResearchContentCache | None:
if not research_id:
return None
with _registry_lock:
return _registry.get(research_id)
def get_current_cache() -> ResearchContentCache | None:
"""Resolve the active research's cache via the thread-local context."""
try:
from .thread_context import get_search_context
except Exception: # pragma: no cover - defensive
return None
try:
ctx = get_search_context()
except Exception: # pragma: no cover - defensive
return None
if not isinstance(ctx, dict):
return None
research_id = ctx.get("research_id")
if not research_id or not isinstance(research_id, str):
# Non-string ids (int legacy) are still valid scopes.
if research_id is None or isinstance(research_id, bool):
return None
research_id = str(research_id)
return get_cache_for_research(research_id)
def resolve_cache(
explicit: ResearchContentCache | None,
) -> ResearchContentCache | None:
"""Prefer *explicit*, else the registry-bound cache for this thread."""
if explicit is not None:
return explicit
return get_current_cache()
__all__ = [
"ResearchContentCache",
"bind_cache_to_research",
"content_cache_key",
"get_cache_for_research",
"get_current_cache",
"normalize_collection_scope",
"resolve_cache",
"unbind_cache_from_research",
]