~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(advanced-search-system): resolve /library/document/<uuid> URLs and [N] citation markers in fetch_content (#5266)
S445ec8a1bee9
diff --git a/changelog.d/5266.bugfix.md b/changelog.d/5266.bugfix.mdnew file mode 100644index 0000000000..9f2bba8205--- /dev/null+++ b/changelog.d/5266.bugfix.md@@ -0,0 +1,8 @@+langgraph-agent `fetch_content` tool: pre-resolve `/library/document/<uuid>[/pdf]`+URLs and bare `[N]` citation markers before the egress gate so library-only research+runs can actually read their sources. The library RAG and collection search engines emit+`/library/document/<uuid>` as a result's citation URL, and the LLM sometimes pastes the+`[N]` citation marker back into the tool; the egress policy correctly rejected both as+`unsupported_scheme`, so every "Reading the page" milestone on a library-only run produced+no content. The pre-resolution passes library URLs through a local DB read of+`Document.text_content` and rewrites citation markers via `SearchResultsCollector.find_by_index`.diff --git a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.pyindex 9c03c52be5..bbb3c669d5 100644--- a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py+++ b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py@@ -26,7 +26,7 @@ from ...citation_handler import CitationHandlerfrom ...security.egress import EngineClassification, classify_enginefrom ...security import sanitize_error_for_clientfrom ...utilities.thread_context import get_search_context, search_context-from ..tools.fetch import FETCH_MODES, build_fetch_tool+from ..tools.fetch import FETCH_MODES, build_fetch_tool, make_library_resolverfrom .base_strategy import (BaseSearchStrategy,CHECK_CONTEXT_AGENT_STREAM,@@ -156,6 +156,25 @@ class SearchResultsCollector:return Nonereturn None+ def find_by_index(self, idx: int) -> dict | None:+ """Return the result dict for a 1-based citation index, or ``None``.++ Reverse of ``find_by_url``: given ``[N]`` (the citation marker the+ LLM sees in the search-results block), look up the source it+ references so the fetch tool can resolve a confused "fetch [1062]"+ call to the real URL. Thread-safe via the collector lock; uses+ ``_all_links`` (the shared, monotonic list) so a citation registered+ by the lead agent is also resolvable by a pooled subagent.+ """+ if not isinstance(idx, int) or idx < 1:+ return None+ with self._lock:+ for r in self._all_links:+ stored = r.get("index")+ if stored is not None and int(stored) == idx:+ return r+ return None+def reset(self) -> None:"""Clear per-call state. ``_all_links`` is intentionally kept."""with self._lock:@@ -528,6 +547,7 @@ def _make_research_subtopic_tool(overall_query: str = "",egress_context=None,max_subagent_workers: int = MAX_SUBAGENT_WORKERS,+ library_resolver: Any = None,web_search_description: str = NEUTRAL_PRIMARY_SEARCH_DESCRIPTION,):"""Create the ``research_subtopic`` tool that spawns parallel subagents.@@ -543,6 +563,12 @@ def _make_research_subtopic_tool(drain-loop start -- see #5014). The value comes from the user setting``langgraph_agent.max_subagent_workers`` and falls back to``MAX_SUBAGENT_WORKERS`` when unset / invalid.++ ``library_resolver`` is threaded into the subagent's fetch tool so a+ subagent researching a library-derived subtopic can also resolve+ ``/library/document/<uuid>`` URLs and ``[N]`` citation markers instead+ of burning the egress-denial quota on them (A3). When ``None``,+ library / citation URLs fall through to the egress gate unchanged."""@tool@@ -592,6 +618,7 @@ def _make_research_subtopic_tool(overall_query=overall_query,settings_snapshot=settings_snapshot,egress_context=egress_context,+ library_resolver=library_resolver,)if sub_fetch is not None:sub_tools.append(sub_fetch)@@ -1300,6 +1327,28 @@ class LangGraphAgentStrategy(BaseSearchStrategy):)return None+ def _build_library_resolver(self):+ """Build a ``library_resolver`` callable for the fetch tool.++ Returns ``None`` when no user is associated with the run (programmatic+ mode, benchmarks, news). The fetch tool then behaves exactly as it+ did before the A3 fix: library / citation URLs fall through to the+ egress gate and are rejected as ``unsupported_scheme``. Returning+ ``None`` keeps those callers' behaviour identical.+ """+ username = None+ if self.settings_snapshot:+ # The snapshot carries the username under the ``_username`` key+ # injected by ``AdvancedSearchSystem._ensure_snapshot_username``;+ # the strategy also checks ``self._username`` attribute as a fallback+ # for non-snapshot callers (tests, programmatic API).+ username = self.settings_snapshot.get("_username")+ if not username:+ username = getattr(self, "_username", None)+ if not username:+ return None+ return make_library_resolver(username)+def _build_tools(self, overall_query: str = "") -> list:"""Build the LangChain tool list for the lead agent.@@ -1313,6 +1362,13 @@ class LangGraphAgentStrategy(BaseSearchStrategy):# every tool builder so subagent threads — which don't inherit# thread-local state — get the same context as the lead agent.policy_ctx = self._build_egress_context()+ # Same lifetime rule as policy_ctx: build ONCE, thread through every+ # tool so the lead agent and pooled subagents resolve the same+ # library documents. Without this, every fetch call on a library+ # doc URL is rejected by the egress policy as ``unsupported_scheme``+ # (A3 — 26 of 26 "Reading the page" milestones produced no content+ # in the f3045c5b run).+ library_resolver = self._build_library_resolver()primary_search_description = NEUTRAL_PRIMARY_SEARCH_DESCRIPTION# Web search (always present if we have a search engine)@@ -1403,6 +1459,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):overall_query=overall_query,settings_snapshot=self.settings_snapshot,egress_context=policy_ctx,+ library_resolver=library_resolver,)if fetch is not None:tools.append(fetch)@@ -1451,6 +1508,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):overall_query=overall_query,egress_context=policy_ctx,max_subagent_workers=self.max_subagent_workers,+ library_resolver=library_resolver,web_search_description=primary_search_description,))diff --git a/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py b/src/local_deep_research/advanced_search_system/tools/fetch/__init__.pyindex bfae8f5965..e27180317e 100644--- a/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py+++ b/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py@@ -40,6 +40,11 @@ from local_deep_research.security import (sanitize_error_for_client,)+from .library_resolver import (+ is_citation_reference,+ make_library_resolver,+ resolve_citation_reference as _resolve_citation_reference,+)from .prompts import SUMMARY_FOCUS_PROMPT, SUMMARY_FOCUS_QUERY_PROMPT@@ -124,51 +129,223 @@ _EGRESS_DENIAL_HINT = ("skip external URLs.")+# ---------------------------------------------------------------------------+# Pre-resolution: rewrite non-network URLs BEFORE the egress gate.+#+# Two URL-shaped strings reach the fetch tool that aren't actually network+# fetches and that the egress policy correctly rejects as+# ``unsupported_scheme``. Resolving them here lets the agent get the page+# content (or a helpful error) instead of a generic denial — A3 from+# research_f3045c5b_issue_analysis.md.+# ---------------------------------------------------------------------------++# ``_KIND_RESULT`` marks a pre-resolved local document; the fetch tool+# consumes the dict directly. ``_KIND_ERROR`` short-circuits the fetch+# tool with the message in ``payload`` (already routed through the+# credential scrubber by the caller). ``_KIND_REWRITTEN`` means "this+# URL was rewritten to a new one — use the new URL going forward but+# keep running the normal HTTP flow" (e.g. a ``[N]`` citation marker+# that resolves to an external URL). ``None`` means "no pre-resolution;+# run the normal HTTP path with the original URL."+_KIND_RESULT = "result"+_KIND_ERROR = "error"+_KIND_REWRITTEN = "rewritten"+++def _try_resolve_url(+ url: str,+ library_resolver: Any,+ collector: Any,+ _visited: set[str] | None = None,+) -> tuple[str, Any] | None:+ """Pre-resolve a fetch URL that isn't a network URL.++ Returns one of:++ - ``(``_KIND_RESULT``, dict)`` — local-content payload shaped like a+ ``ContentFetcher.fetch()`` success result, ready for the post-fetch+ pipeline. The caller MUST skip the egress gate (it's a local read).+ - ``(``_KIND_ERROR``, str)`` — short-circuit with this error message.+ - ``(``_KIND_REWRITTEN``, str)`` — *url* was rewritten (e.g. a+ citation marker ``[N]`` that resolved to an external URL); use+ the new URL but otherwise run the normal HTTP flow (the egress+ gate still applies).+ - ``None`` — no pre-resolution; run the normal HTTP path with *url*.++ Handles:++ - ``[N]`` citation markers → ``SearchResultsCollector.find_by_index``,+ then recurse with the resolved URL (so a citation whose source is a+ library doc still hits the local fast path, while a citation whose+ source is external surfaces a ``_KIND_REWRITTEN`` URL for the gate).+ - ``/library/document/<uuid>[/pdf]`` → ``Document.text_content`` read+ from the user DB via the library resolver.+ """+ if not isinstance(url, str):+ return None++ if _visited is None:+ _visited = set()+ if url in _visited or len(_visited) >= 5:+ return (+ _KIND_ERROR,+ f"Circular citation reference detected for '{url}'.",+ )+ _visited.add(url)++ # 1. Citation marker [N] → resolve to citation's URL.+ citation = _resolve_citation_reference(url, collector)+ if citation is not None:+ resolved_url = citation.get("link") or citation.get("url") or ""+ if not resolved_url:+ return (+ _KIND_ERROR,+ (+ f"Citation {url} has no URL field. "+ "Use the citation's URL or a different source instead."+ ),+ )+ # First, check if the citation's URL is itself a library doc+ # (the recursive call sees the library path and returns _KIND_RESULT+ # directly). Otherwise surface the citation's URL as a rewrite so+ # the fetch tool updates its ``url`` variable before the egress+ # gate / HTTP fetch runs against the resolved URL — not the marker.+ recursive = _try_resolve_url(+ resolved_url, library_resolver, collector, _visited=_visited+ )+ if recursive is not None:+ return recursive+ return (_KIND_REWRITTEN, resolved_url)++ # 2. Citation marker that doesn't match a tracked citation.+ # ``_resolve_citation_reference`` only returns the citation for a+ # well-formed ``[N]`` marker; anything else falls through to the+ # library-resolver check below.+ if is_citation_reference(url) is not None:+ return (+ _KIND_ERROR,+ (+ f"No registered citation matches {url}. The agent's "+ "search results use citation markers; use the source URL "+ "(the link next to the marker) or a tracked citation "+ "instead of a raw marker."+ ),+ )++ # 3. Library document URL.+ if library_resolver is not None:+ content = library_resolver(url)+ if content is not None:+ text = content.get("content") or ""+ if len(text) > CONTENT_MAX_LENGTH:+ text = text[:CONTENT_MAX_LENGTH]+ return (+ _KIND_RESULT,+ {+ "status": "success",+ "title": content.get("title") or "",+ "content": text,+ "url": content.get("url") or url,+ },+ )++ return None+++def _fetch_raw_content(+ url: str,+ library_resolver: Any,+ collector: Any,+ egress_context: Any,+ settings_snapshot: dict | None,+ mode_label: str,+) -> tuple[dict | None, str, str | None]:+ """Pre-resolve non-network URL shapes or fetch via HTTP.++ Returns:+ (result, final_url, error_string)+ If error_string is not None, a pre-resolution error occurred (already+ scrubbed) and should be returned immediately by the fetch tool.+ """+ from local_deep_research.content_fetcher import ContentFetcher++ pre = _try_resolve_url(url, library_resolver, collector)+ result: dict | None = None+ if pre is not None:+ kind, payload = pre+ if kind == _KIND_ERROR:+ return None, url, _scrub_tool_error(payload)+ if kind == _KIND_RESULT:+ # A local library document was resolved directly from the user DB.+ # Skip both the egress gate (local read) and ContentFetcher (no HTTP).+ result = payload+ url = result.get("url") or url+ logger.info(+ f"[FETCH] mode={mode_label} source=library url={url} — "+ "resolved local library document directly"+ )+ else: # _KIND_REWRITTEN — citation marker resolved to a URL+ url = payload++ if result is None:+ # Either no pre-resolution or a citation-marker rewrite —+ # run the normal HTTP path. Per-URL egress gate (pre-fetch)+ # + ContentFetcher's own per-redirect gate both raise+ # PolicyDeniedError on an out-of-scope URL. Run the gate+ # INSIDE the try so the denial is returned as a recoverable+ # tool message.+ _enforce_url_policy(url, egress_context)+ enable_js = _read_js_rendering_setting(settings_snapshot)+ with ContentFetcher(+ timeout=CONTENT_FETCH_TIMEOUT,+ enable_js_rendering=enable_js,+ egress_context=egress_context,+ ) as fetcher:+ result = fetcher.fetch(url, max_length=CONTENT_MAX_LENGTH)++ return result, url, None+def _make_full_fetch_tool(collector: Any,settings_snapshot: dict | None = None,egress_context: Any = None,+ library_resolver: Any = None,):mode_label = "full"@tooldef fetch_content(url: str) -> str:"""Download and read the full text content from a URL. Use when search snippets aren't detailed enough."""- from local_deep_research.content_fetcher import ContentFetcherfrom local_deep_research.security.egress.policy import (PolicyDeniedError,)- enable_js = _read_js_rendering_setting(settings_snapshot)try:- # Per-URL egress gate (pre-fetch) + ContentFetcher's own- # per-redirect gate both raise PolicyDeniedError on an out-of-scope- # URL. Run the gate INSIDE the try so the denial is returned as a- # recoverable tool message (see the except below).- _enforce_url_policy(url, egress_context)- with ContentFetcher(- timeout=CONTENT_FETCH_TIMEOUT,- enable_js_rendering=enable_js,- egress_context=egress_context,- ) as fetcher:- result = fetcher.fetch(url, max_length=CONTENT_MAX_LENGTH)- if result.get("status") == "success":- title = result.get("title", "")- content = result.get("content", "")- cite_idx = _register_in_collector(- collector, url, title, content- )- return (- f"[{cite_idx}] Title: {title}\nURL: {url}\n\n{content}"- )- # result['error'] comes from ContentFetcher, which returns a- # raw str(exception) — scrub it (and the url) before this- # reaches the agent/LLM and user-visible output (#4633).- return _scrub_tool_error(- f"Failed to fetch {url}: "- f"{result.get('error', 'unknown error')}"+ result, url, err_msg = _fetch_raw_content(+ url,+ library_resolver,+ collector,+ egress_context,+ settings_snapshot,+ mode_label,+ )+ if err_msg is not None:+ return err_msg++ if result.get("status") == "success":+ title = result.get("title", "")+ content = result.get("content", "")+ cite_idx = _register_in_collector(+ collector, url, title, content)+ return f"[{cite_idx}] Title: {title}\nURL: {url}\n\n{content}"+ # result['error'] comes from ContentFetcher, which returns a+ # raw str(exception) — scrub it (and the url) before this+ # reaches the agent/LLM and user-visible output (#4633).+ return _scrub_tool_error(+ f"Failed to fetch {url}: {result.get('error', 'unknown error')}"+ )except PolicyDeniedError as exc:# An out-of-scope URL is a RECOVERABLE, per-call decision (the agent# picked one bad URL among many). Return it as a tool message — like@@ -177,8 +354,9 @@ def _make_full_fetch_tool(# of re-raising (which aborts a subagent and depends on each agent's# tool-error layer). The URL was already NOT fetched; the policy# already enforced — only the REPORTING changes, not security.+ target_url = getattr(exc, "target", "") or urlreturn _scrub_tool_error(- f"Cannot fetch {url}: blocked by egress policy "+ f"Cannot fetch {target_url}: blocked by egress policy "f"({_denial_reason(exc)}). {_EGRESS_DENIAL_HINT}")except Exception as exc:@@ -188,12 +366,13 @@ def _make_full_fetch_tool(# The traceback follows the sink's diagnose setting (off by default;# see utilities/log_utils). The agent/user-facing return is scrubbed# separately below.+ target_url = getattr(exc, "target", "") or urllogger.exception("fetch_content tool error (mode={}, url={})",mode_label,- redact_url_for_log(url),+ redact_url_for_log(target_url),)- return _scrub_tool_error(f"Error fetching {url}: {exc}")+ return _scrub_tool_error(f"Error fetching {target_url}: {exc}")return fetch_content@@ -204,6 +383,7 @@ def _make_summary_fetch_tool(overall_query: str | None,settings_snapshot: dict | None = None,egress_context: Any = None,+ library_resolver: Any = None,):"""Build the summary-mode fetch tool.@@ -218,121 +398,119 @@ def _make_summary_fetch_tool(@tooldef fetch_content(url: str, focus: str) -> str:"""Fetch a URL and return only the spans of text relevant to ``focus``.- Pass the specific question or claim you want answered as ``focus`` — the- tool will quote relevant facts verbatim and discard unrelated content.+ Pass the specific question or claim you want answered as ``focus`` —+ the tool will quote relevant facts verbatim and discard unrelated content."""- from local_deep_research.content_fetcher import ContentFetcherfrom local_deep_research.security.egress.policy import (PolicyDeniedError,)- enable_js = _read_js_rendering_setting(settings_snapshot)try:- # Run the per-URL egress gate INSIDE the try so an out-of-scope URL- # is returned as a recoverable tool message (see the except below),- # not re-raised.- _enforce_url_policy(url, egress_context)- with ContentFetcher(- timeout=CONTENT_FETCH_TIMEOUT,- enable_js_rendering=enable_js,- egress_context=egress_context,- ) as fetcher:- result = fetcher.fetch(url, max_length=CONTENT_MAX_LENGTH)- if result.get("status") != "success":- # result['error'] comes from ContentFetcher, which- # returns a raw str(exception) — scrub it (and the url)- # before this reaches the agent/LLM / user output (#4633).- return _scrub_tool_error(- f"Failed to fetch {url}: "- f"{result.get('error', 'unknown error')}"- )-- title = result.get("title") or ""- content = result.get("content") or ""-- # Guard 1 — empty page content (paywalls, JS-only SPAs that- # static fetch can't render, deleted pages with HTTP 200).- # Skipping the LLM call here means we don't pay the round-trip- # to summarise nothing, AND we don't register an empty- # citation in the collector — `_register_in_collector` caches- # by URL, so an empty snippet would lock the URL in as- # "already fetched, nothing here" and the agent would never- # retry it under a different focus.- if not content.strip():- logger.info(- f"[FETCH] mode={mode_label} url={url} — "- "empty page content, returning NOT RELEVANT without "- "LLM call or collector registration"- )- return f"NOT RELEVANT (no extractable content at {url})"-- fmt_kwargs = {- "focus": focus,- "title": title,- "url": url,- "content": content,- }- if use_query:- fmt_kwargs["overall_query"] = overall_query- prompt = template.format(**fmt_kwargs)-- try:- summary_msg = model.invoke(prompt)- summary = getattr(- summary_msg, "content", str(summary_msg)- ).strip()- except Exception as exc:- # Redacted scheme://host + mode only — no page content,- # focus, or credentials in the message. Traceback follows- # the sink's diagnose setting (off by default).- logger.exception(- "fetch_content summary LLM error (mode={}, url={})",- mode_label,- redact_url_for_log(url),- )- return _scrub_tool_error(f"Error summarizing {url}: {exc}")-- # Diagnostic log: per-fetch input/output for evaluating the- # summariser. Single multi-line block so it's atomic per call- # and easy to grep with ``grep -A1000 "[FETCH] mode="``.- log_lines = [- f"[FETCH] mode={mode_label} url={url}",- f"[FETCH] focus: {focus}",- ]- if use_query:- log_lines.append(f"[FETCH] overall_query: {overall_query}")- log_lines.extend(- [- f"[FETCH] title: {title}",- f"[FETCH] page_text ({len(content)} chars):",- content,- f"[FETCH] summary returned ({len(summary)} chars):",- summary or "(empty)",- "[FETCH] ---",- ]- )- logger.info("\n".join(log_lines))+ result, url, err_msg = _fetch_raw_content(+ url,+ library_resolver,+ collector,+ egress_context,+ settings_snapshot,+ mode_label,+ )+ if err_msg is not None:+ return err_msg- # Guard 2 — empty LLM summary. The model decided nothing on- # the page answers the focus (or it returned a malformed/empty- # response). Treat as NOT RELEVANT and skip collector- # registration: the agent should be free to re-fetch the URL- # later with a different focus instead of seeing it as- # already-cached with an empty body.- if not summary:- return f"NOT RELEVANT (no spans matched focus at {url})"+ if result.get("status") != "success":+ # result['error'] comes from ContentFetcher, which returns a+ # raw str(exception) — scrub it (and the url) before this+ # reaches the agent/LLM / user output (#4633).+ return _scrub_tool_error(+ f"Failed to fetch {url}: "+ f"{result.get('error', 'unknown error')}"+ )- cite_idx = _register_in_collector(- collector, url, title, summary+ title = result.get("title") or ""+ content = result.get("content") or ""++ # Guard 1 — empty page content (paywalls, JS-only SPAs that+ # static fetch can't render, deleted pages with HTTP 200).+ # Skipping the LLM call here means we don't pay the round-trip+ # to summarise nothing, AND we don't register an empty+ # citation in the collector — `_register_in_collector` caches+ # by URL, so an empty snippet would lock the URL in as+ # "already fetched, nothing here" and the agent would never+ # retry it under a different focus.+ if not content.strip():+ logger.info(+ f"[FETCH] mode={mode_label} url={url} — "+ "empty page content, returning NOT RELEVANT without "+ "LLM call or collector registration"+ )+ return f"NOT RELEVANT (no extractable content at {url})"++ fmt_kwargs = {+ "focus": focus,+ "title": title,+ "url": url,+ "content": content,+ }+ if use_query:+ fmt_kwargs["overall_query"] = overall_query+ prompt = template.format(**fmt_kwargs)++ try:+ summary_msg = model.invoke(prompt)+ summary = getattr(+ summary_msg, "content", str(summary_msg)+ ).strip()+ except Exception as exc:+ # Redacted scheme://host + mode only — no page content,+ # focus, or credentials in the message. Traceback follows+ # the sink's diagnose setting (off by default).+ logger.exception(+ "fetch_content summary LLM error (mode={}, url={})",+ mode_label,+ redact_url_for_log(url),)- return f"[{cite_idx}] Title: {title}\nURL: {url}\n\n{summary}"+ return _scrub_tool_error(f"Error summarizing {url}: {exc}")++ # Diagnostic log: per-fetch input/output for evaluating the+ # summariser. Single multi-line block so it's atomic per call+ # and easy to grep with ``grep -A1000 "[FETCH] mode="``.+ log_lines = [+ f"[FETCH] mode={mode_label} url={url}",+ f"[FETCH] focus: {focus}",+ ]+ if use_query:+ log_lines.append(f"[FETCH] overall_query: {overall_query}")+ log_lines.extend(+ [+ f"[FETCH] title: {title}",+ f"[FETCH] page_text ({len(content)} chars):",+ content,+ f"[FETCH] summary returned ({len(summary)} chars):",+ summary or "(empty)",+ "[FETCH] ---",+ ]+ )+ logger.info("\n".join(log_lines))++ # Guard 2 — empty LLM summary. The model decided nothing on+ # the page answers the focus (or it returned a malformed/empty+ # response). Treat as NOT RELEVANT and skip collector+ # registration: the agent should be free to re-fetch the URL+ # later with a different focus instead of seeing it as+ # already-cached with an empty body.+ if not summary:+ return f"NOT RELEVANT (no spans matched focus at {url})"++ cite_idx = _register_in_collector(collector, url, title, summary)+ return f"[{cite_idx}] Title: {title}\nURL: {url}\n\n{summary}"except PolicyDeniedError as exc:# Recoverable per-URL denial — return a tool message so both the# lead agent and pooled subagents handle it identically and the# agent stays in-scope. The URL was already NOT fetched; only the# reporting changes, not security. (See the full-fetch variant.)+ target_url = getattr(exc, "target", "") or urlreturn _scrub_tool_error(- f"Cannot fetch {url}: blocked by egress policy "+ f"Cannot fetch {target_url}: blocked by egress policy "f"({_denial_reason(exc)}). {_EGRESS_DENIAL_HINT}")except Exception as exc:@@ -342,12 +520,13 @@ def _make_summary_fetch_tool(# The traceback follows the sink's diagnose setting (off by default;# see utilities/log_utils). The agent/user-facing return is scrubbed# separately below.+ target_url = getattr(exc, "target", "") or urllogger.exception("fetch_content tool error (mode={}, url={})",mode_label,- redact_url_for_log(url),+ redact_url_for_log(target_url),)- return _scrub_tool_error(f"Error fetching {url}: {exc}")+ return _scrub_tool_error(f"Error fetching {target_url}: {exc}")return fetch_content@@ -360,6 +539,7 @@ def build_fetch_tool(overall_query: str = "",settings_snapshot: dict | None = None,egress_context: Any = None,+ library_resolver: Any = None,):"""Build the agent-facing ``fetch_content`` tool for *mode*.@@ -375,6 +555,14 @@ def build_fetch_tool(``egress_context`` is captured by the closure so the per-call URLcan be policy-gated; when ``None``, no policy enforcement runs(preserves legacy non-LangGraph callers).++ ``library_resolver`` is captured by the closure so a fetch call can+ short-circuit on ``/library/document/<uuid>[/pdf]`` URLs (a local DB+ read, not a network fetch) and on bare ``[N]`` citation markers+ (rewritten to the citation's URL via ``SearchResultsCollector``).+ When ``None``, the tool falls through to the egress gate unchanged+ (which, when ``egress_context`` is configured, rejects library / citation+ URLs as ``unsupported_scheme``) — same as the pre-fix behaviour."""if mode == "disabled":return None@@ -383,6 +571,7 @@ def build_fetch_tool(collector,settings_snapshot=settings_snapshot,egress_context=egress_context,+ library_resolver=library_resolver,)if mode == "summary_focus":if model is None:@@ -393,6 +582,7 @@ def build_fetch_tool(overall_query=None,settings_snapshot=settings_snapshot,egress_context=egress_context,+ library_resolver=library_resolver,)if mode == "summary_focus_query":if model is None:@@ -405,10 +595,11 @@ def build_fetch_tool(overall_query=overall_query or None,settings_snapshot=settings_snapshot,egress_context=egress_context,+ library_resolver=library_resolver,)raise ValueError(f"Unknown fetch mode {mode!r}; expected one of {FETCH_MODES}")-__all__ = ["FETCH_MODES", "build_fetch_tool"]+__all__ = ["FETCH_MODES", "build_fetch_tool", "make_library_resolver"]diff --git a/src/local_deep_research/advanced_search_system/tools/fetch/library_resolver.py b/src/local_deep_research/advanced_search_system/tools/fetch/library_resolver.pynew file mode 100644index 0000000000..cbb2804633--- /dev/null+++ b/src/local_deep_research/advanced_search_system/tools/fetch/library_resolver.py@@ -0,0 +1,201 @@+"""Resolve ``fetch_content`` URLs that aren't actually network URLs.++The LangGraph agent's ``fetch_content`` tool receives two URL-shaped+strings that the egress policy correctly rejects as+``unsupported_scheme`` (and that the original ``ContentFetcher`` would+not know how to fetch even without a policy gate):++1. **Local library document references** like+ ``/library/document/<uuid>`` or ``/library/document/<uuid>/pdf``.+ The library RAG search engine and the collection search engine emit+ these as the citation URL of a search hit (see+ ``web_search_engines.engines.search_engine_library.LibraryRAGSearchEngine.search``+ and ``...search_engine_collection.CollectionSearchEngine._get_document_url``).+ The URL points at the user's own library — a local DB read, not a+ network fetch — and resolving it via ``Document.text_content`` is what+ the human user would see if they clicked the link in the UI.++2. **Bare citation markers** like ``[1062]``, ``[1084]``. The agent+ sometimes pastes a citation marker it saw in the search-results block+ back into the tool instead of the actual URL. ``SearchResultsCollector``+ already maps URL → index; the inverse ``find_by_index`` lets the tool+ resolve ``[N]`` back to its source.++Both shapes are intercepted here BEFORE the egress policy gate, so the+agent gets the real page content (or a helpful error) instead of a+generic "blocked by egress policy" denial that wastes a fetch slot+(A3 in ``research_f3045c5b_issue_analysis.md``).+"""++from __future__ import annotations++import re+from typing import Any, Callable, Optional++from loguru import logger+++# Match ``/library/document/<uuid>`` and ``/library/document/<uuid>/pdf``+# (with optional trailing slash). The same regex is used by the download+# service in ``research_library.services.download_service`` — keep them+# in sync if either gains a new suffix.+_LIBRARY_PATH_RE = re.compile(+ r"^/library/document/(?P<doc_id>[^/?#]+)(?:/(?P<suffix>pdf))?/?$"+)++# Match ``[N]`` — a 1-based citation marker with no surrounding chars.+# The agent emits this verbatim when it confuses the citation marker for+# a URL; we resolve it via SearchResultsCollector.find_by_index.+_CITATION_REF_RE = re.compile(r"^\[(\d+)\]$")+++def parse_library_url(url: str) -> tuple[str, Optional[str]] | None:+ """Return ``(doc_id, suffix)`` for a ``/library/document/...`` URL, else ``None``.++ ``suffix`` is ``"pdf"`` for the PDF-direct form and ``None`` for the+ root document page. Both forms map to the same ``Document`` row; the+ suffix is captured so callers can decide how to render the result.+ """+ if not isinstance(url, str):+ return None+ m = _LIBRARY_PATH_RE.match(url.strip())+ if not m:+ return None+ return m.group("doc_id"), m.group("suffix")+++def is_citation_reference(url: str) -> int | None:+ """If *url* is a bare ``[N]`` citation marker, return ``N`` (int), else ``None``."""+ if not isinstance(url, str):+ return None+ m = _CITATION_REF_RE.match(url.strip())+ if not m:+ return None+ return int(m.group(1))+++def resolve_library_document(+ url: str, username: Optional[str]+) -> Optional[dict[str, Any]]:+ """Look up a local library document by ``/library/document/<uuid>[/pdf]`` URL.++ Returns a dict shaped like the fetch tool's success payload (``title``,+ ``content``) so callers can pass it straight to ``_register_in_collector``+ + the ``[N] Title:\\nURL:\\n\\n<body>`` formatter. The ``url`` field of+ the dict is the *original* library URL, so a downstream re-citation still+ reads "library/document/…" instead of something more confusing.++ Returns ``None`` for:++ - URLs that don't match ``/library/document/...``+ - Documents the user can't access (no ``username`` or no row found).+ Callers should fall through to the egress policy / HTTP fetcher+ for these so a malformed UUID produces the normal "blocked by+ egress policy" outcome instead of a misleading custom error.++ (Documents with empty text content return a result payload with+ ``content=""``, which summary-mode handles as NOT RELEVANT without an+ LLM call.)+ """+ parsed = parse_library_url(url)+ if parsed is None:+ return None+ doc_id, _suffix = parsed++ if not username:+ # No user context → no library. Caller falls through to the+ # egress policy, which rejects the URL as ``unsupported_scheme``.+ return None++ # Lazy imports: avoid pulling SQLAlchemy / user-DB modules into the+ # module-load path for the (common) no-library case.+ try:+ from local_deep_research.database.models.library import Document+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ except Exception:+ logger.exception(+ "library_resolver: failed to import DB modules for document "+ "lookup (url={!r})",+ url,+ )+ return None++ try:+ with get_user_db_session(username) as session:+ document = session.query(Document).filter_by(id=doc_id).first()+ if document is None:+ return None+ text = document.text_content or ""+ return {+ "title": document.title or f"Document {doc_id}",+ "content": text,+ "url": url,+ # ``snippet`` is what the collector registers as the+ # citation preview. Use the title fallback when text is+ # empty so the agent still gets *something* descriptive.+ "snippet": (+ text[:200].strip() if text else document.title or ""+ )+ or "",+ }+ except Exception:+ # Match the existing LibraryRAGSearchEngine behaviour: a per-document+ # DB hiccup is non-fatal; the tool returns a clean "not found" via+ # None and the caller can fall through.+ logger.exception(+ "library_resolver: failed to load document (doc_id={!r}, username={!r})",+ doc_id,+ username,+ )+ return None+++def make_library_resolver(+ username: Optional[str],+) -> Callable[[str], Optional[dict[str, Any]]]:+ """Build a one-arg URL → dict resolver suitable for the fetch tool.++ Captures *username* in the closure so subagent workers (which don't+ inherit thread-local Flask session state) can resolve library URLs+ using the run's user, the same way the LangGraph strategy threads+ ``settings_snapshot`` and ``egress_context`` through tool closures.++ Returns the resolved dict, or ``None`` for URLs that aren't library+ document refs — the fetch tool then falls through to the egress gate+ unchanged.+ """+ # Bind the username once so the inner function doesn't re-resolve on+ # every call (cheap, but keeps the per-call hot path branch-free).+ _username = username++ def _resolve(url: str) -> Optional[dict[str, Any]]:+ return resolve_library_document(url, _username)++ return _resolve+++def resolve_citation_reference(+ url: str,+ collector: Any,+) -> Optional[dict[str, Any]]:+ """Resolve a bare ``[N]`` citation marker to its source result dict.++ Returns the result dict (with at least ``link``/``url`` and ``title``+ populated, mirroring ``SearchResultsCollector.find_by_index``) when+ the marker matches a tracked citation, else ``None``. The fetch tool+ uses ``None`` as a signal to return a helpful "no such citation"+ error to the agent instead of falling through to the egress gate+ (where the marker would also be rejected as ``unsupported_scheme``,+ but with a less actionable message).+ """+ idx = is_citation_reference(url)+ if idx is None:+ return None+ if collector is None:+ return None+ find = getattr(collector, "find_by_index", None)+ if not callable(find):+ return None+ return find(idx)diff --git a/tests/advanced_search_system/tools/test_fetch_modes.py b/tests/advanced_search_system/tools/test_fetch_modes.pyindex ebdfd8f77e..94218819dc 100644--- a/tests/advanced_search_system/tools/test_fetch_modes.py+++ b/tests/advanced_search_system/tools/test_fetch_modes.py@@ -372,3 +372,596 @@ def test_fetch_error_log_redacts_url_and_adds_mode():assert "SECRET123" not in flat # userinfo droppedassert "ABCXYZ" not in flat # query token droppedassert "secretpath" not in flat # path dropped+++# ---------------------------------------------------------------------------+# Library-document and citation-marker URL pre-resolution (A3).+#+# The library RAG engine and the collection engine emit+# ``/library/document/<uuid>[/pdf]`` as a result's citation URL; agents+# sometimes paste a bare ``[N]`` citation marker back into ``fetch_content``+# instead of the actual URL. Both shapes used to be rejected by the egress+# policy as ``unsupported_scheme`` and the agent saw no page content. The+# pre-resolution pass routes library URLs to a local DB read and rewrites+# citation markers via SearchResultsCollector.find_by_index.+# ---------------------------------------------------------------------------+++def _library_doc_resolver(url):+ """Minimal library resolver for tests: returns content for one URL."""+ if url == "/library/document/doc-1":+ return {+ "title": "Doc One",+ "content": "Body of document one.",+ "url": url,+ "snippet": "Body of document one.",+ }+ if url == "/library/document/doc-1/pdf":+ return {+ "title": "Doc One (PDF)",+ "content": "Body of document one.",+ "url": url,+ "snippet": "Body of document one.",+ }+ return None+++def test_full_mode_resolves_library_document_url_locally():+ """A ``/library/document/<uuid>`` URL fetches from the resolver and+ registers the citation; ContentFetcher is NOT called and the egress+ policy is NOT consulted (the fetch is a local DB read)."""+ collector = SearchResultsCollector([])+ tool = build_fetch_tool(+ "full",+ collector,+ library_resolver=_library_doc_resolver,+ egress_context=object(),+ )++ cm = _fetcher_cm() # would fail if ContentFetcher were called+ with (+ patch(+ "local_deep_research.content_fetcher.ContentFetcher",+ return_value=cm,+ ) as factory,+ patch(+ "local_deep_research.security.egress.policy.evaluate_url"+ ) as mock_evaluate_url,+ ):+ out = tool.invoke({"url": "/library/document/doc-1"})++ factory.assert_not_called() # local read, no HTTP+ mock_evaluate_url.assert_not_called() # egress bypass locked directly+ assert "Title: Doc One" in out+ assert "URL: /library/document/doc-1" in out+ assert "Body of document one." in out+ # The library URL is registered as a citation so the agent can cite it.+ assert len(collector.results) == 1+ assert collector.results[0]["link"] == "/library/document/doc-1"+++def test_full_mode_resolves_library_pdf_url_locally():+ """The ``/pdf`` suffix is just a hint to render the doc as PDF; the+ fetch tool reads the same Document row regardless of suffix."""+ collector = SearchResultsCollector([])+ tool = build_fetch_tool(+ "full", collector, library_resolver=_library_doc_resolver+ )++ cm = _fetcher_cm()+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke({"url": "/library/document/doc-1/pdf"})++ assert "Title: Doc One (PDF)" in out+ assert "URL: /library/document/doc-1/pdf" in out+ assert "Body of document one." in out+++def test_full_mode_falls_through_when_library_resolver_returns_none():+ """An unknown library UUID (resolver returns None) falls through to+ the egress policy and is rejected as ``unsupported_scheme`` — same as+ the pre-fix behaviour for malformed library URLs."""+ from local_deep_research.security.egress.policy import Decision++ collector = SearchResultsCollector([])++ def _always_none(_url):+ return None++ tool = build_fetch_tool(+ "full",+ collector,+ library_resolver=_always_none,+ egress_context=object(),+ )++ with patch(+ "local_deep_research.security.egress.policy.evaluate_url",+ return_value=Decision(False, "unsupported_scheme"),+ ):+ out = tool.invoke({"url": "/library/document/missing"})++ assert "Cannot fetch" in out+ assert "unsupported_scheme" in out+++def test_full_mode_without_library_resolver_falls_through_to_egress():+ """When no library_resolver is passed, the tool preserves the+ pre-A3 behaviour: library URLs hit the egress policy unchanged."""+ from local_deep_research.security.egress.policy import Decision++ collector = SearchResultsCollector([])+ tool = build_fetch_tool("full", collector, egress_context=object())++ with patch(+ "local_deep_research.security.egress.policy.evaluate_url",+ return_value=Decision(False, "unsupported_scheme"),+ ):+ out = tool.invoke({"url": "/library/document/abc"})++ assert "Cannot fetch" in out+ assert "unsupported_scheme" in out+++def test_full_mode_citation_marker_resolves_to_citation_url():+ """A bare ``[N]`` citation marker is rewritten to the citation's+ stored URL and the rewritten URL is what ContentFetcher sees (and+ what the egress gate evaluates, if any)."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [+ {+ "title": "Some Paper",+ "link": "https://example.com/paper",+ "snippet": "snippet",+ }+ ],+ engine_name="web",+ )++ tool = build_fetch_tool("full", collector)++ cm = _fetcher_cm(title="Some Paper", content="body")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke({"url": "[1]"})++ # The resolved URL (not the citation marker) was fetched.+ cm.__enter__.return_value.fetch.assert_called_once()+ fetched_url = cm.__enter__.return_value.fetch.call_args.args[0]+ assert fetched_url == "https://example.com/paper"+ assert "Title: Some Paper" in out+++def test_full_mode_unknown_citation_marker_returns_helpful_error():+ """A well-formed ``[N]`` with no matching citation must produce an+ actionable error, not the generic egress-denial message. The error+ must mention the marker and steer the agent toward the citation's URL."""+ collector = SearchResultsCollector([]) # no citations tracked+ tool = build_fetch_tool("full", collector)++ cm = _fetcher_cm()+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ) as factory:+ out = tool.invoke({"url": "[9999]"})++ factory.assert_not_called()+ assert "No registered citation matches [9999]" in out+ assert "URL" in out # tells the agent what to do instead+++def test_full_mode_citation_with_library_url_resolves_locally():+ """A citation whose source is a library doc URL resolves through the+ library resolver (recursive: [N] → /library/document/<uuid> → content).+ ContentFetcher is NOT called."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [+ {+ "title": "Library Doc",+ "link": "/library/document/doc-1",+ "snippet": "snippet",+ }+ ],+ engine_name="library",+ )+ tool = build_fetch_tool(+ "full", collector, library_resolver=_library_doc_resolver+ )++ cm = _fetcher_cm()+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ) as factory:+ out = tool.invoke({"url": "[1]"})++ factory.assert_not_called()+ assert "Title: Doc One" in out+ assert "URL: /library/document/doc-1" in out+++def test_summary_mode_resolves_library_document_url_locally():+ """Summary mode routes the resolved document content through the LLM+ summariser the same way a fetched HTTP page would."""+ collector = SearchResultsCollector([])+ model = _model_returning("summary of doc 1")+ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ library_resolver=_library_doc_resolver,+ egress_context=object(),+ )++ cm = _fetcher_cm()+ with (+ patch(+ "local_deep_research.content_fetcher.ContentFetcher",+ return_value=cm,+ ) as factory,+ patch(+ "local_deep_research.security.egress.policy.evaluate_url"+ ) as mock_evaluate_url,+ ):+ out = tool.invoke(+ {"url": "/library/document/doc-1", "focus": "what does it say"}+ )++ factory.assert_not_called()+ mock_evaluate_url.assert_not_called()+ model.invoke.assert_called_once()+ # The prompt was built from the library document content.+ prompt = model.invoke.call_args.args[0]+ assert "Body of document one." in prompt+ assert "Page title: Doc One" in prompt+ assert "Page URL: /library/document/doc-1" in prompt+ assert "summary of doc 1" in out+++def test_summary_mode_citation_marker_resolves_to_citation_url():+ """Summary mode also rewrites citation markers to the citation URL;+ the egress gate (if any) then evaluates the resolved URL."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [+ {+ "title": "Some Paper",+ "link": "https://example.com/paper",+ "snippet": "snippet",+ }+ ],+ engine_name="web",+ )+ model = _model_returning("summary")+ tool = build_fetch_tool("summary_focus", collector, model=model)++ cm = _fetcher_cm(title="T", content="body")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke({"url": "[1]", "focus": "year"})++ cm.__enter__.return_value.fetch.assert_called_once()+ fetched_url = cm.__enter__.return_value.fetch.call_args.args[0]+ assert fetched_url == "https://example.com/paper"+ assert "summary" in out+++def test_summary_mode_unknown_citation_marker_short_circuits_with_error():+ """Summary mode also short-circuits on an unknown citation marker so+ no LLM round-trip is spent on a guaranteed-fail call."""+ collector = SearchResultsCollector([])+ model = _model_returning("should-not-be-called")+ tool = build_fetch_tool("summary_focus", collector, model=model)++ cm = _fetcher_cm()+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke({"url": "[42]", "focus": "anything"})++ cm.__enter__.return_value.fetch.assert_not_called()+ model.invoke.assert_not_called()+ assert "No registered citation matches [42]" in out+++def test_summary_mode_library_doc_with_empty_content_returns_not_relevant():+ """A library document with no extractable text mirrors the existing+ HTTP-fetcher behaviour: NOT RELEVANT, no LLM call, no registration+ (so the agent can retry with a different focus)."""++ def _empty_resolver(_url):+ return {+ "title": "Empty Doc",+ "content": "",+ "url": _url,+ "snippet": "",+ }++ collector = SearchResultsCollector([])+ model = _model_returning("should-not-be-called")+ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ library_resolver=_empty_resolver,+ )++ cm = _fetcher_cm()+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke(+ {"url": "/library/document/empty", "focus": "anything"}+ )++ model.invoke.assert_not_called()+ assert "NOT RELEVANT" in out+ assert "/library/document/empty" in out+ # Empty citation must NOT be registered — the agent should be free to+ # retry with a different focus instead of seeing it as already-cached+ # with an empty body.+ assert collector.results == []+++def test_inputs_that_look_like_citation_but_are_not_pass_through():+ """Inputs that LOOK like a citation but aren't a well-formed ``[N]``+ (e.g. ``[1, 2]`` or ``[]`` or ``[abc]``) must NOT short-circuit with+ the citation error. The fetch tool should fall through to the egress+ policy and produce a normal denial."""+ from local_deep_research.security.egress.policy import Decision++ collector = SearchResultsCollector([])+ tool = build_fetch_tool("full", collector, egress_context=object())++ for bad in ("[1, 2]", "[]", "[abc]"):+ with patch(+ "local_deep_research.security.egress.policy.evaluate_url",+ return_value=Decision(False, "unsupported_scheme"),+ ):+ out = tool.invoke({"url": bad})+ assert "Cannot fetch" in out, (+ f"expected denial for {bad!r}, got {out!r}"+ )+++def test_library_doc_content_length_capped_at_max_length():+ """A library document whose content exceeds CONTENT_MAX_LENGTH is truncated."""+ from local_deep_research.advanced_search_system.tools.fetch import (+ CONTENT_MAX_LENGTH,+ )++ long_text = "A" * (CONTENT_MAX_LENGTH + 5000)++ def _large_resolver(_url):+ return {+ "title": "Large Doc",+ "content": long_text,+ "url": _url,+ "snippet": "A" * 200,+ }++ collector = SearchResultsCollector([])+ tool = build_fetch_tool("full", collector, library_resolver=_large_resolver)++ out = tool.invoke({"url": "/library/document/large-doc"})+ assert out.startswith(+ "[1] Title: Large Doc\nURL: /library/document/large-doc\n\n"+ )+ body = out.split("\n\n", 1)[1]+ assert len(body) == CONTENT_MAX_LENGTH+++def test_circular_citation_marker_prevents_unbounded_recursion():+ """If a citation marker points to a link that cycles back to itself, circular detection prevents recursion."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [{"title": "Loop", "link": "[1]", "snippet": "Looping citation"}],+ engine_name="fetch",+ )+ tool = build_fetch_tool("full", collector)++ out = tool.invoke({"url": "[1]"})+ assert "Circular citation reference detected" in out+++def test_citation_marker_rewrite_denied_by_egress_policy_names_resolved_url():+ """When a citation marker [N] rewrites to an external URL and that resolved URL is+ denied by policy, the denial message names the resolved URL, not the marker."""+ from local_deep_research.security.egress.policy import Decision++ collector = SearchResultsCollector([])+ collector.add_results(+ [+ {+ "title": "Secret Doc",+ "link": "https://forbidden-external.com/secret",+ "snippet": "snip",+ }+ ],+ engine_name="web",+ )+ tool = build_fetch_tool("full", collector, egress_context=object())++ with patch(+ "local_deep_research.security.egress.policy.evaluate_url",+ return_value=Decision(False, "scope_mismatch_private_only"),+ ):+ out = tool.invoke({"url": "[1]"})++ assert "Cannot fetch https://forbidden-external.com/secret" in out+ assert "Cannot fetch [1]" not in out+ assert "scope_mismatch_private_only" in out+++def test_citation_marker_with_empty_url_field_returns_error():+ """A citation marker [N] whose source citation lacks a link/url field returns a clear error."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [{"title": "No URL Paper", "snippet": "no link given"}],+ engine_name="web",+ )+ tool = build_fetch_tool("full", collector)++ out = tool.invoke({"url": "[1]"})+ assert "Citation [1] has no URL field." in out+++def test_summary_mode_citation_with_library_url_resolves_locally():+ """Summary mode citation marker [N] pointing to a library doc resolves locally without HTTP or egress gate."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [+ {+ "title": "Library Doc Citation",+ "link": "/library/document/doc-1",+ "snippet": "snip",+ }+ ],+ engine_name="library",+ )+ model = _model_returning("summary of cited doc")+ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ library_resolver=_library_doc_resolver,+ egress_context=object(),+ )++ cm = _fetcher_cm()+ with (+ patch(+ "local_deep_research.content_fetcher.ContentFetcher",+ return_value=cm,+ ) as factory,+ patch(+ "local_deep_research.security.egress.policy.evaluate_url"+ ) as mock_evaluate_url,+ ):+ out = tool.invoke({"url": "[1]", "focus": "facts"})++ factory.assert_not_called()+ mock_evaluate_url.assert_not_called()+ model.invoke.assert_called_once()+ assert "summary of cited doc" in out+++def test_multi_level_citation_chain_resolves_deepest_target():+ """Multi-level citation chains ([1] -> [2] -> external URL or library doc) resolve recursively."""+ collector = SearchResultsCollector([])+ collector.add_results(+ [{"title": "First", "link": "[2]", "snippet": "points to 2"}],+ engine_name="web",+ )+ collector.add_results(+ [+ {+ "title": "Second",+ "link": "https://example.com/deep-page",+ "snippet": "deep",+ }+ ],+ engine_name="web",+ )++ tool = build_fetch_tool("full", collector)++ cm = _fetcher_cm(title="Deep Page", content="deep content")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke({"url": "[1]"})++ cm.__enter__.return_value.fetch.assert_called_once()+ fetched_url = cm.__enter__.return_value.fetch.call_args.args[0]+ assert fetched_url == "https://example.com/deep-page"+ assert "Title: Deep Page" in out+++def test_summary_fetch_tool_handles_llm_exception():+ """Summary-mode fetch tool catches LLM model.invoke exceptions and returns a scrubbed error string."""+ collector = SearchResultsCollector([])+ model = MagicMock()+ model.invoke.side_effect = RuntimeError("LLM rate limit reached")++ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ )++ cm = _fetcher_cm(title="Page Title", content="Page text to summarize.")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke(+ {"url": "https://example.com/page", "focus": "key facts"}+ )++ assert "Error summarizing https://example.com/page" in out+ assert "LLM rate limit reached" in out+++def test_summary_fetch_tool_handles_empty_summary():+ """Summary-mode fetch tool returns NOT RELEVANT when LLM returns an empty summary."""+ collector = SearchResultsCollector([])+ model = _model_returning("")++ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ )++ cm = _fetcher_cm(title="Unrelated Page", content="Unrelated page content.")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke(+ {"url": "https://example.com/unrelated", "focus": "specific topic"}+ )++ assert (+ "NOT RELEVANT (no spans matched focus at https://example.com/unrelated)"+ in out+ )+++def test_summary_fetch_tool_handles_empty_content():+ """Summary-mode fetch tool returns NOT RELEVANT without invoking LLM when page content is empty."""+ collector = SearchResultsCollector([])+ model = MagicMock()++ tool = build_fetch_tool(+ "summary_focus",+ collector,+ model=model,+ )++ cm = _fetcher_cm(title="Empty Page", content=" \n ")+ with patch(+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm+ ):+ out = tool.invoke(+ {"url": "https://example.com/empty", "focus": "anything"}+ )++ model.invoke.assert_not_called()+ assert (+ "NOT RELEVANT (no extractable content at https://example.com/empty)"+ in out+ )+++def test_non_string_url_returns_none_in_try_resolve_url():+ """_try_resolve_url returns None immediately if url is not a string."""+ from local_deep_research.advanced_search_system.tools.fetch import (+ _try_resolve_url,+ )++ assert _try_resolve_url(12345, None, None) is Nonediff --git a/tests/advanced_search_system/tools/test_library_resolver.py b/tests/advanced_search_system/tools/test_library_resolver.pynew file mode 100644index 0000000000..39ff160d1d--- /dev/null+++ b/tests/advanced_search_system/tools/test_library_resolver.py@@ -0,0 +1,370 @@+"""Unit tests for ``advanced_search_system.tools.fetch.library_resolver``.++The library resolver powers the A3 fix: a ``fetch_content`` call on a+``/library/document/<uuid>[/pdf]`` URL or a bare ``[N]`` citation marker+short-circuits the egress policy and reads the document / rewrites the+URL locally. Tests here pin the parsing, the lookup shape, and the+``None``-fall-through contract so a regression can't silently break the+fix (which previously turned 26 of 26 fetches into ``unsupported_scheme``+denials).+"""++from __future__ import annotations++from unittest.mock import MagicMock, patch+++# ---------------------------------------------------------------------------+# Pure parsing+# ---------------------------------------------------------------------------+++def test_parse_library_url_root_form():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ parse_library_url,+ )++ assert parse_library_url("/library/document/abc-123") == ("abc-123", None)+++def test_parse_library_url_pdf_suffix():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ parse_library_url,+ )++ assert parse_library_url("/library/document/abc-123/pdf") == (+ "abc-123",+ "pdf",+ )+++def test_parse_library_url_trailing_slash():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ parse_library_url,+ )++ assert parse_library_url("/library/document/abc-123/") == (+ "abc-123",+ None,+ )+++def test_parse_library_url_rejects_non_matching_shapes():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ parse_library_url,+ )++ for bad in (+ "http://example.com/library/document/abc",+ "/library/document/",+ "/library/document",+ "/library/abc", # not under document/+ "/library/document/abc/txt", # txt suffix is not library-DB route+ "library/document/abc", # missing leading slash+ "",+ None,+ 42,+ ):+ assert parse_library_url(bad) is None, f"expected None for {bad!r}"+++def test_is_citation_reference_matches_well_formed_marker():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ is_citation_reference,+ )++ assert is_citation_reference("[1]") == 1+ assert is_citation_reference("[1062]") == 1062+ # Marker is strict: no whitespace tolerance — a malformed marker+ # must NOT be silently treated as a valid citation reference.+ assert is_citation_reference("[ 42 ]") is None+++def test_is_citation_reference_rejects_lookalikes():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ is_citation_reference,+ )++ for bad in (+ "[1, 2]", # list marker+ "[]",+ "[abc]", # non-digit+ "1", # no brackets+ "[1", # missing close+ "1]",+ "",+ None,+ ):+ assert is_citation_reference(bad) is None, f"expected None for {bad!r}"+++# ---------------------------------------------------------------------------+# resolve_library_document — DB interaction+# ---------------------------------------------------------------------------+++def _fake_document(text_content="Body text", title="Doc Title"):+ doc = MagicMock()+ doc.id = "abc-123"+ doc.title = title+ doc.text_content = text_content+ return doc+++def test_resolve_library_document_returns_shape_for_full_text():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ document = _fake_document(text_content="full body", title="T")++ # Patch both the model import and the session so the test runs without+ # SQLAlchemy fixtures.+ with (+ patch(+ "local_deep_research.database.models.library.Document",+ return_value=document,+ ),+ patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm,+ ):+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ document+ )+ session_cm.return_value.__enter__.return_value = session++ result = resolve_library_document(+ "/library/document/abc-123", username="alice"+ )++ assert result is not None+ assert result["title"] == "T"+ assert result["content"] == "full body"+ assert result["url"] == "/library/document/abc-123"+ # Snippet is the first ~200 chars of the content.+ assert result["snippet"] == "full body"+++def test_resolve_library_document_handles_empty_text():+ """An empty ``text_content`` is still a successful resolve — the+ fetch tool's existing NOT RELEVANT guard handles the empty case."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ document = _fake_document(text_content="", title="T")++ with (+ patch(+ "local_deep_research.database.models.library.Document",+ return_value=document,+ ),+ patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm,+ ):+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ document+ )+ session_cm.return_value.__enter__.return_value = session++ result = resolve_library_document(+ "/library/document/abc-123", username="alice"+ )++ assert result is not None+ assert result["content"] == ""+ assert result["snippet"] == "T" # falls back to title+++def test_resolve_library_document_returns_none_for_unknown_doc():+ """An unknown document UUID returns ``None`` so the caller falls+ through to the egress policy and produces the standard+ ``unsupported_scheme`` denial."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm:+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ None+ )+ session_cm.return_value.__enter__.return_value = session++ assert (+ resolve_library_document(+ "/library/document/missing", username="alice"+ )+ is None+ )+++def test_resolve_library_document_returns_none_when_no_username():+ """No username (programmatic mode, benchmarks) means no library —+ return ``None`` so the call falls through to the egress policy."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ assert (+ resolve_library_document("/library/document/abc-123", username=None)+ is None+ )+++def test_resolve_library_document_swallows_db_errors():+ """A per-document DB hiccup is non-fatal (mirrors the+ LibraryRAGSearchEngine behaviour). Return ``None`` so the caller+ falls through."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm:+ session_cm.return_value.__enter__.side_effect = RuntimeError("db down")++ assert (+ resolve_library_document(+ "/library/document/abc-123", username="alice"+ )+ is None+ )+++def test_resolve_library_document_handles_import_error():+ """An import failure when loading DB models returns None and logs an exception."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ with patch.dict(+ "sys.modules", {"local_deep_research.database.models.library": None}+ ):+ assert (+ resolve_library_document(+ "/library/document/abc-123", username="alice"+ )+ is None+ )+++def test_resolve_library_document_returns_none_for_non_matching_url():+ """A URL that isn't ``/library/document/...`` returns ``None``+ immediately (no DB call)."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_library_document,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm:+ assert (+ resolve_library_document("http://example.com/", username="alice")+ is None+ )+ session_cm.assert_not_called()+++# ---------------------------------------------------------------------------+# resolve_citation_reference — collector interaction+# ---------------------------------------------------------------------------+++def test_resolve_citation_reference_returns_dict_for_known_index():+ """A bare ``[N]`` marker returns the citation dict from the+ collector (so the fetch tool can extract its URL and recurse)."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_citation_reference,+ )++ citation = {"link": "http://example.com/", "title": "T", "index": "1"}+ collector = MagicMock()+ collector.find_by_index.return_value = citation++ result = resolve_citation_reference("[1]", collector)+ assert result == citation+ collector.find_by_index.assert_called_once_with(1)+++def test_resolve_citation_reference_returns_none_for_unknown():+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_citation_reference,+ )++ collector = MagicMock()+ collector.find_by_index.return_value = None++ assert resolve_citation_reference("[42]", collector) is None+++def test_resolve_citation_reference_returns_none_for_non_marker():+ """A non-marker input doesn't even hit the collector."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_citation_reference,+ )++ collector = MagicMock()+ assert resolve_citation_reference("http://example.com/", collector) is None+ collector.find_by_index.assert_not_called()+++def test_resolve_citation_reference_returns_none_without_find_by_index():+ """A duck-typed collector without ``find_by_index`` doesn't crash+ (mirrors the langgraph module's own defensive getattr in+ ``SearchResultsCollector``)."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ resolve_citation_reference,+ )++ collector = MagicMock(spec=[]) # no find_by_index attribute+ assert resolve_citation_reference("[1]", collector) is None+++# ---------------------------------------------------------------------------+# make_library_resolver — closure binding+# ---------------------------------------------------------------------------+++def test_make_library_resolver_binds_username():+ """The closure captures the username so subagent threads can resolve+ library URLs without inheriting Flask session state."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ make_library_resolver,+ )++ document = _fake_document(text_content="hello")++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as session_cm:+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ document+ )+ session_cm.return_value.__enter__.return_value = session++ resolver = make_library_resolver("alice")+ result = resolver("/library/document/abc-123")++ assert result is not None+ assert result["content"] == "hello"+++def test_make_library_resolver_returns_none_for_no_username():+ """``make_library_resolver(None)`` returns a resolver that always+ returns ``None`` — used in programmatic mode where no user library+ exists."""+ from local_deep_research.advanced_search_system.tools.fetch.library_resolver import (+ make_library_resolver,+ )++ resolver = make_library_resolver(None)+ assert resolver("/library/document/abc-123") is Nonediff --git a/tests/strategies/test_langgraph_agent_strategy.py b/tests/strategies/test_langgraph_agent_strategy.pyindex b4e7518d8b..c2dd52c0e4 100644--- a/tests/strategies/test_langgraph_agent_strategy.py+++ b/tests/strategies/test_langgraph_agent_strategy.py@@ -182,6 +182,51 @@ class TestSearchResultsCollector:indices = [r["index"] for r in all_results]assert len(indices) == len(set(indices)), "Duplicate indices found!"+ def test_find_by_index_returns_result_dict_when_present(self):+ """``find_by_index(N)`` returns the dict stored at citation N so the+ fetch tool can resolve a bare ``[N]`` marker to its source URL+ (A3 follow-up)."""+ collector, _ = self._make_collector()+ collector.add_results(+ [+ {"title": "A", "link": "http://a.com", "snippet": "a"},+ {"title": "B", "link": "http://b.com", "snippet": "b"},+ ]+ )++ result = collector.find_by_index(1)+ assert result is not None+ assert result["title"] == "A"+ assert result["link"] == "http://a.com"++ result = collector.find_by_index(2)+ assert result["title"] == "B"++ def test_find_by_index_returns_none_when_absent(self):+ collector, _ = self._make_collector()+ collector.add_results(+ [{"title": "A", "link": "http://a.com", "snippet": "a"}]+ )+ assert collector.find_by_index(9999) is None+ assert collector.find_by_index(0) is None+ assert collector.find_by_index(-1) is None++ def test_find_by_index_uses_all_links_across_resets(self):+ """Citation indices survive a ``reset()`` because they live on+ ``_all_links``, not on the per-call ``_results`` list. The fetch+ tool's citation resolution depends on this — ``reset()`` runs+ before every subsection in detailed-report mode."""+ collector, _ = self._make_collector()+ collector.add_results(+ [{"title": "A", "link": "http://a.com", "snippet": "a"}]+ )+ collector.reset()++ # Citation 1 still resolvable after reset.+ result = collector.find_by_index(1)+ assert result is not None+ assert result["link"] == "http://a.com"+# ---------------------------------------------------------------------------# Format results helper@@ -785,7 +830,80 @@ class TestPrimarySearchDescriptionText:# ----------------------------------------------------------------------------# Tool-call progress formatting+# Library resolver wiring (A3)+#+# The strategy threads a library_resolver into both the lead-agent's fetch+# tool and the subagent's fetch tool so a /library/document/<uuid> URL or+# a [N] citation marker doesn't get rejected by the egress policy. Without+# the resolver, every fetch in a library-only run returns+# ``unsupported_scheme`` (the f3045c5b run produced zero usable pages).+# ---------------------------------------------------------------------------+++class TestBuildLibraryResolver:+ """``_build_library_resolver`` returns a callable for the fetch tool.++ Returns ``None`` for callers without a username (programmatic mode,+ benchmarks, news) — those callers preserve the pre-A3 behaviour so+ the policy gate still rejects library URLs as ``unsupported_scheme``.+ """++ def _make_strategy(self, **overrides):+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (+ LangGraphAgentStrategy,+ )++ defaults = {+ "model": MagicMock(),+ "search": MagicMock(),+ "all_links_of_system": [],+ "settings_snapshot": {"search.tool": {"value": "duckduckgo"}},+ }+ defaults.update(overrides)+ return LangGraphAgentStrategy(**defaults)++ def test_returns_resolver_when_username_present_in_snapshot(self):+ """The username injected by ``_ensure_snapshot_username`` (via the+ ``_username`` snapshot key) drives the resolver build. The web+ run calls this; programmatic mode without a username does not."""+ from local_deep_research.advanced_search_system.tools.fetch import (+ build_fetch_tool,+ )++ strategy = self._make_strategy(+ settings_snapshot={+ "search.tool": {"value": "duckduckgo"},+ "_username": "alice",+ }+ )+ resolver = strategy._build_library_resolver()+ assert resolver is not None+ # Round-trip: the returned callable is wired into the fetch tool.+ tool = build_fetch_tool("full", MagicMock(), library_resolver=resolver)+ assert tool is not None++ def test_returns_none_when_snapshot_has_no_username(self):+ """No ``_username`` key (programmatic mode, benchmarks) — preserve+ the pre-A3 behaviour so the egress policy rejects library URLs+ unchanged."""+ strategy = self._make_strategy(+ settings_snapshot={"search.tool": {"value": "duckduckgo"}}+ )+ assert strategy._build_library_resolver() is None++ def test_returns_none_when_snapshot_is_empty(self):+ strategy = self._make_strategy(settings_snapshot={})+ assert strategy._build_library_resolver() is None++ def test_returns_resolver_when_username_attr_set_without_snapshot(self):+ """When settings_snapshot is empty or None, but _username is set on the strategy,+ _build_library_resolver returns a resolver bound to that username."""+ strategy = self._make_strategy(settings_snapshot={})+ strategy._username = "bob"+ resolver = strategy._build_library_resolver()+ assert resolver is not None++## Regression coverage for the ``fetch_url`` → ``fetch_content`` rename:# prior to the fix, the display-renderer branch in ``analyze_topic``@@ -2443,8 +2561,13 @@ class TestResearchSubtopicToolTruncation:# Progress metadata surfaces the truncation to the UI.assert captured["meta"].get("truncated_from") == 8- # A warning is emitted at the truncation point (#5012).- assert log.warning.called+ # A warning is emitted at the truncation point (#5012) with matching placeholders.+ log.warning.assert_called_once()+ warning_args = log.warning.call_args.args+ assert len(warning_args) == 3+ assert warning_args[1] == 8+ assert warning_args[2] == 5+ assert warning_args[0].count("{}") == 2def test_truncation_above_limit_notes_dropped_in_output(self):subtopics = [f"topic {i}" for i in range(8)]