~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(ux): report bulk page-fetch progress with throttled milestones (#7193)
S69eb0aecce52
diff --git a/changelog.d/+fetch-progress-heartbeats.bugfix.md b/changelog.d/+fetch-progress-heartbeats.bugfix.mdnew file mode 100644index 0000000000..ed9c21dbee--- /dev/null+++ b/changelog.d/+fetch-progress-heartbeats.bugfix.md@@ -0,0 +1 @@+Report bulk page-fetch progress to the UI with throttled milestones and per-page cancellation checks (live via pipeline-strategy FullSearchResults; agent tools stay snippets-only post-#7189 as dormant groundwork), guard the progress hook so flaky callbacks cannot corrupt results, and quiet per-URL extraction success logs.diff --git a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.pyindex 972ce85f52..5013f5509a 100644--- a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py+++ b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py@@ -88,6 +88,10 @@ from .primary_search_metadata import (classify_primary_source,format_primary_search_description,)+from local_deep_research.web_search_engines.engines.full_search import (+ _FETCH_HEARTBEAT_SECONDS as _FETCH_HEARTBEAT_SECONDS,+ make_fetch_progress_adapter as _make_fetch_progress_adapter,+)# ---------------------------------------------------------------------------# Constants@@ -1425,6 +1429,14 @@ def _format_results(results: list[dict], start_idx: int) -> str:return "\n\n".join(lines) if lines else "No results."+# Bulk-fetch progress adapter lives in the full-search engine module so+# the pipeline-strategy wiring (``AdvancedSearchSystem`` → ``get_search``+# → ``FullSearchResults``) and the agent tool factories below share one+# throttling implementation. Imported at top as+# ``_make_fetch_progress_adapter`` for backwards compatibility with+# existing tests.++def _make_web_search_tool(search_engine_name: str,model: BaseChatModel,@@ -1433,6 +1445,7 @@ def _make_web_search_tool(programmatic_mode: bool = False,description: str = NEUTRAL_PRIMARY_SEARCH_DESCRIPTION,availability: SearchEngineAvailability | None = None,+ progress_callback=None,):"""Create a primary search tool with research-scoped availability."""availability = availability or SearchEngineAvailability()@@ -1444,6 +1457,11 @@ def _make_web_search_tool(create_search_engine,)+ # Per-invocation adapter (not shared): subagents reuse tool+ # objects across pool workers, so heartbeat state must not+ # leak between concurrent searches.+ fetch_progress = _make_fetch_progress_adapter(progress_callback)+results = run_search(search_engine_name,query,@@ -1452,6 +1470,21 @@ def _make_web_search_tool(llm=model,settings_snapshot=settings_snapshot,programmatic_mode=programmatic_mode,+ progress_callback=fetch_progress,+ # Documented agent invariant (CONFIGURATION.md+ # ``search.snippets_only``): autonomous agents always+ # search snippets and fetch full pages on demand via+ # fetch_content. Force it here so a global+ # ``search.snippets_only=false`` (meant for pipeline+ # strategies) cannot silently turn every web_search+ # into a FullSearchResults bulk page-fetch. Post-#7189+ # this keeps the hook above as dormant groundwork for a+ # future agent full-search mode; the live bulk-fetch+ # progress path is the pipeline strategy via+ # ``get_search(..., progress_callback=...)`` →+ # ``FullSearchResults``.+ search_snippets_only=True,+ use_full_search=False,),availability,programmatic_mode=programmatic_mode,@@ -1480,6 +1513,7 @@ def _make_specialized_search_tool(collector: SearchResultsCollector,programmatic_mode: bool = False,availability: SearchEngineAvailability | None = None,+ progress_callback=None,):"""Create a specialized search tool with research-scoped availability."""availability = availability or SearchEngineAvailability()@@ -1491,6 +1525,9 @@ def _make_specialized_search_tool(create_search_engine,)+ # Per-invocation adapter (not shared): see web_search above.+ fetch_progress = _make_fetch_progress_adapter(progress_callback)+results = run_search(engine_name,query,@@ -1499,6 +1536,13 @@ def _make_specialized_search_tool(llm=model,settings_snapshot=settings_snapshot,programmatic_mode=programmatic_mode,+ progress_callback=fetch_progress,+ # Same documented agent invariant as web_search above:+ # specialized agent tools stay snippets-only even when+ # the global search.snippets_only toggle is off.+ # Dormant groundwork post-#7189; see web_search comment.+ search_snippets_only=True,+ use_full_search=False,),availability,programmatic_mode=programmatic_mode,@@ -1525,6 +1569,7 @@ def _load_specialized_engine_tools(programmatic_mode: bool = False,egress_context=None,availability: SearchEngineAvailability | None = None,+ progress_callback=None,) -> list:"""Load tools for all available specialized search engines, filtered byegress policy and per-engine ``agent_enabled`` flag.@@ -1673,6 +1718,7 @@ def _load_specialized_engine_tools(collector,programmatic_mode=programmatic_mode,availability=availability,+ progress_callback=progress_callback,))except Exception:@@ -1813,6 +1859,7 @@ def _make_research_subtopic_tool(programmatic_mode=programmatic_mode,description=web_search_description,availability=availability,+ progress_callback=progress_callback,))sub_fetch = build_fetch_tool(@@ -1845,6 +1892,7 @@ def _make_research_subtopic_tool(programmatic_mode=programmatic_mode,egress_context=egress_context,availability=availability,+ progress_callback=progress_callback,))@@ -2909,6 +2957,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):programmatic_mode=self.programmatic_mode,description=primary_search_description,availability=self.engine_availability,+ progress_callback=self.progress_callback,))@@ -2948,6 +2997,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):programmatic_mode=self.programmatic_mode,egress_context=policy_ctx,availability=self.engine_availability,+ progress_callback=self.progress_callback,))diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.pyindex d9069aae69..b275e0d330 100644--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py@@ -14,7 +14,7 @@ specialized downloaders first. Generic HTML runs only when that route permits it"""from importlib import import_module-from typing import Any, Dict, List, NamedTuple, Optional+from typing import Any, Callable, Dict, List, NamedTuple, Optionalfrom bs4 import BeautifulSoup, NavigableStringfrom loguru import logger@@ -655,6 +655,7 @@ def batch_fetch_and_extract(enable_js_rendering: bool = False,allow_private_ips: bool = False,block_link_local: bool = False,+ progress_callback: Optional[Callable[[int, int], None]] = None,) -> Dict[str, Optional[str]]:"""Fetch multiple URLs and extract clean text from each.@@ -688,6 +689,17 @@ def batch_fetch_and_extract(block_link_local: Keep the whole link-local range blocked even when``allow_private_ips`` is True (forwarded to the downloader'sstatic-fetch ``SafeSession``). No effect without the grant.+ progress_callback: Optional ``(done, total)`` hook invoked once+ up front with ``(0, len(urls))`` and after each URL+ completes. Lets a long serial fetch report liveness (and+ notice cancellation) instead of going silent for minutes.+ A regular ``Exception`` raised by the hook is swallowed+ (debug-logged) so a flaky UI callback can neither corrupt an+ already-fetched result nor abort the remaining batch;+ ``BaseException`` — including ``ResearchTerminatedException``,+ which derives ``BaseException`` and therefore passes through+ the per-URL ``except Exception`` handlers untouched — still+ propagates so cancellation takes effect promptly.Returns:Dict mapping URL → extracted text (or None if failed).@@ -695,6 +707,28 @@ def batch_fetch_and_extract(from ..playwright_html import AutoHTMLDownloaderresults: Dict[str, Optional[str]] = {}+ total = len(urls)+ done = 0++ def _report_progress() -> None:+ if progress_callback is not None and total:+ try:+ progress_callback(done, total)+ except Exception:+ # A flaky UI callback must not corrupt already-fetched+ # results (specialized loop: the ``except Exception``+ # below would otherwise mistake it for a downloader+ # failure and overwrite good content with the HTML+ # fallback) nor abort the remaining batch (HTML loop).+ # ``except Exception`` deliberately lets ``BaseException``+ # — including ``ResearchTerminatedException`` — propagate+ # for prompt cancellation.+ logger.debug(+ "batch_fetch_and_extract progress_callback failed; "+ "continuing batch"+ )++ _report_progress()# Try specialized downloaders first — collect URLs that need HTML fallbackhtml_urls: List[str] = []@@ -703,6 +737,8 @@ def batch_fetch_and_extract(specialized = _try_specialized_downloader(url, timeout=timeout)if specialized.content:results[url] = specialized.content+ done += 1+ _report_progress()continueif not specialized.fallback_allowed:logger.warning(@@ -713,6 +749,8 @@ def batch_fetch_and_extract(),)results[url] = None+ done += 1+ _report_progress()continueexcept Exception as e:logger.opt(exception=False).debug(@@ -721,6 +759,8 @@ def batch_fetch_and_extract()if is_arxiv_paper_url(url):results[url] = None+ done += 1+ _report_progress()continuehtml_urls.append(url)@@ -747,6 +787,8 @@ def batch_fetch_and_extract(f"{redact_url_for_log(url)} ({type(e).__name__})")results[url] = None+ done += 1+ _report_progress()finally:try:downloader.close()diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.pyindex 7a47daf9b1..727760aadc 100644--- a/src/local_deep_research/research_library/downloaders/html.py+++ b/src/local_deep_research/research_library/downloaders/html.py@@ -696,7 +696,7 @@ class HTMLDownloader(BaseDownloader):title = result.get("title")content = result["content"]- logger.info(+ logger.debug("html.extract_succeeded url={url} content_length={content_length}",url=redact_url_for_log(url),content_length=len(content),diff --git a/src/local_deep_research/search_system.py b/src/local_deep_research/search_system.pyindex 27d46bf778..c4f53aca4b 100644--- a/src/local_deep_research/search_system.py+++ b/src/local_deep_research/search_system.py@@ -319,6 +319,28 @@ class AdvancedSearchSystem:self.progress_callback = callbackif hasattr(self, "strategy"):self.strategy.set_progress_callback(callback)+ # Pipeline-strategy live path for bulk-fetch progress (#7193):+ # adapt the 3-arg UI callback to the 2-arg ``(done, total)`` hook+ # and attach it to the search engine so ``FullSearchResults``+ # bulk fetches emit throttled milestones + per-page cancellation+ # checks. Late-binds so engines created via ``get_search`` before+ # the callback existed still get progress; agents stay snippets-+ # only post-#7189, so their hook remains dormant groundwork.+ try:+ search = getattr(self, "search", None)+ if search is not None and callback is not None:+ from .web_search_engines.engines.full_search import (+ make_fetch_progress_adapter,+ )++ hook = make_fetch_progress_adapter(callback)+ setter = getattr(search, "set_fetch_progress_callback", None)+ if callable(setter):+ setter(hook)+ elif hasattr(search, "progress_callback"):+ search.progress_callback = hook+ except Exception:+ logger.debug("Failed to wire fetch progress to search engine")def analyze_topic(self,diff --git a/src/local_deep_research/web_search_engines/engines/full_search.py b/src/local_deep_research/web_search_engines/engines/full_search.pyindex d22ff5292c..c54bacae6a 100644--- a/src/local_deep_research/web_search_engines/engines/full_search.py+++ b/src/local_deep_research/web_search_engines/engines/full_search.py@@ -1,5 +1,14 @@+import timefrom datetime import datetime, UTC-from typing import Any, Dict, List, Optional, Protocol, runtime_checkable+from typing import (+ Any,+ Callable,+ Dict,+ List,+ Optional,+ Protocol,+ runtime_checkable,+)from langchain_core.language_models import BaseLLM@@ -25,6 +34,77 @@ class _Invokable(Protocol):def invoke(self, query: str) -> Any: ...+# Minimum seconds between bulk-fetch heartbeat milestones. Each UI+# emit is a DB row + socket event, so per-page emits would spam the+# research log on a 46-page fetch; the start announcement is always+# emitted and the completion milestone ("From the web …") follows the+# fetch, so heartbeats only fill the silence in between.+_FETCH_HEARTBEAT_SECONDS = 15.0+++def make_fetch_progress_adapter(progress_callback):+ """Adapt bulk-fetch ``(done, total)`` progress to UI milestones.++ Shared by the pipeline-strategy wiring (``AdvancedSearchSystem`` →+ ``FullSearchResults``) and the langgraph-agent tool factories, so+ both surfaces throttle identically.++ Returns a hook for engine page-fetch progress that:+ * announces the fetch upfront (``done == 0``) so a minutes-long+ fetch never looks stalled,+ * emits a heartbeat at most every ``_FETCH_HEARTBEAT_SECONDS``,+ * stays silent on the final page — the completion milestone+ ("From the web …") follows immediately,+ * performs a silent termination check on every page (no UI+ emission): each check doubles as a cancellation check, because+ the UI callback raises ``ResearchTerminatedException`` when the+ user cancels — and that derives ``BaseException``, so it+ propagates through the fetch pipeline's ``except Exception``+ handlers untouched. Cancellation during a long fetch therefore+ takes effect promptly instead of only after the whole batch.++ ``progress_callback`` is the 3-arg UI callback+ ``(message, percent, metadata)``; ``None`` disables everything.+ """+ state = {"last_emit": 0.0}++ def _on_fetch_progress(done: int, total: int) -> None:+ if progress_callback is None or total <= 0:+ return+ # Silent cancellation check on every page — no UI/logging.+ progress_callback(+ "Checking termination status",+ None,+ {"phase": "termination_check"},+ )+ if done >= total:+ return+ now = time.monotonic()+ if done > 0 and now - state["last_emit"] < _FETCH_HEARTBEAT_SECONDS:+ return+ state["last_emit"] = now+ if done <= 0:+ progress_callback(+ f"📄 Fetching full content for {total} pages — "+ "this can take a minute…",+ None,+ {"phase": "fetch", "type": "milestone", "total": total},+ )+ else:+ progress_callback(+ f"📄 Fetching page content ({done}/{total})…",+ None,+ {+ "phase": "fetch",+ "type": "milestone",+ "done": done,+ "total": total,+ },+ )++ return _on_fetch_progress++# Egress scopes whose ``evaluate_url`` admits every http(s) host the SSRF# validator admits (UNPROTECTED lifts only the scope restriction; the hard# SSRF and cloud-metadata blocks stay and the download pipeline enforces@@ -52,6 +132,7 @@ class FullSearchResults:settings_snapshot: Optional[Dict] = None,egress_context: Optional[Any] = None,allow_private_ips: bool = False,+ progress_callback: Optional[Callable[[int, int], None]] = None,):self.llm = llmself.output_format = output_format@@ -75,10 +156,24 @@ class FullSearchResults:# Link-local and the cloud-metadata literals in# ALWAYS_BLOCKED_METADATA_IPS stay blocked regardless (issue #2477).self.allow_private_ips = allow_private_ips+ # Optional ``(done, total)`` liveness hook for the bulk page+ # fetch below (see ``batch_fetch_and_extract``). Threaded in by+ # the pipeline strategy (``AdvancedSearchSystem`` → ``get_search``)+ # and by agent search tools so a minutes-long fetch emits progress+ # instead of silence; ``None`` disables it with zero overhead.+ self.progress_callback = progress_callback# Set by the factory when the parent engine is gated against a# specific scope; used to evaluate per-URL fetches below.self.egress_context = egress_context+ def set_fetch_progress_callback(self, hook) -> None:+ """Late-bind the ``(done, total)`` bulk-fetch hook.++ Used by ``AdvancedSearchSystem.set_progress_callback`` when the UI+ callback arrives after engine construction.+ """+ self.progress_callback = hook+@propertydef last_search_failure(self) -> SearchFailure | None:"""Expose provider failures through the optional full-content wrapper."""@@ -330,6 +425,7 @@ class FullSearchResults:),allow_private_ips=self._downloader_allow_private_ips(),block_link_local=True,+ progress_callback=self.progress_callback,)nr_full_text = sum(1 for v in url_to_content.values() if v)@@ -391,6 +487,7 @@ class FullSearchResults:),allow_private_ips=self._downloader_allow_private_ips(),block_link_local=True,+ progress_callback=self.progress_callback,)except Exception as e:safe_msg = scrub_error(e)diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.pyindex c88ee64f02..0f276c708d 100644--- a/src/local_deep_research/web_search_engines/search_engine_base.py+++ b/src/local_deep_research/web_search_engines/search_engine_base.py@@ -403,6 +403,7 @@ class BaseSearchEngine(ABC):include_full_content: bool = False,settings_snapshot: Optional[Dict[str, Any]] = None,programmatic_mode: bool = False,+ progress_callback=None,**kwargs,):"""@@ -420,6 +421,10 @@ class BaseSearchEngine(ABC):include_full_content: Whether to use FullSearchResults for full webpage contentsettings_snapshot: Settings snapshot for configurationprogrammatic_mode: If True, disables database operations and uses memory-only tracking+ progress_callback: Optional ``(done, total)`` bulk-fetch liveness+ hook forwarded to ``FullSearchResults`` via+ ``_init_full_search`` / ``set_fetch_progress_callback``+ (pipeline-strategy fetch progress, #7193).**kwargs: Additional engine-specific parameters"""if max_filtered_results is None:@@ -445,6 +450,12 @@ class BaseSearchEngine(ABC):self.settings_snapshot = (settings_snapshot or {}) # Store settings snapshot+ # Bulk-fetch liveness hook (``(done, total)``) for full-search+ # progress. Set at construction via the factory or later via+ # ``set_fetch_progress_callback`` (e.g. when+ # ``AdvancedSearchSystem.set_progress_callback`` arrives after+ # engine creation). Forwarded to ``FullSearchResults``.+ self._fetch_progress_callback = progress_callbackself.engine_type = self.__class__.__name__self._engine_name: str = "" # set by the factory after construction@@ -1380,6 +1391,23 @@ class BaseSearchEngine(ABC):self.include_full_content = Falsereturn None+ def set_fetch_progress_callback(self, hook) -> None:+ """Attach a ``(done, total)`` bulk-fetch liveness hook.++ Stores the hook for future ``_init_full_search`` calls and, when+ this engine already holds a ``FullSearchResults`` wrapper (legacy+ ``include_full_content`` seam), forwards it immediately. Used by+ the factory stamp and by ``AdvancedSearchSystem`` late-binding so+ pipeline-strategy full-search progress works regardless of whether+ the engine was created before the UI callback existed.+ """+ self._fetch_progress_callback = hook+ full_search = getattr(self, "full_search", None)+ if full_search is not None and hasattr(+ full_search, "progress_callback"+ ):+ full_search.progress_callback = hook+def _init_full_search(self,web_search=None,@@ -1388,6 +1416,7 @@ class BaseSearchEngine(ABC):region=None,time_period=None,safe_search=None,+ progress_callback=None,):"""Initialize FullSearchResults if include_full_content is True.@@ -1400,6 +1429,9 @@ class BaseSearchEngine(ABC):region: Region/country code for resultstime_period: Time period filtersafe_search: Safe search setting (string value for FullSearchResults)+ progress_callback: Optional ``(done, total)`` liveness hook for+ the bulk page fetch. Defaults to the hook stored on+ ``self`` (construction-time or ``set_fetch_progress_callback``)."""if self.include_full_content and self.llm:try:@@ -1425,6 +1457,11 @@ class BaseSearchEngine(ABC):# explicit boolean grant, nothing else, so an engine# built through this seam does not lose it silently.allow_private_ips=self.allow_private_result_fetch is True,+ progress_callback=(+ progress_callback+ if progress_callback is not None+ else self._fetch_progress_callback+ ),)except ImportError:logger.warning(diff --git a/src/local_deep_research/web_search_engines/search_engine_factory.py b/src/local_deep_research/web_search_engines/search_engine_factory.pyindex 313d8a5cb3..e5dab2dc43 100644--- a/src/local_deep_research/web_search_engines/search_engine_factory.py+++ b/src/local_deep_research/web_search_engines/search_engine_factory.py@@ -53,6 +53,7 @@ _BASE_FORWARDED_PARAMS = frozenset("language","time_period","include_full_content",+ "progress_callback",})@@ -681,6 +682,22 @@ def create_search_engine(all_params["search_snippets_only"])+ # Stamp the bulk-fetch liveness hook for engines built through+ # this seam (pipeline-strategy full-search progress). The base+ # engine itself does not fetch, but `_init_full_search` and+ # `set_fetch_progress_callback` forward it to FullSearchResults.+ if isinstance(engine, BaseSearchEngine) and (+ "progress_callback" in all_params+ ):+ try:+ engine.set_fetch_progress_callback(+ all_params["progress_callback"]+ )+ except Exception:+ logger.debug(+ "Failed to stamp progress_callback on engine",+ )+# Determine if this engine should use LLM relevance filtering# Priority: per-engine setting > needs_llm_relevance_filter > global setting#@@ -983,6 +1000,7 @@ def get_search(max_filtered_results: Optional[int] = None,settings_snapshot: Dict[str, Any] | None = None,programmatic_mode: bool = False,+ progress_callback=None,):"""Get search tool instance based on the provided parameters.@@ -999,6 +1017,10 @@ def get_search(search_language: Language for search resultsmax_filtered_results: Maximum number of results to keep after filteringprogrammatic_mode: If True, disables database operations and metrics tracking+ progress_callback: Optional ``(done, total)`` bulk-fetch liveness hook,+ forwarded to ``FullSearchResults`` when the engine is wrapped for+ full search. This is the pipeline-strategy live path for fetch+ progress (see #7193); ``None`` disables it.Returns:Initialized search engine instance@@ -1017,6 +1039,12 @@ def get_search(if max_filtered_results is not None:params["max_filtered_results"] = max_filtered_results+ # Forward the bulk-fetch liveness hook to the FullSearchResults+ # wrapper below (via create_search_engine kwargs → signature+ # filtering). Only matters when use_full_search resolves True.+ if progress_callback is not None:+ params["progress_callback"] = progress_callback+# Resolve use_full_search for engines that support full search wrapping.# When search_snippets_only is not explicitly passed, defer to settings_snapshot# if present; otherwise default to False (matching BaseSearchEngine's snippet-only default).diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.pyindex fdf14fb111..1b1869aea1 100644--- a/tests/research_library/downloaders/test_extraction_pipeline.py+++ b/tests/research_library/downloaders/test_extraction_pipeline.py@@ -1069,6 +1069,181 @@ class TestBatchFetchAndExtract:mock_downloader.download.assert_called_once_with(unowned_url)+class TestBatchFetchProgress:+ """The ``progress_callback`` liveness hook."""++ def _html_only(self, monkeypatch):+ monkeypatch.setattr(+ pipeline,+ "_try_specialized_downloader",+ lambda url, timeout=30: _specialized(),+ )+ mock_downloader = Mock()+ mock_downloader.download.return_value = b"content"+ mock_downloader.close = Mock()+ monkeypatch.setattr(+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",+ Mock(return_value=mock_downloader),+ )++ def test_reports_start_and_each_url(self, monkeypatch):+ self._html_only(monkeypatch)+ calls = []+ urls = ["https://a.example/1", "https://a.example/2"]++ results = pipeline.batch_fetch_and_extract(+ urls,+ progress_callback=lambda done, total: calls.append((done, total)),+ )++ assert len(results) == 2+ assert calls == [(0, 2), (1, 2), (2, 2)]++ def test_specialized_hits_count_as_done(self, monkeypatch):+ def mock_specialized(url, timeout=30):+ if "arxiv" in url:+ return _specialized("arXiv paper content", False)+ return _specialized()++ monkeypatch.setattr(+ pipeline, "_try_specialized_downloader", mock_specialized+ )+ mock_downloader = Mock()+ mock_downloader.download.return_value = b"generic"+ mock_downloader.close = Mock()+ monkeypatch.setattr(+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",+ Mock(return_value=mock_downloader),+ )+ calls = []+ urls = ["https://arxiv.org/abs/1234", "https://example.com/page"]++ pipeline.batch_fetch_and_extract(+ urls,+ progress_callback=lambda done, total: calls.append((done, total)),+ )++ assert calls == [(0, 2), (1, 2), (2, 2)]++ def test_empty_list_reports_nothing(self, monkeypatch):+ self._html_only(monkeypatch)+ calls = []+ assert (+ pipeline.batch_fetch_and_extract([], progress_callback=calls.append)+ == {}+ )+ assert calls == []++ def test_termination_propagates_through_per_url_handler(self, monkeypatch):+ """A termination signal must not be swallowed as a URL failure."""+ from local_deep_research.exceptions import (+ ResearchTerminatedException,+ )++ self._html_only(monkeypatch)+ calls = []++ def stop_on_second(done, total):+ calls.append((done, total))+ if done >= 1:+ raise ResearchTerminatedException()++ with pytest.raises(ResearchTerminatedException):+ pipeline.batch_fetch_and_extract(+ ["https://a.example/1", "https://a.example/2"],+ progress_callback=stop_on_second,+ )+ assert calls[0] == (0, 2)++ def test_flaky_callback_does_not_corrupt_specialized_hit(self, monkeypatch):+ """A one-shot callback failure must not overwrite good content.++ Reproduces review item 1: URL 1 resolves to good specialized+ content, the callback raises once at ``done == 1``, and the+ unguarded ``except Exception`` used to mistake it for a+ downloader failure — falling through to the HTML path and+ overwriting the result with the fallback.+ """++ def mock_specialized(url, timeout=30):+ if "special" in url:+ return _specialized("good specialized content", False)+ return _specialized()++ monkeypatch.setattr(+ pipeline, "_try_specialized_downloader", mock_specialized+ )+ mock_downloader = Mock()+ mock_downloader.download.return_value = b"html fallback"+ mock_downloader.close = Mock()+ monkeypatch.setattr(+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",+ Mock(return_value=mock_downloader),+ )+ calls = []+ raised = {"done": False}++ def flaky(done, total):+ calls.append((done, total))+ if done == 1 and not raised["done"]:+ raised["done"] = True+ raise RuntimeError("flaky UI callback")++ urls = ["https://special.example/1", "https://example.com/2"]+ results = pipeline.batch_fetch_and_extract(+ urls, progress_callback=flaky+ )++ assert results["https://special.example/1"] == (+ "good specialized content"+ )+ assert results["https://example.com/2"] == "html fallback"+ assert calls[0] == (0, 2)+ assert calls[-1] == (2, 2)++ def test_flaky_callback_does_not_abort_html_batch(self, monkeypatch):+ """A flaky callback in the HTML loop must not fail the search."""+ self._html_only(monkeypatch)+ calls = []+ raised = {"done": False}++ def flaky(done, total):+ calls.append((done, total))+ if done == 1 and not raised["done"]:+ raised["done"] = True+ raise RuntimeError("flaky UI callback")++ results = pipeline.batch_fetch_and_extract(+ ["https://a.example/1", "https://a.example/2"],+ progress_callback=flaky,+ )++ assert results == {+ "https://a.example/1": "content",+ "https://a.example/2": "content",+ }+ assert calls[0] == (0, 2)+ assert calls[-1] == (2, 2)++ def test_initial_callback_failure_does_not_abort_batch(self, monkeypatch):+ """An exception on the upfront ``(0, total)`` report is swallowed."""+ self._html_only(monkeypatch)+ calls = []++ def flaky(done, total):+ calls.append((done, total))+ if done == 0:+ raise RuntimeError("flaky upfront callback")++ results = pipeline.batch_fetch_and_extract(+ ["https://a.example/1"],+ progress_callback=flaky,+ )++ assert results == {"https://a.example/1": "content"}+ assert calls[0] == (0, 1)++# ---------------------------------------------------------------------------# enable_js_rendering plumbing through fetch_and_extract /# batch_fetch_and_extract — issue #3826diff --git a/tests/strategies/test_langgraph_agent_strategy.py b/tests/strategies/test_langgraph_agent_strategy.pyindex 36638ceed2..a4611db484 100644--- a/tests/strategies/test_langgraph_agent_strategy.py+++ b/tests/strategies/test_langgraph_agent_strategy.py@@ -1920,6 +1920,193 @@ class TestSearchToolMakers:assert call_kwargs["settings_snapshot"] == snapshot+class TestFetchProgressAdapter:+ """Bulk-fetch ``(done, total)`` hook → throttled UI milestones."""++ def _adapter(self, callback):+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (+ _make_fetch_progress_adapter,+ )++ return _make_fetch_progress_adapter(callback)++ def test_none_callback_is_noop(self):+ hook = self._adapter(None)+ hook(0, 5)+ hook(3, 5)++ def test_start_announces_fetch(self):+ callback = MagicMock()+ hook = self._adapter(callback)++ hook(0, 46)++ milestones = [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 1+ msg, percent, metadata = milestones[0].args+ assert "46" in msg+ assert metadata["phase"] == "fetch"+ assert metadata["type"] == "milestone"+ assert metadata["total"] == 46++ def test_rapid_calls_are_throttled(self):+ callback = MagicMock()+ hook = self._adapter(callback)++ hook(0, 46)+ hook(1, 46)+ hook(2, 46)++ milestones = [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 1++ def test_heartbeat_after_interval(self, monkeypatch):+ import time as time_module++ callback = MagicMock()+ hook = self._adapter(callback)+ now = [1000.0]+ monkeypatch.setattr(time_module, "monotonic", lambda: now[0])++ hook(0, 46)+ hook(10, 46)+ assert (+ len(+ [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ )+ == 1+ )+ now[0] += 16.0+ hook(20, 46)++ milestones = [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 2+ assert "(20/46)" in milestones[1].args[0]++ def test_final_page_stays_silent(self):+ callback = MagicMock()+ hook = self._adapter(callback)++ hook(0, 46)+ hook(46, 46)++ milestones = [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 1++ def test_empty_total_stays_silent(self):+ callback = MagicMock()+ hook = self._adapter(callback)++ hook(0, 0)++ callback.assert_not_called()++ def test_termination_propagates(self):+ from local_deep_research.exceptions import (+ ResearchTerminatedException,+ )++ callback = MagicMock(side_effect=ResearchTerminatedException())+ hook = self._adapter(callback)++ with pytest.raises(ResearchTerminatedException):+ hook(0, 5)++ def test_web_search_tool_forwards_progress_callback(self):+ from unittest.mock import MagicMock, patch+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (+ SearchResultsCollector,+ _make_web_search_tool,+ )++ mock_engine = MagicMock()+ mock_engine.run.return_value = [+ {"title": "Result 1", "link": "http://a.com", "snippet": "s1"}+ ]+ collector = SearchResultsCollector()+ progress_callback = MagicMock()++ with patch(+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine",+ return_value=mock_engine,+ ) as mock_create:+ tool_fn = _make_web_search_tool(+ search_engine_name="duckduckgo",+ model=MagicMock(),+ settings_snapshot={},+ collector=collector,+ progress_callback=progress_callback,+ )+ tool_fn.invoke({"query": "test query"})++ _, call_kwargs = mock_create.call_args+ fetch_progress = call_kwargs["progress_callback"]+ assert callable(fetch_progress)+ # Post-#7189 agent invariant: snippets-only even when the global+ # toggle is off; the hook above is dormant groundwork.+ assert call_kwargs["search_snippets_only"] is True+ assert call_kwargs["use_full_search"] is False+ # The forwarded hook drives UI milestones.+ fetch_progress(0, 3)+ assert progress_callback.called+ milestone = [+ c+ for c in progress_callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestone) == 1++ def test_specialized_tool_forwards_progress_callback(self):+ from unittest.mock import MagicMock, patch+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (+ SearchResultsCollector,+ _make_specialized_search_tool,+ )++ mock_engine = MagicMock()+ mock_engine.run.return_value = []+ collector = SearchResultsCollector()++ with patch(+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine",+ return_value=mock_engine,+ ) as mock_create:+ tool_fn = _make_specialized_search_tool(+ engine_name="arxiv",+ description="Arxiv search",+ model=MagicMock(),+ settings_snapshot={},+ collector=collector,+ progress_callback=MagicMock(),+ )+ tool_fn.invoke({"query": "physics"})++ _, call_kwargs = mock_create.call_args+ assert callable(call_kwargs["progress_callback"])+ assert call_kwargs["search_snippets_only"] is True+ assert call_kwargs["use_full_search"] is False++# ---------------------------------------------------------------------------# Format results helper# ---------------------------------------------------------------------------diff --git a/tests/test_search_system.py b/tests/test_search_system.pyindex 9c40a6483a..156bbc40a9 100644--- a/tests/test_search_system.py+++ b/tests/test_search_system.py@@ -206,6 +206,45 @@ def test_set_progress_callback(monkeypatch):assert system.strategy.progress_callback == mock_callback+def test_set_progress_callback_wires_fetch_progress_to_engine(monkeypatch):+ """Pipeline live path: UI callback becomes a (done, total) engine hook."""+ from local_deep_research.search_system import AdvancedSearchSystem++ mock_llm_instance = Mock()+ mock_search_instance = Mock()+ # Engine exposes the late-binding setter.+ mock_search_instance.set_fetch_progress_callback = Mock()++ monkeypatch.setattr(+ "local_deep_research.config.llm_config.get_llm",+ lambda **kwargs: mock_llm_instance,+ )+ monkeypatch.setattr(+ "local_deep_research.config.search_config.get_search",+ lambda llm_instance=None, **kwargs: mock_search_instance,+ )++ system = AdvancedSearchSystem(+ llm=mock_llm_instance,+ search=mock_search_instance,+ settings_snapshot={},+ )+ ui_callback = Mock()+ system.set_progress_callback(ui_callback)++ mock_search_instance.set_fetch_progress_callback.assert_called_once()+ (hook,), _ = mock_search_instance.set_fetch_progress_callback.call_args+ assert callable(hook)+ # The hook drives throttled UI milestones.+ hook(0, 5)+ milestones = [+ c+ for c in ui_callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 1++@pytest.mark.requires_llmdef test_analyze_topic(monkeypatch):"""Test analyzing a topic."""diff --git a/tests/web_search_engines/engines/test_full_search.py b/tests/web_search_engines/engines/test_full_search.pyindex ca625a2b25..74a253d387 100644--- a/tests/web_search_engines/engines/test_full_search.py+++ b/tests/web_search_engines/engines/test_full_search.py@@ -481,6 +481,115 @@ class TestJSRenderingForwardingFromSettingsSnapshot:assert mock_batch.call_args.kwargs.get("enable_js_rendering") is True+class TestProgressCallbackForwarding:+ """``FullSearchResults`` forwards its ``(done, total)`` liveness hook+ to every ``batch_fetch_and_extract`` call, defaulting to ``None``."""++ def _patched_run(self, **engine_kwargs):+ """Run engine.run() with batch_fetch_and_extract patched and+ return the captured kwargs from that call."""+ from local_deep_research.web_search_engines.engines.full_search import (+ FullSearchResults,+ )++ mock_web_search = Mock()+ mock_web_search.invoke.return_value = [+ {"link": "https://example.com/1", "title": "Result 1"},+ ]++ with (+ patch(+ "local_deep_research.web_search_engines.engines.full_search.QUALITY_CHECK_DDG_URLS",+ False,+ ),+ patch(+ "local_deep_research.web_search_engines.engines.full_search.validate_url",+ return_value=True,+ ),+ patch(+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",+ return_value={"https://example.com/1": "content"},+ ) as mock_batch,+ ):+ FullSearchResults(+ llm=Mock(), web_search=mock_web_search, **engine_kwargs+ ).run("test query")+ assert mock_batch.call_args is not None+ return mock_batch.call_args.kwargs++ def test_run_defaults_to_no_progress_callback(self):+ kwargs = self._patched_run()+ assert kwargs.get("progress_callback") is None++ def test_run_forwards_progress_callback(self):+ hook = Mock()+ kwargs = self._patched_run(progress_callback=hook)+ assert kwargs.get("progress_callback") is hook++ def test_get_full_content_forwards_progress_callback(self):+ from local_deep_research.web_search_engines.engines.full_search import (+ FullSearchResults,+ )++ hook = Mock()+ with (+ patch(+ "local_deep_research.web_search_engines.engines.full_search.validate_url",+ return_value=True,+ ),+ patch(+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",+ return_value={"https://example.com/1": "content"},+ ) as mock_batch,+ ):+ FullSearchResults(+ llm=Mock(), web_search=Mock(), progress_callback=hook+ )._get_full_content([{"link": "https://example.com/1"}])+ assert mock_batch.call_args.kwargs.get("progress_callback") is hook+++class TestFetchProgressAdapterShared:+ """Pipeline + agent surfaces share one throttling implementation."""++ def test_adapter_lives_in_full_search_and_langgraph_reexports(self):+ from local_deep_research.web_search_engines.engines.full_search import (+ make_fetch_progress_adapter,+ )+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (+ _make_fetch_progress_adapter,+ )++ assert _make_fetch_progress_adapter is make_fetch_progress_adapter++ def test_adapter_announces_and_throttles(self):+ from local_deep_research.web_search_engines.engines.full_search import (+ make_fetch_progress_adapter,+ )++ callback = Mock()+ hook = make_fetch_progress_adapter(callback)+ hook(0, 10)+ hook(1, 10)+ milestones = [+ c+ for c in callback.call_args_list+ if c.args[2].get("type") == "milestone"+ ]+ assert len(milestones) == 1+ assert "10" in milestones[0].args[0]++ def test_set_fetch_progress_callback_late_binds(self):+ from local_deep_research.web_search_engines.engines.full_search import (+ FullSearchResults,+ )++ engine = FullSearchResults(llm=Mock(), web_search=Mock())+ assert engine.progress_callback is None+ hook = Mock()+ engine.set_fetch_progress_callback(hook)+ assert engine.progress_callback is hook++class TestCheckUrlsSyncAsyncSplit:"""The sync/async split of the URL-quality LLM call (#5854).diff --git a/tests/web_search_engines/test_factory_full_search_wrapper.py b/tests/web_search_engines/test_factory_full_search_wrapper.pyindex b043cde82a..e5f4cd8b99 100644--- a/tests/web_search_engines/test_factory_full_search_wrapper.py+++ b/tests/web_search_engines/test_factory_full_search_wrapper.py@@ -738,3 +738,97 @@ class TestWrapperPrivateResultFetchGrant:entry.module_path, entry.class_name)assert engine_class.allow_private_result_fetch is False, name+++class TestFetchProgressPipelineWiring:+ """Pipeline-strategy live path for bulk-fetch progress (#7193)."""++ def test_get_search_forwards_progress_callback_to_wrapper(self, mock_llm):+ """get_search(..., progress_callback=hook) reaches FullSearchResults."""+ hook = Mock()+ settings_snapshot = {+ "search.tool": {"value": "searxng"},+ "search.engine.web.searxng.supports_full_search": {+ "value": True,+ "ui_element": "checkbox",+ },+ "search.engine.web.searxng.api_key": {"value": "mock-api-key"},+ }+ # Patch the engine class resolution to avoid live imports.+ from local_deep_research.web_search_engines import (+ search_engine_factory,+ )++ real_gsmc = search_engine_factory.get_safe_module_class++ def _fake_gsmc(module_path: str, class_name: str):+ if class_name == "FullSearchResults":+ return real_gsmc(module_path, class_name)+ return _MockBaseEngine++ with patch(+ "local_deep_research.web_search_engines.search_engine_factory.get_safe_module_class",+ _fake_gsmc,+ ):+ result = get_search(+ search_tool="searxng",+ llm_instance=mock_llm,+ search_snippets_only=False,+ settings_snapshot=settings_snapshot,+ programmatic_mode=True,+ progress_callback=hook,+ )+ assert isinstance(result, FullSearchResults)+ assert result.progress_callback is hook++ def test_base_engine_set_fetch_progress_forwards_to_full_search(+ self, mock_llm+ ):+ """BaseSearchEngine late-binding reaches an existing wrapper."""+ engine = _MockBaseEngine(llm=mock_llm, programmatic_mode=True)+ engine.include_full_content = True+ engine.full_search = FullSearchResults(llm=mock_llm, web_search=Mock())+ hook = Mock()+ engine.set_fetch_progress_callback(hook)+ assert engine._fetch_progress_callback is hook+ assert engine.full_search.progress_callback is hook++ def test_create_search_engine_stamps_base_engine_hook(self, mock_llm):+ """Direct create_search_engine(..., progress_callback) stamps base."""+ from local_deep_research.web_search_engines import (+ search_engine_factory,+ )++ hook = Mock()+ settings_snapshot = {+ "search.tool": {"value": "searxng"},+ "search.engine.web.searxng.supports_full_search": {+ "value": True,+ "ui_element": "checkbox",+ },+ "search.engine.web.searxng.api_key": {"value": "mock-api-key"},+ }+ real_gsmc = search_engine_factory.get_safe_module_class++ def _fake_gsmc(module_path: str, class_name: str):+ if class_name == "FullSearchResults":+ return real_gsmc(module_path, class_name)+ return _MockBaseEngine++ with patch(+ "local_deep_research.web_search_engines.search_engine_factory.get_safe_module_class",+ _fake_gsmc,+ ):+ result = create_search_engine(+ "searxng",+ llm=mock_llm,+ settings_snapshot=settings_snapshot,+ programmatic_mode=True,+ search_snippets_only=True,+ use_full_search=False,+ progress_callback=hook,+ )+ # Snippets-only: no wrapper, but the base engine keeps the hook+ # for a future _init_full_search call.+ assert not isinstance(result, FullSearchResults)+ assert result._fetch_progress_callback is hook