Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(ux): report bulk page-fetch progress with throttled milestones (#7193)

S69eb0aecce52
diff --git a/changelog.d/+fetch-progress-heartbeats.bugfix.md b/changelog.d/+fetch-progress-heartbeats.bugfix.md
new file mode 100644
index 0000000000..ed9c21dbee
--- /dev/null
+++ b/changelog.d/+fetch-progress-heartbeats.bugfix.md
@@ -0,0 +1 @@
+Report bulk page-fetch progress to the UI with throttled milestones and per-page cancellation checks (live via pipeline-strategy FullSearchResults; agent tools stay snippets-only post-#7189 as dormant groundwork), guard the progress hook so flaky callbacks cannot corrupt results, and quiet per-URL extraction success logs.
diff --git a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
index 972ce85f52..5013f5509a 100644
--- a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
+++ b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
@@ -88,6 +88,10 @@ from .primary_search_metadata import (
classify_primary_source,
format_primary_search_description,
)
+from local_deep_research.web_search_engines.engines.full_search import (
+ _FETCH_HEARTBEAT_SECONDS as _FETCH_HEARTBEAT_SECONDS,
+ make_fetch_progress_adapter as _make_fetch_progress_adapter,
+)
# ---------------------------------------------------------------------------
# Constants
@@ -1425,6 +1429,14 @@ def _format_results(results: list[dict], start_idx: int) -> str:
return "\n\n".join(lines) if lines else "No results."
+# Bulk-fetch progress adapter lives in the full-search engine module so
+# the pipeline-strategy wiring (``AdvancedSearchSystem`` → ``get_search``
+# → ``FullSearchResults``) and the agent tool factories below share one
+# throttling implementation. Imported at top as
+# ``_make_fetch_progress_adapter`` for backwards compatibility with
+# existing tests.
+
+
def _make_web_search_tool(
search_engine_name: str,
model: BaseChatModel,
@@ -1433,6 +1445,7 @@ def _make_web_search_tool(
programmatic_mode: bool = False,
description: str = NEUTRAL_PRIMARY_SEARCH_DESCRIPTION,
availability: SearchEngineAvailability | None = None,
+ progress_callback=None,
):
"""Create a primary search tool with research-scoped availability."""
availability = availability or SearchEngineAvailability()
@@ -1444,6 +1457,11 @@ def _make_web_search_tool(
create_search_engine,
)
+ # Per-invocation adapter (not shared): subagents reuse tool
+ # objects across pool workers, so heartbeat state must not
+ # leak between concurrent searches.
+ fetch_progress = _make_fetch_progress_adapter(progress_callback)
+
results = run_search(
search_engine_name,
query,
@@ -1452,6 +1470,21 @@ def _make_web_search_tool(
llm=model,
settings_snapshot=settings_snapshot,
programmatic_mode=programmatic_mode,
+ progress_callback=fetch_progress,
+ # Documented agent invariant (CONFIGURATION.md
+ # ``search.snippets_only``): autonomous agents always
+ # search snippets and fetch full pages on demand via
+ # fetch_content. Force it here so a global
+ # ``search.snippets_only=false`` (meant for pipeline
+ # strategies) cannot silently turn every web_search
+ # into a FullSearchResults bulk page-fetch. Post-#7189
+ # this keeps the hook above as dormant groundwork for a
+ # future agent full-search mode; the live bulk-fetch
+ # progress path is the pipeline strategy via
+ # ``get_search(..., progress_callback=...)`` →
+ # ``FullSearchResults``.
+ search_snippets_only=True,
+ use_full_search=False,
),
availability,
programmatic_mode=programmatic_mode,
@@ -1480,6 +1513,7 @@ def _make_specialized_search_tool(
collector: SearchResultsCollector,
programmatic_mode: bool = False,
availability: SearchEngineAvailability | None = None,
+ progress_callback=None,
):
"""Create a specialized search tool with research-scoped availability."""
availability = availability or SearchEngineAvailability()
@@ -1491,6 +1525,9 @@ def _make_specialized_search_tool(
create_search_engine,
)
+ # Per-invocation adapter (not shared): see web_search above.
+ fetch_progress = _make_fetch_progress_adapter(progress_callback)
+
results = run_search(
engine_name,
query,
@@ -1499,6 +1536,13 @@ def _make_specialized_search_tool(
llm=model,
settings_snapshot=settings_snapshot,
programmatic_mode=programmatic_mode,
+ progress_callback=fetch_progress,
+ # Same documented agent invariant as web_search above:
+ # specialized agent tools stay snippets-only even when
+ # the global search.snippets_only toggle is off.
+ # Dormant groundwork post-#7189; see web_search comment.
+ search_snippets_only=True,
+ use_full_search=False,
),
availability,
programmatic_mode=programmatic_mode,
@@ -1525,6 +1569,7 @@ def _load_specialized_engine_tools(
programmatic_mode: bool = False,
egress_context=None,
availability: SearchEngineAvailability | None = None,
+ progress_callback=None,
) -> list:
"""Load tools for all available specialized search engines, filtered by
egress policy and per-engine ``agent_enabled`` flag.
@@ -1673,6 +1718,7 @@ def _load_specialized_engine_tools(
collector,
programmatic_mode=programmatic_mode,
availability=availability,
+ progress_callback=progress_callback,
)
)
except Exception:
@@ -1813,6 +1859,7 @@ def _make_research_subtopic_tool(
programmatic_mode=programmatic_mode,
description=web_search_description,
availability=availability,
+ progress_callback=progress_callback,
)
)
sub_fetch = build_fetch_tool(
@@ -1845,6 +1892,7 @@ def _make_research_subtopic_tool(
programmatic_mode=programmatic_mode,
egress_context=egress_context,
availability=availability,
+ progress_callback=progress_callback,
)
)
@@ -2909,6 +2957,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
programmatic_mode=self.programmatic_mode,
description=primary_search_description,
availability=self.engine_availability,
+ progress_callback=self.progress_callback,
)
)
@@ -2948,6 +2997,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
programmatic_mode=self.programmatic_mode,
egress_context=policy_ctx,
availability=self.engine_availability,
+ progress_callback=self.progress_callback,
)
)
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
index d9069aae69..b275e0d330 100644
--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
@@ -14,7 +14,7 @@ specialized downloaders first. Generic HTML runs only when that route permits it
"""
from importlib import import_module
-from typing import Any, Dict, List, NamedTuple, Optional
+from typing import Any, Callable, Dict, List, NamedTuple, Optional
from bs4 import BeautifulSoup, NavigableString
from loguru import logger
@@ -655,6 +655,7 @@ def batch_fetch_and_extract(
enable_js_rendering: bool = False,
allow_private_ips: bool = False,
block_link_local: bool = False,
+ progress_callback: Optional[Callable[[int, int], None]] = None,
) -> Dict[str, Optional[str]]:
"""Fetch multiple URLs and extract clean text from each.
@@ -688,6 +689,17 @@ def batch_fetch_and_extract(
block_link_local: Keep the whole link-local range blocked even when
``allow_private_ips`` is True (forwarded to the downloader's
static-fetch ``SafeSession``). No effect without the grant.
+ progress_callback: Optional ``(done, total)`` hook invoked once
+ up front with ``(0, len(urls))`` and after each URL
+ completes. Lets a long serial fetch report liveness (and
+ notice cancellation) instead of going silent for minutes.
+ A regular ``Exception`` raised by the hook is swallowed
+ (debug-logged) so a flaky UI callback can neither corrupt an
+ already-fetched result nor abort the remaining batch;
+ ``BaseException`` — including ``ResearchTerminatedException``,
+ which derives ``BaseException`` and therefore passes through
+ the per-URL ``except Exception`` handlers untouched — still
+ propagates so cancellation takes effect promptly.
Returns:
Dict mapping URL → extracted text (or None if failed).
@@ -695,6 +707,28 @@ def batch_fetch_and_extract(
from ..playwright_html import AutoHTMLDownloader
results: Dict[str, Optional[str]] = {}
+ total = len(urls)
+ done = 0
+
+ def _report_progress() -> None:
+ if progress_callback is not None and total:
+ try:
+ progress_callback(done, total)
+ except Exception:
+ # A flaky UI callback must not corrupt already-fetched
+ # results (specialized loop: the ``except Exception``
+ # below would otherwise mistake it for a downloader
+ # failure and overwrite good content with the HTML
+ # fallback) nor abort the remaining batch (HTML loop).
+ # ``except Exception`` deliberately lets ``BaseException``
+ # — including ``ResearchTerminatedException`` — propagate
+ # for prompt cancellation.
+ logger.debug(
+ "batch_fetch_and_extract progress_callback failed; "
+ "continuing batch"
+ )
+
+ _report_progress()
# Try specialized downloaders first — collect URLs that need HTML fallback
html_urls: List[str] = []
@@ -703,6 +737,8 @@ def batch_fetch_and_extract(
specialized = _try_specialized_downloader(url, timeout=timeout)
if specialized.content:
results[url] = specialized.content
+ done += 1
+ _report_progress()
continue
if not specialized.fallback_allowed:
logger.warning(
@@ -713,6 +749,8 @@ def batch_fetch_and_extract(
),
)
results[url] = None
+ done += 1
+ _report_progress()
continue
except Exception as e:
logger.opt(exception=False).debug(
@@ -721,6 +759,8 @@ def batch_fetch_and_extract(
)
if is_arxiv_paper_url(url):
results[url] = None
+ done += 1
+ _report_progress()
continue
html_urls.append(url)
@@ -747,6 +787,8 @@ def batch_fetch_and_extract(
f"{redact_url_for_log(url)} ({type(e).__name__})"
)
results[url] = None
+ done += 1
+ _report_progress()
finally:
try:
downloader.close()
diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.py
index 7a47daf9b1..727760aadc 100644
--- a/src/local_deep_research/research_library/downloaders/html.py
+++ b/src/local_deep_research/research_library/downloaders/html.py
@@ -696,7 +696,7 @@ class HTMLDownloader(BaseDownloader):
title = result.get("title")
content = result["content"]
- logger.info(
+ logger.debug(
"html.extract_succeeded url={url} content_length={content_length}",
url=redact_url_for_log(url),
content_length=len(content),
diff --git a/src/local_deep_research/search_system.py b/src/local_deep_research/search_system.py
index 27d46bf778..c4f53aca4b 100644
--- a/src/local_deep_research/search_system.py
+++ b/src/local_deep_research/search_system.py
@@ -319,6 +319,28 @@ class AdvancedSearchSystem:
self.progress_callback = callback
if hasattr(self, "strategy"):
self.strategy.set_progress_callback(callback)
+ # Pipeline-strategy live path for bulk-fetch progress (#7193):
+ # adapt the 3-arg UI callback to the 2-arg ``(done, total)`` hook
+ # and attach it to the search engine so ``FullSearchResults``
+ # bulk fetches emit throttled milestones + per-page cancellation
+ # checks. Late-binds so engines created via ``get_search`` before
+ # the callback existed still get progress; agents stay snippets-
+ # only post-#7189, so their hook remains dormant groundwork.
+ try:
+ search = getattr(self, "search", None)
+ if search is not None and callback is not None:
+ from .web_search_engines.engines.full_search import (
+ make_fetch_progress_adapter,
+ )
+
+ hook = make_fetch_progress_adapter(callback)
+ setter = getattr(search, "set_fetch_progress_callback", None)
+ if callable(setter):
+ setter(hook)
+ elif hasattr(search, "progress_callback"):
+ search.progress_callback = hook
+ except Exception:
+ logger.debug("Failed to wire fetch progress to search engine")
def analyze_topic(
self,
diff --git a/src/local_deep_research/web_search_engines/engines/full_search.py b/src/local_deep_research/web_search_engines/engines/full_search.py
index d22ff5292c..c54bacae6a 100644
--- a/src/local_deep_research/web_search_engines/engines/full_search.py
+++ b/src/local_deep_research/web_search_engines/engines/full_search.py
@@ -1,5 +1,14 @@
+import time
from datetime import datetime, UTC
-from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ List,
+ Optional,
+ Protocol,
+ runtime_checkable,
+)
from langchain_core.language_models import BaseLLM
@@ -25,6 +34,77 @@ class _Invokable(Protocol):
def invoke(self, query: str) -> Any: ...
+# Minimum seconds between bulk-fetch heartbeat milestones. Each UI
+# emit is a DB row + socket event, so per-page emits would spam the
+# research log on a 46-page fetch; the start announcement is always
+# emitted and the completion milestone ("From the web …") follows the
+# fetch, so heartbeats only fill the silence in between.
+_FETCH_HEARTBEAT_SECONDS = 15.0
+
+
+def make_fetch_progress_adapter(progress_callback):
+ """Adapt bulk-fetch ``(done, total)`` progress to UI milestones.
+
+ Shared by the pipeline-strategy wiring (``AdvancedSearchSystem`` →
+ ``FullSearchResults``) and the langgraph-agent tool factories, so
+ both surfaces throttle identically.
+
+ Returns a hook for engine page-fetch progress that:
+ * announces the fetch upfront (``done == 0``) so a minutes-long
+ fetch never looks stalled,
+ * emits a heartbeat at most every ``_FETCH_HEARTBEAT_SECONDS``,
+ * stays silent on the final page — the completion milestone
+ ("From the web …") follows immediately,
+ * performs a silent termination check on every page (no UI
+ emission): each check doubles as a cancellation check, because
+ the UI callback raises ``ResearchTerminatedException`` when the
+ user cancels — and that derives ``BaseException``, so it
+ propagates through the fetch pipeline's ``except Exception``
+ handlers untouched. Cancellation during a long fetch therefore
+ takes effect promptly instead of only after the whole batch.
+
+ ``progress_callback`` is the 3-arg UI callback
+ ``(message, percent, metadata)``; ``None`` disables everything.
+ """
+ state = {"last_emit": 0.0}
+
+ def _on_fetch_progress(done: int, total: int) -> None:
+ if progress_callback is None or total <= 0:
+ return
+ # Silent cancellation check on every page — no UI/logging.
+ progress_callback(
+ "Checking termination status",
+ None,
+ {"phase": "termination_check"},
+ )
+ if done >= total:
+ return
+ now = time.monotonic()
+ if done > 0 and now - state["last_emit"] < _FETCH_HEARTBEAT_SECONDS:
+ return
+ state["last_emit"] = now
+ if done <= 0:
+ progress_callback(
+ f"📄 Fetching full content for {total} pages — "
+ "this can take a minute…",
+ None,
+ {"phase": "fetch", "type": "milestone", "total": total},
+ )
+ else:
+ progress_callback(
+ f"📄 Fetching page content ({done}/{total})…",
+ None,
+ {
+ "phase": "fetch",
+ "type": "milestone",
+ "done": done,
+ "total": total,
+ },
+ )
+
+ return _on_fetch_progress
+
+
# Egress scopes whose ``evaluate_url`` admits every http(s) host the SSRF
# validator admits (UNPROTECTED lifts only the scope restriction; the hard
# SSRF and cloud-metadata blocks stay and the download pipeline enforces
@@ -52,6 +132,7 @@ class FullSearchResults:
settings_snapshot: Optional[Dict] = None,
egress_context: Optional[Any] = None,
allow_private_ips: bool = False,
+ progress_callback: Optional[Callable[[int, int], None]] = None,
):
self.llm = llm
self.output_format = output_format
@@ -75,10 +156,24 @@ class FullSearchResults:
# Link-local and the cloud-metadata literals in
# ALWAYS_BLOCKED_METADATA_IPS stay blocked regardless (issue #2477).
self.allow_private_ips = allow_private_ips
+ # Optional ``(done, total)`` liveness hook for the bulk page
+ # fetch below (see ``batch_fetch_and_extract``). Threaded in by
+ # the pipeline strategy (``AdvancedSearchSystem`` → ``get_search``)
+ # and by agent search tools so a minutes-long fetch emits progress
+ # instead of silence; ``None`` disables it with zero overhead.
+ self.progress_callback = progress_callback
# Set by the factory when the parent engine is gated against a
# specific scope; used to evaluate per-URL fetches below.
self.egress_context = egress_context
+ def set_fetch_progress_callback(self, hook) -> None:
+ """Late-bind the ``(done, total)`` bulk-fetch hook.
+
+ Used by ``AdvancedSearchSystem.set_progress_callback`` when the UI
+ callback arrives after engine construction.
+ """
+ self.progress_callback = hook
+
@property
def last_search_failure(self) -> SearchFailure | None:
"""Expose provider failures through the optional full-content wrapper."""
@@ -330,6 +425,7 @@ class FullSearchResults:
),
allow_private_ips=self._downloader_allow_private_ips(),
block_link_local=True,
+ progress_callback=self.progress_callback,
)
nr_full_text = sum(1 for v in url_to_content.values() if v)
@@ -391,6 +487,7 @@ class FullSearchResults:
),
allow_private_ips=self._downloader_allow_private_ips(),
block_link_local=True,
+ progress_callback=self.progress_callback,
)
except Exception as e:
safe_msg = scrub_error(e)
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index c88ee64f02..0f276c708d 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -403,6 +403,7 @@ class BaseSearchEngine(ABC):
include_full_content: bool = False,
settings_snapshot: Optional[Dict[str, Any]] = None,
programmatic_mode: bool = False,
+ progress_callback=None,
**kwargs,
):
"""
@@ -420,6 +421,10 @@ class BaseSearchEngine(ABC):
include_full_content: Whether to use FullSearchResults for full webpage content
settings_snapshot: Settings snapshot for configuration
programmatic_mode: If True, disables database operations and uses memory-only tracking
+ progress_callback: Optional ``(done, total)`` bulk-fetch liveness
+ hook forwarded to ``FullSearchResults`` via
+ ``_init_full_search`` / ``set_fetch_progress_callback``
+ (pipeline-strategy fetch progress, #7193).
**kwargs: Additional engine-specific parameters
"""
if max_filtered_results is None:
@@ -445,6 +450,12 @@ class BaseSearchEngine(ABC):
self.settings_snapshot = (
settings_snapshot or {}
) # Store settings snapshot
+ # Bulk-fetch liveness hook (``(done, total)``) for full-search
+ # progress. Set at construction via the factory or later via
+ # ``set_fetch_progress_callback`` (e.g. when
+ # ``AdvancedSearchSystem.set_progress_callback`` arrives after
+ # engine creation). Forwarded to ``FullSearchResults``.
+ self._fetch_progress_callback = progress_callback
self.engine_type = self.__class__.__name__
self._engine_name: str = "" # set by the factory after construction
@@ -1380,6 +1391,23 @@ class BaseSearchEngine(ABC):
self.include_full_content = False
return None
+ def set_fetch_progress_callback(self, hook) -> None:
+ """Attach a ``(done, total)`` bulk-fetch liveness hook.
+
+ Stores the hook for future ``_init_full_search`` calls and, when
+ this engine already holds a ``FullSearchResults`` wrapper (legacy
+ ``include_full_content`` seam), forwards it immediately. Used by
+ the factory stamp and by ``AdvancedSearchSystem`` late-binding so
+ pipeline-strategy full-search progress works regardless of whether
+ the engine was created before the UI callback existed.
+ """
+ self._fetch_progress_callback = hook
+ full_search = getattr(self, "full_search", None)
+ if full_search is not None and hasattr(
+ full_search, "progress_callback"
+ ):
+ full_search.progress_callback = hook
+
def _init_full_search(
self,
web_search=None,
@@ -1388,6 +1416,7 @@ class BaseSearchEngine(ABC):
region=None,
time_period=None,
safe_search=None,
+ progress_callback=None,
):
"""Initialize FullSearchResults if include_full_content is True.
@@ -1400,6 +1429,9 @@ class BaseSearchEngine(ABC):
region: Region/country code for results
time_period: Time period filter
safe_search: Safe search setting (string value for FullSearchResults)
+ progress_callback: Optional ``(done, total)`` liveness hook for
+ the bulk page fetch. Defaults to the hook stored on
+ ``self`` (construction-time or ``set_fetch_progress_callback``).
"""
if self.include_full_content and self.llm:
try:
@@ -1425,6 +1457,11 @@ class BaseSearchEngine(ABC):
# explicit boolean grant, nothing else, so an engine
# built through this seam does not lose it silently.
allow_private_ips=self.allow_private_result_fetch is True,
+ progress_callback=(
+ progress_callback
+ if progress_callback is not None
+ else self._fetch_progress_callback
+ ),
)
except ImportError:
logger.warning(
diff --git a/src/local_deep_research/web_search_engines/search_engine_factory.py b/src/local_deep_research/web_search_engines/search_engine_factory.py
index 313d8a5cb3..e5dab2dc43 100644
--- a/src/local_deep_research/web_search_engines/search_engine_factory.py
+++ b/src/local_deep_research/web_search_engines/search_engine_factory.py
@@ -53,6 +53,7 @@ _BASE_FORWARDED_PARAMS = frozenset(
"language",
"time_period",
"include_full_content",
+ "progress_callback",
}
)
@@ -681,6 +682,22 @@ def create_search_engine(
all_params["search_snippets_only"]
)
+ # Stamp the bulk-fetch liveness hook for engines built through
+ # this seam (pipeline-strategy full-search progress). The base
+ # engine itself does not fetch, but `_init_full_search` and
+ # `set_fetch_progress_callback` forward it to FullSearchResults.
+ if isinstance(engine, BaseSearchEngine) and (
+ "progress_callback" in all_params
+ ):
+ try:
+ engine.set_fetch_progress_callback(
+ all_params["progress_callback"]
+ )
+ except Exception:
+ logger.debug(
+ "Failed to stamp progress_callback on engine",
+ )
+
# Determine if this engine should use LLM relevance filtering
# Priority: per-engine setting > needs_llm_relevance_filter > global setting
#
@@ -983,6 +1000,7 @@ def get_search(
max_filtered_results: Optional[int] = None,
settings_snapshot: Dict[str, Any] | None = None,
programmatic_mode: bool = False,
+ progress_callback=None,
):
"""
Get search tool instance based on the provided parameters.
@@ -999,6 +1017,10 @@ def get_search(
search_language: Language for search results
max_filtered_results: Maximum number of results to keep after filtering
programmatic_mode: If True, disables database operations and metrics tracking
+ progress_callback: Optional ``(done, total)`` bulk-fetch liveness hook,
+ forwarded to ``FullSearchResults`` when the engine is wrapped for
+ full search. This is the pipeline-strategy live path for fetch
+ progress (see #7193); ``None`` disables it.
Returns:
Initialized search engine instance
@@ -1017,6 +1039,12 @@ def get_search(
if max_filtered_results is not None:
params["max_filtered_results"] = max_filtered_results
+ # Forward the bulk-fetch liveness hook to the FullSearchResults
+ # wrapper below (via create_search_engine kwargs → signature
+ # filtering). Only matters when use_full_search resolves True.
+ if progress_callback is not None:
+ params["progress_callback"] = progress_callback
+
# Resolve use_full_search for engines that support full search wrapping.
# When search_snippets_only is not explicitly passed, defer to settings_snapshot
# if present; otherwise default to False (matching BaseSearchEngine's snippet-only default).
diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.py
index fdf14fb111..1b1869aea1 100644
--- a/tests/research_library/downloaders/test_extraction_pipeline.py
+++ b/tests/research_library/downloaders/test_extraction_pipeline.py
@@ -1069,6 +1069,181 @@ class TestBatchFetchAndExtract:
mock_downloader.download.assert_called_once_with(unowned_url)
+class TestBatchFetchProgress:
+ """The ``progress_callback`` liveness hook."""
+
+ def _html_only(self, monkeypatch):
+ monkeypatch.setattr(
+ pipeline,
+ "_try_specialized_downloader",
+ lambda url, timeout=30: _specialized(),
+ )
+ mock_downloader = Mock()
+ mock_downloader.download.return_value = b"content"
+ mock_downloader.close = Mock()
+ monkeypatch.setattr(
+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
+ Mock(return_value=mock_downloader),
+ )
+
+ def test_reports_start_and_each_url(self, monkeypatch):
+ self._html_only(monkeypatch)
+ calls = []
+ urls = ["https://a.example/1", "https://a.example/2"]
+
+ results = pipeline.batch_fetch_and_extract(
+ urls,
+ progress_callback=lambda done, total: calls.append((done, total)),
+ )
+
+ assert len(results) == 2
+ assert calls == [(0, 2), (1, 2), (2, 2)]
+
+ def test_specialized_hits_count_as_done(self, monkeypatch):
+ def mock_specialized(url, timeout=30):
+ if "arxiv" in url:
+ return _specialized("arXiv paper content", False)
+ return _specialized()
+
+ monkeypatch.setattr(
+ pipeline, "_try_specialized_downloader", mock_specialized
+ )
+ mock_downloader = Mock()
+ mock_downloader.download.return_value = b"generic"
+ mock_downloader.close = Mock()
+ monkeypatch.setattr(
+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
+ Mock(return_value=mock_downloader),
+ )
+ calls = []
+ urls = ["https://arxiv.org/abs/1234", "https://example.com/page"]
+
+ pipeline.batch_fetch_and_extract(
+ urls,
+ progress_callback=lambda done, total: calls.append((done, total)),
+ )
+
+ assert calls == [(0, 2), (1, 2), (2, 2)]
+
+ def test_empty_list_reports_nothing(self, monkeypatch):
+ self._html_only(monkeypatch)
+ calls = []
+ assert (
+ pipeline.batch_fetch_and_extract([], progress_callback=calls.append)
+ == {}
+ )
+ assert calls == []
+
+ def test_termination_propagates_through_per_url_handler(self, monkeypatch):
+ """A termination signal must not be swallowed as a URL failure."""
+ from local_deep_research.exceptions import (
+ ResearchTerminatedException,
+ )
+
+ self._html_only(monkeypatch)
+ calls = []
+
+ def stop_on_second(done, total):
+ calls.append((done, total))
+ if done >= 1:
+ raise ResearchTerminatedException()
+
+ with pytest.raises(ResearchTerminatedException):
+ pipeline.batch_fetch_and_extract(
+ ["https://a.example/1", "https://a.example/2"],
+ progress_callback=stop_on_second,
+ )
+ assert calls[0] == (0, 2)
+
+ def test_flaky_callback_does_not_corrupt_specialized_hit(self, monkeypatch):
+ """A one-shot callback failure must not overwrite good content.
+
+ Reproduces review item 1: URL 1 resolves to good specialized
+ content, the callback raises once at ``done == 1``, and the
+ unguarded ``except Exception`` used to mistake it for a
+ downloader failure — falling through to the HTML path and
+ overwriting the result with the fallback.
+ """
+
+ def mock_specialized(url, timeout=30):
+ if "special" in url:
+ return _specialized("good specialized content", False)
+ return _specialized()
+
+ monkeypatch.setattr(
+ pipeline, "_try_specialized_downloader", mock_specialized
+ )
+ mock_downloader = Mock()
+ mock_downloader.download.return_value = b"html fallback"
+ mock_downloader.close = Mock()
+ monkeypatch.setattr(
+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
+ Mock(return_value=mock_downloader),
+ )
+ calls = []
+ raised = {"done": False}
+
+ def flaky(done, total):
+ calls.append((done, total))
+ if done == 1 and not raised["done"]:
+ raised["done"] = True
+ raise RuntimeError("flaky UI callback")
+
+ urls = ["https://special.example/1", "https://example.com/2"]
+ results = pipeline.batch_fetch_and_extract(
+ urls, progress_callback=flaky
+ )
+
+ assert results["https://special.example/1"] == (
+ "good specialized content"
+ )
+ assert results["https://example.com/2"] == "html fallback"
+ assert calls[0] == (0, 2)
+ assert calls[-1] == (2, 2)
+
+ def test_flaky_callback_does_not_abort_html_batch(self, monkeypatch):
+ """A flaky callback in the HTML loop must not fail the search."""
+ self._html_only(monkeypatch)
+ calls = []
+ raised = {"done": False}
+
+ def flaky(done, total):
+ calls.append((done, total))
+ if done == 1 and not raised["done"]:
+ raised["done"] = True
+ raise RuntimeError("flaky UI callback")
+
+ results = pipeline.batch_fetch_and_extract(
+ ["https://a.example/1", "https://a.example/2"],
+ progress_callback=flaky,
+ )
+
+ assert results == {
+ "https://a.example/1": "content",
+ "https://a.example/2": "content",
+ }
+ assert calls[0] == (0, 2)
+ assert calls[-1] == (2, 2)
+
+ def test_initial_callback_failure_does_not_abort_batch(self, monkeypatch):
+ """An exception on the upfront ``(0, total)`` report is swallowed."""
+ self._html_only(monkeypatch)
+ calls = []
+
+ def flaky(done, total):
+ calls.append((done, total))
+ if done == 0:
+ raise RuntimeError("flaky upfront callback")
+
+ results = pipeline.batch_fetch_and_extract(
+ ["https://a.example/1"],
+ progress_callback=flaky,
+ )
+
+ assert results == {"https://a.example/1": "content"}
+ assert calls[0] == (0, 1)
+
+
# ---------------------------------------------------------------------------
# enable_js_rendering plumbing through fetch_and_extract /
# batch_fetch_and_extract — issue #3826
diff --git a/tests/strategies/test_langgraph_agent_strategy.py b/tests/strategies/test_langgraph_agent_strategy.py
index 36638ceed2..a4611db484 100644
--- a/tests/strategies/test_langgraph_agent_strategy.py
+++ b/tests/strategies/test_langgraph_agent_strategy.py
@@ -1920,6 +1920,193 @@ class TestSearchToolMakers:
assert call_kwargs["settings_snapshot"] == snapshot
+class TestFetchProgressAdapter:
+ """Bulk-fetch ``(done, total)`` hook → throttled UI milestones."""
+
+ def _adapter(self, callback):
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ _make_fetch_progress_adapter,
+ )
+
+ return _make_fetch_progress_adapter(callback)
+
+ def test_none_callback_is_noop(self):
+ hook = self._adapter(None)
+ hook(0, 5)
+ hook(3, 5)
+
+ def test_start_announces_fetch(self):
+ callback = MagicMock()
+ hook = self._adapter(callback)
+
+ hook(0, 46)
+
+ milestones = [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 1
+ msg, percent, metadata = milestones[0].args
+ assert "46" in msg
+ assert metadata["phase"] == "fetch"
+ assert metadata["type"] == "milestone"
+ assert metadata["total"] == 46
+
+ def test_rapid_calls_are_throttled(self):
+ callback = MagicMock()
+ hook = self._adapter(callback)
+
+ hook(0, 46)
+ hook(1, 46)
+ hook(2, 46)
+
+ milestones = [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 1
+
+ def test_heartbeat_after_interval(self, monkeypatch):
+ import time as time_module
+
+ callback = MagicMock()
+ hook = self._adapter(callback)
+ now = [1000.0]
+ monkeypatch.setattr(time_module, "monotonic", lambda: now[0])
+
+ hook(0, 46)
+ hook(10, 46)
+ assert (
+ len(
+ [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ )
+ == 1
+ )
+ now[0] += 16.0
+ hook(20, 46)
+
+ milestones = [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 2
+ assert "(20/46)" in milestones[1].args[0]
+
+ def test_final_page_stays_silent(self):
+ callback = MagicMock()
+ hook = self._adapter(callback)
+
+ hook(0, 46)
+ hook(46, 46)
+
+ milestones = [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 1
+
+ def test_empty_total_stays_silent(self):
+ callback = MagicMock()
+ hook = self._adapter(callback)
+
+ hook(0, 0)
+
+ callback.assert_not_called()
+
+ def test_termination_propagates(self):
+ from local_deep_research.exceptions import (
+ ResearchTerminatedException,
+ )
+
+ callback = MagicMock(side_effect=ResearchTerminatedException())
+ hook = self._adapter(callback)
+
+ with pytest.raises(ResearchTerminatedException):
+ hook(0, 5)
+
+ def test_web_search_tool_forwards_progress_callback(self):
+ from unittest.mock import MagicMock, patch
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ SearchResultsCollector,
+ _make_web_search_tool,
+ )
+
+ mock_engine = MagicMock()
+ mock_engine.run.return_value = [
+ {"title": "Result 1", "link": "http://a.com", "snippet": "s1"}
+ ]
+ collector = SearchResultsCollector()
+ progress_callback = MagicMock()
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine",
+ return_value=mock_engine,
+ ) as mock_create:
+ tool_fn = _make_web_search_tool(
+ search_engine_name="duckduckgo",
+ model=MagicMock(),
+ settings_snapshot={},
+ collector=collector,
+ progress_callback=progress_callback,
+ )
+ tool_fn.invoke({"query": "test query"})
+
+ _, call_kwargs = mock_create.call_args
+ fetch_progress = call_kwargs["progress_callback"]
+ assert callable(fetch_progress)
+ # Post-#7189 agent invariant: snippets-only even when the global
+ # toggle is off; the hook above is dormant groundwork.
+ assert call_kwargs["search_snippets_only"] is True
+ assert call_kwargs["use_full_search"] is False
+ # The forwarded hook drives UI milestones.
+ fetch_progress(0, 3)
+ assert progress_callback.called
+ milestone = [
+ c
+ for c in progress_callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestone) == 1
+
+ def test_specialized_tool_forwards_progress_callback(self):
+ from unittest.mock import MagicMock, patch
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ SearchResultsCollector,
+ _make_specialized_search_tool,
+ )
+
+ mock_engine = MagicMock()
+ mock_engine.run.return_value = []
+ collector = SearchResultsCollector()
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine",
+ return_value=mock_engine,
+ ) as mock_create:
+ tool_fn = _make_specialized_search_tool(
+ engine_name="arxiv",
+ description="Arxiv search",
+ model=MagicMock(),
+ settings_snapshot={},
+ collector=collector,
+ progress_callback=MagicMock(),
+ )
+ tool_fn.invoke({"query": "physics"})
+
+ _, call_kwargs = mock_create.call_args
+ assert callable(call_kwargs["progress_callback"])
+ assert call_kwargs["search_snippets_only"] is True
+ assert call_kwargs["use_full_search"] is False
+
+
# ---------------------------------------------------------------------------
# Format results helper
# ---------------------------------------------------------------------------
diff --git a/tests/test_search_system.py b/tests/test_search_system.py
index 9c40a6483a..156bbc40a9 100644
--- a/tests/test_search_system.py
+++ b/tests/test_search_system.py
@@ -206,6 +206,45 @@ def test_set_progress_callback(monkeypatch):
assert system.strategy.progress_callback == mock_callback
+def test_set_progress_callback_wires_fetch_progress_to_engine(monkeypatch):
+ """Pipeline live path: UI callback becomes a (done, total) engine hook."""
+ from local_deep_research.search_system import AdvancedSearchSystem
+
+ mock_llm_instance = Mock()
+ mock_search_instance = Mock()
+ # Engine exposes the late-binding setter.
+ mock_search_instance.set_fetch_progress_callback = Mock()
+
+ monkeypatch.setattr(
+ "local_deep_research.config.llm_config.get_llm",
+ lambda **kwargs: mock_llm_instance,
+ )
+ monkeypatch.setattr(
+ "local_deep_research.config.search_config.get_search",
+ lambda llm_instance=None, **kwargs: mock_search_instance,
+ )
+
+ system = AdvancedSearchSystem(
+ llm=mock_llm_instance,
+ search=mock_search_instance,
+ settings_snapshot={},
+ )
+ ui_callback = Mock()
+ system.set_progress_callback(ui_callback)
+
+ mock_search_instance.set_fetch_progress_callback.assert_called_once()
+ (hook,), _ = mock_search_instance.set_fetch_progress_callback.call_args
+ assert callable(hook)
+ # The hook drives throttled UI milestones.
+ hook(0, 5)
+ milestones = [
+ c
+ for c in ui_callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 1
+
+
@pytest.mark.requires_llm
def test_analyze_topic(monkeypatch):
"""Test analyzing a topic."""
diff --git a/tests/web_search_engines/engines/test_full_search.py b/tests/web_search_engines/engines/test_full_search.py
index ca625a2b25..74a253d387 100644
--- a/tests/web_search_engines/engines/test_full_search.py
+++ b/tests/web_search_engines/engines/test_full_search.py
@@ -481,6 +481,115 @@ class TestJSRenderingForwardingFromSettingsSnapshot:
assert mock_batch.call_args.kwargs.get("enable_js_rendering") is True
+class TestProgressCallbackForwarding:
+ """``FullSearchResults`` forwards its ``(done, total)`` liveness hook
+ to every ``batch_fetch_and_extract`` call, defaulting to ``None``."""
+
+ def _patched_run(self, **engine_kwargs):
+ """Run engine.run() with batch_fetch_and_extract patched and
+ return the captured kwargs from that call."""
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ mock_web_search = Mock()
+ mock_web_search.invoke.return_value = [
+ {"link": "https://example.com/1", "title": "Result 1"},
+ ]
+
+ with (
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.QUALITY_CHECK_DDG_URLS",
+ False,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.validate_url",
+ return_value=True,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
+ return_value={"https://example.com/1": "content"},
+ ) as mock_batch,
+ ):
+ FullSearchResults(
+ llm=Mock(), web_search=mock_web_search, **engine_kwargs
+ ).run("test query")
+ assert mock_batch.call_args is not None
+ return mock_batch.call_args.kwargs
+
+ def test_run_defaults_to_no_progress_callback(self):
+ kwargs = self._patched_run()
+ assert kwargs.get("progress_callback") is None
+
+ def test_run_forwards_progress_callback(self):
+ hook = Mock()
+ kwargs = self._patched_run(progress_callback=hook)
+ assert kwargs.get("progress_callback") is hook
+
+ def test_get_full_content_forwards_progress_callback(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ hook = Mock()
+ with (
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.validate_url",
+ return_value=True,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
+ return_value={"https://example.com/1": "content"},
+ ) as mock_batch,
+ ):
+ FullSearchResults(
+ llm=Mock(), web_search=Mock(), progress_callback=hook
+ )._get_full_content([{"link": "https://example.com/1"}])
+ assert mock_batch.call_args.kwargs.get("progress_callback") is hook
+
+
+class TestFetchProgressAdapterShared:
+ """Pipeline + agent surfaces share one throttling implementation."""
+
+ def test_adapter_lives_in_full_search_and_langgraph_reexports(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ make_fetch_progress_adapter,
+ )
+ from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
+ _make_fetch_progress_adapter,
+ )
+
+ assert _make_fetch_progress_adapter is make_fetch_progress_adapter
+
+ def test_adapter_announces_and_throttles(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ make_fetch_progress_adapter,
+ )
+
+ callback = Mock()
+ hook = make_fetch_progress_adapter(callback)
+ hook(0, 10)
+ hook(1, 10)
+ milestones = [
+ c
+ for c in callback.call_args_list
+ if c.args[2].get("type") == "milestone"
+ ]
+ assert len(milestones) == 1
+ assert "10" in milestones[0].args[0]
+
+ def test_set_fetch_progress_callback_late_binds(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ engine = FullSearchResults(llm=Mock(), web_search=Mock())
+ assert engine.progress_callback is None
+ hook = Mock()
+ engine.set_fetch_progress_callback(hook)
+ assert engine.progress_callback is hook
+
+
class TestCheckUrlsSyncAsyncSplit:
"""The sync/async split of the URL-quality LLM call (#5854).
diff --git a/tests/web_search_engines/test_factory_full_search_wrapper.py b/tests/web_search_engines/test_factory_full_search_wrapper.py
index b043cde82a..e5f4cd8b99 100644
--- a/tests/web_search_engines/test_factory_full_search_wrapper.py
+++ b/tests/web_search_engines/test_factory_full_search_wrapper.py
@@ -738,3 +738,97 @@ class TestWrapperPrivateResultFetchGrant:
entry.module_path, entry.class_name
)
assert engine_class.allow_private_result_fetch is False, name
+
+
+class TestFetchProgressPipelineWiring:
+ """Pipeline-strategy live path for bulk-fetch progress (#7193)."""
+
+ def test_get_search_forwards_progress_callback_to_wrapper(self, mock_llm):
+ """get_search(..., progress_callback=hook) reaches FullSearchResults."""
+ hook = Mock()
+ settings_snapshot = {
+ "search.tool": {"value": "searxng"},
+ "search.engine.web.searxng.supports_full_search": {
+ "value": True,
+ "ui_element": "checkbox",
+ },
+ "search.engine.web.searxng.api_key": {"value": "mock-api-key"},
+ }
+ # Patch the engine class resolution to avoid live imports.
+ from local_deep_research.web_search_engines import (
+ search_engine_factory,
+ )
+
+ real_gsmc = search_engine_factory.get_safe_module_class
+
+ def _fake_gsmc(module_path: str, class_name: str):
+ if class_name == "FullSearchResults":
+ return real_gsmc(module_path, class_name)
+ return _MockBaseEngine
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_factory.get_safe_module_class",
+ _fake_gsmc,
+ ):
+ result = get_search(
+ search_tool="searxng",
+ llm_instance=mock_llm,
+ search_snippets_only=False,
+ settings_snapshot=settings_snapshot,
+ programmatic_mode=True,
+ progress_callback=hook,
+ )
+ assert isinstance(result, FullSearchResults)
+ assert result.progress_callback is hook
+
+ def test_base_engine_set_fetch_progress_forwards_to_full_search(
+ self, mock_llm
+ ):
+ """BaseSearchEngine late-binding reaches an existing wrapper."""
+ engine = _MockBaseEngine(llm=mock_llm, programmatic_mode=True)
+ engine.include_full_content = True
+ engine.full_search = FullSearchResults(llm=mock_llm, web_search=Mock())
+ hook = Mock()
+ engine.set_fetch_progress_callback(hook)
+ assert engine._fetch_progress_callback is hook
+ assert engine.full_search.progress_callback is hook
+
+ def test_create_search_engine_stamps_base_engine_hook(self, mock_llm):
+ """Direct create_search_engine(..., progress_callback) stamps base."""
+ from local_deep_research.web_search_engines import (
+ search_engine_factory,
+ )
+
+ hook = Mock()
+ settings_snapshot = {
+ "search.tool": {"value": "searxng"},
+ "search.engine.web.searxng.supports_full_search": {
+ "value": True,
+ "ui_element": "checkbox",
+ },
+ "search.engine.web.searxng.api_key": {"value": "mock-api-key"},
+ }
+ real_gsmc = search_engine_factory.get_safe_module_class
+
+ def _fake_gsmc(module_path: str, class_name: str):
+ if class_name == "FullSearchResults":
+ return real_gsmc(module_path, class_name)
+ return _MockBaseEngine
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_factory.get_safe_module_class",
+ _fake_gsmc,
+ ):
+ result = create_search_engine(
+ "searxng",
+ llm=mock_llm,
+ settings_snapshot=settings_snapshot,
+ programmatic_mode=True,
+ search_snippets_only=True,
+ use_full_search=False,
+ progress_callback=hook,
+ )
+ # Snippets-only: no wrapper, but the base engine keeps the hook
+ # for a future _init_full_search call.
+ assert not isinstance(result, FullSearchResults)
+ assert result._fetch_progress_callback is hook