Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(extraction): preserve specialist skip reasons (#6745)

Mateeh Ullahc9ae107dde96
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
index de1dc79efe..fc133ddcfd 100644
--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
@@ -417,6 +417,7 @@ def extract_content_with_metadata(
class _SpecializedResult(NamedTuple):
content: str | None = None
fallback_allowed: bool = True
+ skip_reason: str | None = None
def _try_specialized_downloader(
@@ -424,13 +425,17 @@ def _try_specialized_downloader(
) -> _SpecializedResult:
"""Try a specialized downloader (arXiv, PubMed, etc.) for the URL.
- The result carries extracted text and whether generic HTML fallback is safe.
+ The result carries extracted text, whether generic HTML fallback is safe,
+ and the specialist's failure reason for observability.
"""
# Terminal ownership follows the paper identifier, not the host: a
# non-paper arXiv page has no canonical downloader source, so denying it
# the generic fallback would return nothing at all.
arxiv_url = is_arxiv_paper_url(url)
- failure = _SpecializedResult(fallback_allowed=not arxiv_url)
+ failure = _SpecializedResult(
+ fallback_allowed=not arxiv_url,
+ skip_reason="URL classifier is unavailable",
+ )
try:
from local_deep_research.content_fetcher.url_classifier import (
URLClassifier,
@@ -464,14 +469,18 @@ def _try_specialized_downloader(
if url_type == URLType.ARXIV:
package = __package__
if package is None:
- return failure
+ return failure._replace(
+ skip_reason="Downloader package is unavailable"
+ )
downloaders_package = package.rpartition(".")[0]
arxiv_module = import_module(f"{downloaders_package}.arxiv")
downloader_type = getattr(arxiv_module, "ArxivDownloader", None)
if not isinstance(downloader_type, type) or not issubclass(
downloader_type, BaseDownloader
):
- return failure
+ return failure._replace(
+ skip_reason="arXiv downloader is unavailable"
+ )
downloader = downloader_type(timeout=timeout)
elif url_type in (URLType.PUBMED, URLType.PMC):
from ..pubmed import PubMedDownloader
@@ -489,11 +498,16 @@ def _try_specialized_downloader(
logger.debug(
f"Pipeline: specialized downloader not available for {url_type.value}"
)
- return failure
+ return failure._replace(
+ skip_reason=f"{url_type.value} downloader is unavailable"
+ )
if not downloader:
- return failure
+ return failure._replace(
+ skip_reason=f"{url_type.value} downloader is unavailable"
+ )
+ skip_reason: str | None = None
try:
result = downloader.download_with_result(url, ContentType.TEXT)
if result.is_success and result.content:
@@ -508,7 +522,12 @@ def _try_specialized_downloader(
content=text,
fallback_allowed=not arxiv_terminal,
)
+ skip_reason = (
+ result.skip_reason
+ or "Specialized downloader returned no usable content"
+ )
except Exception:
+ skip_reason = "Specialized downloader failed"
logger.opt(exception=False).debug(
f"Pipeline: specialized downloader failed for "
f"{redact_url_for_log(url)}"
@@ -519,12 +538,12 @@ def _try_specialized_downloader(
except Exception: # noqa: silent-exception
pass
+ failure = failure._replace(skip_reason=skip_reason)
if failure.fallback_allowed:
- # Specialized downloader didn't produce content — fall back to HTML
logger.debug(
f"Pipeline: specialized downloader ({url_type.value}) returned "
- f"no content for {redact_url_for_log(url)}, "
- f"falling back to HTML pipeline"
+ f"no content for {redact_url_for_log(url)} "
+ f"({failure.skip_reason}), falling back to HTML pipeline"
)
return failure
@@ -586,6 +605,11 @@ def fetch_and_extract(
if specialized.content:
return specialized.content
if not specialized.fallback_allowed:
+ logger.warning(
+ "Pipeline: terminal specialized download failed for {}: {}",
+ redact_url_for_log(url),
+ specialized.skip_reason or "reason unavailable",
+ )
return None
# Generic HTML pipeline
@@ -675,6 +699,11 @@ def batch_fetch_and_extract(
results[url] = specialized.content
continue
if not specialized.fallback_allowed:
+ logger.warning(
+ "Pipeline: terminal specialized download failed for {}: {}",
+ redact_url_for_log(url),
+ specialized.skip_reason or "reason unavailable",
+ )
results[url] = None
continue
except Exception:
diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.py
index 58e9b6cc1c..fdf14fb111 100644
--- a/tests/research_library/downloaders/test_extraction_pipeline.py
+++ b/tests/research_library/downloaders/test_extraction_pipeline.py
@@ -32,12 +32,15 @@ def _html(body: str, head: str = "") -> str:
class _SpecializedState(NamedTuple):
content: str | None
fallback_allowed: bool
+ skip_reason: str | None = None
def _specialized(
- content: str | None = None, fallback_allowed: bool = True
+ content: str | None = None,
+ fallback_allowed: bool = True,
+ skip_reason: str | None = None,
) -> _SpecializedState:
- return _SpecializedState(content, fallback_allowed)
+ return _SpecializedState(content, fallback_allowed, skip_reason)
LONG_TEXT = "A " * 600 # 1200 chars — above METADATA_ENRICHMENT_THRESHOLD
@@ -554,6 +557,29 @@ class TestTrySpecializedDownloader:
)
assert result.content is None
assert result.fallback_allowed is False
+ assert result.skip_reason == "unavailable"
+
+ def test_terminal_failure_logs_specialist_reason(self, monkeypatch):
+ reason = "Invalid arXiv URL - could not extract article ID"
+ monkeypatch.setattr(
+ pipeline,
+ "_try_specialized_downloader",
+ lambda url, timeout=30: _specialized(
+ fallback_allowed=False, skip_reason=reason
+ ),
+ )
+
+ warning = Mock()
+ monkeypatch.setattr(pipeline.logger, "warning", warning)
+
+ assert (
+ pipeline.fetch_and_extract(
+ "https://arxiv.org/abs/not-an-identifier"
+ )
+ is None
+ )
+
+ assert warning.call_args.args[-1] == reason
def test_import_error_returns_none(self, monkeypatch):
"""If url_classifier can't be imported, returns None."""