~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(extraction): preserve specialist skip reasons (#6745)
Mateeh Ullahc9ae107dde96
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.pyindex de1dc79efe..fc133ddcfd 100644--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py@@ -417,6 +417,7 @@ def extract_content_with_metadata(class _SpecializedResult(NamedTuple):content: str | None = Nonefallback_allowed: bool = True+ skip_reason: str | None = Nonedef _try_specialized_downloader(@@ -424,13 +425,17 @@ def _try_specialized_downloader() -> _SpecializedResult:"""Try a specialized downloader (arXiv, PubMed, etc.) for the URL.- The result carries extracted text and whether generic HTML fallback is safe.+ The result carries extracted text, whether generic HTML fallback is safe,+ and the specialist's failure reason for observability."""# Terminal ownership follows the paper identifier, not the host: a# non-paper arXiv page has no canonical downloader source, so denying it# the generic fallback would return nothing at all.arxiv_url = is_arxiv_paper_url(url)- failure = _SpecializedResult(fallback_allowed=not arxiv_url)+ failure = _SpecializedResult(+ fallback_allowed=not arxiv_url,+ skip_reason="URL classifier is unavailable",+ )try:from local_deep_research.content_fetcher.url_classifier import (URLClassifier,@@ -464,14 +469,18 @@ def _try_specialized_downloader(if url_type == URLType.ARXIV:package = __package__if package is None:- return failure+ return failure._replace(+ skip_reason="Downloader package is unavailable"+ )downloaders_package = package.rpartition(".")[0]arxiv_module = import_module(f"{downloaders_package}.arxiv")downloader_type = getattr(arxiv_module, "ArxivDownloader", None)if not isinstance(downloader_type, type) or not issubclass(downloader_type, BaseDownloader):- return failure+ return failure._replace(+ skip_reason="arXiv downloader is unavailable"+ )downloader = downloader_type(timeout=timeout)elif url_type in (URLType.PUBMED, URLType.PMC):from ..pubmed import PubMedDownloader@@ -489,11 +498,16 @@ def _try_specialized_downloader(logger.debug(f"Pipeline: specialized downloader not available for {url_type.value}")- return failure+ return failure._replace(+ skip_reason=f"{url_type.value} downloader is unavailable"+ )if not downloader:- return failure+ return failure._replace(+ skip_reason=f"{url_type.value} downloader is unavailable"+ )+ skip_reason: str | None = Nonetry:result = downloader.download_with_result(url, ContentType.TEXT)if result.is_success and result.content:@@ -508,7 +522,12 @@ def _try_specialized_downloader(content=text,fallback_allowed=not arxiv_terminal,)+ skip_reason = (+ result.skip_reason+ or "Specialized downloader returned no usable content"+ )except Exception:+ skip_reason = "Specialized downloader failed"logger.opt(exception=False).debug(f"Pipeline: specialized downloader failed for "f"{redact_url_for_log(url)}"@@ -519,12 +538,12 @@ def _try_specialized_downloader(except Exception: # noqa: silent-exceptionpass+ failure = failure._replace(skip_reason=skip_reason)if failure.fallback_allowed:- # Specialized downloader didn't produce content — fall back to HTMLlogger.debug(f"Pipeline: specialized downloader ({url_type.value}) returned "- f"no content for {redact_url_for_log(url)}, "- f"falling back to HTML pipeline"+ f"no content for {redact_url_for_log(url)} "+ f"({failure.skip_reason}), falling back to HTML pipeline")return failure@@ -586,6 +605,11 @@ def fetch_and_extract(if specialized.content:return specialized.contentif not specialized.fallback_allowed:+ logger.warning(+ "Pipeline: terminal specialized download failed for {}: {}",+ redact_url_for_log(url),+ specialized.skip_reason or "reason unavailable",+ )return None# Generic HTML pipeline@@ -675,6 +699,11 @@ def batch_fetch_and_extract(results[url] = specialized.contentcontinueif not specialized.fallback_allowed:+ logger.warning(+ "Pipeline: terminal specialized download failed for {}: {}",+ redact_url_for_log(url),+ specialized.skip_reason or "reason unavailable",+ )results[url] = Nonecontinueexcept Exception:diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.pyindex 58e9b6cc1c..fdf14fb111 100644--- a/tests/research_library/downloaders/test_extraction_pipeline.py+++ b/tests/research_library/downloaders/test_extraction_pipeline.py@@ -32,12 +32,15 @@ def _html(body: str, head: str = "") -> str:class _SpecializedState(NamedTuple):content: str | Nonefallback_allowed: bool+ skip_reason: str | None = Nonedef _specialized(- content: str | None = None, fallback_allowed: bool = True+ content: str | None = None,+ fallback_allowed: bool = True,+ skip_reason: str | None = None,) -> _SpecializedState:- return _SpecializedState(content, fallback_allowed)+ return _SpecializedState(content, fallback_allowed, skip_reason)LONG_TEXT = "A " * 600 # 1200 chars — above METADATA_ENRICHMENT_THRESHOLD@@ -554,6 +557,29 @@ class TestTrySpecializedDownloader:)assert result.content is Noneassert result.fallback_allowed is False+ assert result.skip_reason == "unavailable"++ def test_terminal_failure_logs_specialist_reason(self, monkeypatch):+ reason = "Invalid arXiv URL - could not extract article ID"+ monkeypatch.setattr(+ pipeline,+ "_try_specialized_downloader",+ lambda url, timeout=30: _specialized(+ fallback_allowed=False, skip_reason=reason+ ),+ )++ warning = Mock()+ monkeypatch.setattr(pipeline.logger, "warning", warning)++ assert (+ pipeline.fetch_and_extract(+ "https://arxiv.org/abs/not-an-identifier"+ )+ is None+ )++ assert warning.call_args.args[-1] == reasondef test_import_error_returns_none(self, monkeypatch):"""If url_classifier can't be imported, returns None."""