~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor(engines): extract _create_journal_filter helper (DRY) (#4710)
LearningCircuitb4501037adc5
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.pyindex d26f16fda6..08cd3a5e18 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py@@ -4,9 +4,6 @@ import arxivfrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...advanced_search_system.filters.journal_reputation_filter import (- JournalReputationFilter,-)from ...constants import SNIPPET_LENGTH_SHORTfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine@@ -55,10 +52,8 @@ class ArXivSearchEngine(BaseSearchEngine):# are instant data lookups — no point sending irrelevant journals# through the expensive LLM relevance filter.preview_filters = []- journal_filter = JournalReputationFilter.create_default(- model=llm, # type: ignore[arg-type]- engine_name="arxiv",- settings_snapshot=settings_snapshot,+ journal_filter = self._create_journal_filter(+ "arxiv", llm, settings_snapshot)if journal_filter is not None:preview_filters.append(journal_filter)diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.pyindex f65c77b4fd..4d04860cdf 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py@@ -6,9 +6,6 @@ from langchain_core.language_models import BaseLLMfrom loguru import loggerfrom ...constants import SNIPPET_LENGTH_LONG, USER_AGENT-from ...advanced_search_system.filters.journal_reputation_filter import (- JournalReputationFilter,-)from ...security.safe_requests import safe_getfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine@@ -54,10 +51,8 @@ class NasaAdsSearchEngine(BaseSearchEngine):"""# Journal filter runs before LLM relevance (Tiers 1-3 are instant)preview_filters = []- journal_filter = JournalReputationFilter.create_default(- model=llm, # type: ignore[arg-type]- engine_name="nasa_ads",- settings_snapshot=settings_snapshot,+ journal_filter = self._create_journal_filter(+ "nasa_ads", llm, settings_snapshot)if journal_filter is not None:preview_filters.append(journal_filter)diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.pyindex 81eb6ab064..3740923f6a 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py@@ -6,9 +6,6 @@ from langchain_core.language_models import BaseLLMfrom loguru import loggerfrom ...constants import SNIPPET_LENGTH_LONG, USER_AGENT-from ...advanced_search_system.filters.journal_reputation_filter import (- JournalReputationFilter,-)from ...security.safe_requests import safe_getfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine@@ -54,10 +51,8 @@ class OpenAlexSearchEngine(BaseSearchEngine):"""# Journal filter runs before LLM relevance (Tiers 1-3 are instant)preview_filters = []- journal_filter = JournalReputationFilter.create_default(- model=llm, # type: ignore[arg-type]- engine_name="openalex",- settings_snapshot=settings_snapshot,+ journal_filter = self._create_journal_filter(+ "openalex", llm, settings_snapshot)if journal_filter is not None:preview_filters.append(journal_filter)diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.pyindex 70f2219017..c8dfe55d55 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py@@ -8,9 +8,6 @@ from loguru import loggerfrom ...constants import SNIPPET_LENGTH_LONGfrom ...security.safe_requests import safe_get-from ...advanced_search_system.filters.journal_reputation_filter import (- JournalReputationFilter,-)from ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine@@ -73,10 +70,8 @@ class PubMedSearchEngine(BaseSearchEngine):# results are scored against bundled OpenAlex/DOAJ/predatory data# before the (more expensive) LLM relevance pass.preview_filters = []- journal_filter = JournalReputationFilter.create_default(- model=llm, # type: ignore[arg-type]- engine_name="pubmed",- settings_snapshot=settings_snapshot,+ journal_filter = self._create_journal_filter(+ "pubmed", llm, settings_snapshot)if journal_filter is not None:preview_filters.append(journal_filter)diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.pyindex a0b519d114..9436cfc2ff 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py@@ -72,17 +72,10 @@ class SemanticScholarSearchEngine(BaseSearchEngine):settings_snapshot: Settings snapshot for configuration**kwargs: Additional parameters to pass to parent class"""- # Initialize journal reputation filter- from ...advanced_search_system.filters.journal_reputation_filter import (- JournalReputationFilter,- )-# Journal filter runs before LLM relevance (Tiers 1-3 are instant)preview_filters = []- journal_filter = JournalReputationFilter.create_default(- model=llm, # type: ignore[arg-type]- engine_name="semantic_scholar",- settings_snapshot=settings_snapshot,+ journal_filter = self._create_journal_filter(+ "semantic_scholar", llm, settings_snapshot)if journal_filter is not None:preview_filters.append(journal_filter)diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.pyindex 6c6aebfbd8..60dca8ac0e 100644--- a/src/local_deep_research/web_search_engines/search_engine_base.py+++ b/src/local_deep_research/web_search_engines/search_engine_base.py@@ -1,7 +1,7 @@import jsonimport timefrom abc import ABC, abstractmethod-from typing import Any, Dict, List, Optional, Set, Union+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Unionfrom urllib.parse import urlparsefrom langchain_core.language_models import BaseLLM@@ -24,6 +24,11 @@ from ..utilities.thread_context import clear_search_context, set_search_contextfrom .rate_limiting import RateLimitError, get_trackerfrom ..constants import DEFAULT_SEARCH_TOOL+if TYPE_CHECKING:+ from ..advanced_search_system.filters.journal_reputation_filter import (+ JournalReputationFilter,+ )+# Common placeholder values that should never be treated as a real API key.# Three call sites previously had inconsistent subsets of this list, which@@ -359,6 +364,47 @@ class BaseSearchEngine(ABC):)self._last_results_count = 0+ def _create_journal_filter(+ self,+ engine_name: str,+ llm: Optional[BaseLLM],+ settings_snapshot: Optional[Dict[str, Any]],+ ) -> Optional["JournalReputationFilter"]:+ """Build the default :class:`JournalReputationFilter` for this engine.++ Wraps the identical 8-line ``JournalReputationFilter.create_default``+ boilerplate that previously lived in every academic subclass. The+ ``engine_name`` is passed explicitly because the auto-derived class+ name does not match the settings key for ``nasa_ads`` (would be+ ``"nasaads"``) or ``semantic_scholar`` (would be+ ``"semanticscholar"``).++ ``llm`` and ``settings_snapshot`` are passed in rather than read from+ ``self`` because subclasses build their preview filters *before*+ calling ``super().__init__()`` (the filter is handed to the parent+ constructor), so ``self.llm`` / ``self.settings_snapshot`` do not+ exist yet at call time.++ Args:+ engine_name: Settings key identifying the engine (e.g.+ ``"arxiv"``, ``"nasa_ads"``).+ llm: Language model used for the filter's relevance pass.+ settings_snapshot: Settings snapshot for configuration lookups.++ Returns:+ A configured :class:`JournalReputationFilter`, or ``None`` if+ filtering is disabled in settings for this engine.+ """+ from ..advanced_search_system.filters.journal_reputation_filter import (+ JournalReputationFilter,+ )++ return JournalReputationFilter.create_default(+ model=llm, # type: ignore[arg-type]+ engine_name=engine_name,+ settings_snapshot=settings_snapshot,+ )+def _configure_programmatic_mode(self, programmatic_mode: bool) -> None:"""Set ``programmatic_mode`` and (re)bind the matching rate tracker.diff --git a/tests/test_search_engines_enhanced.py b/tests/test_search_engines_enhanced.pyindex 944473a024..fc1d02f4e4 100644--- a/tests/test_search_engines_enhanced.py+++ b/tests/test_search_engines_enhanced.py@@ -178,7 +178,7 @@ class TestArxivSearchEnhanced:# Mock JournalReputationFilter.create_default to return None# This avoids the need for LLM initializationmonkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",lambda *args, **kwargs: None,)diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.pyindex 0a4bc7fd95..f22f388620 100644--- a/tests/web_search_engines/engines/test_arxiv_coverage.py+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py@@ -76,7 +76,7 @@ def _make_mock_paper(def engine():"""Create ArXivSearchEngine with mocked dependencies."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = Nonefrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (@@ -91,7 +91,7 @@ def engine():def engine_with_pdf():"""Engine configured for PDF download."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = Nonefrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (@@ -127,7 +127,7 @@ class TestInit:"""Journal filter is added to content_filters when created."""mock_filter = Mock()with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = mock_filterfrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (@@ -140,7 +140,7 @@ class TestInit:def test_init_custom_sort(self):"""Custom sort_by and sort_order are stored."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = Nonefrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (@@ -156,7 +156,7 @@ class TestInit:def test_max_results_at_least_25(self):"""max_results should be at least 25 even if lower value passed."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = Nonefrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (@@ -212,7 +212,7 @@ class TestGetSearchResults:import arxivwith patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter") as mock_jrf:mock_jrf.create_default.return_value = Nonefrom local_deep_research.web_search_engines.engines.search_engine_arxiv import (diff --git a/tests/web_search_engines/engines/test_openalex_new_fields.py b/tests/web_search_engines/engines/test_openalex_new_fields.pyindex df7eaf744a..bd9a2de5b9 100644--- a/tests/web_search_engines/engines/test_openalex_new_fields.py+++ b/tests/web_search_engines/engines/test_openalex_new_fields.py@@ -50,7 +50,7 @@ class TestOpenAlexSourceIdExtraction:)with patch(- f"{MODULE_OA}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):engine = OpenAlexSearchEngine()@@ -67,7 +67,7 @@ class TestOpenAlexSourceIdExtraction:)with patch(- f"{MODULE_OA}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):engine = OpenAlexSearchEngine()@@ -84,7 +84,7 @@ class TestOpenAlexSourceIdExtraction:)with patch(- f"{MODULE_OA}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):engine = OpenAlexSearchEngine()@@ -101,7 +101,7 @@ class TestOpenAlexSourceIdExtraction:)with patch(- f"{MODULE_OA}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):engine = OpenAlexSearchEngine()@@ -121,7 +121,7 @@ class TestOpenAlexSourceIdExtraction:)with patch(- f"{MODULE_OA}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):engine = OpenAlexSearchEngine()diff --git a/tests/web_search_engines/engines/test_search_engine_nasa_ads.py b/tests/web_search_engines/engines/test_search_engine_nasa_ads.pyindex 51758c3044..c0e9828564 100644--- a/tests/web_search_engines/engines/test_search_engine_nasa_ads.py+++ b/tests/web_search_engines/engines/test_search_engine_nasa_ads.py@@ -20,7 +20,7 @@ import pytestdef mock_journal_filter():"""Mock JournalReputationFilter to avoid LLM initialization."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_nasa_ads.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):yielddiff --git a/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py b/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.pyindex b1f6ff499a..1ca570caf0 100644--- a/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py@@ -25,7 +25,7 @@ MODULE = "local_deep_research.web_search_engines.engines.search_engine_nasa_ads"@pytest.fixturedef engine():"""Create engine with mocked init to avoid settings context requirement."""- with patch(f"{MODULE}.JournalReputationFilter"):+ with patch("local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"):eng = NasaAdsSearchEngine(api_key="test-key",max_results=5,diff --git a/tests/web_search_engines/engines/test_search_engine_openalex.py b/tests/web_search_engines/engines/test_search_engine_openalex.pyindex b741ee0e3e..42b5166781 100644--- a/tests/web_search_engines/engines/test_search_engine_openalex.py+++ b/tests/web_search_engines/engines/test_search_engine_openalex.py@@ -20,7 +20,7 @@ import pytestdef mock_journal_filter():"""Mock JournalReputationFilter to avoid LLM initialization."""with patch(- "local_deep_research.web_search_engines.engines.search_engine_openalex.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):yielddiff --git a/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py b/tests/web_search_engines/engines/test_search_engine_openalex_coverage.pyindex 8df22c4141..38e33c95bc 100644--- a/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py@@ -26,7 +26,7 @@ MODULE = "local_deep_research.web_search_engines.engines.search_engine_openalex"@pytest.fixture(autouse=True)def mock_journal_filter():with patch(- f"{MODULE}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=None,):yield@@ -73,7 +73,7 @@ class TestInitExtra:it must be added to content_filters passed to BaseSearchEngine."""fake_filter = Mock()with patch(- f"{MODULE}.JournalReputationFilter.create_default",+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",return_value=fake_filter,):engine = OpenAlexSearchEngine()diff --git a/tests/web_search_engines/test_search_engine_base.py b/tests/web_search_engines/test_search_engine_base.pyindex b7f23f1f75..d7d96fb9f2 100644--- a/tests/web_search_engines/test_search_engine_base.py+++ b/tests/web_search_engines/test_search_engine_base.py@@ -1346,3 +1346,48 @@ class TestEngineNameSetByFactory:assert eng._engine_name == ""eng._engine_name = "collection_abc-123"assert eng._engine_name == "collection_abc-123"+++class TestCreateJournalFilter:+ """Regression tests for ``_create_journal_filter``.++ Academic engine subclasses build their preview filters BEFORE calling+ ``super().__init__()`` (the filter is passed into the parent+ constructor), so ``self.llm`` / ``self.settings_snapshot`` do not exist+ yet at that point. The helper must therefore read ``llm`` and+ ``settings_snapshot`` from its arguments — never from ``self`` — or+ every academic engine crashes at construction with+ ``AttributeError: ... has no attribute 'llm'``.+ """++ def test_reads_args_not_self_and_forwards_them(self):+ from unittest.mock import patch++ from local_deep_research.web_search_engines.search_engine_base import (+ BaseSearchEngine,+ )++ # A bare object with NO .llm / .settings_snapshot attributes,+ # standing in for an engine instance mid-__init__ (pre-super()).+ not_yet_initialized = object()+ sentinel_llm = Mock()+ snapshot = {"journal_reputation.enabled": True}++ with patch(+ "local_deep_research.advanced_search_system.filters."+ "journal_reputation_filter.JournalReputationFilter.create_default"+ ) as mock_create:+ mock_create.return_value = "FILTER"+ result = BaseSearchEngine._create_journal_filter(+ not_yet_initialized,+ "semantic_scholar",+ sentinel_llm,+ snapshot,+ )++ assert result == "FILTER"+ mock_create.assert_called_once_with(+ model=sentinel_llm,+ engine_name="semantic_scholar",+ settings_snapshot=snapshot,+ )