Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor(engines): extract _create_journal_filter helper (DRY) (#4710)

LearningCircuitb4501037adc5
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
index d26f16fda6..08cd3a5e18 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
@@ -4,9 +4,6 @@ import arxiv
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...advanced_search_system.filters.journal_reputation_filter import (
- JournalReputationFilter,
-)
from ...constants import SNIPPET_LENGTH_SHORT
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine
@@ -55,10 +52,8 @@ class ArXivSearchEngine(BaseSearchEngine):
# are instant data lookups — no point sending irrelevant journals
# through the expensive LLM relevance filter.
preview_filters = []
- journal_filter = JournalReputationFilter.create_default(
- model=llm, # type: ignore[arg-type]
- engine_name="arxiv",
- settings_snapshot=settings_snapshot,
+ journal_filter = self._create_journal_filter(
+ "arxiv", llm, settings_snapshot
)
if journal_filter is not None:
preview_filters.append(journal_filter)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
index f65c77b4fd..4d04860cdf 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
@@ -6,9 +6,6 @@ from langchain_core.language_models import BaseLLM
from loguru import logger
from ...constants import SNIPPET_LENGTH_LONG, USER_AGENT
-from ...advanced_search_system.filters.journal_reputation_filter import (
- JournalReputationFilter,
-)
from ...security.safe_requests import safe_get
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine
@@ -54,10 +51,8 @@ class NasaAdsSearchEngine(BaseSearchEngine):
"""
# Journal filter runs before LLM relevance (Tiers 1-3 are instant)
preview_filters = []
- journal_filter = JournalReputationFilter.create_default(
- model=llm, # type: ignore[arg-type]
- engine_name="nasa_ads",
- settings_snapshot=settings_snapshot,
+ journal_filter = self._create_journal_filter(
+ "nasa_ads", llm, settings_snapshot
)
if journal_filter is not None:
preview_filters.append(journal_filter)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
index 81eb6ab064..3740923f6a 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
@@ -6,9 +6,6 @@ from langchain_core.language_models import BaseLLM
from loguru import logger
from ...constants import SNIPPET_LENGTH_LONG, USER_AGENT
-from ...advanced_search_system.filters.journal_reputation_filter import (
- JournalReputationFilter,
-)
from ...security.safe_requests import safe_get
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine
@@ -54,10 +51,8 @@ class OpenAlexSearchEngine(BaseSearchEngine):
"""
# Journal filter runs before LLM relevance (Tiers 1-3 are instant)
preview_filters = []
- journal_filter = JournalReputationFilter.create_default(
- model=llm, # type: ignore[arg-type]
- engine_name="openalex",
- settings_snapshot=settings_snapshot,
+ journal_filter = self._create_journal_filter(
+ "openalex", llm, settings_snapshot
)
if journal_filter is not None:
preview_filters.append(journal_filter)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
index 70f2219017..c8dfe55d55 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
@@ -8,9 +8,6 @@ from loguru import logger
from ...constants import SNIPPET_LENGTH_LONG
from ...security.safe_requests import safe_get
-from ...advanced_search_system.filters.journal_reputation_filter import (
- JournalReputationFilter,
-)
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine
@@ -73,10 +70,8 @@ class PubMedSearchEngine(BaseSearchEngine):
# results are scored against bundled OpenAlex/DOAJ/predatory data
# before the (more expensive) LLM relevance pass.
preview_filters = []
- journal_filter = JournalReputationFilter.create_default(
- model=llm, # type: ignore[arg-type]
- engine_name="pubmed",
- settings_snapshot=settings_snapshot,
+ journal_filter = self._create_journal_filter(
+ "pubmed", llm, settings_snapshot
)
if journal_filter is not None:
preview_filters.append(journal_filter)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
index a0b519d114..9436cfc2ff 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
@@ -72,17 +72,10 @@ class SemanticScholarSearchEngine(BaseSearchEngine):
settings_snapshot: Settings snapshot for configuration
**kwargs: Additional parameters to pass to parent class
"""
- # Initialize journal reputation filter
- from ...advanced_search_system.filters.journal_reputation_filter import (
- JournalReputationFilter,
- )
-
# Journal filter runs before LLM relevance (Tiers 1-3 are instant)
preview_filters = []
- journal_filter = JournalReputationFilter.create_default(
- model=llm, # type: ignore[arg-type]
- engine_name="semantic_scholar",
- settings_snapshot=settings_snapshot,
+ journal_filter = self._create_journal_filter(
+ "semantic_scholar", llm, settings_snapshot
)
if journal_filter is not None:
preview_filters.append(journal_filter)
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index 6c6aebfbd8..60dca8ac0e 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -1,7 +1,7 @@
import json
import time
from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
from urllib.parse import urlparse
from langchain_core.language_models import BaseLLM
@@ -24,6 +24,11 @@ from ..utilities.thread_context import clear_search_context, set_search_context
from .rate_limiting import RateLimitError, get_tracker
from ..constants import DEFAULT_SEARCH_TOOL
+if TYPE_CHECKING:
+ from ..advanced_search_system.filters.journal_reputation_filter import (
+ JournalReputationFilter,
+ )
+
# Common placeholder values that should never be treated as a real API key.
# Three call sites previously had inconsistent subsets of this list, which
@@ -359,6 +364,47 @@ class BaseSearchEngine(ABC):
)
self._last_results_count = 0
+ def _create_journal_filter(
+ self,
+ engine_name: str,
+ llm: Optional[BaseLLM],
+ settings_snapshot: Optional[Dict[str, Any]],
+ ) -> Optional["JournalReputationFilter"]:
+ """Build the default :class:`JournalReputationFilter` for this engine.
+
+ Wraps the identical 8-line ``JournalReputationFilter.create_default``
+ boilerplate that previously lived in every academic subclass. The
+ ``engine_name`` is passed explicitly because the auto-derived class
+ name does not match the settings key for ``nasa_ads`` (would be
+ ``"nasaads"``) or ``semantic_scholar`` (would be
+ ``"semanticscholar"``).
+
+ ``llm`` and ``settings_snapshot`` are passed in rather than read from
+ ``self`` because subclasses build their preview filters *before*
+ calling ``super().__init__()`` (the filter is handed to the parent
+ constructor), so ``self.llm`` / ``self.settings_snapshot`` do not
+ exist yet at call time.
+
+ Args:
+ engine_name: Settings key identifying the engine (e.g.
+ ``"arxiv"``, ``"nasa_ads"``).
+ llm: Language model used for the filter's relevance pass.
+ settings_snapshot: Settings snapshot for configuration lookups.
+
+ Returns:
+ A configured :class:`JournalReputationFilter`, or ``None`` if
+ filtering is disabled in settings for this engine.
+ """
+ from ..advanced_search_system.filters.journal_reputation_filter import (
+ JournalReputationFilter,
+ )
+
+ return JournalReputationFilter.create_default(
+ model=llm, # type: ignore[arg-type]
+ engine_name=engine_name,
+ settings_snapshot=settings_snapshot,
+ )
+
def _configure_programmatic_mode(self, programmatic_mode: bool) -> None:
"""Set ``programmatic_mode`` and (re)bind the matching rate tracker.
diff --git a/tests/test_search_engines_enhanced.py b/tests/test_search_engines_enhanced.py
index 944473a024..fc1d02f4e4 100644
--- a/tests/test_search_engines_enhanced.py
+++ b/tests/test_search_engines_enhanced.py
@@ -178,7 +178,7 @@ class TestArxivSearchEnhanced:
# Mock JournalReputationFilter.create_default to return None
# This avoids the need for LLM initialization
monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
lambda *args, **kwargs: None,
)
diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.py
index 0a4bc7fd95..f22f388620 100644
--- a/tests/web_search_engines/engines/test_arxiv_coverage.py
+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py
@@ -76,7 +76,7 @@ def _make_mock_paper(
def engine():
"""Create ArXivSearchEngine with mocked dependencies."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = None
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
@@ -91,7 +91,7 @@ def engine():
def engine_with_pdf():
"""Engine configured for PDF download."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = None
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
@@ -127,7 +127,7 @@ class TestInit:
"""Journal filter is added to content_filters when created."""
mock_filter = Mock()
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = mock_filter
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
@@ -140,7 +140,7 @@ class TestInit:
def test_init_custom_sort(self):
"""Custom sort_by and sort_order are stored."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = None
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
@@ -156,7 +156,7 @@ class TestInit:
def test_max_results_at_least_25(self):
"""max_results should be at least 25 even if lower value passed."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = None
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
@@ -212,7 +212,7 @@ class TestGetSearchResults:
import arxiv
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.JournalReputationFilter"
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"
) as mock_jrf:
mock_jrf.create_default.return_value = None
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
diff --git a/tests/web_search_engines/engines/test_openalex_new_fields.py b/tests/web_search_engines/engines/test_openalex_new_fields.py
index df7eaf744a..bd9a2de5b9 100644
--- a/tests/web_search_engines/engines/test_openalex_new_fields.py
+++ b/tests/web_search_engines/engines/test_openalex_new_fields.py
@@ -50,7 +50,7 @@ class TestOpenAlexSourceIdExtraction:
)
with patch(
- f"{MODULE_OA}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
engine = OpenAlexSearchEngine()
@@ -67,7 +67,7 @@ class TestOpenAlexSourceIdExtraction:
)
with patch(
- f"{MODULE_OA}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
engine = OpenAlexSearchEngine()
@@ -84,7 +84,7 @@ class TestOpenAlexSourceIdExtraction:
)
with patch(
- f"{MODULE_OA}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
engine = OpenAlexSearchEngine()
@@ -101,7 +101,7 @@ class TestOpenAlexSourceIdExtraction:
)
with patch(
- f"{MODULE_OA}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
engine = OpenAlexSearchEngine()
@@ -121,7 +121,7 @@ class TestOpenAlexSourceIdExtraction:
)
with patch(
- f"{MODULE_OA}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
engine = OpenAlexSearchEngine()
diff --git a/tests/web_search_engines/engines/test_search_engine_nasa_ads.py b/tests/web_search_engines/engines/test_search_engine_nasa_ads.py
index 51758c3044..c0e9828564 100644
--- a/tests/web_search_engines/engines/test_search_engine_nasa_ads.py
+++ b/tests/web_search_engines/engines/test_search_engine_nasa_ads.py
@@ -20,7 +20,7 @@ import pytest
def mock_journal_filter():
"""Mock JournalReputationFilter to avoid LLM initialization."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_nasa_ads.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
yield
diff --git a/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py b/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py
index b1f6ff499a..1ca570caf0 100644
--- a/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_nasa_ads_deep_coverage.py
@@ -25,7 +25,7 @@ MODULE = "local_deep_research.web_search_engines.engines.search_engine_nasa_ads"
@pytest.fixture
def engine():
"""Create engine with mocked init to avoid settings context requirement."""
- with patch(f"{MODULE}.JournalReputationFilter"):
+ with patch("local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter"):
eng = NasaAdsSearchEngine(
api_key="test-key",
max_results=5,
diff --git a/tests/web_search_engines/engines/test_search_engine_openalex.py b/tests/web_search_engines/engines/test_search_engine_openalex.py
index b741ee0e3e..42b5166781 100644
--- a/tests/web_search_engines/engines/test_search_engine_openalex.py
+++ b/tests/web_search_engines/engines/test_search_engine_openalex.py
@@ -20,7 +20,7 @@ import pytest
def mock_journal_filter():
"""Mock JournalReputationFilter to avoid LLM initialization."""
with patch(
- "local_deep_research.web_search_engines.engines.search_engine_openalex.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
yield
diff --git a/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py b/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py
index 8df22c4141..38e33c95bc 100644
--- a/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_openalex_coverage.py
@@ -26,7 +26,7 @@ MODULE = "local_deep_research.web_search_engines.engines.search_engine_openalex"
@pytest.fixture(autouse=True)
def mock_journal_filter():
with patch(
- f"{MODULE}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=None,
):
yield
@@ -73,7 +73,7 @@ class TestInitExtra:
it must be added to content_filters passed to BaseSearchEngine."""
fake_filter = Mock()
with patch(
- f"{MODULE}.JournalReputationFilter.create_default",
+ "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
return_value=fake_filter,
):
engine = OpenAlexSearchEngine()
diff --git a/tests/web_search_engines/test_search_engine_base.py b/tests/web_search_engines/test_search_engine_base.py
index b7f23f1f75..d7d96fb9f2 100644
--- a/tests/web_search_engines/test_search_engine_base.py
+++ b/tests/web_search_engines/test_search_engine_base.py
@@ -1346,3 +1346,48 @@ class TestEngineNameSetByFactory:
assert eng._engine_name == ""
eng._engine_name = "collection_abc-123"
assert eng._engine_name == "collection_abc-123"
+
+
+class TestCreateJournalFilter:
+ """Regression tests for ``_create_journal_filter``.
+
+ Academic engine subclasses build their preview filters BEFORE calling
+ ``super().__init__()`` (the filter is passed into the parent
+ constructor), so ``self.llm`` / ``self.settings_snapshot`` do not exist
+ yet at that point. The helper must therefore read ``llm`` and
+ ``settings_snapshot`` from its arguments — never from ``self`` — or
+ every academic engine crashes at construction with
+ ``AttributeError: ... has no attribute 'llm'``.
+ """
+
+ def test_reads_args_not_self_and_forwards_them(self):
+ from unittest.mock import patch
+
+ from local_deep_research.web_search_engines.search_engine_base import (
+ BaseSearchEngine,
+ )
+
+ # A bare object with NO .llm / .settings_snapshot attributes,
+ # standing in for an engine instance mid-__init__ (pre-super()).
+ not_yet_initialized = object()
+ sentinel_llm = Mock()
+ snapshot = {"journal_reputation.enabled": True}
+
+ with patch(
+ "local_deep_research.advanced_search_system.filters."
+ "journal_reputation_filter.JournalReputationFilter.create_default"
+ ) as mock_create:
+ mock_create.return_value = "FILTER"
+ result = BaseSearchEngine._create_journal_filter(
+ not_yet_initialized,
+ "semantic_scholar",
+ sentinel_llm,
+ snapshot,
+ )
+
+ assert result == "FILTER"
+ mock_create.assert_called_once_with(
+ model=sentinel_llm,
+ engine_name="semantic_scholar",
+ settings_snapshot=snapshot,
+ )