Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(content-fetcher): disable JS rendering by default (#3826) (#3971)

LearningCircuit15a3df4aff67
diff --git a/changelog.d/3826.bugfix.md b/changelog.d/3826.bugfix.md
new file mode 100644
index 0000000000..d98b54b709
--- /dev/null
+++ b/changelog.d/3826.bugfix.md
@@ -0,0 +1 @@
+**JavaScript rendering disabled by default in the production Docker image (#3826).** The headless-browser fallback in the content fetcher (Crawl4AI/Playwright) was previously attempted on every fetch even though the default Docker production image ships without a Chromium binary - each attempt failed loudly and contributed to the memory growth reported in issue #3826. A new `web.enable_javascript_rendering` setting (default `false`) gates the fallback. In limited (mostly accidental) internal benchmark comparisons - between dev instances that happened to have Chromium installed and routine Docker runs that did not - JS rendering did not measurably improve research quality, and most regular benchmark runs are on Docker without Chromium anyway, so the change does not regress observed quality. To enable: install Chromium in the runtime environment (`playwright install --with-deps chromium`) and toggle the setting in the UI.
diff --git a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
index ae56f4f2af..4a58d95643 100644
--- a/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
+++ b/src/local_deep_research/advanced_search_system/strategies/langgraph_agent_strategy.py
@@ -300,6 +300,7 @@ def _make_research_subtopic_tool(
collector,
model=model,
overall_query=overall_query,
+ settings_snapshot=settings_snapshot,
)
if sub_fetch is not None:
sub_tools.append(sub_fetch)
@@ -491,6 +492,7 @@ class LangGraphAgentStrategy(BaseSearchStrategy):
self.collector,
model=self.model,
overall_query=overall_query,
+ settings_snapshot=self.settings_snapshot,
)
if fetch is not None:
tools.append(fetch)
diff --git a/src/local_deep_research/advanced_search_system/strategies/mcp_strategy.py b/src/local_deep_research/advanced_search_system/strategies/mcp_strategy.py
index d60b8e2c85..f382a8446c 100644
--- a/src/local_deep_research/advanced_search_system/strategies/mcp_strategy.py
+++ b/src/local_deep_research/advanced_search_system/strategies/mcp_strategy.py
@@ -1142,9 +1142,15 @@ class MCPSearchStrategy(BaseSearchStrategy):
try:
from local_deep_research.content_fetcher import ContentFetcher
+ from local_deep_research.utilities.js_rendering import (
+ read_js_rendering_setting,
+ )
with ContentFetcher(
- timeout=self.CONTENT_FETCHER_TIMEOUT
+ timeout=self.CONTENT_FETCHER_TIMEOUT,
+ enable_js_rendering=read_js_rendering_setting(
+ self.settings_snapshot
+ ),
) as fetcher:
result = fetcher.fetch(url, max_length=max_length)
diff --git a/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py b/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py
index ba4bf4a1f5..46fdf23d07 100644
--- a/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py
+++ b/src/local_deep_research/advanced_search_system/tools/fetch/__init__.py
@@ -32,8 +32,13 @@ from langchain_core.language_models import BaseChatModel
from langchain_core.tools import tool
from loguru import logger
+from local_deep_research.utilities.js_rendering import (
+ read_js_rendering_setting as _read_js_rendering_setting,
+)
+
from .prompts import SUMMARY_FOCUS_PROMPT, SUMMARY_FOCUS_QUERY_PROMPT
+
# Per-call timeouts and caps. Kept here rather than in the strategy file
# because they are properties of the fetch tool, not of agent
# orchestration.
@@ -72,14 +77,20 @@ def _register_in_collector(
return start + 1
-def _make_full_fetch_tool(collector: Any):
+def _make_full_fetch_tool(
+ collector: Any, settings_snapshot: dict | None = None
+):
@tool
def fetch_content(url: str) -> str:
"""Download and read the full text content from a URL. Use when search snippets aren't detailed enough."""
from local_deep_research.content_fetcher import ContentFetcher
+ enable_js = _read_js_rendering_setting(settings_snapshot)
try:
- with ContentFetcher(timeout=CONTENT_FETCH_TIMEOUT) as fetcher:
+ with ContentFetcher(
+ timeout=CONTENT_FETCH_TIMEOUT,
+ enable_js_rendering=enable_js,
+ ) as fetcher:
result = fetcher.fetch(url, max_length=CONTENT_MAX_LENGTH)
if result.get("status") == "success":
title = result.get("title", "")
@@ -102,6 +113,7 @@ def _make_summary_fetch_tool(
collector: Any,
model: BaseChatModel,
overall_query: str | None,
+ settings_snapshot: dict | None = None,
):
"""Build the summary-mode fetch tool.
@@ -121,8 +133,12 @@ def _make_summary_fetch_tool(
"""
from local_deep_research.content_fetcher import ContentFetcher
+ enable_js = _read_js_rendering_setting(settings_snapshot)
try:
- with ContentFetcher(timeout=CONTENT_FETCH_TIMEOUT) as fetcher:
+ with ContentFetcher(
+ timeout=CONTENT_FETCH_TIMEOUT,
+ enable_js_rendering=enable_js,
+ ) as fetcher:
result = fetcher.fetch(url, max_length=CONTENT_MAX_LENGTH)
if result.get("status") != "success":
return f"Failed to fetch {url}: {result.get('error', 'unknown error')}"
@@ -187,6 +203,7 @@ def build_fetch_tool(
*,
model: BaseChatModel | None = None,
overall_query: str = "",
+ settings_snapshot: dict | None = None,
):
"""Build the agent-facing ``fetch_content`` tool for *mode*.
@@ -194,22 +211,36 @@ def build_fetch_tool(
register the tool with the agent in that case (and the system prompt
should also drop the corresponding instruction line so the agent
isn't told to use a tool that doesn't exist).
+
+ ``settings_snapshot`` is captured by the tool closure so the per-call
+ JS-rendering toggle can be read on a worker thread (where
+ ``threading.local`` context does not propagate).
"""
if mode == "disabled":
return None
if mode == "full":
- return _make_full_fetch_tool(collector)
+ return _make_full_fetch_tool(
+ collector, settings_snapshot=settings_snapshot
+ )
if mode == "summary_focus":
if model is None:
raise ValueError("summary_focus fetch mode requires a model")
- return _make_summary_fetch_tool(collector, model, overall_query=None)
+ return _make_summary_fetch_tool(
+ collector,
+ model,
+ overall_query=None,
+ settings_snapshot=settings_snapshot,
+ )
if mode == "summary_focus_query":
if model is None:
raise ValueError("summary_focus_query fetch mode requires a model")
# Empty overall_query falls back to focus-only behaviour at format
# time; we keep the *_query mode label so logs stay diagnostic.
return _make_summary_fetch_tool(
- collector, model, overall_query=overall_query or None
+ collector,
+ model,
+ overall_query=overall_query or None,
+ settings_snapshot=settings_snapshot,
)
raise ValueError(
f"Unknown fetch mode {mode!r}; expected one of {FETCH_MODES}"
diff --git a/src/local_deep_research/content_fetcher/fetcher.py b/src/local_deep_research/content_fetcher/fetcher.py
index af87f125d6..2aa8b087b5 100644
--- a/src/local_deep_research/content_fetcher/fetcher.py
+++ b/src/local_deep_research/content_fetcher/fetcher.py
@@ -29,16 +29,34 @@ class ContentFetcher:
Automatically detects URL type and uses the best downloader.
"""
- def __init__(self, timeout: int = 30, language: str = "English"):
+ def __init__(
+ self,
+ timeout: int = 30,
+ language: str = "English",
+ enable_js_rendering: bool = False,
+ ):
"""
Initialize the content fetcher.
Args:
timeout: Request timeout in seconds
language: Language for justext stoplist (passed to HTML downloader)
+ enable_js_rendering: When True, the HTML/DOI downloader falls back
+ to a headless browser (Crawl4AI/Playwright) for pages that need
+ JavaScript to render. Defaults to False because the default
+ Docker production image ships without Chromium and the fallback
+ otherwise wastes work on every fetch. In limited (mostly
+ accidental) internal benchmark comparisons between dev
+ instances that happened to have Chromium installed and routine
+ Docker runs that did not, JS rendering did not measurably
+ improve research quality, and most regular benchmark runs are
+ on Docker without Chromium anyway — so disabling by default
+ does not regress observed quality. The user-facing toggle is
+ the ``web.enable_javascript_rendering`` setting.
"""
self.timeout = timeout
self.language = language
+ self.enable_js_rendering = enable_js_rendering
self._downloaders: Dict[URLType, Any] = {}
def _get_downloader(self, url_type: URLType):
@@ -103,7 +121,9 @@ class ContentFetcher:
)
downloader = HTMLDownloader(
- timeout=self.timeout, language=self.language
+ timeout=self.timeout,
+ language=self.language,
+ enable_js_rendering=self.enable_js_rendering,
)
except ImportError:
logger.warning("HTMLDownloader not available")
@@ -117,7 +137,9 @@ class ContentFetcher:
)
downloader = HTMLDownloader(
- timeout=self.timeout, language=self.language
+ timeout=self.timeout,
+ language=self.language,
+ enable_js_rendering=self.enable_js_rendering,
)
except ImportError:
logger.warning("HTMLDownloader not available")
diff --git a/src/local_deep_research/defaults/default_settings.json b/src/local_deep_research/defaults/default_settings.json
index ce0e162241..b7dd8e094f 100644
--- a/src/local_deep_research/defaults/default_settings.json
+++ b/src/local_deep_research/defaults/default_settings.json
@@ -3205,6 +3205,20 @@
"value": true,
"visible": false
},
+ "web.enable_javascript_rendering": {
+ "category": null,
+ "description": "When enabled, falls back to a headless browser (Crawl4AI/Playwright) for pages that need JavaScript to render. Requires Chromium to be installed - the default Docker production image does NOT include it. To enable: install Chromium in the runtime environment (e.g. 'playwright install --with-deps chromium'), then toggle this setting. Note: in our limited (mostly accidental) benchmark comparisons - between dev instances that happened to have Chromium installed and routine Docker runs that did not - JS rendering did not measurably improve research quality, and most of our regular benchmark runs are on Docker without Chromium anyway. Leave disabled unless you have a specific need to scrape JavaScript-only sites.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Enable JavaScript rendering",
+ "options": null,
+ "step": null,
+ "type": "APP",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ },
"search.engine.web.wikipedia.display_name": {
"category": "wikipedia",
"description": "Display name to use in the U.I. for this search engine.",
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
index 710c67487f..060f4e8002 100644
--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
@@ -393,18 +393,28 @@ def fetch_and_extract(
url: str,
timeout: int = 30,
language: str = "English",
+ enable_js_rendering: bool = False,
) -> Optional[str]:
"""Fetch a URL and extract clean text content.
Pipeline:
1. Specialized downloader (arXiv PDF, PubMed API, etc.) if URL matches
- 2. Static HTTP fetch → Playwright fallback (if JS needed)
- → trafilatura → readability → justext
+ 2. Static HTTP fetch → Playwright fallback (if JS needed and
+ ``enable_js_rendering`` is True) → trafilatura → readability →
+ justext
Args:
url: The URL to fetch.
timeout: Request timeout in seconds.
language: Language for justext stoplist.
+ enable_js_rendering: When True, the HTML pipeline falls back to a
+ headless browser for pages that need JavaScript. Defaults to
+ False because the default Docker production image ships without
+ Chromium. Limited internal benchmark comparisons (dev instances
+ with Chromium vs Docker without) showed no measurable
+ research-quality improvement from JS rendering, and most regular
+ benchmark runs are on Docker without Chromium anyway. The
+ user-facing toggle is ``web.enable_javascript_rendering``.
Returns:
Extracted plain text, or None if fetch or extraction failed.
@@ -420,6 +430,7 @@ def fetch_and_extract(
downloader = AutoHTMLDownloader(
timeout=timeout,
language=language,
+ enable_js_rendering=enable_js_rendering,
)
try:
# download() returns extracted text as UTF-8 bytes (not raw HTML):
@@ -443,6 +454,7 @@ def batch_fetch_and_extract(
urls: List[str],
timeout: int = 30,
language: str = "English",
+ enable_js_rendering: bool = False,
) -> Dict[str, Optional[str]]:
"""Fetch multiple URLs and extract clean text from each.
@@ -457,6 +469,14 @@ def batch_fetch_and_extract(
urls: List of URLs to fetch.
timeout: Request timeout in seconds per URL.
language: Language for justext stoplist.
+ enable_js_rendering: When True, the HTML pipeline falls back to a
+ headless browser for pages that need JavaScript. Defaults to
+ False because the default Docker production image ships without
+ Chromium. Limited internal benchmark comparisons (dev instances
+ with Chromium vs Docker without) showed no measurable
+ research-quality improvement from JS rendering, and most regular
+ benchmark runs are on Docker without Chromium anyway. The
+ user-facing toggle is ``web.enable_javascript_rendering``.
Returns:
Dict mapping URL → extracted text (or None if failed).
@@ -485,6 +505,7 @@ def batch_fetch_and_extract(
downloader = AutoHTMLDownloader(
timeout=timeout,
language=language,
+ enable_js_rendering=enable_js_rendering,
)
try:
for url in html_urls:
diff --git a/src/local_deep_research/research_library/downloaders/playwright_html.py b/src/local_deep_research/research_library/downloaders/playwright_html.py
index 700fd9c5b1..c8e8164974 100644
--- a/src/local_deep_research/research_library/downloaders/playwright_html.py
+++ b/src/local_deep_research/research_library/downloaders/playwright_html.py
@@ -311,10 +311,18 @@ class AutoHTMLDownloader(HTMLDownloader):
timeout: int = 30,
language: str = "English",
min_content_length: int = 200,
+ # Disabled by default to match the production Docker image, which
+ # ships without Chromium — every JS-rendering fallback attempt
+ # would otherwise fail loudly (see issue #3826). Callers running
+ # outside Docker with Chromium installed opt in via the
+ # ``web.enable_javascript_rendering`` setting, or pass ``True``
+ # explicitly when constructing the downloader.
+ enable_js_rendering: bool = False,
**kwargs,
):
super().__init__(timeout=timeout, language=language)
self.min_content_length = min_content_length
+ self.enable_js_rendering = enable_js_rendering
self._playwright_downloader = None
def _get_playwright_downloader(self) -> PlaywrightHTMLDownloader:
@@ -382,6 +390,13 @@ class AutoHTMLDownloader(HTMLDownloader):
no_content = result is None or len(result) < self.min_content_length
if needs_js or no_content:
+ if not self.enable_js_rendering:
+ logger.debug(
+ f"Auto: would fall back to JS rendering for {url}, "
+ "but JS rendering is disabled "
+ "(setting: web.enable_javascript_rendering)"
+ )
+ return result
reason = "SPA signals" if needs_js else "no/short content"
logger.info(
f"Auto: {reason} for {url}, falling back to JS rendering"
@@ -428,6 +443,13 @@ class AutoHTMLDownloader(HTMLDownloader):
)
if needs_js or no_content:
+ if not self.enable_js_rendering:
+ logger.debug(
+ f"Auto: would fall back to JS rendering for {url}, "
+ "but JS rendering is disabled "
+ "(setting: web.enable_javascript_rendering)"
+ )
+ return result
reason = "SPA signals" if needs_js else "no/short content"
logger.info(
f"Auto: {reason} for {url}, falling back to JS rendering"
diff --git a/src/local_deep_research/utilities/js_rendering.py b/src/local_deep_research/utilities/js_rendering.py
new file mode 100644
index 0000000000..f517cec8ea
--- /dev/null
+++ b/src/local_deep_research/utilities/js_rendering.py
@@ -0,0 +1,49 @@
+"""Shared accessor for the ``web.enable_javascript_rendering`` setting.
+
+The setting is read from a thread-local settings context (set on the
+research's main thread) or from an explicit snapshot dict (passed
+through closures so it works on LangGraph ``ToolNode`` worker threads
+where ``threading.local()`` state from the research thread does not
+propagate). Defaults to ``False`` when neither is available.
+
+Why disabled by default: the production Docker image ships without
+Chromium, so the headless-browser fallback (Crawl4AI/Playwright)
+cannot succeed for the majority of users; before this gate landed each
+attempt failed loudly and contributed to the memory growth reported
+in issue #3826. In limited (mostly accidental) internal benchmark
+comparisons between dev instances that happened to have Chromium
+installed and routine Docker runs that did not, JS rendering did not
+measurably improve research quality, and most regular benchmark runs
+are on Docker without Chromium anyway. Users who specifically need
+JS rendering can install Chromium (``playwright install --with-deps
+chromium``) and toggle the setting in the UI.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from ..config.thread_settings import get_bool_setting_from_snapshot
+
+
+def read_js_rendering_setting(settings_snapshot: Optional[dict]) -> bool:
+ """Return the current value of ``web.enable_javascript_rendering``.
+
+ Args:
+ settings_snapshot: Optional dict captured at the boundary where
+ the calling code crosses into a worker thread. Pass the
+ strategy's ``self.settings_snapshot`` if available; pass
+ ``None`` when the caller has no snapshot — the helper will
+ try thread-local context and finally fall back to ``False``.
+
+ Returns:
+ ``True`` only when the setting is explicitly enabled. ``bool(...)``
+ coerces ``Any`` to a definite bool for mypy ``warn_return_any``.
+ """
+ return bool(
+ get_bool_setting_from_snapshot(
+ "web.enable_javascript_rendering",
+ default=False,
+ settings_snapshot=settings_snapshot,
+ )
+ )
diff --git a/src/local_deep_research/web_search_engines/engines/full_search.py b/src/local_deep_research/web_search_engines/engines/full_search.py
index 25554f9f6d..4f535b51c9 100644
--- a/src/local_deep_research/web_search_engines/engines/full_search.py
+++ b/src/local_deep_research/web_search_engines/engines/full_search.py
@@ -9,6 +9,9 @@ from ...research_library.downloaders.extraction import (
batch_fetch_and_extract,
)
from ...security.ssrf_validator import validate_url
+from ...utilities.js_rendering import (
+ read_js_rendering_setting as _read_js_rendering_setting,
+)
from ...utilities.json_utils import extract_json, get_llm_response_text
@@ -28,6 +31,7 @@ class FullSearchResults:
region: str = "wt-wt",
time: Optional[str] = "y",
safesearch: str | int = "Moderate",
+ settings_snapshot: Optional[Dict] = None,
):
self.llm = llm
self.output_format = output_format
@@ -37,6 +41,7 @@ class FullSearchResults:
self.time = time
self.safesearch = safesearch
self.web_search = web_search
+ self.settings_snapshot = settings_snapshot
def check_urls(self, results: List[Dict], query: str) -> List[Dict]:
if not results:
@@ -125,7 +130,11 @@ class FullSearchResults:
# Fetch and extract all pages — specialized downloaders (arXiv,
# PubMed, etc.) are tried first, with HTML crawling as fallback.
url_to_content = batch_fetch_and_extract(
- safe_urls, language=self.language
+ safe_urls,
+ language=self.language,
+ enable_js_rendering=_read_js_rendering_setting(
+ self.settings_snapshot
+ ),
)
nr_full_text = sum(1 for v in url_to_content.values() if v)
@@ -157,7 +166,11 @@ class FullSearchResults:
try:
url_to_content = batch_fetch_and_extract(
- urls, language=self.language
+ urls,
+ language=self.language,
+ enable_js_rendering=_read_js_rendering_setting(
+ self.settings_snapshot
+ ),
)
except Exception:
logger.exception("Error fetching full content")
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index f3be58f463..d5eb3d9e75 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -926,6 +926,7 @@ class BaseSearchEngine(ABC):
region=region,
time=time_period,
safesearch=safe_search,
+ settings_snapshot=self.settings_snapshot,
)
except ImportError:
logger.warning(
diff --git a/tests/advanced_search_system/tools/test_fetch_modes.py b/tests/advanced_search_system/tools/test_fetch_modes.py
index d29c48c3da..1f7ad27ffe 100644
--- a/tests/advanced_search_system/tools/test_fetch_modes.py
+++ b/tests/advanced_search_system/tools/test_fetch_modes.py
@@ -146,3 +146,87 @@ def test_summary_focus_query_with_empty_overall_query_falls_back_to_focus_only()
def test_unknown_mode_raises_with_valid_modes_listed():
with pytest.raises(ValueError, match="Unknown fetch mode"):
build_fetch_tool("magic", SearchResultsCollector([]))
+
+
+# ---- issue #3826: web.enable_javascript_rendering plumbing ----
+
+
+def _captured_content_fetcher_kwargs(invoke_target):
+ """Run *invoke_target* with ContentFetcher patched and return its
+ constructor kwargs from the call.
+
+ The patched factory returns a context manager whose ``fetch`` succeeds
+ so the tool body runs end-to-end up to the call we care about.
+ """
+ cm = _fetcher_cm()
+ with patch(
+ "local_deep_research.content_fetcher.ContentFetcher", return_value=cm
+ ) as factory:
+ invoke_target()
+ assert factory.call_args is not None
+ return factory.call_args.kwargs
+
+
+def test_full_mode_passes_js_off_when_snapshot_disables_it():
+ """Full-mode fetch tool must pass enable_js_rendering=False to
+ ContentFetcher when the snapshot disables JS rendering."""
+ collector = SearchResultsCollector([])
+ snapshot = {
+ "web.enable_javascript_rendering": {
+ "value": False,
+ "ui_element": "checkbox",
+ }
+ }
+ tool = build_fetch_tool("full", collector, settings_snapshot=snapshot)
+ kwargs = _captured_content_fetcher_kwargs(
+ lambda: tool.invoke({"url": "http://example.com/"})
+ )
+ assert kwargs.get("enable_js_rendering") is False
+
+
+def test_full_mode_passes_js_on_when_snapshot_enables_it():
+ """When the snapshot opts in, JS rendering is forwarded to ContentFetcher."""
+ collector = SearchResultsCollector([])
+ snapshot = {
+ "web.enable_javascript_rendering": {
+ "value": True,
+ "ui_element": "checkbox",
+ }
+ }
+ tool = build_fetch_tool("full", collector, settings_snapshot=snapshot)
+ kwargs = _captured_content_fetcher_kwargs(
+ lambda: tool.invoke({"url": "http://example.com/"})
+ )
+ assert kwargs.get("enable_js_rendering") is True
+
+
+def test_full_mode_defaults_to_js_off_without_snapshot():
+ """No snapshot, no thread-local context → JS disabled (safe default)."""
+ collector = SearchResultsCollector([])
+ tool = build_fetch_tool("full", collector)
+ kwargs = _captured_content_fetcher_kwargs(
+ lambda: tool.invoke({"url": "http://example.com/"})
+ )
+ assert kwargs.get("enable_js_rendering") is False
+
+
+def test_summary_mode_forwards_js_setting():
+ """Summary-mode tool also forwards the JS toggle from the snapshot."""
+ collector = SearchResultsCollector([])
+ model = _model_returning("ok")
+ snapshot = {
+ "web.enable_javascript_rendering": {
+ "value": False,
+ "ui_element": "checkbox",
+ }
+ }
+ tool = build_fetch_tool(
+ "summary_focus",
+ collector,
+ model=model,
+ settings_snapshot=snapshot,
+ )
+ kwargs = _captured_content_fetcher_kwargs(
+ lambda: tool.invoke({"url": "http://example.com/", "focus": "x"})
+ )
+ assert kwargs.get("enable_js_rendering") is False
diff --git a/tests/content_fetcher/test_fetcher.py b/tests/content_fetcher/test_fetcher.py
index 7968759fd1..f89a626ee3 100644
--- a/tests/content_fetcher/test_fetcher.py
+++ b/tests/content_fetcher/test_fetcher.py
@@ -22,6 +22,35 @@ class TestContentFetcherInit:
fetcher = ContentFetcher(timeout=60)
assert fetcher.timeout == 60
+ def test_init_default_js_rendering_disabled(self):
+ """JS rendering must be off by default — issue #3826."""
+ fetcher = ContentFetcher()
+ assert fetcher.enable_js_rendering is False
+
+ def test_init_explicit_js_rendering_enabled(self):
+ """Caller can opt in to JS rendering explicitly."""
+ fetcher = ContentFetcher(enable_js_rendering=True)
+ assert fetcher.enable_js_rendering is True
+
+ def test_html_downloader_inherits_js_rendering_flag(self):
+ """The HTML downloader receives the JS rendering toggle."""
+ fetcher = ContentFetcher(enable_js_rendering=False)
+ downloader = fetcher._get_downloader(URLType.HTML)
+ assert downloader is not None
+ assert downloader.enable_js_rendering is False
+
+ fetcher_on = ContentFetcher(enable_js_rendering=True)
+ downloader_on = fetcher_on._get_downloader(URLType.HTML)
+ assert downloader_on is not None
+ assert downloader_on.enable_js_rendering is True
+
+ def test_doi_downloader_inherits_js_rendering_flag(self):
+ """The DOI downloader (also HTML-based) receives the toggle."""
+ fetcher = ContentFetcher(enable_js_rendering=False)
+ downloader = fetcher._get_downloader(URLType.DOI)
+ assert downloader is not None
+ assert downloader.enable_js_rendering is False
+
class TestContentFetcherGetDownloader:
"""Test downloader selection."""
diff --git a/tests/content_fetcher/test_fetcher_coverage.py b/tests/content_fetcher/test_fetcher_coverage.py
index 3b339cd6fb..1a9c587dda 100644
--- a/tests/content_fetcher/test_fetcher_coverage.py
+++ b/tests/content_fetcher/test_fetcher_coverage.py
@@ -787,7 +787,11 @@ class TestLanguageParam:
) as dl_cls:
dl_cls.return_value = MagicMock()
downloader = fetcher._get_downloader(URLType.HTML)
- dl_cls.assert_called_once_with(timeout=30, language="French")
+ dl_cls.assert_called_once_with(
+ timeout=30,
+ language="French",
+ enable_js_rendering=False,
+ )
assert downloader is not None
diff --git a/tests/mcp/test_mcp_strategy.py b/tests/mcp/test_mcp_strategy.py
index 4e937981f4..c228e6a9c7 100644
--- a/tests/mcp/test_mcp_strategy.py
+++ b/tests/mcp/test_mcp_strategy.py
@@ -1719,3 +1719,72 @@ class TestArgumentValidation:
assert "valid" in sanitized
assert "also_valid" in sanitized
assert 123 not in sanitized
+
+
+class TestDownloadContentJSRendering:
+ """The MCP ``download_content`` tool must read
+ ``web.enable_javascript_rendering`` from the strategy's settings
+ snapshot and forward it to ``ContentFetcher`` (issue #3826).
+ """
+
+ @staticmethod
+ def _snapshot(value: bool) -> dict:
+ return {
+ "web.enable_javascript_rendering": {
+ "value": value,
+ "ui_element": "checkbox",
+ }
+ }
+
+ def _capture_content_fetcher_kwargs(
+ self, settings_snapshot, monkeypatch
+ ) -> dict:
+ """Drive ``_execute_download_content`` with a patched
+ ``ContentFetcher`` and return the constructor kwargs from the
+ captured call."""
+ from local_deep_research.advanced_search_system.strategies.mcp_strategy import (
+ MCPSearchStrategy,
+ )
+
+ cm = MagicMock()
+ cm.__enter__.return_value.fetch.return_value = {
+ "status": "success",
+ "title": "T",
+ "content": "body",
+ "source_type": "html",
+ }
+ cm.__exit__.return_value = False
+
+ factory = MagicMock(return_value=cm)
+ monkeypatch.setattr(
+ "local_deep_research.content_fetcher.ContentFetcher",
+ factory,
+ )
+
+ strategy = MCPSearchStrategy(
+ model=MagicMock(),
+ search=MagicMock(),
+ settings_snapshot=settings_snapshot,
+ )
+ result = strategy._execute_download_content(
+ {"url": "https://example.com/"}
+ )
+ assert result["status"] == "success"
+ assert factory.call_args is not None
+ return factory.call_args.kwargs
+
+ def test_passes_js_off_when_snapshot_disables(self, monkeypatch):
+ kwargs = self._capture_content_fetcher_kwargs(
+ self._snapshot(False), monkeypatch
+ )
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_passes_js_on_when_snapshot_enables(self, monkeypatch):
+ kwargs = self._capture_content_fetcher_kwargs(
+ self._snapshot(True), monkeypatch
+ )
+ assert kwargs.get("enable_js_rendering") is True
+
+ def test_defaults_to_js_off_without_snapshot(self, monkeypatch):
+ kwargs = self._capture_content_fetcher_kwargs(None, monkeypatch)
+ assert kwargs.get("enable_js_rendering") is False
diff --git a/tests/performance/content_fetcher/test_extraction_benchmark.py b/tests/performance/content_fetcher/test_extraction_benchmark.py
index e2a213ba37..0fddb3cfed 100644
--- a/tests/performance/content_fetcher/test_extraction_benchmark.py
+++ b/tests/performance/content_fetcher/test_extraction_benchmark.py
@@ -428,7 +428,9 @@ class TestExtractionBenchmark:
@pytest.fixture(autouse=True)
def _setup(self):
self.static_dl = HTMLDownloader(timeout=20)
- self.auto_dl = AutoHTMLDownloader(timeout=20)
+ # Benchmark exercises the JS-rendering fallback path explicitly,
+ # so opt in regardless of the disable-by-default ctor.
+ self.auto_dl = AutoHTMLDownloader(timeout=20, enable_js_rendering=True)
yield
self.static_dl.close()
self.auto_dl.close()
diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.py
index be152dcf8a..f733d3cbfd 100644
--- a/tests/research_library/downloaders/test_extraction_pipeline.py
+++ b/tests/research_library/downloaders/test_extraction_pipeline.py
@@ -637,3 +637,67 @@ class TestBatchFetchAndExtract:
results = pipeline.batch_fetch_and_extract(["https://example.com"])
assert results["https://example.com"] == "fallback content"
+
+
+# ---------------------------------------------------------------------------
+# enable_js_rendering plumbing through fetch_and_extract /
+# batch_fetch_and_extract — issue #3826
+# ---------------------------------------------------------------------------
+
+
+class TestFetchAndExtractJSRenderingPlumbing:
+ """The ``enable_js_rendering`` flag must be forwarded into the
+ ``AutoHTMLDownloader`` constructor when the HTML pipeline is used."""
+
+ def _patched_downloader(self, monkeypatch):
+ """Stub specialized + AutoHTMLDownloader; return the captured Mock class."""
+ monkeypatch.setattr(
+ pipeline,
+ "_try_specialized_downloader",
+ lambda url, timeout=30: None,
+ )
+ mock_downloader = Mock()
+ mock_downloader.download.return_value = b"x"
+ mock_downloader.close = Mock()
+ mock_class = Mock(return_value=mock_downloader)
+ monkeypatch.setattr(
+ "local_deep_research.research_library.downloaders.playwright_html.AutoHTMLDownloader",
+ mock_class,
+ )
+ return mock_class
+
+ def test_fetch_and_extract_default_disables_js(self, monkeypatch):
+ mock_class = self._patched_downloader(monkeypatch)
+ pipeline.fetch_and_extract("https://example.com")
+ kwargs = mock_class.call_args.kwargs
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_fetch_and_extract_forwards_explicit_true(self, monkeypatch):
+ mock_class = self._patched_downloader(monkeypatch)
+ pipeline.fetch_and_extract(
+ "https://example.com", enable_js_rendering=True
+ )
+ kwargs = mock_class.call_args.kwargs
+ assert kwargs.get("enable_js_rendering") is True
+
+ def test_fetch_and_extract_forwards_explicit_false(self, monkeypatch):
+ mock_class = self._patched_downloader(monkeypatch)
+ pipeline.fetch_and_extract(
+ "https://example.com", enable_js_rendering=False
+ )
+ kwargs = mock_class.call_args.kwargs
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_batch_default_disables_js(self, monkeypatch):
+ mock_class = self._patched_downloader(monkeypatch)
+ pipeline.batch_fetch_and_extract(["https://example.com"])
+ kwargs = mock_class.call_args.kwargs
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_batch_forwards_explicit_true(self, monkeypatch):
+ mock_class = self._patched_downloader(monkeypatch)
+ pipeline.batch_fetch_and_extract(
+ ["https://example.com"], enable_js_rendering=True
+ )
+ kwargs = mock_class.call_args.kwargs
+ assert kwargs.get("enable_js_rendering") is True
diff --git a/tests/research_library/downloaders/test_no_browser_when_js_disabled.py b/tests/research_library/downloaders/test_no_browser_when_js_disabled.py
new file mode 100644
index 0000000000..174a74b5b5
--- /dev/null
+++ b/tests/research_library/downloaders/test_no_browser_when_js_disabled.py
@@ -0,0 +1,160 @@
+"""End-to-end regression: when ``enable_js_rendering=False``, no
+browser-spawning code path can run, even via the lazy imports inside
+``_fetch_with_crawl4ai`` and ``_fetch_with_playwright``.
+
+This complements the unit tests in ``test_playwright_html.py`` (which
+patch ``_get_playwright_downloader`` to assert it's never called) by
+mocking *the actual library symbols* — ``crawl4ai.AsyncWebCrawler`` and
+``playwright.sync_api.sync_playwright``. If anyone ever adds a code
+path that bypasses the ``self.enable_js_rendering`` gate and reaches
+into Crawl4AI/Playwright directly, this test catches it.
+
+Issue #3826 / PR #3971.
+"""
+
+from unittest.mock import MagicMock, patch
+
+from local_deep_research.research_library.downloaders.html import HTMLDownloader
+from local_deep_research.research_library.downloaders.playwright_html import (
+ AutoHTMLDownloader,
+)
+
+
+LONG = "x" * 1000
+SHORT_HTML = "<html><body><p>tiny</p></body></html>"
+SPA_HTML = (
+ '<html><body><div id="root"></div>'
+ "<noscript>You need to enable JavaScript</noscript>"
+ "</body></html>"
+)
+
+
+class TestNoBrowserSpawnsWhenJSDisabled:
+ """With JS disabled, neither Crawl4AI nor Playwright must be
+ instantiated, regardless of how the static fetch path resolves
+ (short content / SPA signals / JSON content type)."""
+
+ def _patch_browsers(self):
+ """Return paired mocks for Crawl4AI and Playwright at their
+ actual import sites. Both are lazy-imported inside the
+ downloader's fetch methods (``_fetch_with_crawl4ai`` line 112,
+ ``_fetch_with_playwright`` line 216), so patching them here
+ intercepts the import."""
+ crawl4ai_mock = MagicMock(name="AsyncWebCrawler")
+ playwright_mock = MagicMock(name="sync_playwright")
+ return (
+ patch("crawl4ai.AsyncWebCrawler", crawl4ai_mock),
+ patch("playwright.sync_api.sync_playwright", playwright_mock),
+ crawl4ai_mock,
+ playwright_mock,
+ )
+
+ def test_no_browser_when_static_returns_short_content(self):
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+
+ cm_crawl, cm_pw, crawl_mock, pw_mock = self._patch_browsers()
+ with (
+ cm_crawl,
+ cm_pw,
+ patch.object(
+ HTMLDownloader, "_fetch_html", return_value=SHORT_HTML
+ ),
+ ):
+ dl.download("https://example.com")
+
+ assert crawl_mock.call_count == 0
+ assert pw_mock.call_count == 0
+ assert dl._playwright_downloader is None
+ dl.close()
+
+ def test_no_browser_when_spa_signals_present(self):
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+ # Simulate a 403 SPA shell response (matches the path that
+ # would normally trigger JS rendering)
+ mock_resp = MagicMock(status_code=403, text=SPA_HTML)
+ dl.session = MagicMock()
+ dl.session.get.return_value = mock_resp
+
+ cm_crawl, cm_pw, crawl_mock, pw_mock = self._patch_browsers()
+ with cm_crawl, cm_pw:
+ dl.download("https://spa.example.com")
+
+ assert crawl_mock.call_count == 0
+ assert pw_mock.call_count == 0
+ assert dl._playwright_downloader is None
+ dl.close()
+
+ def test_no_browser_when_static_returns_none(self):
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+
+ cm_crawl, cm_pw, crawl_mock, pw_mock = self._patch_browsers()
+ with (
+ cm_crawl,
+ cm_pw,
+ patch.object(HTMLDownloader, "_fetch_html", return_value=None),
+ ):
+ dl.download("https://example.com")
+
+ assert crawl_mock.call_count == 0
+ assert pw_mock.call_count == 0
+ dl.close()
+
+ def test_no_browser_in_download_with_result_path(self):
+ """Both ``download()`` and ``download_with_result()`` must
+ honor the gate. The latter is the path the agent actually
+ uses via ``ContentFetcher.fetch``."""
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+
+ cm_crawl, cm_pw, crawl_mock, pw_mock = self._patch_browsers()
+ with (
+ cm_crawl,
+ cm_pw,
+ patch.object(
+ HTMLDownloader, "_fetch_html", return_value=SHORT_HTML
+ ),
+ ):
+ dl.download_with_result("https://example.com")
+
+ assert crawl_mock.call_count == 0
+ assert pw_mock.call_count == 0
+ dl.close()
+
+
+class TestContentFetcherEndToEndNoBrowser:
+ """Same guarantee at the public ``ContentFetcher`` boundary, which
+ is what the agent fetch tool and the MCP download tool actually use.
+ """
+
+ def test_no_browser_when_content_fetcher_disabled(self):
+ from local_deep_research.content_fetcher import ContentFetcher
+ from local_deep_research.content_fetcher.url_classifier import (
+ URLType,
+ )
+
+ crawl4ai_mock = MagicMock(name="AsyncWebCrawler")
+ playwright_mock = MagicMock(name="sync_playwright")
+
+ with (
+ patch("crawl4ai.AsyncWebCrawler", crawl4ai_mock),
+ patch("playwright.sync_api.sync_playwright", playwright_mock),
+ ContentFetcher(timeout=5, enable_js_rendering=False) as fetcher,
+ ):
+ downloader = fetcher._get_downloader(URLType.HTML)
+ assert downloader is not None
+ # Drive the static path and force "no content" to give the
+ # downloader the strongest motivation to fall back to JS:
+ with patch.object(
+ HTMLDownloader, "_fetch_html", return_value=SHORT_HTML
+ ):
+ downloader.download_with_result("https://example.com")
+
+ assert crawl4ai_mock.call_count == 0
+ assert playwright_mock.call_count == 0
diff --git a/tests/research_library/downloaders/test_playwright_html.py b/tests/research_library/downloaders/test_playwright_html.py
index c653491ef0..3a8e1ab354 100644
--- a/tests/research_library/downloaders/test_playwright_html.py
+++ b/tests/research_library/downloaders/test_playwright_html.py
@@ -108,7 +108,9 @@ class TestAutoDownloaderFallbackLogic:
def test_short_content_with_spa_triggers_playwright(self):
"""When content is short and SPA signals present, tries Playwright."""
- dl = AutoHTMLDownloader(timeout=5, min_content_length=200)
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=True
+ )
spa_html = '<html><body><div id="root"></div><noscript>You need to enable JavaScript</noscript></body></html>'
pw_content = f"<html><body><p>{LONG_PARA}</p></body></html>"
@@ -144,3 +146,132 @@ class TestPlaywrightDownloaderClose:
"""close() works when Playwright fallback was never used."""
dl = AutoHTMLDownloader(timeout=5)
dl.close() # Should not raise
+
+
+class TestAutoDownloaderJSRenderingToggle:
+ """Regression tests for ``enable_js_rendering`` (issue #3826).
+
+ The default Docker production image ships without Chromium, so the
+ JS-rendering fallback must be skippable to avoid wasted work and
+ confusing tracebacks on every fetch. With the toggle off, the
+ downloader must never construct or invoke its Playwright fallback,
+ even when content is short or SPA signals are present.
+ """
+
+ def test_disabled_skips_js_when_content_short(self):
+ """JS off + short content → static result returned, no Playwright."""
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+ short_html = "<html><body><p>tiny</p></body></html>"
+
+ with patch.object(
+ HTMLDownloader, "_fetch_html", return_value=short_html
+ ):
+ with patch.object(
+ AutoHTMLDownloader,
+ "_get_playwright_downloader",
+ side_effect=AssertionError(
+ "JS fallback must not run when disabled"
+ ),
+ ):
+ result = dl.download("https://example.com")
+
+ # Static result is returned (or None) without raising
+ assert dl._playwright_downloader is None
+ # Result is the static (short) extraction result
+ assert result is None or len(result) < 200
+ dl.close()
+
+ def test_disabled_skips_js_with_spa_signals(self):
+ """JS off + SPA signals → static result returned, no Playwright."""
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+ spa_html = (
+ '<html><body><div id="root"></div>'
+ "<noscript>You need to enable JavaScript</noscript>"
+ "</body></html>"
+ )
+ mock_response = Mock()
+ mock_response.status_code = 403
+ mock_response.text = spa_html
+ dl.session = Mock()
+ dl.session.get.return_value = mock_response
+
+ with patch.object(
+ AutoHTMLDownloader,
+ "_get_playwright_downloader",
+ side_effect=AssertionError(
+ "JS fallback must not run when disabled"
+ ),
+ ):
+ result = dl.download("https://spa-app.com")
+
+ # Static path returned None (SPA shell), but no Playwright was used
+ assert dl._playwright_downloader is None
+ assert result is None or len(result) < 200
+ dl.close()
+
+ def test_disabled_also_skips_in_download_with_result(self):
+ """download_with_result must honor the toggle as well."""
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=False
+ )
+ short_html = "<html><body><p>tiny</p></body></html>"
+
+ with patch.object(
+ HTMLDownloader, "_fetch_html", return_value=short_html
+ ):
+ with patch.object(
+ AutoHTMLDownloader,
+ "_get_playwright_downloader",
+ side_effect=AssertionError(
+ "JS fallback must not run when disabled"
+ ),
+ ):
+ result = dl.download_with_result("https://example.com")
+
+ # Static DownloadResult was returned without invoking JS
+ assert dl._playwright_downloader is None
+ assert result is not None
+ dl.close()
+
+ def test_enabled_still_falls_back_to_js(self):
+ """Regression guard: when explicitly enabled, the JS fallback
+ is still triggered for short SPA pages (existing behaviour)."""
+ dl = AutoHTMLDownloader(
+ timeout=5, min_content_length=200, enable_js_rendering=True
+ )
+ spa_html = (
+ '<html><body><div id="root"></div>'
+ "<noscript>You need to enable JavaScript</noscript>"
+ "</body></html>"
+ )
+ pw_content = f"<html><body><p>{LONG_PARA}</p></body></html>"
+ mock_response = Mock()
+ mock_response.status_code = 403
+ mock_response.text = spa_html
+ dl.session = Mock()
+ dl.session.get.return_value = mock_response
+
+ mock_pw = Mock()
+ mock_pw.download.return_value = pw_content.encode("utf-8")
+ dl._playwright_downloader = mock_pw
+
+ result = dl.download("https://spa-app.com")
+
+ assert result is not None
+ mock_pw.download.assert_called_once()
+ dl.close()
+
+ def test_default_is_false_for_direct_callers(self):
+ """The constructor defaults to ``enable_js_rendering=False``,
+ consistent with every other layer in the stack
+ (``ContentFetcher``, ``fetch_and_extract``, ``FullSearchResults``,
+ and the user-facing setting). Callers that explicitly need JS
+ rendering opt in by passing ``enable_js_rendering=True``.
+ """
+ dl = AutoHTMLDownloader(timeout=5)
+ assert dl.enable_js_rendering is False
+ dl.close()
diff --git a/tests/settings/golden_master_settings.json b/tests/settings/golden_master_settings.json
index 583a95b77c..1bf862abab 100644
--- a/tests/settings/golden_master_settings.json
+++ b/tests/settings/golden_master_settings.json
@@ -7977,6 +7977,20 @@
"value": 2,
"visible": true
},
+ "web.enable_javascript_rendering": {
+ "category": null,
+ "description": "When enabled, falls back to a headless browser (Crawl4AI/Playwright) for pages that need JavaScript to render. Requires Chromium to be installed - the default Docker production image does NOT include it. To enable: install Chromium in the runtime environment (e.g. 'playwright install --with-deps chromium'), then toggle this setting. Note: in our limited (mostly accidental) benchmark comparisons - between dev instances that happened to have Chromium installed and routine Docker runs that did not - JS rendering did not measurably improve research quality, and most of our regular benchmark runs are on Docker without Chromium anyway. Leave disabled unless you have a specific need to scrape JavaScript-only sites.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Enable JavaScript rendering",
+ "options": null,
+ "step": null,
+ "type": "APP",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ },
"web.host": {
"category": null,
"description": "The host address to listen on. Configured via LDR_WEB_HOST environment variable. Requires server restart.",
diff --git a/tests/settings/test_settings_defaults_integrity.py b/tests/settings/test_settings_defaults_integrity.py
index 1b76ff29bd..c225934094 100644
--- a/tests/settings/test_settings_defaults_integrity.py
+++ b/tests/settings/test_settings_defaults_integrity.py
@@ -186,6 +186,7 @@ def collect_consumed_setting_keys() -> set:
r'_get_setting\([^,]+,\s*["\']([^"\']+)["\']',
r'get_setting_from_snapshot\(\s*["\']([^"\']+)["\']',
r'get_bool_setting\(\s*["\']([^"\']+)["\']',
+ r'get_bool_setting_from_snapshot\(\s*["\']([^"\']+)["\']',
r'extract_setting_value\([^,]+,\s*["\']([^"\']+)["\']',
r'extract_bool_setting\([^,]+,\s*["\']([^"\']+)["\']',
rf'{_deprecated_fn}\(\s*["\']([^"\']+)["\']',
diff --git a/tests/utilities/test_js_rendering.py b/tests/utilities/test_js_rendering.py
new file mode 100644
index 0000000000..7e78cd584a
--- /dev/null
+++ b/tests/utilities/test_js_rendering.py
@@ -0,0 +1,47 @@
+"""Unit tests for ``utilities.js_rendering.read_js_rendering_setting``.
+
+The helper is the single read point for ``web.enable_javascript_rendering``
+shared by the agent fetch tool, the ``FullSearchResults`` web-search
+"include full content" path, and the MCP strategy's download tool.
+"""
+
+from local_deep_research.utilities.js_rendering import (
+ read_js_rendering_setting,
+)
+
+
+def _snapshot(value):
+ return {
+ "web.enable_javascript_rendering": {
+ "value": value,
+ "ui_element": "checkbox",
+ }
+ }
+
+
+def test_default_off_when_no_snapshot_no_context():
+ assert read_js_rendering_setting(None) is False
+
+
+def test_returns_true_when_snapshot_enables():
+ assert read_js_rendering_setting(_snapshot(True)) is True
+
+
+def test_returns_false_when_snapshot_disables():
+ assert read_js_rendering_setting(_snapshot(False)) is False
+
+
+def test_coerces_string_true_to_bool():
+ """Snapshots loaded from the DB sometimes carry string-typed values."""
+ assert read_js_rendering_setting(_snapshot("true")) is True
+
+
+def test_coerces_string_false_to_bool():
+ assert read_js_rendering_setting(_snapshot("false")) is False
+
+
+def test_returns_concrete_bool_not_truthy_value():
+ """``bool(...)`` coercion pins the return type — never returns Any."""
+ result = read_js_rendering_setting(_snapshot(True))
+ assert result is True or result is False # not just truthy
+ assert isinstance(result, bool)
diff --git a/tests/web_search_engines/engines/test_full_search.py b/tests/web_search_engines/engines/test_full_search.py
index 25998e5c52..a8d4b7b290 100644
--- a/tests/web_search_engines/engines/test_full_search.py
+++ b/tests/web_search_engines/engines/test_full_search.py
@@ -317,3 +317,141 @@ class TestCallable:
mock_invoke.assert_called_once_with("test query")
assert result == [{"result": "test"}]
+
+
+class TestJSRenderingForwardingFromSettingsSnapshot:
+ """``FullSearchResults`` must read ``web.enable_javascript_rendering``
+ from its ``settings_snapshot`` and forward the boolean to every
+ ``batch_fetch_and_extract`` call (issue #3826).
+
+ Both code paths into ``batch_fetch_and_extract`` are exercised:
+ ``run()`` and ``_get_full_content()``.
+ """
+
+ @staticmethod
+ def _snapshot(value: bool) -> dict:
+ return {
+ "web.enable_javascript_rendering": {
+ "value": value,
+ "ui_element": "checkbox",
+ }
+ }
+
+ def _patched_run(self, snapshot, mock_llm=None, mock_web_search=None):
+ """Run engine.run() with batch_fetch_and_extract patched and
+ return the captured kwargs from that call."""
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ mock_llm = mock_llm or Mock()
+ mock_web_search = mock_web_search or Mock()
+ mock_web_search.invoke.return_value = [
+ {"link": "https://example.com/1", "title": "Result 1"},
+ ]
+
+ with (
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.QUALITY_CHECK_DDG_URLS",
+ False,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.validate_url",
+ return_value=True,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
+ return_value={"https://example.com/1": "content"},
+ ) as mock_batch,
+ ):
+ engine = FullSearchResults(
+ llm=mock_llm,
+ web_search=mock_web_search,
+ settings_snapshot=snapshot,
+ )
+ engine.run("test query")
+ assert mock_batch.call_args is not None
+ return mock_batch.call_args.kwargs
+
+ def test_init_default_snapshot_is_none(self):
+ """Existing callers (no snapshot) keep working — attribute defaults."""
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ engine = FullSearchResults(llm=Mock(), web_search=Mock())
+ assert engine.settings_snapshot is None
+
+ def test_init_stores_snapshot(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ snap = self._snapshot(True)
+ engine = FullSearchResults(
+ llm=Mock(), web_search=Mock(), settings_snapshot=snap
+ )
+ assert engine.settings_snapshot is snap
+
+ def test_run_passes_js_off_when_snapshot_disables(self):
+ kwargs = self._patched_run(self._snapshot(False))
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_run_passes_js_on_when_snapshot_enables(self):
+ kwargs = self._patched_run(self._snapshot(True))
+ assert kwargs.get("enable_js_rendering") is True
+
+ def test_run_defaults_to_js_off_without_snapshot(self):
+ """No snapshot, no thread-local context → JS off (safe default)."""
+ kwargs = self._patched_run(None)
+ assert kwargs.get("enable_js_rendering") is False
+
+ def test_get_full_content_passes_js_off_when_snapshot_disables(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ with (
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.validate_url",
+ return_value=True,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
+ return_value={"https://example.com/1": "content"},
+ ) as mock_batch,
+ ):
+ engine = FullSearchResults(
+ llm=Mock(),
+ web_search=Mock(),
+ settings_snapshot=self._snapshot(False),
+ )
+ engine._get_full_content(
+ [{"link": "https://example.com/1", "title": "T"}]
+ )
+ assert mock_batch.call_args.kwargs.get("enable_js_rendering") is False
+
+ def test_get_full_content_passes_js_on_when_snapshot_enables(self):
+ from local_deep_research.web_search_engines.engines.full_search import (
+ FullSearchResults,
+ )
+
+ with (
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.validate_url",
+ return_value=True,
+ ),
+ patch(
+ "local_deep_research.web_search_engines.engines.full_search.batch_fetch_and_extract",
+ return_value={"https://example.com/1": "content"},
+ ) as mock_batch,
+ ):
+ engine = FullSearchResults(
+ llm=Mock(),
+ web_search=Mock(),
+ settings_snapshot=self._snapshot(True),
+ )
+ engine._get_full_content(
+ [{"link": "https://example.com/1", "title": "T"}]
+ )
+ assert mock_batch.call_args.kwargs.get("enable_js_rendering") is True
diff --git a/tests/web_search_engines/test_search_engine_base.py b/tests/web_search_engines/test_search_engine_base.py
index 34548f90cc..69363791b8 100644
--- a/tests/web_search_engines/test_search_engine_base.py
+++ b/tests/web_search_engines/test_search_engine_base.py
@@ -836,3 +836,62 @@ class TestDoiEnrichmentOrdering:
assert called["count"] == 0, (
"non-scientific engines should not trigger DOI enrichment"
)
+
+
+class TestInitFullSearchForwardsSettingsSnapshot:
+ """Issue #3826: ``_init_full_search`` must forward
+ ``self.settings_snapshot`` so ``FullSearchResults`` can read the
+ ``web.enable_javascript_rendering`` toggle."""
+
+ def _make_engine(self, settings_snapshot):
+ """Build a minimal BaseSearchEngine subclass that calls
+ ``_init_full_search``. The caller is responsible for patching
+ ``FullSearchResults`` around the invocation when it needs to
+ capture the constructor kwargs."""
+ from local_deep_research.web_search_engines.search_engine_base import (
+ BaseSearchEngine,
+ )
+
+ class _Engine(BaseSearchEngine):
+ def _get_previews(self, query):
+ return []
+
+ def _get_full_content(self, items):
+ return items
+
+ engine = _Engine(
+ llm=Mock(),
+ include_full_content=True,
+ settings_snapshot=settings_snapshot,
+ )
+ engine._init_full_search(web_search=Mock())
+ return engine
+
+ def test_forwards_snapshot_with_value(self):
+ from unittest.mock import patch
+
+ snapshot = {
+ "web.enable_javascript_rendering": {
+ "value": True,
+ "ui_element": "checkbox",
+ }
+ }
+ with patch(
+ "local_deep_research.web_search_engines.engines.full_search.FullSearchResults"
+ ) as mock_full_search:
+ self._make_engine(snapshot)
+ # FullSearchResults must have been constructed with the snapshot
+ kwargs = mock_full_search.call_args.kwargs
+ assert kwargs.get("settings_snapshot") is snapshot
+
+ def test_forwards_none_when_no_snapshot(self):
+ from unittest.mock import patch
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.full_search.FullSearchResults"
+ ) as mock_full_search:
+ self._make_engine(None)
+ kwargs = mock_full_search.call_args.kwargs
+ # The base class normalizes missing snapshot to {} not None — accept either
+ snap = kwargs.get("settings_snapshot")
+ assert snap is None or snap == {}