Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Read DNS and thread pool settings from the first spider (#7923)

Adriane16200029e2c
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 142dedbad..e1bcb57df 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -334,6 +334,12 @@ Reactor settings
**Reactor settings** are settings tied to the :doc:`Twisted reactor
<twisted:core/howto/reactor-basics>`.
+.. versionchanged:: VERSION
+ :setting:`TWISTED_DNS_RESOLVER`, the settings of the resolver and
+ :setting:`REACTOR_THREADPOOL_MAXSIZE` are now read from the first spider,
+ instead of being read from the project settings and ignored in
+ :ref:`per-spider settings <spider-settings>`.
+
Because only 1 reactor can be used per process, these settings cannot use a
different value per spider when :ref:`running multiple spiders in the same
process <run-multiple-spiders>`.
@@ -363,11 +369,10 @@ These settings are applied when starting the reactor:
- :setting:`REACTOR_THREADPOOL_MAXSIZE`
-They are read from the settings of the
-:class:`~scrapy.crawler.CrawlerProcess` or
-:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a
-spider or an :ref:`add-on <topics-addons>` has no effect. They are ignored
-altogether when using :class:`~scrapy.crawler.CrawlerRunner` or
+They can also be :ref:`set from a spider <spider-settings>`, but only the
+values from the first spider that runs are used; if a later spider defines a
+different value, a warning is issued. They are ignored altogether when using
+:class:`~scrapy.crawler.CrawlerRunner` or
:class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor.
There is an additional restriction for :setting:`TWISTED_REACTOR` and
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 07630817e..3da43d88f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -57,6 +57,17 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
+# Reactor settings that are applied when starting the reactor, i.e. once per
+# process, and so can only take the value of one of the crawlers.
+_REACTOR_SETTINGS = (
+ "DNSCACHE_ENABLED",
+ "DNSCACHE_SIZE",
+ "DNS_RESOLVER",
+ "DNS_TIMEOUT",
+ "REACTOR_THREADPOOL_MAXSIZE",
+ "TWISTED_DNS_RESOLVER",
+)
+
class _LateAttribute(Generic[_T]):
"""Descriptor for a :class:`Crawler` attribute that only gets a value once
@@ -721,6 +732,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
install_root_handler: bool | None = None,
):
super().__init__(settings)
+ self._reactor_crawler: Crawler | None = None
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@@ -730,6 +742,30 @@ class CrawlerProcessBase(CrawlerRunnerBase):
) -> None:
raise NotImplementedError
+ def create_crawler(
+ self, crawler_or_spidercls: type[Spider] | str | Crawler
+ ) -> Crawler:
+ crawler = super().create_crawler(crawler_or_spidercls)
+ if self._reactor_crawler is None:
+ self._reactor_crawler = crawler
+ else:
+ ignored = [
+ setting
+ for setting in _REACTOR_SETTINGS
+ if self._reactor_crawler.settings[setting] != crawler.settings[setting]
+ ]
+ if ignored:
+ warnings.warn(
+ f"Spider {crawler.spidercls.__name__} defines a different "
+ f"value than spider "
+ f"{self._reactor_crawler.spidercls.__name__} for the "
+ f"following reactor settings: {', '.join(ignored)}. Only "
+ f"the value of the first spider is used, since the "
+ f"reactor is shared by every spider in a process.",
+ stacklevel=2,
+ )
+ return crawler
+
def _signal_shutdown(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
@@ -767,7 +803,13 @@ class CrawlerProcessBase(CrawlerRunnerBase):
def _setup_reactor(self, install_signal_handlers: bool) -> None:
from twisted.internet import reactor
- dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0
+ # Reactor settings are read from the first crawler, so that they can be
+ # defined from a spider, and fall back to the process settings when no
+ # crawler has been created yet.
+ crawler: Crawler | CrawlerProcessBase = self._reactor_crawler or self
+ settings = crawler.settings
+
+ dns_priority = settings.getpriority("DNS_RESOLVER") or 0
default_priority = SETTINGS_PRIORITIES["default"]
if dns_priority > default_priority:
@@ -778,24 +820,20 @@ class CrawlerProcessBase(CrawlerRunnerBase):
stacklevel=2,
)
- twisted_dns_priority = (
- self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0
- )
+ twisted_dns_priority = settings.getpriority("TWISTED_DNS_RESOLVER") or 0
if twisted_dns_priority > dns_priority:
- resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+ resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]
else:
- resolver_cls_path = self.settings["DNS_RESOLVER"]
+ resolver_cls_path = settings["DNS_RESOLVER"]
else:
- resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+ resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]
resolver_class = load_object(resolver_cls_path)
- # We pass self, which is CrawlerProcess, instead of Crawler here,
- # which works because the default resolvers only use crawler.settings.
- resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload]
+ resolver = build_from_crawler(resolver_class, crawler, reactor=reactor) # type: ignore[call-overload]
resolver.install_on_reactor()
tp = reactor.getThreadPool()
- tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
+ tp.adjustPoolsize(maxthreads=settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd)
if install_signal_handlers:
reactor.addSystemEventTrigger(
diff --git a/tests/AsyncCrawlerProcess/reactor_settings.py b/tests/AsyncCrawlerProcess/reactor_settings.py
new file mode 100644
index 000000000..3b64c22a2
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/reactor_settings.py
@@ -0,0 +1,35 @@
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.resolver import dnscache
+
+
+class Spider1(scrapy.Spider):
+ name = "spider1"
+ custom_settings = {
+ "DNSCACHE_ENABLED": False,
+ "DNS_TIMEOUT": 11,
+ "REACTOR_THREADPOOL_MAXSIZE": 42,
+ }
+
+ async def start(self):
+ from twisted.internet import reactor
+
+ self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")
+ self.logger.info(f"DNS cache limit: {dnscache.limit}")
+ self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")
+ return
+ yield
+
+
+class Spider2(Spider1):
+ name = "spider2"
+ custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}
+
+
+process = AsyncCrawlerProcess()
+process.crawl(Spider1)
+if len(sys.argv) > 1 and sys.argv[1] == "conflict":
+ process.crawl(Spider2)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_settings.py b/tests/CrawlerProcess/reactor_settings.py
new file mode 100644
index 000000000..8a4848114
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_settings.py
@@ -0,0 +1,35 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.resolver import dnscache
+
+
+class Spider1(scrapy.Spider):
+ name = "spider1"
+ custom_settings = {
+ "DNSCACHE_ENABLED": False,
+ "DNS_TIMEOUT": 11,
+ "REACTOR_THREADPOOL_MAXSIZE": 42,
+ }
+
+ async def start(self):
+ from twisted.internet import reactor
+
+ self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")
+ self.logger.info(f"DNS cache limit: {dnscache.limit}")
+ self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")
+ return
+ yield
+
+
+class Spider2(Spider1):
+ name = "spider2"
+ custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}
+
+
+process = CrawlerProcess()
+process.crawl(Spider1)
+if len(sys.argv) > 1 and sys.argv[1] == "conflict":
+ process.crawl(Spider2)
+process.start()
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index d2b6f6b6a..7f9831958 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -137,6 +137,23 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "Spider closed (finished)" in log
assert "The DNS_RESOLVER setting is deprecated" in log
+ def test_reactor_settings(self) -> None:
+ log = self.run_script("reactor_settings.py")
+ assert "Spider closed (finished)" in log
+ assert "DNS timeout: 11.0" in log
+ assert "DNS cache limit: 0" in log
+ assert "Thread pool size: 42" in log
+ assert "reactor settings" not in log
+
+ def test_reactor_settings_conflict(self) -> None:
+ log = self.run_script("reactor_settings.py", "conflict")
+ assert "Spider closed (finished)" in log
+ assert "DNS timeout: 11.0" in log
+ assert (
+ "Spider Spider2 defines a different value than spider Spider1 for "
+ "the following reactor settings: DNS_TIMEOUT" in log
+ )
+
def test_twisted_reactor_asyncio(self) -> None:
log = self.run_script("twisted_reactor_asyncio.py")
assert "Spider closed (finished)" in log