Scrapy, a fast high-level web crawling & scraping framework for Python.
Read DNS and thread pool settings from the first spider (#7923)
Adriane16200029e2c
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 142dedbad..e1bcb57df 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -334,6 +334,12 @@ Reactor settings**Reactor settings** are settings tied to the :doc:`Twisted reactor<twisted:core/howto/reactor-basics>`.+.. versionchanged:: VERSION+ :setting:`TWISTED_DNS_RESOLVER`, the settings of the resolver and+ :setting:`REACTOR_THREADPOOL_MAXSIZE` are now read from the first spider,+ instead of being read from the project settings and ignored in+ :ref:`per-spider settings <spider-settings>`.+Because only 1 reactor can be used per process, these settings cannot use adifferent value per spider when :ref:`running multiple spiders in the sameprocess <run-multiple-spiders>`.@@ -363,11 +369,10 @@ These settings are applied when starting the reactor:- :setting:`REACTOR_THREADPOOL_MAXSIZE`-They are read from the settings of the-:class:`~scrapy.crawler.CrawlerProcess` or-:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a-spider or an :ref:`add-on <topics-addons>` has no effect. They are ignored-altogether when using :class:`~scrapy.crawler.CrawlerRunner` or+They can also be :ref:`set from a spider <spider-settings>`, but only the+values from the first spider that runs are used; if a later spider defines a+different value, a warning is issued. They are ignored altogether when using+:class:`~scrapy.crawler.CrawlerRunner` or:class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor.There is an additional restriction for :setting:`TWISTED_REACTOR` anddiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 07630817e..3da43d88f 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -57,6 +57,17 @@ logger = logging.getLogger(__name__)_T = TypeVar("_T")+# Reactor settings that are applied when starting the reactor, i.e. once per+# process, and so can only take the value of one of the crawlers.+_REACTOR_SETTINGS = (+ "DNSCACHE_ENABLED",+ "DNSCACHE_SIZE",+ "DNS_RESOLVER",+ "DNS_TIMEOUT",+ "REACTOR_THREADPOOL_MAXSIZE",+ "TWISTED_DNS_RESOLVER",+)+class _LateAttribute(Generic[_T]):"""Descriptor for a :class:`Crawler` attribute that only gets a value once@@ -721,6 +732,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):install_root_handler: bool | None = None,):super().__init__(settings)+ self._reactor_crawler: Crawler | None = Noneconfigure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)@@ -730,6 +742,30 @@ class CrawlerProcessBase(CrawlerRunnerBase):) -> None:raise NotImplementedError+ def create_crawler(+ self, crawler_or_spidercls: type[Spider] | str | Crawler+ ) -> Crawler:+ crawler = super().create_crawler(crawler_or_spidercls)+ if self._reactor_crawler is None:+ self._reactor_crawler = crawler+ else:+ ignored = [+ setting+ for setting in _REACTOR_SETTINGS+ if self._reactor_crawler.settings[setting] != crawler.settings[setting]+ ]+ if ignored:+ warnings.warn(+ f"Spider {crawler.spidercls.__name__} defines a different "+ f"value than spider "+ f"{self._reactor_crawler.spidercls.__name__} for the "+ f"following reactor settings: {', '.join(ignored)}. Only "+ f"the value of the first spider is used, since the "+ f"reactor is shared by every spider in a process.",+ stacklevel=2,+ )+ return crawler+def _signal_shutdown(self, signum: int, _: Any) -> None:from twisted.internet import reactor@@ -767,7 +803,13 @@ class CrawlerProcessBase(CrawlerRunnerBase):def _setup_reactor(self, install_signal_handlers: bool) -> None:from twisted.internet import reactor- dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0+ # Reactor settings are read from the first crawler, so that they can be+ # defined from a spider, and fall back to the process settings when no+ # crawler has been created yet.+ crawler: Crawler | CrawlerProcessBase = self._reactor_crawler or self+ settings = crawler.settings++ dns_priority = settings.getpriority("DNS_RESOLVER") or 0default_priority = SETTINGS_PRIORITIES["default"]if dns_priority > default_priority:@@ -778,24 +820,20 @@ class CrawlerProcessBase(CrawlerRunnerBase):stacklevel=2,)- twisted_dns_priority = (- self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0- )+ twisted_dns_priority = settings.getpriority("TWISTED_DNS_RESOLVER") or 0if twisted_dns_priority > dns_priority:- resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]+ resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]else:- resolver_cls_path = self.settings["DNS_RESOLVER"]+ resolver_cls_path = settings["DNS_RESOLVER"]else:- resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]+ resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]resolver_class = load_object(resolver_cls_path)- # We pass self, which is CrawlerProcess, instead of Crawler here,- # which works because the default resolvers only use crawler.settings.- resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload]+ resolver = build_from_crawler(resolver_class, crawler, reactor=reactor) # type: ignore[call-overload]resolver.install_on_reactor()tp = reactor.getThreadPool()- tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))+ tp.adjustPoolsize(maxthreads=settings.getint("REACTOR_THREADPOOL_MAXSIZE"))reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd)if install_signal_handlers:reactor.addSystemEventTrigger(diff --git a/tests/AsyncCrawlerProcess/reactor_settings.py b/tests/AsyncCrawlerProcess/reactor_settings.pynew file mode 100644index 000000000..3b64c22a2--- /dev/null+++ b/tests/AsyncCrawlerProcess/reactor_settings.py@@ -0,0 +1,35 @@+import sys++import scrapy+from scrapy.crawler import AsyncCrawlerProcess+from scrapy.resolver import dnscache+++class Spider1(scrapy.Spider):+ name = "spider1"+ custom_settings = {+ "DNSCACHE_ENABLED": False,+ "DNS_TIMEOUT": 11,+ "REACTOR_THREADPOOL_MAXSIZE": 42,+ }++ async def start(self):+ from twisted.internet import reactor++ self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")+ self.logger.info(f"DNS cache limit: {dnscache.limit}")+ self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")+ return+ yield+++class Spider2(Spider1):+ name = "spider2"+ custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}+++process = AsyncCrawlerProcess()+process.crawl(Spider1)+if len(sys.argv) > 1 and sys.argv[1] == "conflict":+ process.crawl(Spider2)+process.start()diff --git a/tests/CrawlerProcess/reactor_settings.py b/tests/CrawlerProcess/reactor_settings.pynew file mode 100644index 000000000..8a4848114--- /dev/null+++ b/tests/CrawlerProcess/reactor_settings.py@@ -0,0 +1,35 @@+import sys++import scrapy+from scrapy.crawler import CrawlerProcess+from scrapy.resolver import dnscache+++class Spider1(scrapy.Spider):+ name = "spider1"+ custom_settings = {+ "DNSCACHE_ENABLED": False,+ "DNS_TIMEOUT": 11,+ "REACTOR_THREADPOOL_MAXSIZE": 42,+ }++ async def start(self):+ from twisted.internet import reactor++ self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")+ self.logger.info(f"DNS cache limit: {dnscache.limit}")+ self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")+ return+ yield+++class Spider2(Spider1):+ name = "spider2"+ custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}+++process = CrawlerProcess()+process.crawl(Spider1)+if len(sys.argv) > 1 and sys.argv[1] == "conflict":+ process.crawl(Spider2)+process.start()diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex d2b6f6b6a..7f9831958 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -137,6 +137,23 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):assert "Spider closed (finished)" in logassert "The DNS_RESOLVER setting is deprecated" in log+ def test_reactor_settings(self) -> None:+ log = self.run_script("reactor_settings.py")+ assert "Spider closed (finished)" in log+ assert "DNS timeout: 11.0" in log+ assert "DNS cache limit: 0" in log+ assert "Thread pool size: 42" in log+ assert "reactor settings" not in log++ def test_reactor_settings_conflict(self) -> None:+ log = self.run_script("reactor_settings.py", "conflict")+ assert "Spider closed (finished)" in log+ assert "DNS timeout: 11.0" in log+ assert (+ "Spider Spider2 defines a different value than spider Spider1 for "+ "the following reactor settings: DNS_TIMEOUT" in log+ )+def test_twisted_reactor_asyncio(self) -> None:log = self.run_script("twisted_reactor_asyncio.py")assert "Spider closed (finished)" in log