Scrapy, a fast high-level web crawling & scraping framework for Python.
Rename DNS_RESOLVER to TWISTED_DNS_RESOLVER (#7361)
Kailesha4377f9a4fc6
diff --git a/docs/faq.rst b/docs/faq.rstindex fdf0582de..e117d2a5f 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -360,7 +360,7 @@ method for this purpose. For example:Does Scrapy support IPv6 addresses?------------------------------------Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.+Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.Note that by doing so, you lose the ability to set a specific timeout for DNS requests(the value of the :setting:`DNS_TIMEOUT` setting is ignored).diff --git a/docs/news.rst b/docs/news.rstindex 356220c3f..e445b0498 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -4731,7 +4731,7 @@ Highlights:* :ref:`FTP support <media-pipeline-ftp>` for media pipelines* New :attr:`Response.certificate <scrapy.http.Response.certificate>`attribute-* IPv6 support through :setting:`DNS_RESOLVER`+* IPv6 support through ``DNS_RESOLVER``Backward-incompatible changes~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@@ -4839,7 +4839,7 @@ New features:class:`twisted.internet.ssl.Certificate` object for HTTPS responses(:issue:`2726`, :issue:`4054`)-* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support+* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support(:issue:`1031`, :issue:`4227`)* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuringdiff --git a/docs/topics/components.rst b/docs/topics/components.rstindex d8a604ea7..c0df86922 100644--- a/docs/topics/components.rst+++ b/docs/topics/components.rst@@ -11,7 +11,7 @@ That includes the classes that you may assign to the following settings:- :setting:`ADDONS`-- :setting:`DNS_RESOLVER`+- :setting:`TWISTED_DNS_RESOLVER`- :setting:`DOWNLOAD_HANDLERS`diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex f0456b4d5..97c8f41de 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -332,7 +332,7 @@ These settings are:- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using:class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)-- :setting:`DNS_RESOLVER` and settings used by the corresponding+- :setting:`TWISTED_DNS_RESOLVER` and settings used by the correspondingcomponent, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`and :setting:`DNS_TIMEOUT` for the default one.@@ -671,10 +671,10 @@ Default: ``10000``DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.-.. setting:: DNS_RESOLVER+.. setting:: TWISTED_DNS_RESOLVER-DNS_RESOLVER-------------+TWISTED_DNS_RESOLVER+--------------------Default: ``'scrapy.resolver.CachingThreadedResolver'``diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 500139566..0a19e9985 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -16,7 +16,7 @@ from scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEnginefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManager-from scrapy.settings import Settings, overridden_settings+from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loaderfrom scrapy.utils.defer import deferred_from_coro@@ -660,7 +660,29 @@ class CrawlerProcessBase(CrawlerRunnerBase):def _setup_reactor(self, install_signal_handlers: bool) -> None:from twisted.internet import reactor- resolver_class = load_object(self.settings["DNS_RESOLVER"])+ dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0+ default_priority = SETTINGS_PRIORITIES["default"]++ if dns_priority > default_priority:+ warnings.warn(+ "The DNS_RESOLVER setting is deprecated, please use "+ "TWISTED_DNS_RESOLVER instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )++ twisted_dns_priority = (+ self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0+ )+ if twisted_dns_priority > dns_priority:+ resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]+ else:+ resolver_cls_path = self.settings["DNS_RESOLVER"]+ else:+ resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]++ resolver_class = load_object(resolver_cls_path)+# We pass self, which is CrawlerProcess, instead of Crawler here,# which works because the default resolvers only use crawler.settings.resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload]diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex d5c4b2dd3..456e90e77 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -162,6 +162,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):stacklevel=2,)+ if name == "DNS_RESOLVER":+ warnings.warn(+ "The DNS_RESOLVER setting is deprecated, please use "+ "TWISTED_DNS_RESOLVER instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+return self[name] if self[name] is not None else defaultdef getbool(self, name: _SettingsKey, default: bool = False) -> bool:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 462240133..b80e48601 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -48,7 +48,6 @@ __all__ = ["DEPTH_STATS_VERBOSE","DNSCACHE_ENABLED","DNSCACHE_SIZE",- "DNS_RESOLVER","DNS_TIMEOUT","DOWNLOADER","DOWNLOADER_CLIENTCONTEXTFACTORY",@@ -185,6 +184,7 @@ __all__ = ["TELNETCONSOLE_PORT","TELNETCONSOLE_USERNAME","TEMPLATES_DIR",+ "TWISTED_DNS_RESOLVER","TWISTED_REACTOR","TWISTED_REACTOR_ENABLED","URLLENGTH_LIMIT",@@ -526,6 +526,8 @@ TELNETCONSOLE_PASSWORD = NoneTEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())+TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"+TWISTED_REACTOR_ENABLED = TrueTWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.pyindex 5f75d5e17..a6520ca22 100644--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py@@ -28,7 +28,7 @@ if __name__ == "__main__":process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False,- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",})process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.pyindex 07e2d3684..55d2ef711 100644--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py@@ -15,7 +15,7 @@ if __name__ == "__main__":process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False,- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",})process.crawl(CachingHostnameResolverSpider)diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.pyindex 53d427061..7f687e7d9 100644--- a/tests/CrawlerProcess/caching_hostname_resolver.py+++ b/tests/CrawlerProcess/caching_hostname_resolver.py@@ -28,7 +28,7 @@ if __name__ == "__main__":process = CrawlerProcess(settings={"RETRY_ENABLED": False,- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",})process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.pyindex 5cca94bed..da9c16cb8 100644--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py@@ -15,7 +15,7 @@ if __name__ == "__main__":process = CrawlerProcess(settings={"RETRY_ENABLED": False,- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",})process.crawl(CachingHostnameResolverSpider)