Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rename DNS_RESOLVER to TWISTED_DNS_RESOLVER (#7361)

Kailesha4377f9a4fc6
diff --git a/docs/faq.rst b/docs/faq.rst
index fdf0582de..e117d2a5f 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -360,7 +360,7 @@ method for this purpose. For example:
Does Scrapy support IPv6 addresses?
-----------------------------------
-Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
diff --git a/docs/news.rst b/docs/news.rst
index 356220c3f..e445b0498 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -4731,7 +4731,7 @@ Highlights:
* :ref:`FTP support <media-pipeline-ftp>` for media pipelines
* New :attr:`Response.certificate <scrapy.http.Response.certificate>`
attribute
-* IPv6 support through :setting:`DNS_RESOLVER`
+* IPv6 support through ``DNS_RESOLVER``
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -4839,7 +4839,7 @@ New features
:class:`twisted.internet.ssl.Certificate` object for HTTPS responses
(:issue:`2726`, :issue:`4054`)
-* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support
+* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support
(:issue:`1031`, :issue:`4227`)
* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index d8a604ea7..c0df86922 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -11,7 +11,7 @@ That includes the classes that you may assign to the following settings:
- :setting:`ADDONS`
-- :setting:`DNS_RESOLVER`
+- :setting:`TWISTED_DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f0456b4d5..97c8f41de 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -332,7 +332,7 @@ These settings are:
- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using
:class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)
-- :setting:`DNS_RESOLVER` and settings used by the corresponding
+- :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding
component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`
and :setting:`DNS_TIMEOUT` for the default one.
@@ -671,10 +671,10 @@ Default: ``10000``
DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
-.. setting:: DNS_RESOLVER
+.. setting:: TWISTED_DNS_RESOLVER
-DNS_RESOLVER
-------------
+TWISTED_DNS_RESOLVER
+--------------------
Default: ``'scrapy.resolver.CachingThreadedResolver'``
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 500139566..0a19e9985 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -16,7 +16,7 @@ from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
-from scrapy.settings import Settings, overridden_settings
+from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils.defer import deferred_from_coro
@@ -660,7 +660,29 @@ class CrawlerProcessBase(CrawlerRunnerBase):
def _setup_reactor(self, install_signal_handlers: bool) -> None:
from twisted.internet import reactor
- resolver_class = load_object(self.settings["DNS_RESOLVER"])
+ dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0
+ default_priority = SETTINGS_PRIORITIES["default"]
+
+ if dns_priority > default_priority:
+ warnings.warn(
+ "The DNS_RESOLVER setting is deprecated, please use "
+ "TWISTED_DNS_RESOLVER instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
+ twisted_dns_priority = (
+ self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0
+ )
+ if twisted_dns_priority > dns_priority:
+ resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+ else:
+ resolver_cls_path = self.settings["DNS_RESOLVER"]
+ else:
+ resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+
+ resolver_class = load_object(resolver_cls_path)
+
# We pass self, which is CrawlerProcess, instead of Crawler here,
# which works because the default resolvers only use crawler.settings.
resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload]
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d5c4b2dd3..456e90e77 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -162,6 +162,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
stacklevel=2,
)
+ if name == "DNS_RESOLVER":
+ warnings.warn(
+ "The DNS_RESOLVER setting is deprecated, please use "
+ "TWISTED_DNS_RESOLVER instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
return self[name] if self[name] is not None else default
def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 462240133..b80e48601 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -48,7 +48,6 @@ __all__ = [
"DEPTH_STATS_VERBOSE",
"DNSCACHE_ENABLED",
"DNSCACHE_SIZE",
- "DNS_RESOLVER",
"DNS_TIMEOUT",
"DOWNLOADER",
"DOWNLOADER_CLIENTCONTEXTFACTORY",
@@ -185,6 +184,7 @@ __all__ = [
"TELNETCONSOLE_PORT",
"TELNETCONSOLE_USERNAME",
"TEMPLATES_DIR",
+ "TWISTED_DNS_RESOLVER",
"TWISTED_REACTOR",
"TWISTED_REACTOR_ENABLED",
"URLLENGTH_LIMIT",
@@ -526,6 +526,8 @@ TELNETCONSOLE_PASSWORD = None
TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
+TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
+
TWISTED_REACTOR_ENABLED = True
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
index 5f75d5e17..a6520ca22 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
@@ -28,7 +28,7 @@ if __name__ == "__main__":
process = AsyncCrawlerProcess(
settings={
"RETRY_ENABLED": False,
- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
}
)
process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
index 07e2d3684..55d2ef711 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -15,7 +15,7 @@ if __name__ == "__main__":
process = AsyncCrawlerProcess(
settings={
"RETRY_ENABLED": False,
- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
}
)
process.crawl(CachingHostnameResolverSpider)
diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py
index 53d427061..7f687e7d9 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver.py
@@ -28,7 +28,7 @@ if __name__ == "__main__":
process = CrawlerProcess(
settings={
"RETRY_ENABLED": False,
- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
}
)
process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
index 5cca94bed..da9c16cb8 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -15,7 +15,7 @@ if __name__ == "__main__":
process = CrawlerProcess(
settings={
"RETRY_ENABLED": False,
- "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+ "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
}
)
process.crawl(CachingHostnameResolverSpider)