Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Ability to choose name resolver

Eugenio Lacuestae69cf415c832
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 61851acc3..c5351c08f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -305,23 +305,13 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
- reactor.installNameResolver(self._get_dns_resolver())
+ resolver_class = load_object(self.settings["DNS_RESOLVER"])
+ resolver_class.install(reactor, self.settings)
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
- def _get_dns_resolver(self):
- from twisted.internet import reactor
- if self.settings.getbool('DNSCACHE_ENABLED'):
- cache_size = self.settings.getint('DNSCACHE_SIZE')
- else:
- cache_size = 0
- return CachingHostnameResolver(
- resolver=reactor.nameResolver,
- cache_size=cache_size,
- )
-
def _graceful_stop_reactor(self):
d = self.stop()
d.addBoth(self._stop_reactor)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index ddbae61a9..2bef9f1b8 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,4 +1,6 @@
-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
@@ -8,8 +10,58 @@ from scrapy.utils.datatypes import LocalCache
dnscache = LocalCache(10000)
+@implementer(IResolverSimple)
+class CachingThreadedResolver(ThreadedResolver):
+ """
+ Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests
+ """
+
+ @classmethod
+ def install(cls, reactor, settings):
+ if settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = settings.getint('DNSCACHE_SIZE')
+ else:
+ cache_size = 0
+ resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))
+ reactor.installResolver(resolver)
+
+ def __init__(self, reactor, cache_size, timeout):
+ super(CachingThreadedResolver, self).__init__(reactor)
+ dnscache.limit = cache_size
+ self.timeout = timeout
+
+ def getHostByName(self, name, timeout=None):
+ if name in dnscache:
+ return defer.succeed(dnscache[name])
+ # in Twisted<=16.6, getHostByName() is always called with
+ # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
+ # so the input argument above is simply overridden
+ # to enforce Scrapy's DNS_TIMEOUT setting's value
+ timeout = (self.timeout,)
+ d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+ if dnscache.limit:
+ d.addCallback(self._cache_result, name)
+ return d
+
+ def _cache_result(self, result, name):
+ dnscache[name] = result
+ return result
+
+
@implementer(IHostnameResolver)
class CachingHostnameResolver:
+ """
+ Experimental caching resolver, supporting IPv4 and IPv6
+ """
+
+ @classmethod
+ def install(cls, reactor, settings):
+ if settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = settings.getint('DNSCACHE_SIZE')
+ else:
+ cache_size = 0
+ resolver = cls(reactor.nameResolver, cache_size)
+ reactor.installNameResolver(resolver)
def __init__(self, resolver, cache_size):
self.resolver = resolver
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d03fd37b0..46ed3be96 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0
DNSCACHE_ENABLED = True
DNSCACHE_SIZE = 10000
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
DNS_TIMEOUT = 60
DOWNLOAD_DELAY = 0