Scrapy, a fast high-level web crawling & scraping framework for Python.
Ability to choose name resolver
Eugenio Lacuestae69cf415c832
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 61851acc3..c5351c08f 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -305,23 +305,13 @@ class CrawlerProcess(CrawlerRunner):returnd.addBoth(self._stop_reactor)- reactor.installNameResolver(self._get_dns_resolver())+ resolver_class = load_object(self.settings["DNS_RESOLVER"])+ resolver_class.install(reactor, self.settings)tp = reactor.getThreadPool()tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))reactor.addSystemEventTrigger('before', 'shutdown', self.stop)reactor.run(installSignalHandlers=False) # blocking call- def _get_dns_resolver(self):- from twisted.internet import reactor- if self.settings.getbool('DNSCACHE_ENABLED'):- cache_size = self.settings.getint('DNSCACHE_SIZE')- else:- cache_size = 0- return CachingHostnameResolver(- resolver=reactor.nameResolver,- cache_size=cache_size,- )-def _graceful_stop_reactor(self):d = self.stop()d.addBoth(self._stop_reactor)diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex ddbae61a9..2bef9f1b8 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -1,4 +1,6 @@-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver+from twisted.internet import defer+from twisted.internet.base import ThreadedResolver+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimplefrom zope.interface.declarations import implementer, providerfrom scrapy.utils.datatypes import LocalCache@@ -8,8 +10,58 @@ from scrapy.utils.datatypes import LocalCachednscache = LocalCache(10000)+@implementer(IResolverSimple)+class CachingThreadedResolver(ThreadedResolver):+ """+ Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests+ """++ @classmethod+ def install(cls, reactor, settings):+ if settings.getbool('DNSCACHE_ENABLED'):+ cache_size = settings.getint('DNSCACHE_SIZE')+ else:+ cache_size = 0+ resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))+ reactor.installResolver(resolver)++ def __init__(self, reactor, cache_size, timeout):+ super(CachingThreadedResolver, self).__init__(reactor)+ dnscache.limit = cache_size+ self.timeout = timeout++ def getHostByName(self, name, timeout=None):+ if name in dnscache:+ return defer.succeed(dnscache[name])+ # in Twisted<=16.6, getHostByName() is always called with+ # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),+ # so the input argument above is simply overridden+ # to enforce Scrapy's DNS_TIMEOUT setting's value+ timeout = (self.timeout,)+ d = super(CachingThreadedResolver, self).getHostByName(name, timeout)+ if dnscache.limit:+ d.addCallback(self._cache_result, name)+ return d++ def _cache_result(self, result, name):+ dnscache[name] = result+ return result++@implementer(IHostnameResolver)class CachingHostnameResolver:+ """+ Experimental caching resolver, supporting IPv4 and IPv6+ """++ @classmethod+ def install(cls, reactor, settings):+ if settings.getbool('DNSCACHE_ENABLED'):+ cache_size = settings.getint('DNSCACHE_SIZE')+ else:+ cache_size = 0+ resolver = cls(reactor.nameResolver, cache_size)+ reactor.installNameResolver(resolver)def __init__(self, resolver, cache_size):self.resolver = resolverdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d03fd37b0..46ed3be96 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0DNSCACHE_ENABLED = TrueDNSCACHE_SIZE = 10000+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'DNS_TIMEOUT = 60DOWNLOAD_DELAY = 0