Scrapy, a fast high-level web crawling & scraping framework for Python.
from_crawler support for name resolvers
Eugenio Lacuestaf45b4c7f8d19
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 5658e264b..4531c3aac 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -305,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):d.addBoth(self._stop_reactor)resolver_class = load_object(self.settings["DNS_RESOLVER"])- resolver_class.install(reactor, self.settings)+ resolver_class.install_on_reactor(reactor, crawler=self)tp = reactor.getThreadPool()tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))reactor.addSystemEventTrigger('before', 'shutdown', self.stop)diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 2bef9f1b8..792563e79 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -4,6 +4,7 @@ from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver,from zope.interface.declarations import implementer, providerfrom scrapy.utils.datatypes import LocalCache+from scrapy.utils.misc import create_instance# TODO: cache misses@@ -13,23 +14,27 @@ dnscache = LocalCache(10000)@implementer(IResolverSimple)class CachingThreadedResolver(ThreadedResolver):"""- Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests+ Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests."""- @classmethod- def install(cls, reactor, settings):- if settings.getbool('DNSCACHE_ENABLED'):- cache_size = settings.getint('DNSCACHE_SIZE')- else:- cache_size = 0- resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))- reactor.installResolver(resolver)-def __init__(self, reactor, cache_size, timeout):super(CachingThreadedResolver, self).__init__(reactor)dnscache.limit = cache_sizeself.timeout = timeout+ @classmethod+ def from_crawler(cls, crawler, reactor):+ if crawler.settings.getbool('DNSCACHE_ENABLED'):+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')+ else:+ cache_size = 0+ return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))++ @classmethod+ def install_on_reactor(cls, reactor, crawler):+ resolver = create_instance(cls, None, crawler, reactor=reactor)+ reactor.installResolver(resolver)+def getHostByName(self, name, timeout=None):if name in dnscache:return defer.succeed(dnscache[name])@@ -51,21 +56,26 @@ class CachingThreadedResolver(ThreadedResolver):@implementer(IHostnameResolver)class CachingHostnameResolver:"""- Experimental caching resolver, supporting IPv4 and IPv6+ Experimental caching resolver. Resolves IPv4 and IPv6 addresses,+ does not support setting a timeout value for DNS requests."""+ def __init__(self, reactor, cache_size):+ self.resolver = reactor.nameResolver+ dnscache.limit = cache_size+@classmethod- def install(cls, reactor, settings):- if settings.getbool('DNSCACHE_ENABLED'):- cache_size = settings.getint('DNSCACHE_SIZE')+ def from_crawler(cls, crawler, reactor):+ if crawler.settings.getbool('DNSCACHE_ENABLED'):+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')else:cache_size = 0- resolver = cls(reactor.nameResolver, cache_size)- reactor.installNameResolver(resolver)+ return cls(reactor, cache_size)- def __init__(self, resolver, cache_size):- self.resolver = resolver- dnscache.limit = cache_size+ @classmethod+ def install_on_reactor(cls, reactor, crawler):+ resolver = create_instance(cls, None, crawler, reactor=reactor)+ reactor.installNameResolver(resolver)def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,addressTypes=None, transportSemantics='TCP'):