Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

from_crawler support for name resolvers

Eugenio Lacuestaf45b4c7f8d19
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 5658e264b..4531c3aac 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -305,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):
d.addBoth(self._stop_reactor)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
- resolver_class.install(reactor, self.settings)
+ resolver_class.install_on_reactor(reactor, crawler=self)
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 2bef9f1b8..792563e79 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,6 +4,7 @@ from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver,
from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
+from scrapy.utils.misc import create_instance
# TODO: cache misses
@@ -13,23 +14,27 @@ dnscache = LocalCache(10000)
@implementer(IResolverSimple)
class CachingThreadedResolver(ThreadedResolver):
"""
- Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests
+ Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
"""
- @classmethod
- def install(cls, reactor, settings):
- if settings.getbool('DNSCACHE_ENABLED'):
- cache_size = settings.getint('DNSCACHE_SIZE')
- else:
- cache_size = 0
- resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))
- reactor.installResolver(resolver)
-
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
+ @classmethod
+ def from_crawler(cls, crawler, reactor):
+ if crawler.settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+ else:
+ cache_size = 0
+ return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
+
+ @classmethod
+ def install_on_reactor(cls, reactor, crawler):
+ resolver = create_instance(cls, None, crawler, reactor=reactor)
+ reactor.installResolver(resolver)
+
def getHostByName(self, name, timeout=None):
if name in dnscache:
return defer.succeed(dnscache[name])
@@ -51,21 +56,26 @@ class CachingThreadedResolver(ThreadedResolver):
@implementer(IHostnameResolver)
class CachingHostnameResolver:
"""
- Experimental caching resolver, supporting IPv4 and IPv6
+ Experimental caching resolver. Resolves IPv4 and IPv6 addresses,
+ does not support setting a timeout value for DNS requests.
"""
+ def __init__(self, reactor, cache_size):
+ self.resolver = reactor.nameResolver
+ dnscache.limit = cache_size
+
@classmethod
- def install(cls, reactor, settings):
- if settings.getbool('DNSCACHE_ENABLED'):
- cache_size = settings.getint('DNSCACHE_SIZE')
+ def from_crawler(cls, crawler, reactor):
+ if crawler.settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')
else:
cache_size = 0
- resolver = cls(reactor.nameResolver, cache_size)
- reactor.installNameResolver(resolver)
+ return cls(reactor, cache_size)
- def __init__(self, resolver, cache_size):
- self.resolver = resolver
- dnscache.limit = cache_size
+ @classmethod
+ def install_on_reactor(cls, reactor, crawler):
+ resolver = create_instance(cls, None, crawler, reactor=reactor)
+ reactor.installNameResolver(resolver)
def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
addressTypes=None, transportSemantics='TCP'):