Scrapy, a fast high-level web crawling & scraping framework for Python.
Custom name resolver implementing twisted.internet.interfaces.IHostnameResolver
Eugenio Lacuesta735c0ceb7890
diff --git a/pytest.ini b/pytest.iniindex c3f3292bb..a0e89f0a9 100644--- a/pytest.ini+++ b/pytest.ini@@ -158,6 +158,7 @@ flake8-ignore =scrapy/mail.py E402 E128 E501 E502scrapy/middleware.py E128 E501scrapy/pqueues.py E501+ scrapy/resolver.py E501scrapy/responsetypes.py E128 E501 E305scrapy/robotstxt.py E501scrapy/shell.py E501diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex f87e67d93..1350ea84f 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -4,34 +4,36 @@ import signalimport warningsfrom twisted.internet import defer-from zope.interface.verify import verifyClass, DoesNotImplement+from zope.interface.verify import DoesNotImplement, verifyClass-from scrapy import Spider+from scrapy import signals, Spiderfrom scrapy.core.engine import ExecutionEngine-from scrapy.resolver import CachingThreadedResolver-from scrapy.interfaces import ISpiderLoader+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManager+from scrapy.interfaces import ISpiderLoader+from scrapy.resolver import CachingHostnameResolverfrom scrapy.settings import overridden_settings, Settingsfrom scrapy.signalmanager import SignalManager-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names-from scrapy.utils.misc import load_objectfrom scrapy.utils.log import (- LogCounterHandler, configure_logging, log_scrapy_info,- get_scrapy_root_handler, install_scrapy_root_handler)-from scrapy import signals+ configure_logging,+ get_scrapy_root_handler,+ install_scrapy_root_handler,+ log_scrapy_info,+ LogCounterHandler,+)+from scrapy.utils.misc import load_object+from scrapy.utils.ossignal import install_shutdown_handlers, signal_nameslogger = logging.getLogger(__name__)-class Crawler(object):+class Crawler:def __init__(self, spidercls, settings=None):if isinstance(spidercls, Spider):- raise ValueError(- 'The spidercls argument must be a class, not an object')+ raise ValueError('The spidercls argument must be a class, not an object')if isinstance(settings, dict) or settings is None:settings = Settings(settings)@@ -110,7 +112,7 @@ class Crawler(object):yield defer.maybeDeferred(self.engine.stop)-class CrawlerRunner(object):+class CrawlerRunner:"""This is a convenient helper class that keeps track of, manages and runscrawlers inside an already setup :mod:`~twisted.internet.reactor`.@@ -303,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):returnd.addBoth(self._stop_reactor)- reactor.installResolver(self._get_dns_resolver())+ reactor.installNameResolver(self._get_dns_resolver())tp = reactor.getThreadPool()tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))reactor.addSystemEventTrigger('before', 'shutdown', self.stop)@@ -315,10 +317,10 @@ class CrawlerProcess(CrawlerRunner):cache_size = self.settings.getint('DNSCACHE_SIZE')else:cache_size = 0- return CachingThreadedResolver(- reactor=reactor,+ return CachingHostnameResolver(+ resolver=reactor.nameResolver,cache_size=cache_size,- timeout=self.settings.getfloat('DNS_TIMEOUT')+ timeout=self.settings.getfloat('DNS_TIMEOUT'),)def _graceful_stop_reactor(self):diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 4df949015..03964f269 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -1,32 +1,39 @@-from twisted.internet import defer-from twisted.internet.base import ThreadedResolver+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver+from zope.interface.declarations import implementer, providerfrom scrapy.utils.datatypes import LocalCache-# TODO: cache misses+# TODO: cache missesdnscache = LocalCache(10000)-class CachingThreadedResolver(ThreadedResolver):- def __init__(self, reactor, cache_size, timeout):- super(CachingThreadedResolver, self).__init__(reactor)- dnscache.limit = cache_size+@implementer(IHostnameResolver)+class CachingHostnameResolver(object):++ def __init__(self, resolver, cache_size, timeout):+ self.resolver = resolverself.timeout = timeout+ dnscache.limit = cache_size++ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,+ addressTypes=None, transportSemantics='TCP'):++ @provider(IResolutionReceiver)+ class CachingResolutionReceiver(resolutionReceiver):+ def resolutionBegan(self, resolution):+ super(CachingResolutionReceiver, self).resolutionBegan(resolution)+ self.resolution = resolution++ def resolutionComplete(self):+ super(CachingResolutionReceiver, self).resolutionComplete()+ dnscache[hostName] = self.resolution- def getHostByName(self, name, timeout=None):- if name in dnscache:- return defer.succeed(dnscache[name])- # in Twisted<=16.6, getHostByName() is always called with- # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),- # so the input argument above is simply overridden- # to enforce Scrapy's DNS_TIMEOUT setting's value- timeout = (self.timeout,)- d = super(CachingThreadedResolver, self).getHostByName(name, timeout)- if dnscache.limit:- d.addCallback(self._cache_result, name)- return d-- def _cache_result(self, result, name):- dnscache[name] = result- return result+ try:+ result = dnscache[hostName]+ except KeyError:+ result = self.resolver.resolveHostName(+ CachingResolutionReceiver(), hostName, portNumber, addressTypes, transportSemantics+ )+ finally:+ return result