Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Custom name resolver implementing twisted.internet.interfaces.IHostnameResolver

Eugenio Lacuesta735c0ceb7890
diff --git a/pytest.ini b/pytest.ini
index c3f3292bb..a0e89f0a9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -158,6 +158,7 @@ flake8-ignore =
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
+ scrapy/resolver.py E501
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E501
scrapy/shell.py E501
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f87e67d93..1350ea84f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,34 +4,36 @@ import signal
import warnings
from twisted.internet import defer
-from zope.interface.verify import verifyClass, DoesNotImplement
+from zope.interface.verify import DoesNotImplement, verifyClass
-from scrapy import Spider
+from scrapy import signals, Spider
from scrapy.core.engine import ExecutionEngine
-from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderLoader
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
+from scrapy.interfaces import ISpiderLoader
+from scrapy.resolver import CachingHostnameResolver
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.misc import load_object
from scrapy.utils.log import (
- LogCounterHandler, configure_logging, log_scrapy_info,
- get_scrapy_root_handler, install_scrapy_root_handler)
-from scrapy import signals
+ configure_logging,
+ get_scrapy_root_handler,
+ install_scrapy_root_handler,
+ log_scrapy_info,
+ LogCounterHandler,
+)
+from scrapy.utils.misc import load_object
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
logger = logging.getLogger(__name__)
-class Crawler(object):
+class Crawler:
def __init__(self, spidercls, settings=None):
if isinstance(spidercls, Spider):
- raise ValueError(
- 'The spidercls argument must be a class, not an object')
+ raise ValueError('The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
@@ -110,7 +112,7 @@ class Crawler(object):
yield defer.maybeDeferred(self.engine.stop)
-class CrawlerRunner(object):
+class CrawlerRunner:
"""
This is a convenient helper class that keeps track of, manages and runs
crawlers inside an already setup :mod:`~twisted.internet.reactor`.
@@ -303,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
- reactor.installResolver(self._get_dns_resolver())
+ reactor.installNameResolver(self._get_dns_resolver())
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
@@ -315,10 +317,10 @@ class CrawlerProcess(CrawlerRunner):
cache_size = self.settings.getint('DNSCACHE_SIZE')
else:
cache_size = 0
- return CachingThreadedResolver(
- reactor=reactor,
+ return CachingHostnameResolver(
+ resolver=reactor.nameResolver,
cache_size=cache_size,
- timeout=self.settings.getfloat('DNS_TIMEOUT')
+ timeout=self.settings.getfloat('DNS_TIMEOUT'),
)
def _graceful_stop_reactor(self):
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 4df949015..03964f269 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,32 +1,39 @@
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
-# TODO: cache misses
+# TODO: cache misses
dnscache = LocalCache(10000)
-class CachingThreadedResolver(ThreadedResolver):
- def __init__(self, reactor, cache_size, timeout):
- super(CachingThreadedResolver, self).__init__(reactor)
- dnscache.limit = cache_size
+@implementer(IHostnameResolver)
+class CachingHostnameResolver(object):
+
+ def __init__(self, resolver, cache_size, timeout):
+ self.resolver = resolver
self.timeout = timeout
+ dnscache.limit = cache_size
+
+ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+ addressTypes=None, transportSemantics='TCP'):
+
+ @provider(IResolutionReceiver)
+ class CachingResolutionReceiver(resolutionReceiver):
+ def resolutionBegan(self, resolution):
+ super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+ self.resolution = resolution
+
+ def resolutionComplete(self):
+ super(CachingResolutionReceiver, self).resolutionComplete()
+ dnscache[hostName] = self.resolution
- def getHostByName(self, name, timeout=None):
- if name in dnscache:
- return defer.succeed(dnscache[name])
- # in Twisted<=16.6, getHostByName() is always called with
- # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
- # so the input argument above is simply overridden
- # to enforce Scrapy's DNS_TIMEOUT setting's value
- timeout = (self.timeout,)
- d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
- if dnscache.limit:
- d.addCallback(self._cache_result, name)
- return d
-
- def _cache_result(self, result, name):
- dnscache[name] = result
- return result
+ try:
+ result = dnscache[hostName]
+ except KeyError:
+ result = self.resolver.resolveHostName(
+ CachingResolutionReceiver(), hostName, portNumber, addressTypes, transportSemantics
+ )
+ finally:
+ return result