Scrapy, a fast high-level web crawling & scraping framework for Python.
Cache resolution only if the DNS request was successful
Eugenio Lacuesta55babf9acd6c
diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 8792ed6ab..0ba22ed0d 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -10,7 +10,7 @@ dnscache = LocalCache(10000)@implementer(IHostnameResolver)-class CachingHostnameResolver(object):+class CachingHostnameResolver:def __init__(self, resolver, cache_size, timeout):self.resolver = resolver@@ -25,15 +25,21 @@ class CachingHostnameResolver(object):def __init__(self, timeout):self.timeout = timeout+ self.resolved = Falsedef resolutionBegan(self, resolution):super(CachingResolutionReceiver, self).resolutionBegan(resolution)self.resolution = resolution# reactor.callLater(self.timeout, resolution.cancel)+ def addressResolved(self, address):+ super(CachingResolutionReceiver, self).addressResolved(address)+ self.resolved = True+def resolutionComplete(self):super(CachingResolutionReceiver, self).resolutionComplete()- dnscache[hostName] = self.resolution+ if self.resolved:+ dnscache[hostName] = self.resolutiontry:result = dnscache[hostName]