Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enforce DNS resolution timeout

Paul Tremberthd2e9ea0c88b7
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 3954fd977..4f4f0b04f 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -16,8 +16,11 @@ class CachingThreadedResolver(ThreadedResolver):
def getHostByName(self, name, timeout=None):
if name in dnscache:
return defer.succeed(dnscache[name])
- if not timeout:
- timeout = self.timeout
+ # in Twisted<=16.6, getHostByName() is always called with
+ # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
+ # so the input argument above is simply overridden
+ # to enforce Scrapy's DNS_TIMEOUT setting's value
+ timeout = (self.timeout,)
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
d.addCallback(self._cache_result, name)
return d