Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix handling of already failed deferreds when downloading page in robots.txt middleware.
Artur Gasparb2beb3e85d2e
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex c061c2407..7f6f0d012 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -8,7 +8,9 @@ import loggingfrom six.moves.urllib import robotparser+from twisted.internet import reactorfrom twisted.internet.defer import Deferred, maybeDeferred+from twisted.internet.task import deferLaterfrom scrapy.exceptions import NotConfigured, IgnoreRequestfrom scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cached@@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object):priority=self.DOWNLOAD_PRIORITY,meta={'dont_obey_robotstxt': True})- dfd = self.crawler.engine.download(robotsreq, spider)+ # engine.download() can return an already-called deferred, e.g. if a+ # middleware returns a response in process_request(). Using+ # deferLater() ensures that the error callback isn't called+ # immediately upon being added, so that it doesn't remove the key+ # before we check for it.+ dfd = deferLater(reactor, 0, self.crawler.engine.download,+ robotsreq, spider)dfd.addCallback(self._parse_robots, netloc)dfd.addErrback(self._logerror, robotsreq, spider)dfd.addErrback(self._robots_error, netloc)