Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix handling of already failed deferreds when downloading page in robots.txt middleware.

Artur Gasparb2beb3e85d2e
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c061c2407..7f6f0d012 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -8,7 +8,9 @@ import logging
from six.moves.urllib import robotparser
+from twisted.internet import reactor
from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.task import deferLater
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
@@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object):
priority=self.DOWNLOAD_PRIORITY,
meta={'dont_obey_robotstxt': True}
)
- dfd = self.crawler.engine.download(robotsreq, spider)
+ # engine.download() can return an already-called deferred, e.g. if a
+ # middleware returns a response in process_request(). Using
+ # deferLater() ensures that the error callback isn't called
+ # immediately upon being added, so that it doesn't remove the key
+ # before we check for it.
+ dfd = deferLater(reactor, 0, self.crawler.engine.download,
+ robotsreq, spider)
dfd.addCallback(self._parse_robots, netloc)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)