Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into master
Konstantin Lopuhin936dbc7bf624
diff --cc scrapy/downloadermiddlewares/robotstxt.pyindex 4f3b5d64f,b86c09c14..200245210--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@@ -41,11 -41,13 +41,13 @@@ class RobotsTxtMiddleware(object)return ddef process_request_2(self, rp, request, spider):- if rp is not None and not rp.can_fetch(- to_native_str(self._useragent), request.url):+ if rp is None:+ return+ if not rp.can_fetch(to_native_str(self._useragent), request.url):logger.debug("Forbidden by robots.txt: %(request)s",{'request': request}, extra={'spider': spider})+ self.crawler.stats.inc_value('robotstxt/forbidden')- raise IgnoreRequest()+ raise IgnoreRequest("Forbidden by robots.txt")def robot_parser(self, request, spider):url = urlparse_cached(request)