Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into master

Konstantin Lopuhin936dbc7bf624
diff --cc scrapy/downloadermiddlewares/robotstxt.py
index 4f3b5d64f,b86c09c14..200245210
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@@ -41,11 -41,13 +41,13 @@@ class RobotsTxtMiddleware(object)
return d
def process_request_2(self, rp, request, spider):
- if rp is not None and not rp.can_fetch(
- to_native_str(self._useragent), request.url):
+ if rp is None:
+ return
+ if not rp.can_fetch(to_native_str(self._useragent), request.url):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
+ self.crawler.stats.inc_value('robotstxt/forbidden')
- raise IgnoreRequest()
+ raise IgnoreRequest("Forbidden by robots.txt")
def robot_parser(self, request, spider):
url = urlparse_cached(request)