Scrapy, a fast high-level web crawling & scraping framework for Python.
Parse robots.txt content as native str
Paul Tremberth28155dfcccc0
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 698f394ad..e26c22a09 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequestfrom scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_info+from scrapy.utils.python import to_native_strlogger = logging.getLogger(__name__)@@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object):# Running rp.parse() will set rp state from# 'disallow all' to 'allow any'.pass- rp.parse(body.splitlines())+ # stdlib's robotparser expects native 'str' ;+ # with unicode input, non-ASCII encoded bytes decoding fails in Python2+ rp.parse(to_native_str(body).splitlines())rp_dfd = self._parsers[netloc]self._parsers[netloc] = rpdiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex f2e94e171..95208c41f 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,3 +1,4 @@+# -*- coding: utf-8 -*-from __future__ import absolute_importimport refrom twisted.internet import reactor, error@@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):def _get_successful_crawler(self):crawler = self.crawlercrawler.settings.set('ROBOTSTXT_OBEY', True)- ROBOTS = re.sub(b'^\s+(?m)', b'', b'''+ ROBOTS = re.sub(b'^\s+(?m)', b'', u'''User-Agent: *Disallow: /admin/Disallow: /static/- ''')++ # taken from https://en.wikipedia.org/robots.txt+ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:+ Disallow: /wiki/Käyttäjä:+ '''.encode('utf-8'))response = TextResponse('http://site.local/robots.txt', body=ROBOTS)def return_response(request, spider):deferred = Deferred()@@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):return DeferredList([self.assertNotIgnored(Request('http://site.local/allowed'), middleware),self.assertIgnored(Request('http://site.local/admin/main'), middleware),- self.assertIgnored(Request('http://site.local/static/'), middleware)+ self.assertIgnored(Request('http://site.local/static/'), middleware),+ self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),+ self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)], fireOnOneErrback=True)def test_robotstxt_ready_parser(self):