Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Parse robots.txt content as native str

Paul Tremberth28155dfcccc0
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 698f394ad..e26c22a09 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import to_native_str
logger = logging.getLogger(__name__)
@@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object):
# Running rp.parse() will set rp state from
# 'disallow all' to 'allow any'.
pass
- rp.parse(body.splitlines())
+ # stdlib's robotparser expects native 'str' ;
+ # with unicode input, non-ASCII encoded bytes decoding fails in Python2
+ rp.parse(to_native_str(body).splitlines())
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f2e94e171..95208c41f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
from __future__ import absolute_import
import re
from twisted.internet import reactor, error
@@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def _get_successful_crawler(self):
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
- ROBOTS = re.sub(b'^\s+(?m)', b'', b'''
+ ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
User-Agent: *
Disallow: /admin/
Disallow: /static/
- ''')
+
+ # taken from https://en.wikipedia.org/robots.txt
+ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+ Disallow: /wiki/Käyttäjä:
+ '''.encode('utf-8'))
response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()
@@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
return DeferredList([
self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
self.assertIgnored(Request('http://site.local/admin/main'), middleware),
- self.assertIgnored(Request('http://site.local/static/'), middleware)
+ self.assertIgnored(Request('http://site.local/static/'), middleware),
+ self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
+ self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
], fireOnOneErrback=True)
def test_robotstxt_ready_parser(self):