Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle robots.txt files not UTF-8 encoded
Lorenzo Verardo7b37dcd80d37
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 6ea2bfd97..ad06137e2 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):if to_native_str_type:robotstxt_body = to_unicode(robotstxt_body)else:- robotstxt_body = robotstxt_body.decode("utf-8")+ robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")except UnicodeDecodeError:# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.# Switch to 'allow all' state.diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex d7a923085..6ad30deed 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -1,5 +1,7 @@from twisted.trial import unittest+from scrapy.robotstxt import decode_robotstxt+def reppy_available():# check if reppy parser is installed@@ -141,6 +143,25 @@ class BaseRobotParserTest:)+class DecodeRobotsTxtTest(unittest.TestCase):+ def test_native_string_conversion(self):+ robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")+ decoded_content = decode_robotstxt(+ robotstxt_body, spider=None, to_native_str_type=True+ )+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")++ def test_decode_utf8(self):+ robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")++ def test_decode_non_utf8(self):+ robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n"+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")++class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):def setUp(self):from scrapy.robotstxt import PythonRobotParser