Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle robots.txt files not UTF-8 encoded

Lorenzo Verardo7b37dcd80d37
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 6ea2bfd97..ad06137e2 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
if to_native_str_type:
robotstxt_body = to_unicode(robotstxt_body)
else:
- robotstxt_body = robotstxt_body.decode("utf-8")
+ robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
except UnicodeDecodeError:
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
# Switch to 'allow all' state.
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index d7a923085..6ad30deed 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,5 +1,7 @@
from twisted.trial import unittest
+from scrapy.robotstxt import decode_robotstxt
+
def reppy_available():
# check if reppy parser is installed
@@ -141,6 +143,25 @@ class BaseRobotParserTest:
)
+class DecodeRobotsTxtTest(unittest.TestCase):
+ def test_native_string_conversion(self):
+ robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+ decoded_content = decode_robotstxt(
+ robotstxt_body, spider=None, to_native_str_type=True
+ )
+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+ def test_decode_utf8(self):
+ robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+ def test_decode_non_utf8(self):
+ robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n"
+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+ self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+
class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import PythonRobotParser