Scrapy, a fast high-level web crawling & scraping framework for Python.
UTF-8 BOM at the beginning of the file ignored (#7095)
Vasiliy Kiryanov7b215c6578de
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex e1a12be05..18b622546 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -28,7 +28,7 @@ def decode_robotstxt(if to_native_str_type:body_decoded = to_unicode(robotstxt_body)else:- body_decoded = robotstxt_body.decode("utf-8", errors="ignore")+ body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore")except UnicodeDecodeError:# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.# Switch to 'allow all' state.diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 6a24d2e90..bc79e3a17 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -129,6 +129,12 @@ class TestDecodeRobotsTxt:decoded_content = decode_robotstxt(robotstxt_body, spider=None)assert decoded_content == "User-agent: *\nDisallow: /\n"+ # UTF-8 BOM at the beginning of the file ignored+ def test_decode_utf8_bom(self):+ robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)+ assert decoded_content == "User-agent: *\nDisallow: /\n"+class TestPythonRobotParser(BaseRobotParserTest):def setup_method(self):