Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

UTF-8 BOM at the beginning of the file ignored (#7095)

Vasiliy Kiryanov7b215c6578de
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index e1a12be05..18b622546 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -28,7 +28,7 @@ def decode_robotstxt(
if to_native_str_type:
body_decoded = to_unicode(robotstxt_body)
else:
- body_decoded = robotstxt_body.decode("utf-8", errors="ignore")
+ body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore")
except UnicodeDecodeError:
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
# Switch to 'allow all' state.
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 6a24d2e90..bc79e3a17 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -129,6 +129,12 @@ class TestDecodeRobotsTxt:
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
+ # UTF-8 BOM at the beginning of the file ignored
+ def test_decode_utf8_bom(self):
+ robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"
+ decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+ assert decoded_content == "User-agent: *\nDisallow: /\n"
+
class TestPythonRobotParser(BaseRobotParserTest):
def setup_method(self):