Scrapy, a fast high-level web crawling & scraping framework for Python.
BOM should take precedence over Content-Type header when detecting the encoding closes #5601
Mohammadtaher Abbasie411ea94eb85
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 89516b9b6..bfcde878d 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -11,8 +11,13 @@ from typing import Generator, Tuplefrom urllib.parse import urljoinimport parsel-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,- http_content_type_encoding, resolve_encoding)+from w3lib.encoding import (+ html_body_declared_encoding,+ html_to_unicode,+ http_content_type_encoding,+ resolve_encoding,+ read_bom,+)from w3lib.html import strip_html5_whitespacefrom scrapy.http import Request@@ -60,6 +65,7 @@ class TextResponse(Response):def _declared_encoding(self):return (self._encoding+ or self._bom_encoding()or self._headers_encoding()or self._body_declared_encoding())@@ -117,6 +123,10 @@ class TextResponse(Response):def _body_declared_encoding(self):return html_body_declared_encoding(self.body)+ @memoizemethod_noargs+ def _bom_encoding(self):+ return read_bom(self.body)[0]+@propertydef selector(self):from scrapy.selector import Selector