Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

BOM should take precedence over Content-Type header when detecting the encoding closes #5601

Mohammadtaher Abbasie411ea94eb85
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 89516b9b6..bfcde878d 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,8 +11,13 @@ from typing import Generator, Tuple
from urllib.parse import urljoin
import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
- http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+ html_body_declared_encoding,
+ html_to_unicode,
+ http_content_type_encoding,
+ resolve_encoding,
+ read_bom,
+)
from w3lib.html import strip_html5_whitespace
from scrapy.http import Request
@@ -60,6 +65,7 @@ class TextResponse(Response):
def _declared_encoding(self):
return (
self._encoding
+ or self._bom_encoding()
or self._headers_encoding()
or self._body_declared_encoding()
)
@@ -117,6 +123,10 @@ class TextResponse(Response):
def _body_declared_encoding(self):
return html_body_declared_encoding(self.body)
+ @memoizemethod_noargs
+ def _bom_encoding(self):
+ return read_bom(self.body)[0]
+
@property
def selector(self):
from scrapy.selector import Selector