Scrapy, a fast high-level web crawling & scraping framework for Python.
Use charset-normalizer for undeclared encodings (#8054)
Adriana0233152681f
diff --git a/pyproject.toml b/pyproject.tomlindex b97af0319..51e29e72e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -9,6 +9,7 @@ description = "A high-level Web Crawling and Web Scraping framework"dependencies = ["Twisted>=21.7.0","aiohttp>=3.13.3",+ "charset-normalizer>=3.4.0","cryptography>=41.0.5","cssselect>=0.9.1","defusedxml>=0.7.1",diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 525043918..c4c48db1c 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, castfrom urllib.parse import urljoinimport parsel+from charset_normalizer import from_bytesfrom w3lib.encoding import (html_body_declared_encoding,html_to_unicode,@@ -139,13 +140,14 @@ class TextResponse(Response):return self._cached_bencdef _auto_detect_fun(self, text: bytes) -> str | None:- for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):+ for enc in (self._DEFAULT_ENCODING, "utf-8"):try:text.decode(enc)except UnicodeError:continuereturn resolve_encoding(enc)- return None+ match = from_bytes(text).best()+ return resolve_encoding(match.encoding) if match else None@memoizemethod_noargsdef _body_declared_encoding(self) -> str | None:diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex f705dbcee..5745dfa0c 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -144,15 +144,15 @@ class TestTextResponse(TestResponseBase):assert r9._declared_encoding() is Noneself._assert_response_encoding(r5, "utf-8")self._assert_response_encoding(r8, "utf-8")- self._assert_response_encoding(r9, "cp1252")assert r4._body_inferred_encoding() is not Noneassert r4._body_inferred_encoding() != "ascii"+ assert r9._body_inferred_encoding() is not None+ assert r9._body_inferred_encoding() != "ascii"self._assert_response_values(r1, "utf-8", "\xa3")self._assert_response_values(r2, "utf-8", "\xa3")self._assert_response_values(r3, "iso-8859-1", "\xa3")self._assert_response_values(r6, "gb18030", "\u2015")self._assert_response_values(r7, "gb18030", "\u2015")- self._assert_response_values(r9, "cp1252", "€")# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodieswith pytest.raises(TypeError):@@ -168,6 +168,13 @@ class TestTextResponse(TestResponseBase):assert r._declared_encoding() is Noneself._assert_response_values(r, "utf-8", "\xa3")+ def test_body_inferred_encoding_of_undeclared_legacy_page(self):+ body = "ICD10 国際疾病分類第10版2013年版 病名マスター".encode("shift-jis")+ r = self.response_class("http://www.example.com", body=body)+ assert r._declared_encoding() is None+ assert r.encoding in {"shift_jis", "cp932"}+ assert r.text == body.decode(r.encoding)+def test_utf16(self):"""Test utf-16 because UnicodeDammit is known to have problems with"""body = b"\xff\xfeh\x00i\x00"diff --git a/tox.ini b/tox.iniindex c988573bb..bea257c7b 100644--- a/tox.ini+++ b/tox.ini@@ -146,6 +146,7 @@ deps =backports.zstd==1.3.0brotli==1.2.0; implementation_name != "pypy"brotlicffi==1.2.0.0; implementation_name == "pypy"+ charset-normalizer==3.4.0cryptography==41.0.5cssselect==0.9.1httpx2==2.0.0@@ -326,6 +327,7 @@ deps =Twisted==21.7.0backports.zstd==1.3.0brotlicffi==1.2.0.0+ charset-normalizer==3.4.0cryptography==44.0.2cssselect==0.9.1httpx2==2.0.0