Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use charset-normalizer for undeclared encodings (#8054)

Adriana0233152681f
diff --git a/pyproject.toml b/pyproject.toml
index b97af0319..51e29e72e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,6 +9,7 @@ description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
"Twisted>=21.7.0",
"aiohttp>=3.13.3",
+ "charset-normalizer>=3.4.0",
"cryptography>=41.0.5",
"cssselect>=0.9.1",
"defusedxml>=0.7.1",
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 525043918..c4c48db1c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
import parsel
+from charset_normalizer import from_bytes
from w3lib.encoding import (
html_body_declared_encoding,
html_to_unicode,
@@ -139,13 +140,14 @@ class TextResponse(Response):
return self._cached_benc
def _auto_detect_fun(self, text: bytes) -> str | None:
- for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
+ for enc in (self._DEFAULT_ENCODING, "utf-8"):
try:
text.decode(enc)
except UnicodeError:
continue
return resolve_encoding(enc)
- return None
+ match = from_bytes(text).best()
+ return resolve_encoding(match.encoding) if match else None
@memoizemethod_noargs
def _body_declared_encoding(self) -> str | None:
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index f705dbcee..5745dfa0c 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -144,15 +144,15 @@ class TestTextResponse(TestResponseBase):
assert r9._declared_encoding() is None
self._assert_response_encoding(r5, "utf-8")
self._assert_response_encoding(r8, "utf-8")
- self._assert_response_encoding(r9, "cp1252")
assert r4._body_inferred_encoding() is not None
assert r4._body_inferred_encoding() != "ascii"
+ assert r9._body_inferred_encoding() is not None
+ assert r9._body_inferred_encoding() != "ascii"
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")
self._assert_response_values(r6, "gb18030", "\u2015")
self._assert_response_values(r7, "gb18030", "\u2015")
- self._assert_response_values(r9, "cp1252", "€")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
with pytest.raises(TypeError):
@@ -168,6 +168,13 @@ class TestTextResponse(TestResponseBase):
assert r._declared_encoding() is None
self._assert_response_values(r, "utf-8", "\xa3")
+ def test_body_inferred_encoding_of_undeclared_legacy_page(self):
+ body = "ICD10 国際疾病分類第10版2013年版 病名マスター".encode("shift-jis")
+ r = self.response_class("http://www.example.com", body=body)
+ assert r._declared_encoding() is None
+ assert r.encoding in {"shift_jis", "cp932"}
+ assert r.text == body.decode(r.encoding)
+
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
body = b"\xff\xfeh\x00i\x00"
diff --git a/tox.ini b/tox.ini
index c988573bb..bea257c7b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -146,6 +146,7 @@ deps =
backports.zstd==1.3.0
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
+ charset-normalizer==3.4.0
cryptography==41.0.5
cssselect==0.9.1
httpx2==2.0.0
@@ -326,6 +327,7 @@ deps =
Twisted==21.7.0
backports.zstd==1.3.0
brotlicffi==1.2.0.0
+ charset-normalizer==3.4.0
cryptography==44.0.2
cssselect==0.9.1
httpx2==2.0.0