🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: parse quoted charset values in content-type headers
Bortlesboat6390c0af3d0a
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 8d624e0..cf303c8 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -10,7 +10,7 @@ from scrapling.core.utils import logfrom .custom import Response, StatusTextfrom scrapling.core._types import Dict, List, Optional-__CHARSET_RE__ = re_compile(r"charset=([\w-]+)")+__CHARSET_RE__ = re_compile(r"""charset=["']?([\w-]+)""")class ResponseFactory:diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 942835f..4679478 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -1,5 +1,6 @@import pytest+from scrapling.engines.toolbelt.convertor import ResponseFactoryfrom scrapling.engines.toolbelt.custom import StatusText, Responsefrom scrapling.engines.toolbelt.navigation import (construct_proxy_dict,@@ -139,6 +140,32 @@ def test_unknown_status_code():assert StatusText.get(1000) == "Unknown Status Code"+# The private classmethod is name-mangled; resolve it once for the tests below.+_extract_encoding = getattr(ResponseFactory, "_ResponseFactory__extract_browser_encoding")+++def test_browser_encoding_unquoted_charset():+ """A charset declared without quotes is returned verbatim."""+ assert _extract_encoding("text/html; charset=utf-8") == "utf-8"+ assert _extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"+ assert _extract_encoding("text/html;charset=windows-1252") == "windows-1252"+++def test_browser_encoding_quoted_charset():+ """A quoted charset value (RFC 7231 allows quoting) is unwrapped, not dropped."""+ assert _extract_encoding('text/html; charset="utf-8"') == "utf-8"+ assert _extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"+ assert _extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"+ assert _extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"+++def test_browser_encoding_defaults_when_missing():+ """Fall back to the default when no charset is present or the header is empty."""+ assert _extract_encoding("text/html") == "utf-8"+ assert _extract_encoding("") == "utf-8"+ assert _extract_encoding(None) == "utf-8"++class TestConstructProxyDict:"""Test proxy dictionary construction"""