🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Handling problematic encodings
Karim shoair14ffdbd6fc49
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex f0053d7..0cdc65e 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -39,7 +39,7 @@ class ResponseEncoding:@classmethod@cache(maxsize=None)- def get_value(cls, content_type: Optional[str]) -> str:+ def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:"""Determine the appropriate character encoding from a content-type header.The encoding is determined by these rules in order:@@ -50,26 +50,30 @@ class ResponseEncoding:5. Default to UTF-8 if nothing else matches:param content_type: Content-Type header value or None+ :param text: A text to test the encoding on it:return: String naming the character encoding"""if not content_type:return cls.__DEFAULT_ENCODINGtry:+ encoding = Nonecontent_type, params = cls.__parse_content_type(content_type)# First check for explicit charset parameterif "charset" in params:encoding = params["charset"].strip("'\"")- "test".encode(encoding) # Validate encoding- return encoding# Apply content-type specific rulesif content_type in cls.__ISO_8859_1_CONTENT_TYPES:- return "ISO-8859-1"+ encoding = "ISO-8859-1"if content_type == "application/json":- return cls.__DEFAULT_ENCODING+ encoding = cls.__DEFAULT_ENCODING++ if encoding:+ _ = text.encode(encoding) # Validate encoding and validate it can encode the given text+ return encodingreturn cls.__DEFAULT_ENCODING@@ -87,7 +91,7 @@ class Response(Adaptor):self.cookies = cookiesself.headers = headersself.request_headers = request_headers- encoding = ResponseEncoding.get_value(encoding)+ encoding = ResponseEncoding.get_value(encoding, text)super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)# For back-ward compatibilityself.adaptor = self