🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: replace `raw_response` property with smart content detection
Karim shoairff19dd5bf5e6
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex de705a9..7a2e50e 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -115,7 +115,10 @@ class ResponseFactory:history = cls._process_response_history(first_response, parser_arguments)try:- page_content = cls._get_page_content(page)+ if "html" in final_response.headers.get("content-type", ""):+ page_content = cls._get_page_content(page)+ else:+ page_content = final_response.text()except Exception as e: # pragma: no coverlog.error(f"Error getting page content: {e}")page_content = ""@@ -124,7 +127,6 @@ class ResponseFactory:**{"url": page.url,"content": page_content,- "raw_response": final_response.text(),"status": final_response.status,"reason": status_text,"encoding": encoding,@@ -248,7 +250,10 @@ class ResponseFactory:history = await cls._async_process_response_history(first_response, parser_arguments)try:- page_content = await cls._get_async_page_content(page)+ if "html" in final_response.headers.get("content-type", ""):+ page_content = await cls._get_async_page_content(page)+ else:+ page_content = await final_response.text()except Exception as e: # pragma: no coverlog.error(f"Error getting page content in async: {e}")page_content = ""@@ -258,7 +263,6 @@ class ResponseFactory:"url": page.url,"content": page_content,"status": final_response.status,- "raw_response": await final_response.text(),"reason": status_text,"encoding": encoding,"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 44e550f..1f20038 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -31,7 +31,6 @@ class Response(Selector):request_headers: Dict,encoding: str = "utf-8",method: str = "GET",- raw_response: str | bytes = "",history: List | None = None,**selector_config: Any,):@@ -40,7 +39,6 @@ class Response(Selector):self.reason = reasonself.cookies = cookiesself.headers = headers- self.raw_response = raw_response or contentself.request_headers = request_headersself.history = history or []super().__init__(