Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: replace `raw_response` property with smart content detection

Karim shoairff19dd5bf5e6
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index de705a9..7a2e50e 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -115,7 +115,10 @@ class ResponseFactory:
history = cls._process_response_history(first_response, parser_arguments)
try:
- page_content = cls._get_page_content(page)
+ if "html" in final_response.headers.get("content-type", ""):
+ page_content = cls._get_page_content(page)
+ else:
+ page_content = final_response.text()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}")
page_content = ""
@@ -124,7 +127,6 @@ class ResponseFactory:
**{
"url": page.url,
"content": page_content,
- "raw_response": final_response.text(),
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
@@ -248,7 +250,10 @@ class ResponseFactory:
history = await cls._async_process_response_history(first_response, parser_arguments)
try:
- page_content = await cls._get_async_page_content(page)
+ if "html" in final_response.headers.get("content-type", ""):
+ page_content = await cls._get_async_page_content(page)
+ else:
+ page_content = await final_response.text()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}")
page_content = ""
@@ -258,7 +263,6 @@ class ResponseFactory:
"url": page.url,
"content": page_content,
"status": final_response.status,
- "raw_response": await final_response.text(),
"reason": status_text,
"encoding": encoding,
"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 44e550f..1f20038 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -31,7 +31,6 @@ class Response(Selector):
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
- raw_response: str | bytes = "",
history: List | None = None,
**selector_config: Any,
):
@@ -40,7 +39,6 @@ class Response(Selector):
self.reason = reason
self.cookies = cookies
self.headers = headers
- self.raw_response = raw_response or content
self.request_headers = request_headers
self.history = history or []
super().__init__(