Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: added `raw_response` to separate collected response from rendered response

Karim shoairdf073c468075
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 4ac0558..8d70dbd 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -343,7 +343,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
page_info.page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+ page_info.page, first_response, final_response[0], params.selector_config
)
# Close the page to free up resources
@@ -636,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+ page_info.page, first_response, final_response[0], params.selector_config
)
# Close the page to free up resources
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index bcfd0ac..9fc3490 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -268,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
# Create response object
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+ page_info.page, first_response, final_response[0], params.selector_config
)
# Close the page to free up resources
@@ -492,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+ page_info.page, first_response, final_response[0], params.selector_config
)
# Close the page to free up resources
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 5bb4a49..de705a9 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -85,7 +85,6 @@ class ResponseFactory:
first_response: SyncResponse,
final_response: Optional[SyncResponse],
parser_arguments: Dict,
- automated_page: bool = False,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -101,7 +100,6 @@ class ResponseFactory:
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
- :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
@@ -117,7 +115,7 @@ class ResponseFactory:
history = cls._process_response_history(first_response, parser_arguments)
try:
- page_content = final_response.text() if not automated_page else cls._get_page_content(page)
+ page_content = cls._get_page_content(page)
except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}")
page_content = ""
@@ -126,6 +124,7 @@ class ResponseFactory:
**{
"url": page.url,
"content": page_content,
+ "raw_response": final_response.text(),
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
@@ -219,7 +218,6 @@ class ResponseFactory:
first_response: AsyncResponse,
final_response: Optional[AsyncResponse],
parser_arguments: Dict,
- automated_page: bool = False,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -235,7 +233,6 @@ class ResponseFactory:
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
- :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
@@ -251,7 +248,7 @@ class ResponseFactory:
history = await cls._async_process_response_history(first_response, parser_arguments)
try:
- page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page))
+ page_content = await cls._get_async_page_content(page)
except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}")
page_content = ""
@@ -261,6 +258,7 @@ class ResponseFactory:
"url": page.url,
"content": page_content,
"status": final_response.status,
+ "raw_response": await final_response.text(),
"reason": status_text,
"encoding": encoding,
"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 1f20038..44e550f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -31,6 +31,7 @@ class Response(Selector):
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
+ raw_response: str | bytes = "",
history: List | None = None,
**selector_config: Any,
):
@@ -39,6 +40,7 @@ class Response(Selector):
self.reason = reason
self.cookies = cookies
self.headers = headers
+ self.raw_response = raw_response or content
self.request_headers = request_headers
self.history = history or []
super().__init__(