🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: added `raw_response` to separate collected response from rendered response
Karim shoairdf073c468075
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 4ac0558..8d70dbd 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -343,7 +343,7 @@ class StealthySession(StealthySessionMixin, SyncSession):page_info.page.wait_for_timeout(params.wait)response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config)# Close the page to free up resources@@ -636,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config)# Close the page to free up resourcesdiff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex bcfd0ac..9fc3490 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -268,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):# Create response objectresponse = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config)# Close the page to free up resources@@ -492,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config)# Close the page to free up resourcesdiff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 5bb4a49..de705a9 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -85,7 +85,6 @@ class ResponseFactory:first_response: SyncResponse,final_response: Optional[SyncResponse],parser_arguments: Dict,- automated_page: bool = False,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -101,7 +100,6 @@ class ResponseFactory::param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.- :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -117,7 +115,7 @@ class ResponseFactory:history = cls._process_response_history(first_response, parser_arguments)try:- page_content = final_response.text() if not automated_page else cls._get_page_content(page)+ page_content = cls._get_page_content(page)except Exception as e: # pragma: no coverlog.error(f"Error getting page content: {e}")page_content = ""@@ -126,6 +124,7 @@ class ResponseFactory:**{"url": page.url,"content": page_content,+ "raw_response": final_response.text(),"status": final_response.status,"reason": status_text,"encoding": encoding,@@ -219,7 +218,6 @@ class ResponseFactory:first_response: AsyncResponse,final_response: Optional[AsyncResponse],parser_arguments: Dict,- automated_page: bool = False,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -235,7 +233,6 @@ class ResponseFactory::param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.- :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -251,7 +248,7 @@ class ResponseFactory:history = await cls._async_process_response_history(first_response, parser_arguments)try:- page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page))+ page_content = await cls._get_async_page_content(page)except Exception as e: # pragma: no coverlog.error(f"Error getting page content in async: {e}")page_content = ""@@ -261,6 +258,7 @@ class ResponseFactory:"url": page.url,"content": page_content,"status": final_response.status,+ "raw_response": await final_response.text(),"reason": status_text,"encoding": encoding,"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 1f20038..44e550f 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -31,6 +31,7 @@ class Response(Selector):request_headers: Dict,encoding: str = "utf-8",method: str = "GET",+ raw_response: str | bytes = "",history: List | None = None,**selector_config: Any,):@@ -39,6 +40,7 @@ class Response(Selector):self.reason = reasonself.cookies = cookiesself.headers = headers+ self.raw_response = raw_response or contentself.request_headers = request_headersself.history = history or []super().__init__(