🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(PlaywrightFetcher): Use more dependable response data
Karim shoairf9b85cfbac0e
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 0a6d30d..2cd93e5 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -206,7 +206,7 @@ class PlaywrightEngine:def handle_response(finished_response: PlaywrightResponse):nonlocal final_response- if finished_response.request.resource_type == "document":+ if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():final_response = finished_responsewith sync_playwright() as p:@@ -252,7 +252,6 @@ class PlaywrightEngine:if self.network_idle:page.wait_for_load_state('networkidle')- response_bytes = final_response.body() if final_response else page.content().encode('utf-8')# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_response# This will be parsed inside `Response`@@ -261,15 +260,15 @@ class PlaywrightEngine:status_text = final_response.status_text or StatusText.get(final_response.status)response = Response(- url=final_response.url,+ url=page.url,text=page.content(),- body=response_bytes,+ body=page.content().encode('utf-8'),status=final_response.status,reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},- headers=final_response.all_headers(),- request_headers=final_response.request.all_headers(),+ headers=first_response.all_headers(),+ request_headers=first_response.request.all_headers(),**self.adaptor_arguments)page.close()@@ -293,7 +292,7 @@ class PlaywrightEngine:async def handle_response(finished_response: PlaywrightResponse):nonlocal final_response- if finished_response.request.resource_type == "document":+ if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():final_response = finished_responseasync with async_playwright() as p:@@ -339,7 +338,6 @@ class PlaywrightEngine:if self.network_idle:await page.wait_for_load_state('networkidle')- response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_response# This will be parsed inside `Response`@@ -348,15 +346,15 @@ class PlaywrightEngine:status_text = final_response.status_text or StatusText.get(final_response.status)response = Response(- url=final_response.url,+ url=page.url,text=await page.content(),- body=response_bytes,+ body=(await page.content()).encode('utf-8'),status=final_response.status,reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},- headers=await final_response.all_headers(),- request_headers=await final_response.request.all_headers(),+ headers=await first_response.all_headers(),+ request_headers=await first_response.request.all_headers(),**self.adaptor_arguments)await page.close()