Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(PlaywrightFetcher): Wait for the real response status code

Karim shoair844cb0c6a23d
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index e4c80b7..e434032 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -193,12 +193,21 @@ class PlaywrightEngine:
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
+ from playwright.sync_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.sync_api import sync_playwright
else:
from rebrowser_playwright.sync_api import sync_playwright
+ # Store the final response
+ final_response = None
+
+ def handle_response(finished_response: PlaywrightResponse):
+ nonlocal final_response
+ if finished_response.request.resource_type == "document":
+ final_response = finished_response
+
with sync_playwright() as p:
# Creating the browser
if self.cdp_url:
@@ -212,6 +221,8 @@ class PlaywrightEngine:
page = context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
+ # Listen for all responses
+ page.on("response", handle_response)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
@@ -223,7 +234,7 @@ class PlaywrightEngine:
for script in self.__stealth_scripts():
page.add_init_script(path=script)
- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
@@ -240,21 +251,24 @@ class PlaywrightEngine:
if self.network_idle:
page.wait_for_load_state('networkidle')
+ response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
+ # In case we didn't catch a document type somehow
+ final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
- status_text = res.status_text or StatusText.get(res.status)
+ status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=res.url,
+ url=final_response.url,
text=page.content(),
- body=page.content().encode('utf-8'),
- status=res.status,
+ body=response_bytes,
+ status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
- headers=res.all_headers(),
- request_headers=res.request.all_headers(),
+ headers=final_response.all_headers(),
+ request_headers=final_response.request.all_headers(),
**self.adaptor_arguments
)
page.close()
@@ -266,12 +280,21 @@ class PlaywrightEngine:
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
+ from playwright.async_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.async_api import async_playwright
else:
from rebrowser_playwright.async_api import async_playwright
+ # Store the final response
+ final_response = None
+
+ async def handle_response(finished_response: PlaywrightResponse):
+ nonlocal final_response
+ if finished_response.request.resource_type == "document":
+ final_response = finished_response
+
async with async_playwright() as p:
# Creating the browser
if self.cdp_url:
@@ -285,6 +308,8 @@ class PlaywrightEngine:
page = await context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
+ # Listen for all responses
+ page.on("response", handle_response)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
@@ -296,7 +321,7 @@ class PlaywrightEngine:
for script in self.__stealth_scripts():
await page.add_init_script(path=script)
- res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
@@ -313,21 +338,24 @@ class PlaywrightEngine:
if self.network_idle:
await page.wait_for_load_state('networkidle')
+ response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
+ # In case we didn't catch a document type somehow
+ final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
- status_text = res.status_text or StatusText.get(res.status)
+ status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=res.url,
+ url=final_response.url,
text=await page.content(),
- body=(await page.content()).encode('utf-8'),
- status=res.status,
+ body=response_bytes,
+ status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
- headers=await res.all_headers(),
- request_headers=await res.request.all_headers(),
+ headers=await final_response.all_headers(),
+ request_headers=await final_response.request.all_headers(),
**self.adaptor_arguments
)
await page.close()