Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(StealthyFetcher): Wait for the real response status code

Karim shoair94745b155533
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 1eb9976..2c0ff3d 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -84,6 +84,14 @@ class CamoufoxEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
addons = [] if self.disable_ads else [DefaultAddons.UBO]
+ # Store the final response
+ final_response = None
+
+ def handle_response(finished_response):
+ nonlocal final_response
+ if finished_response.request.resource_type == "document":
+ final_response = finished_response
+
with Camoufox(
geoip=self.geoip,
proxy=self.proxy,
@@ -100,13 +108,15 @@ class CamoufoxEngine:
page = browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
+ # Listen for all responses
+ page.on("response", handle_response)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
@@ -123,21 +133,24 @@ class CamoufoxEngine:
if self.network_idle:
page.wait_for_load_state('networkidle')
+ response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
+ # In case we didn't catch a document type somehow
+ final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
- status_text = res.status_text or StatusText.get(res.status)
+ status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=res.url,
+ url=final_response.url,
text=page.content(),
- body=page.content().encode('utf-8'),
- status=res.status,
+ body=response_bytes,
+ status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
- headers=res.all_headers(),
- request_headers=res.request.all_headers(),
+ headers=final_response.all_headers(),
+ request_headers=final_response.request.all_headers(),
**self.adaptor_arguments
)
page.close()
@@ -151,6 +164,14 @@ class CamoufoxEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
addons = [] if self.disable_ads else [DefaultAddons.UBO]
+ # Store the final response
+ final_response = None
+
+ async def handle_response(finished_response):
+ nonlocal final_response
+ if finished_response.request.resource_type == "document":
+ final_response = finished_response
+
async with AsyncCamoufox(
geoip=self.geoip,
proxy=self.proxy,
@@ -167,13 +188,15 @@ class CamoufoxEngine:
page = await browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
+ # Listen for all responses
+ page.on("response", handle_response)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
- res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
@@ -190,21 +213,24 @@ class CamoufoxEngine:
if self.network_idle:
await page.wait_for_load_state('networkidle')
+ response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
+ # In case we didn't catch a document type somehow
+ final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
- encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
+ encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
- status_text = res.status_text or StatusText.get(res.status)
+ status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=res.url,
+ url=final_response.url,
text=await page.content(),
- body=(await page.content()).encode('utf-8'),
- status=res.status,
+ body=response_bytes,
+ status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
- headers=await res.all_headers(),
- request_headers=await res.request.all_headers(),
+ headers=await final_response.all_headers(),
+ request_headers=await final_response.request.all_headers(),
**self.adaptor_arguments
)
await page.close()