Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(StealthyFetcher): Use more dependable response data

Karim shoair2006be217364
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index f1d8f95..6d28634 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -89,7 +89,7 @@ class CamoufoxEngine:
def handle_response(finished_response):
nonlocal final_response
- if finished_response.request.resource_type == "document":
+ if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
with Camoufox(
@@ -133,7 +133,6 @@ class CamoufoxEngine:
if self.network_idle:
page.wait_for_load_state('networkidle')
- response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
@@ -142,15 +141,15 @@ class CamoufoxEngine:
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=final_response.url,
+ url=page.url,
text=page.content(),
- body=response_bytes,
+ body=page.content().encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
- headers=final_response.all_headers(),
- request_headers=final_response.request.all_headers(),
+ headers=first_response.all_headers(),
+ request_headers=first_response.request.all_headers(),
**self.adaptor_arguments
)
page.close()
@@ -169,7 +168,7 @@ class CamoufoxEngine:
async def handle_response(finished_response):
nonlocal final_response
- if finished_response.request.resource_type == "document":
+ if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
async with AsyncCamoufox(
@@ -213,7 +212,6 @@ class CamoufoxEngine:
if self.network_idle:
await page.wait_for_load_state('networkidle')
- response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
@@ -222,15 +220,15 @@ class CamoufoxEngine:
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
- url=final_response.url,
+ url=page.url,
text=await page.content(),
- body=response_bytes,
+ body=(await page.content()).encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
- headers=await final_response.all_headers(),
- request_headers=await final_response.request.all_headers(),
+ headers=await first_response.all_headers(),
+ request_headers=await first_response.request.all_headers(),
**self.adaptor_arguments
)
await page.close()