Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(StealthyFetcher): Add the redirections history

Karim shoair10ecd414443f
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index e4ee4bb..8766fb1 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -142,6 +142,26 @@ class CamoufoxEngine:
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
+ history = []
+ current_request = first_response.request.redirected_from
+ while current_request:
+ current_response = current_request.response()
+
+ history.insert(0, Response(
+ url=current_request.url,
+ # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+ text='',
+ body=b'',
+ status=current_response.status if current_response else 301,
+ reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+ encoding=current_response.headers.get('content-type', '') or 'utf-8',
+ cookies={},
+ headers=current_response.all_headers() if current_response else {},
+ request_headers=current_request.all_headers(),
+ **self.adaptor_arguments
+ ))
+ current_request = current_request.redirected_from
+
response = Response(
url=page.url,
text=page.content(),
@@ -152,6 +172,7 @@ class CamoufoxEngine:
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
+ history=history,
**self.adaptor_arguments
)
page.close()
@@ -223,6 +244,26 @@ class CamoufoxEngine:
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
+ history = []
+ current_request = first_response.request.redirected_from
+ while current_request:
+ current_response = await current_request.response()
+
+ history.insert(0, Response(
+ url=current_request.url,
+ # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+ text='',
+ body=b'',
+ status=current_response.status if current_response else 301,
+ reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+ encoding=current_response.headers.get('content-type', '') or 'utf-8',
+ cookies={},
+ headers=await current_response.all_headers() if current_response else {},
+ request_headers=await current_request.all_headers(),
+ **self.adaptor_arguments
+ ))
+ current_request = current_request.redirected_from
+
response = Response(
url=page.url,
text=await page.content(),
@@ -233,6 +274,7 @@ class CamoufoxEngine:
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
+ history=history,
**self.adaptor_arguments
)
await page.close()