🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(PlayWrightFetcher): Add the redirections history
Karim shoair8049ac0f280c
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex c0ca34c..6b8003c 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -259,6 +259,26 @@ class PlaywrightEngine:# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)+ history = []+ current_request = first_response.request.redirected_from+ while current_request:+ current_response = current_request.response()++ history.insert(0, Response(+ url=current_request.url,+ # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"+ text='',+ body=b'',+ status=current_response.status if current_response else 301,+ reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),+ encoding=current_response.headers.get('content-type', '') or 'utf-8',+ cookies={},+ headers=current_response.all_headers() if current_response else {},+ request_headers=current_request.all_headers(),+ **self.adaptor_arguments+ ))+ current_request = current_request.redirected_from+response = Response(url=page.url,text=page.content(),@@ -269,6 +289,7 @@ class PlaywrightEngine:cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},headers=first_response.all_headers(),request_headers=first_response.request.all_headers(),+ history=history,**self.adaptor_arguments)page.close()@@ -345,6 +366,26 @@ class PlaywrightEngine:# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)+ history = []+ current_request = first_response.request.redirected_from+ while current_request:+ current_response = await current_request.response()++ history.insert(0, Response(+ url=current_request.url,+ # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"+ text='',+ body=b'',+ status=current_response.status if current_response else 301,+ reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),+ encoding=current_response.headers.get('content-type', '') or 'utf-8',+ cookies={},+ headers=await current_response.all_headers() if current_response else {},+ request_headers=await current_request.all_headers(),+ **self.adaptor_arguments+ ))+ current_request = current_request.redirected_from+response = Response(url=page.url,text=await page.content(),@@ -355,6 +396,7 @@ class PlaywrightEngine:cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},headers=await first_response.all_headers(),request_headers=await first_response.request.all_headers(),+ history=history,**self.adaptor_arguments)await page.close()