🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): Adjusting all cookies returned from each fetcher
Karim shoairc84143129e43
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 98e0747..f3ba36b 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -164,7 +164,7 @@ class CamoufoxEngine:else StatusText.get(301),encoding=current_response.headers.get("content-type", "")or "utf-8",- cookies={},+ cookies=tuple(),headers=current_response.all_headers()if current_responseelse {},@@ -207,7 +207,7 @@ class CamoufoxEngine:else StatusText.get(301),encoding=current_response.headers.get("content-type", "")or "utf-8",- cookies={},+ cookies=tuple(),headers=await current_response.all_headers()if current_responseelse {},@@ -450,9 +450,7 @@ class CamoufoxEngine:status=final_response.status,reason=status_text,encoding=encoding,- cookies={- cookie["name"]: cookie["value"] for cookie in page.context.cookies()- },+ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),headers=first_response.all_headers(),request_headers=first_response.request.all_headers(),history=history,@@ -554,10 +552,7 @@ class CamoufoxEngine:status=final_response.status,reason=status_text,encoding=encoding,- cookies={- cookie["name"]: cookie["value"]- for cookie in await page.context.cookies()- },+ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),headers=await first_response.all_headers(),request_headers=await first_response.request.all_headers(),history=history,diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 521f13f..80b033b 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -242,7 +242,7 @@ class PlaywrightEngine:else StatusText.get(301),encoding=current_response.headers.get("content-type", "")or "utf-8",- cookies={},+ cookies=tuple(),headers=current_response.all_headers()if current_responseelse {},@@ -285,7 +285,7 @@ class PlaywrightEngine:else StatusText.get(301),encoding=current_response.headers.get("content-type", "")or "utf-8",- cookies={},+ cookies=tuple(),headers=await current_response.all_headers()if current_responseelse {},@@ -405,9 +405,7 @@ class PlaywrightEngine:status=final_response.status,reason=status_text,encoding=encoding,- cookies={- cookie["name"]: cookie["value"] for cookie in page.context.cookies()- },+ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),headers=first_response.all_headers(),request_headers=first_response.request.all_headers(),history=history,@@ -519,10 +517,7 @@ class PlaywrightEngine:status=final_response.status,reason=status_text,encoding=encoding,- cookies={- cookie["name"]: cookie["value"]- for cookie in await page.context.cookies()- },+ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),headers=await first_response.all_headers(),request_headers=await first_response.request.all_headers(),history=history,diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 06ee1ee..187d36e 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -17,7 +17,7 @@ class StaticEngine:follow_redirects: bool = True,timeout: Optional[Union[int, float]] = None,retries: Optional[int] = 3,- cookies: Optional[Dict] = None,+ cookies: Optional[Tuple] = None,adaptor_arguments: Tuple = None,):"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex c0e7814..63ea1fc 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -109,7 +109,7 @@ class Response(Adaptor):body: bytes,status: int,reason: str,- cookies: Dict,+ cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]],headers: Dict,request_headers: Dict,encoding: str = "utf-8",@@ -132,7 +132,7 @@ class Response(Adaptor):encoding=encoding,**adaptor_arguments,)- # For back-ward compatibility+ # For backward compatibilityself.adaptor = self# For easier debugging while working from a Python shelllog.info(