Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): Adjusting all cookies returned from each fetcher

Karim shoairc84143129e43
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 98e0747..f3ba36b 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -164,7 +164,7 @@ class CamoufoxEngine:
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
- cookies={},
+ cookies=tuple(),
headers=current_response.all_headers()
if current_response
else {},
@@ -207,7 +207,7 @@ class CamoufoxEngine:
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
- cookies={},
+ cookies=tuple(),
headers=await current_response.all_headers()
if current_response
else {},
@@ -450,9 +450,7 @@ class CamoufoxEngine:
status=final_response.status,
reason=status_text,
encoding=encoding,
- cookies={
- cookie["name"]: cookie["value"] for cookie in page.context.cookies()
- },
+ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
@@ -554,10 +552,7 @@ class CamoufoxEngine:
status=final_response.status,
reason=status_text,
encoding=encoding,
- cookies={
- cookie["name"]: cookie["value"]
- for cookie in await page.context.cookies()
- },
+ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 521f13f..80b033b 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -242,7 +242,7 @@ class PlaywrightEngine:
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
- cookies={},
+ cookies=tuple(),
headers=current_response.all_headers()
if current_response
else {},
@@ -285,7 +285,7 @@ class PlaywrightEngine:
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
- cookies={},
+ cookies=tuple(),
headers=await current_response.all_headers()
if current_response
else {},
@@ -405,9 +405,7 @@ class PlaywrightEngine:
status=final_response.status,
reason=status_text,
encoding=encoding,
- cookies={
- cookie["name"]: cookie["value"] for cookie in page.context.cookies()
- },
+ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
@@ -519,10 +517,7 @@ class PlaywrightEngine:
status=final_response.status,
reason=status_text,
encoding=encoding,
- cookies={
- cookie["name"]: cookie["value"]
- for cookie in await page.context.cookies()
- },
+ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 06ee1ee..187d36e 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -17,7 +17,7 @@ class StaticEngine:
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None,
retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
+ cookies: Optional[Tuple] = None,
adaptor_arguments: Tuple = None,
):
"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index c0e7814..63ea1fc 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -109,7 +109,7 @@ class Response(Adaptor):
body: bytes,
status: int,
reason: str,
- cookies: Dict,
+ cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]],
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
@@ -132,7 +132,7 @@ class Response(Adaptor):
encoding=encoding,
**adaptor_arguments,
)
- # For back-ward compatibility
+ # For backward compatibility
self.adaptor = self
# For easier debugging while working from a Python shell
log.info(