Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): preserve cookies for browser-engine responses in cache (#376)

Amit Vijapur3ed4fde93a20
diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.py
index 0305aef..1aeb154 100644
--- a/scrapling/spiders/cache.py
+++ b/scrapling/spiders/cache.py
@@ -6,7 +6,7 @@ import anyio
from anyio import Path as AsyncPath
from scrapling.core.utils import log
-from scrapling.core._types import Dict, Optional, Any
+from scrapling.core._types import Dict, Optional, Any, Tuple
from scrapling.engines.toolbelt.custom import Response
@@ -28,13 +28,21 @@ class ResponseCacheManager:
async with await anyio.open_file(path, "rb") as f:
data: Dict[str, Any] = orjson.loads(await f.read())
+ # Browser-engine cookies are cached as a JSON array (see `put`) and come back
+ # as a `list`; restore the `tuple` shape `Response` expects. Static-engine
+ # cookies are cached as a JSON object and come back as a `dict` already.
+ cached_cookies = data["cookies"]
+ cookies: Tuple[Dict[str, str], ...] | Dict[str, str] = (
+ tuple(cached_cookies) if isinstance(cached_cookies, list) else cached_cookies
+ )
+
return Response(
url=data["url"],
content=b64decode(data["content"]),
status=data["status"],
reason=data["reason"],
encoding=data["encoding"],
- cookies=data["cookies"],
+ cookies=cookies,
headers=data["headers"],
request_headers=data["request_headers"],
method=data["method"],
@@ -55,7 +63,13 @@ class ResponseCacheManager:
"status": response.status,
"reason": response.reason,
"encoding": response.encoding,
- "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
+ # Browser-engine responses store cookies as a `tuple` of full cookie
+ # dicts; static-engine responses store a flat `dict`. Preserve whichever
+ # shape it is instead of collapsing non-dict cookies to `{}`, which was
+ # silently dropping every cookie from browser-engine responses.
+ "cookies": list(response.cookies)
+ if isinstance(response.cookies, tuple)
+ else dict(response.cookies),
"headers": dict(response.headers),
"request_headers": dict(response.request_headers),
"method": method,
diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py
index eb1b04b..3b990ed 100644
--- a/tests/spiders/test_cache.py
+++ b/tests/spiders/test_cache.py
@@ -14,14 +14,19 @@ from scrapling.engines.toolbelt.custom import Response
from scrapling.core._types import Any, Dict, Set, AsyncGenerator
-def _make_response(url: str = "https://example.com", body: bytes = b"<html>hello</html>", status: int = 200) -> Response:
+def _make_response(
+ url: str = "https://example.com",
+ body: bytes = b"<html>hello</html>",
+ status: int = 200,
+ cookies: Any = None,
+) -> Response:
return Response(
url=url,
content=body,
status=status,
reason="OK",
encoding="utf-8",
- cookies={},
+ cookies={} if cookies is None else cookies,
headers={"content-type": "text/html"},
request_headers={"user-agent": "test"},
method="GET",
@@ -49,6 +54,66 @@ class TestResponseCacheManager:
assert dict(restored.headers) == dict(original.headers)
assert dict(restored.request_headers) == dict(original.request_headers)
+ @pytest.mark.anyio
+ async def test_put_get_roundtrip_preserves_browser_engine_cookies(self):
+ """Regression test for #376.
+
+ Browser engines (Playwright) populate ``Response.cookies`` as a ``tuple`` of
+ full cookie dicts (see ``engines/toolbelt/convertor.py``), unlike the flat
+ ``dict`` the static engine uses. The cache previously only recognized the
+ ``dict`` shape and silently discarded any other cookies, so a cached
+ browser-engine response replayed with no cookies at all.
+ """
+ with tempfile.TemporaryDirectory() as tmpdir:
+ cache = ResponseCacheManager(tmpdir)
+ fp = b"\x06" * 20
+ browser_cookies = (
+ {
+ "name": "session",
+ "value": "abc123",
+ "domain": "example.com",
+ "path": "/",
+ "expires": -1,
+ "httpOnly": True,
+ "secure": True,
+ "sameSite": "Lax",
+ },
+ {
+ "name": "csrftoken",
+ "value": "xyz789",
+ "domain": "example.com",
+ "path": "/",
+ "expires": -1,
+ "httpOnly": False,
+ "secure": True,
+ "sameSite": "Strict",
+ },
+ )
+ original = _make_response(cookies=browser_cookies)
+
+ await cache.put(fp, original, "GET")
+ restored = await cache.get(fp)
+
+ assert restored is not None
+ assert restored.cookies == browser_cookies
+ assert isinstance(restored.cookies, tuple)
+
+ @pytest.mark.anyio
+ async def test_put_get_roundtrip_preserves_static_engine_cookies(self):
+ """Flat-dict cookies (static engine) must still round-trip as a ``dict``."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ cache = ResponseCacheManager(tmpdir)
+ fp = b"\x07" * 20
+ static_cookies = {"session": "abc123", "csrftoken": "xyz789"}
+ original = _make_response(cookies=static_cookies)
+
+ await cache.put(fp, original, "GET")
+ restored = await cache.get(fp)
+
+ assert restored is not None
+ assert restored.cookies == static_cookies
+ assert isinstance(restored.cookies, dict)
+
@pytest.mark.anyio
async def test_put_overwrites_existing_entry(self):
"""Re-caching the same fingerprint must replace the stored response.