🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): preserve cookies for browser-engine responses in cache (#376)
Amit Vijapur3ed4fde93a20
diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.pyindex 0305aef..1aeb154 100644--- a/scrapling/spiders/cache.py+++ b/scrapling/spiders/cache.py@@ -6,7 +6,7 @@ import anyiofrom anyio import Path as AsyncPathfrom scrapling.core.utils import log-from scrapling.core._types import Dict, Optional, Any+from scrapling.core._types import Dict, Optional, Any, Tuplefrom scrapling.engines.toolbelt.custom import Response@@ -28,13 +28,21 @@ class ResponseCacheManager:async with await anyio.open_file(path, "rb") as f:data: Dict[str, Any] = orjson.loads(await f.read())+ # Browser-engine cookies are cached as a JSON array (see `put`) and come back+ # as a `list`; restore the `tuple` shape `Response` expects. Static-engine+ # cookies are cached as a JSON object and come back as a `dict` already.+ cached_cookies = data["cookies"]+ cookies: Tuple[Dict[str, str], ...] | Dict[str, str] = (+ tuple(cached_cookies) if isinstance(cached_cookies, list) else cached_cookies+ )+return Response(url=data["url"],content=b64decode(data["content"]),status=data["status"],reason=data["reason"],encoding=data["encoding"],- cookies=data["cookies"],+ cookies=cookies,headers=data["headers"],request_headers=data["request_headers"],method=data["method"],@@ -55,7 +63,13 @@ class ResponseCacheManager:"status": response.status,"reason": response.reason,"encoding": response.encoding,- "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},+ # Browser-engine responses store cookies as a `tuple` of full cookie+ # dicts; static-engine responses store a flat `dict`. Preserve whichever+ # shape it is instead of collapsing non-dict cookies to `{}`, which was+ # silently dropping every cookie from browser-engine responses.+ "cookies": list(response.cookies)+ if isinstance(response.cookies, tuple)+ else dict(response.cookies),"headers": dict(response.headers),"request_headers": dict(response.request_headers),"method": method,diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.pyindex eb1b04b..3b990ed 100644--- a/tests/spiders/test_cache.py+++ b/tests/spiders/test_cache.py@@ -14,14 +14,19 @@ from scrapling.engines.toolbelt.custom import Responsefrom scrapling.core._types import Any, Dict, Set, AsyncGenerator-def _make_response(url: str = "https://example.com", body: bytes = b"<html>hello</html>", status: int = 200) -> Response:+def _make_response(+ url: str = "https://example.com",+ body: bytes = b"<html>hello</html>",+ status: int = 200,+ cookies: Any = None,+) -> Response:return Response(url=url,content=body,status=status,reason="OK",encoding="utf-8",- cookies={},+ cookies={} if cookies is None else cookies,headers={"content-type": "text/html"},request_headers={"user-agent": "test"},method="GET",@@ -49,6 +54,66 @@ class TestResponseCacheManager:assert dict(restored.headers) == dict(original.headers)assert dict(restored.request_headers) == dict(original.request_headers)+ @pytest.mark.anyio+ async def test_put_get_roundtrip_preserves_browser_engine_cookies(self):+ """Regression test for #376.++ Browser engines (Playwright) populate ``Response.cookies`` as a ``tuple`` of+ full cookie dicts (see ``engines/toolbelt/convertor.py``), unlike the flat+ ``dict`` the static engine uses. The cache previously only recognized the+ ``dict`` shape and silently discarded any other cookies, so a cached+ browser-engine response replayed with no cookies at all.+ """+ with tempfile.TemporaryDirectory() as tmpdir:+ cache = ResponseCacheManager(tmpdir)+ fp = b"\x06" * 20+ browser_cookies = (+ {+ "name": "session",+ "value": "abc123",+ "domain": "example.com",+ "path": "/",+ "expires": -1,+ "httpOnly": True,+ "secure": True,+ "sameSite": "Lax",+ },+ {+ "name": "csrftoken",+ "value": "xyz789",+ "domain": "example.com",+ "path": "/",+ "expires": -1,+ "httpOnly": False,+ "secure": True,+ "sameSite": "Strict",+ },+ )+ original = _make_response(cookies=browser_cookies)++ await cache.put(fp, original, "GET")+ restored = await cache.get(fp)++ assert restored is not None+ assert restored.cookies == browser_cookies+ assert isinstance(restored.cookies, tuple)++ @pytest.mark.anyio+ async def test_put_get_roundtrip_preserves_static_engine_cookies(self):+ """Flat-dict cookies (static engine) must still round-trip as a ``dict``."""+ with tempfile.TemporaryDirectory() as tmpdir:+ cache = ResponseCacheManager(tmpdir)+ fp = b"\x07" * 20+ static_cookies = {"session": "abc123", "csrftoken": "xyz789"}+ original = _make_response(cookies=static_cookies)++ await cache.put(fp, original, "GET")+ restored = await cache.get(fp)++ assert restored is not None+ assert restored.cookies == static_cookies+ assert isinstance(restored.cookies, dict)+@pytest.mark.anyioasync def test_put_overwrites_existing_entry(self):"""Re-caching the same fingerprint must replace the stored response.