Scrapy, a fast high-level web crawling & scraping framework for Python.
Order the Cookie header by path length, then creation time (#8321)
Hygor luz708bc4c3a22b
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex e1b5c6de7..d4b3b2391 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -1,5 +1,6 @@from __future__ import annotations+import itertoolsimport reimport timefrom http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy@@ -19,6 +20,32 @@ if TYPE_CHECKING:# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527IPV4_RE = re.compile(r"\.\d+$", re.ASCII)+# Stands in for the creation time of RFC 6265. A counter, unlike a clock,+# never gives the same value to two cookies.+_creation_indexes = itertools.count()+++class _CreationOrderCookieJar(_CookieJar):+ """Cookie jar that lists cookies in the ``Cookie`` header by descending+ path length, then by ascending creation time, as :rfc:`6265#section-5.4`+ requires.+ """++ def set_cookie(self, cookie: Cookie) -> None:+ try:+ # A cookie that replaces another one keeps its creation time.+ old_cookie = self._cookies[cookie.domain][cookie.path][cookie.name] # type: ignore[attr-defined]+ except KeyError:+ cookie._creation_index = next(_creation_indexes) # type: ignore[attr-defined]+ else:+ cookie._creation_index = getattr(old_cookie, "_creation_index", -1) # type: ignore[attr-defined]+ super().set_cookie(cookie)++ def _cookie_attrs(self, cookies: list[Cookie]) -> list[str]:+ # super() then sorts by path length, and its sort is stable.+ cookies.sort(key=lambda c: getattr(c, "_creation_index", -1))+ return super()._cookie_attrs(cookies) # type: ignore[misc,no-any-return]+class CookieJar:def __init__(@@ -27,7 +54,7 @@ class CookieJar:check_expired_frequency: int = 10000,):self.policy: CookiePolicy = policy or DefaultCookiePolicy()- self.jar: _CookieJar = _CookieJar(self.policy)+ self.jar: _CookieJar = _CreationOrderCookieJar(self.policy)self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined]self.check_expired_frequency: int = check_expired_frequencyself.processed: int = 0diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.pyindex 8840172a5..6f569e30c 100644--- a/tests/test_http_cookies.py+++ b/tests/test_http_cookies.py@@ -2,6 +2,8 @@ from __future__ import annotationsfrom http.cookiejar import DefaultCookiePolicy+import pytest+from scrapy.http import Request, Responsefrom scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponsefrom scrapy.utils.httpobj import urlparse_cached@@ -55,6 +57,73 @@ class TestCookieJar:jar.add_cookie_header(self.request)assert jar.processed == 1+ @pytest.mark.parametrize(+ ("set_cookie_headers", "expected"),+ [+ pytest.param(+ ["a=1; Path=/", "b=2; Path=/a"],+ b"b=2; a=1",+ id="longer-path-first",+ ),+ pytest.param(+ ["id=old; Domain=example.com; Path=/", "id=new; Path=/"],+ b"id=old; id=new",+ id="older-first",+ ),+ pytest.param(+ [+ "id=old; Domain=example.com; Path=/",+ "id=new; Path=/",+ "id=updated; Domain=example.com; Path=/",+ ],+ b"id=updated; id=new",+ id="replacement-keeps-creation-time",+ ),+ pytest.param(+ [+ "id=old; Domain=example.com; Path=/",+ "id=new; Path=/",+ "id=old; Domain=example.com; Path=/; Max-Age=0",+ "id=recreated; Domain=example.com; Path=/",+ ],+ b"id=new; id=recreated",+ id="recreation-gets-new-creation-time",+ ),+ ],+ )+ def test_add_cookie_header_order(+ self, set_cookie_headers: list[str], expected: bytes+ ) -> None:+ jar = CookieJar()+ url = "https://www.example.com/a/b"+ for header in set_cookie_headers:+ response = Response(url, headers={"Set-Cookie": header})+ jar.extract_cookies(response, Request(url))+ request = Request(url)+ jar.add_cookie_header(request)+ assert request.headers["Cookie"] == expected++ def test_cookie_without_creation_index(self) -> None:+ jar = CookieJar()+ url = "https://www.example.com/a/b"+ for header in ["id=old; Domain=example.com; Path=/", "id=new; Path=/"]:+ response = Response(url, headers={"Set-Cookie": header})+ jar.extract_cookies(response, Request(url))+ # e.g. a cookie added to the jar without going through set_cookie()+ old_cookie = jar.jar._cookies[".example.com"]["/"]["id"] # type: ignore[attr-defined]+ del old_cookie._creation_index+ request = Request(url)+ jar.add_cookie_header(request)+ assert request.headers["Cookie"] == b"id=old; id=new"++ response = Response(+ url, headers={"Set-Cookie": "id=updated; Domain=example.com; Path=/"}+ )+ jar.extract_cookies(response, Request(url))+ request = Request(url)+ jar.add_cookie_header(request)+ assert request.headers["Cookie"] == b"id=updated; id=new"+class TestWrappedRequest:def setup_method(self):