Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Order the Cookie header by path length, then creation time (#8321)

Hygor luz708bc4c3a22b
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index e1b5c6de7..d4b3b2391 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import itertools
import re
import time
from http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy
@@ -19,6 +20,32 @@ if TYPE_CHECKING:
# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
+# Stands in for the creation time of RFC 6265. A counter, unlike a clock,
+# never gives the same value to two cookies.
+_creation_indexes = itertools.count()
+
+
+class _CreationOrderCookieJar(_CookieJar):
+ """Cookie jar that lists cookies in the ``Cookie`` header by descending
+ path length, then by ascending creation time, as :rfc:`6265#section-5.4`
+ requires.
+ """
+
+ def set_cookie(self, cookie: Cookie) -> None:
+ try:
+ # A cookie that replaces another one keeps its creation time.
+ old_cookie = self._cookies[cookie.domain][cookie.path][cookie.name] # type: ignore[attr-defined]
+ except KeyError:
+ cookie._creation_index = next(_creation_indexes) # type: ignore[attr-defined]
+ else:
+ cookie._creation_index = getattr(old_cookie, "_creation_index", -1) # type: ignore[attr-defined]
+ super().set_cookie(cookie)
+
+ def _cookie_attrs(self, cookies: list[Cookie]) -> list[str]:
+ # super() then sorts by path length, and its sort is stable.
+ cookies.sort(key=lambda c: getattr(c, "_creation_index", -1))
+ return super()._cookie_attrs(cookies) # type: ignore[misc,no-any-return]
+
class CookieJar:
def __init__(
@@ -27,7 +54,7 @@ class CookieJar:
check_expired_frequency: int = 10000,
):
self.policy: CookiePolicy = policy or DefaultCookiePolicy()
- self.jar: _CookieJar = _CookieJar(self.policy)
+ self.jar: _CookieJar = _CreationOrderCookieJar(self.policy)
self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined]
self.check_expired_frequency: int = check_expired_frequency
self.processed: int = 0
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 8840172a5..6f569e30c 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from http.cookiejar import DefaultCookiePolicy
+import pytest
+
from scrapy.http import Request, Response
from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse
from scrapy.utils.httpobj import urlparse_cached
@@ -55,6 +57,73 @@ class TestCookieJar:
jar.add_cookie_header(self.request)
assert jar.processed == 1
+ @pytest.mark.parametrize(
+ ("set_cookie_headers", "expected"),
+ [
+ pytest.param(
+ ["a=1; Path=/", "b=2; Path=/a"],
+ b"b=2; a=1",
+ id="longer-path-first",
+ ),
+ pytest.param(
+ ["id=old; Domain=example.com; Path=/", "id=new; Path=/"],
+ b"id=old; id=new",
+ id="older-first",
+ ),
+ pytest.param(
+ [
+ "id=old; Domain=example.com; Path=/",
+ "id=new; Path=/",
+ "id=updated; Domain=example.com; Path=/",
+ ],
+ b"id=updated; id=new",
+ id="replacement-keeps-creation-time",
+ ),
+ pytest.param(
+ [
+ "id=old; Domain=example.com; Path=/",
+ "id=new; Path=/",
+ "id=old; Domain=example.com; Path=/; Max-Age=0",
+ "id=recreated; Domain=example.com; Path=/",
+ ],
+ b"id=new; id=recreated",
+ id="recreation-gets-new-creation-time",
+ ),
+ ],
+ )
+ def test_add_cookie_header_order(
+ self, set_cookie_headers: list[str], expected: bytes
+ ) -> None:
+ jar = CookieJar()
+ url = "https://www.example.com/a/b"
+ for header in set_cookie_headers:
+ response = Response(url, headers={"Set-Cookie": header})
+ jar.extract_cookies(response, Request(url))
+ request = Request(url)
+ jar.add_cookie_header(request)
+ assert request.headers["Cookie"] == expected
+
+ def test_cookie_without_creation_index(self) -> None:
+ jar = CookieJar()
+ url = "https://www.example.com/a/b"
+ for header in ["id=old; Domain=example.com; Path=/", "id=new; Path=/"]:
+ response = Response(url, headers={"Set-Cookie": header})
+ jar.extract_cookies(response, Request(url))
+ # e.g. a cookie added to the jar without going through set_cookie()
+ old_cookie = jar.jar._cookies[".example.com"]["/"]["id"] # type: ignore[attr-defined]
+ del old_cookie._creation_index
+ request = Request(url)
+ jar.add_cookie_header(request)
+ assert request.headers["Cookie"] == b"id=old; id=new"
+
+ response = Response(
+ url, headers={"Set-Cookie": "id=updated; Domain=example.com; Path=/"}
+ )
+ jar.extract_cookies(response, Request(url))
+ request = Request(url)
+ jar.add_cookie_header(request)
+ assert request.headers["Cookie"] == b"id=updated; id=new"
+
class TestWrappedRequest:
def setup_method(self):