Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge pull request from GHSA-cjvr-mfj7-j4j8
Adrián Chaves8ce01b3b76d4
diff --git a/docs/news.rst b/docs/news.rstindex 2128f2f0e..aef12d9db 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -5,11 +5,13 @@ Release notes.. _release-2.6.0:-Scrapy 2.6.0 (2022-02-??)+Scrapy 2.6.0 (2022-03-01)-------------------------Highlights:+* :ref:`Security fixes for cookie handling <2.6-security-fixes>`+* Python 3.10 support* :ref:`asyncio support <using-asyncio>` is no longer considered@@ -20,6 +22,37 @@ Highlights::ref:`item filtering <item-filter>` and:ref:`post-processing <post-processing>`+.. _2.6-security-fixes:++Security bug fixes+~~~~~~~~~~~~~~~~~~++- When a :class:`~scrapy.http.Request` object with cookies defined gets a+ redirect response causing a new :class:`~scrapy.http.Request` object to be+ scheduled, the cookies defined in the original+ :class:`~scrapy.http.Request` object are no longer copied into the new+ :class:`~scrapy.http.Request` object.++ If you manually set the ``Cookie`` header on a+ :class:`~scrapy.http.Request` object and the domain name of the redirect+ URL is not an exact match for the domain of the URL of the original+ :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped+ from the new :class:`~scrapy.http.Request` object.++ The old behavior could be exploited by an attacker to gain access to your+ cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more+ information.++ .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8++ .. note:: It is still possible to enable the sharing of cookies between+ different domains with a shared domain suffix (e.g.+ ``example.com`` and any subdomain) by defining the shared domain+ suffix (e.g. ``example.com``) as the cookie domain when defining+ your cookies. See the documentation of the+ :class:`~scrapy.http.Request` class for more information.++Modified requirements~~~~~~~~~~~~~~~~~~~~~@@ -1842,6 +1875,38 @@ affect subclasses:(:issue:`3884`)+.. _release-1.8.2:++Scrapy 1.8.2 (2022-03-01)+-------------------------++**Security bug fixes:**++- When a :class:`~scrapy.http.Request` object with cookies defined gets a+ redirect response causing a new :class:`~scrapy.http.Request` object to be+ scheduled, the cookies defined in the original+ :class:`~scrapy.http.Request` object are no longer copied into the new+ :class:`~scrapy.http.Request` object.++ If you manually set the ``Cookie`` header on a+ :class:`~scrapy.http.Request` object and the domain name of the redirect+ URL is not an exact match for the domain of the URL of the original+ :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped+ from the new :class:`~scrapy.http.Request` object.++ The old behavior could be exploited by an attacker to gain access to your+ cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more+ information.++ .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8++ .. note:: It is still possible to enable the sharing of cookies between+ different domains with a shared domain suffix (e.g.+ ``example.com`` and any subdomain) by defining the shared domain+ suffix (e.g. ``example.com``) as the cookie domain when defining+ your cookies. See the documentation of the+ :class:`~scrapy.http.Request` class for more information.+.. _release-1.8.1:diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 4053fecc5..fcd6c298b 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparsefrom w3lib.url import safe_url_stringfrom scrapy.http import HtmlResponse+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.response import get_meta_refreshfrom scrapy.exceptions import IgnoreRequest, NotConfigured@@ -11,6 +12,21 @@ from scrapy.exceptions import IgnoreRequest, NotConfiguredlogger = logging.getLogger(__name__)+def _build_redirect_request(source_request, *, url, method=None, body=None):+ redirect_request = source_request.replace(+ url=url,+ method=method,+ body=body,+ cookies=None,+ )+ if 'Cookie' in redirect_request.headers:+ source_request_netloc = urlparse_cached(source_request).netloc+ redirect_request_netloc = urlparse_cached(redirect_request).netloc+ if source_request_netloc != redirect_request_netloc:+ del redirect_request.headers['Cookie']+ return redirect_request++class BaseRedirectMiddleware:enabled_setting = 'REDIRECT_ENABLED'@@ -47,10 +63,15 @@ class BaseRedirectMiddleware:raise IgnoreRequest("max redirections reached")def _redirect_request_using_get(self, request, redirect_url):- redirected = request.replace(url=redirect_url, method='GET', body='')- redirected.headers.pop('Content-Type', None)- redirected.headers.pop('Content-Length', None)- return redirected+ redirect_request = _build_redirect_request(+ request,+ url=redirect_url,+ method='GET',+ body='',+ )+ redirect_request.headers.pop('Content-Type', None)+ redirect_request.headers.pop('Content-Length', None)+ return redirect_requestclass RedirectMiddleware(BaseRedirectMiddleware):@@ -80,7 +101,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):redirected_url = urljoin(request.url, location)if response.status in (301, 307, 308) or request.method == 'HEAD':- redirected = request.replace(url=redirected_url)+ redirected = _build_redirect_request(request, url=redirected_url)return self._redirect(redirected, request, spider, response.status)redirected = self._redirect_request_using_get(request, redirected_url)diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 36021bfbf..1747f3b94 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -6,8 +6,10 @@ import pytestfrom scrapy.downloadermiddlewares.cookies import CookiesMiddlewarefrom scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware+from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, Request+from scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler@@ -23,9 +25,11 @@ class CookiesMiddlewareTest(TestCase):def setUp(self):self.spider = Spider('foo')self.mw = CookiesMiddleware()+ self.redirect_middleware = RedirectMiddleware(settings=Settings())def tearDown(self):del self.mw+ del self.redirect_middlewaredef test_basic(self):req = Request('http://scrapytest.org/')@@ -368,3 +372,154 @@ class CookiesMiddlewareTest(TestCase):req4 = Request('http://example.org', cookies={'a': 'b'})assert self.mw.process_request(req4, self.spider) is Noneself.assertCookieValEqual(req4.headers['Cookie'], b'a=b')++ def _test_cookie_redirect(+ self,+ source,+ target,+ *,+ cookies1,+ cookies2,+ ):+ input_cookies = {'a': 'b'}++ if not isinstance(source, dict):+ source = {'url': source}+ if not isinstance(target, dict):+ target = {'url': target}+ target.setdefault('status', 301)++ request1 = Request(cookies=input_cookies, **source)+ self.mw.process_request(request1, self.spider)+ cookies = request1.headers.get('Cookie')+ self.assertEqual(cookies, b"a=b" if cookies1 else None)++ response = Response(+ headers={+ 'Location': target['url'],+ },+ **target,+ )+ self.assertEqual(+ self.mw.process_response(request1, response, self.spider),+ response,+ )++ request2 = self.redirect_middleware.process_response(+ request1,+ response,+ self.spider,+ )+ self.assertIsInstance(request2, Request)++ self.mw.process_request(request2, self.spider)+ cookies = request2.headers.get('Cookie')+ self.assertEqual(cookies, b"a=b" if cookies2 else None)++ def test_cookie_redirect_same_domain(self):+ self._test_cookie_redirect(+ 'https://toscrape.com',+ 'https://toscrape.com',+ cookies1=True,+ cookies2=True,+ )++ def test_cookie_redirect_same_domain_forcing_get(self):+ self._test_cookie_redirect(+ 'https://toscrape.com',+ {'url': 'https://toscrape.com', 'status': 302},+ cookies1=True,+ cookies2=True,+ )++ def test_cookie_redirect_different_domain(self):+ self._test_cookie_redirect(+ 'https://toscrape.com',+ 'https://example.com',+ cookies1=True,+ cookies2=False,+ )++ def test_cookie_redirect_different_domain_forcing_get(self):+ self._test_cookie_redirect(+ 'https://toscrape.com',+ {'url': 'https://example.com', 'status': 302},+ cookies1=True,+ cookies2=False,+ )++ def _test_cookie_header_redirect(+ self,+ source,+ target,+ *,+ cookies2,+ ):+ """Test the handling of a user-defined Cookie header when building a+ redirect follow-up request.++ We follow RFC 6265 for cookie handling. The Cookie header can only+ contain a list of key-value pairs (i.e. no additional cookie+ parameters like Domain or Path). Because of that, we follow the same+ rules that we would follow for the handling of the Set-Cookie response+ header when the Domain is not set: the cookies must be limited to the+ target URL domain (not even subdomains can receive those cookies).++ .. note:: This method tests the scenario where the cookie middleware is+ disabled. Because of known issue #1992, when the cookies+ middleware is enabled we do not need to be concerned about+ the Cookie header getting leaked to unintended domains,+ because the middleware empties the header from every request.+ """+ if not isinstance(source, dict):+ source = {'url': source}+ if not isinstance(target, dict):+ target = {'url': target}+ target.setdefault('status', 301)++ request1 = Request(headers={'Cookie': b'a=b'}, **source)++ response = Response(+ headers={+ 'Location': target['url'],+ },+ **target,+ )++ request2 = self.redirect_middleware.process_response(+ request1,+ response,+ self.spider,+ )+ self.assertIsInstance(request2, Request)++ cookies = request2.headers.get('Cookie')+ self.assertEqual(cookies, b"a=b" if cookies2 else None)++ def test_cookie_header_redirect_same_domain(self):+ self._test_cookie_header_redirect(+ 'https://toscrape.com',+ 'https://toscrape.com',+ cookies2=True,+ )++ def test_cookie_header_redirect_same_domain_forcing_get(self):+ self._test_cookie_header_redirect(+ 'https://toscrape.com',+ {'url': 'https://toscrape.com', 'status': 302},+ cookies2=True,+ )++ def test_cookie_header_redirect_different_domain(self):+ self._test_cookie_header_redirect(+ 'https://toscrape.com',+ 'https://example.com',+ cookies2=False,+ )++ def test_cookie_header_redirect_different_domain_forcing_get(self):+ self._test_cookie_header_redirect(+ 'https://toscrape.com',+ {'url': 'https://example.com', 'status': 302},+ cookies2=False,+ )