Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not process cookies from headers
Eugenio Lacuesta744f352d09a9
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 87f8152a4..d95ed3d38 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -97,35 +97,14 @@ class CookiesMiddleware:def _get_request_cookies(self, jar, request):"""- Extract cookies from a Request. Values from the `Request.cookies` attribute- take precedence over values from the `Cookie` request header.+ Extract cookies from the Request.cookies attribute"""- def get_cookies_from_header(jar, request):- cookie_header = request.headers.get("Cookie")- if not cookie_header:- return []- cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))- cookie_list_unicode = []- for cookie_bytes in cookie_gen_bytes:- try:- cookie_unicode = cookie_bytes.decode("utf8")- except UnicodeDecodeError:- logger.warning("Non UTF-8 encoded cookie found in request %s: %s",- request, cookie_bytes)- cookie_unicode = cookie_bytes.decode("latin1", errors="replace")- cookie_list_unicode.append(cookie_unicode)- response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})- return jar.make_cookies(response, request)-- def get_cookies_from_attribute(jar, request):- if not request.cookies:- return []- elif isinstance(request.cookies, dict):- cookies = ({"name": k, "value": v} for k, v in request.cookies.items())- else:- cookies = request.cookies- formatted = filter(None, (self._format_cookie(c, request) for c in cookies))- response = Response(request.url, headers={"Set-Cookie": formatted})- return jar.make_cookies(response, request)-- return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)+ if not request.cookies:+ return []+ elif isinstance(request.cookies, dict):+ cookies = ({"name": k, "value": v} for k, v in request.cookies.items())+ else:+ cookies = request.cookies+ formatted = filter(None, (self._format_cookie(c, request) for c in cookies))+ response = Response(request.url, headers={"Set-Cookie": formatted})+ return jar.make_cookies(response, request)diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex a3de307ee..aff8542e9 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -2,6 +2,8 @@ import loggingfrom testfixtures import LogCapturefrom unittest import TestCase+import pytest+from scrapy.downloadermiddlewares.cookies import CookiesMiddlewarefrom scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddlewarefrom scrapy.exceptions import NotConfigured@@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase):self.assertIn('Cookie', request.headers)self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])+ @pytest.mark.xfail(reason="Cookie header is not currently being processed")def test_keep_cookie_from_default_request_headers_middleware(self):DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())@@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase):assert self.mw.process_request(req2, self.spider) is Noneself.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')+ @pytest.mark.xfail(reason="Cookie header is not currently being processed")def test_keep_cookie_header(self):# keep only cookies from 'Cookie' request headerreq1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})@@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase):assert self.mw.process_request(req3, self.spider) is Noneself.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')+ @pytest.mark.xfail(reason="Cookie header is not currently being processed")def test_request_headers_cookie_encoding(self):# 1) UTF8-encoded bytesreq1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})