Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix redirection error when the Location header value starts with 3 slashes (#4042)
Amardeep Bhowmick66cbceeb0a91
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 49468a2e4..b73f864dd 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -1,5 +1,5 @@import logging-from six.moves.urllib.parse import urljoin+from six.moves.urllib.parse import urljoin, urlparsefrom w3lib.url import safe_url_string@@ -70,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):if 'Location' not in response.headers or response.status not in allowed_status:return response- location = safe_url_string(response.headers['location'])+ location = safe_url_string(response.headers['Location'])+ if response.headers['Location'].startswith(b'//'):+ request_scheme = urlparse(request.url).scheme+ location = request_scheme + '://' + location.lstrip('/')redirected_url = urljoin(request.url, location)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 0e841489d..e7faf14a7 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -106,6 +106,22 @@ class RedirectMiddlewareTest(unittest.TestCase):del rsp.headers['Location']assert self.mw.process_response(req, rsp, self.spider) is rsp+ def test_redirect_302_relative(self):+ url = 'http://www.example.com/302'+ url2 = '///i8n.example2.com/302'+ url3 = 'http://i8n.example2.com/302'+ req = Request(url, method='HEAD')+ rsp = Response(url, headers={'Location': url2}, status=302)++ req2 = self.mw.process_response(req, rsp, self.spider)+ assert isinstance(req2, Request)+ self.assertEqual(req2.url, url3)+ self.assertEqual(req2.method, 'HEAD')++ # response without Location header but with status code is 3XX should be ignored+ del rsp.headers['Location']+ assert self.mw.process_response(req, rsp, self.spider) is rsp+def test_max_redirect_times(self):self.mw.max_redirect_times = 1