Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix redirection error when the Location header value starts with 3 slashes (#4042)

Amardeep Bhowmick66cbceeb0a91
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 49468a2e4..b73f864dd 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
import logging
-from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@@ -70,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if 'Location' not in response.headers or response.status not in allowed_status:
return response
- location = safe_url_string(response.headers['location'])
+ location = safe_url_string(response.headers['Location'])
+ if response.headers['Location'].startswith(b'//'):
+ request_scheme = urlparse(request.url).scheme
+ location = request_scheme + '://' + location.lstrip('/')
redirected_url = urljoin(request.url, location)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 0e841489d..e7faf14a7 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -106,6 +106,22 @@ class RedirectMiddlewareTest(unittest.TestCase):
del rsp.headers['Location']
assert self.mw.process_response(req, rsp, self.spider) is rsp
+ def test_redirect_302_relative(self):
+ url = 'http://www.example.com/302'
+ url2 = '///i8n.example2.com/302'
+ url3 = 'http://i8n.example2.com/302'
+ req = Request(url, method='HEAD')
+ rsp = Response(url, headers={'Location': url2}, status=302)
+
+ req2 = self.mw.process_response(req, rsp, self.spider)
+ assert isinstance(req2, Request)
+ self.assertEqual(req2.url, url3)
+ self.assertEqual(req2.method, 'HEAD')
+
+ # response without Location header but with status code is 3XX should be ignored
+ del rsp.headers['Location']
+ assert self.mw.process_response(req, rsp, self.spider) is rsp
+
def test_max_redirect_times(self):
self.mw.max_redirect_times = 1