Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle HTTP 308 Permanent Redirect

Paul Tremberth5dc9a88c347d
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 26677e527..30cae3fee 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
request.meta.get('handle_httpstatus_all', False)):
return response
- allowed_status = (301, 302, 303, 307)
+ allowed_status = (301, 302, 303, 307, 308)
if 'Location' not in response.headers or response.status not in allowed_status:
return response
@@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
redirected_url = urljoin(request.url, location)
- if response.status in (301, 307) or request.method == 'HEAD':
+ if response.status in (301, 307, 308) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index e8c92affa..a2da4aa8f 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase):
req2 = self.mw.process_response(req, rsp, self.spider)
assert req2.priority > req.priority
- def test_redirect_301(self):
- def _test(method):
- url = 'http://www.example.com/301'
+ def test_redirect_3xx_permanent(self):
+ def _test(method, status=301):
+ url = 'http://www.example.com/{}'.format(status)
url2 = 'http://www.example.com/redirected'
req = Request(url, method=method)
- rsp = Response(url, headers={'Location': url2}, status=301)
+ rsp = Response(url, headers={'Location': url2}, status=status)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
@@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
_test('POST')
_test('HEAD')
+ _test('GET', status=308)
+ _test('POST', status=308)
+ _test('HEAD', status=308)
+
def test_dont_redirect(self):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'