Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle HTTP 308 Permanent Redirect
Paul Tremberth5dc9a88c347d
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 26677e527..30cae3fee 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):request.meta.get('handle_httpstatus_all', False)):return response- allowed_status = (301, 302, 303, 307)+ allowed_status = (301, 302, 303, 307, 308)if 'Location' not in response.headers or response.status not in allowed_status:return response@@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):redirected_url = urljoin(request.url, location)- if response.status in (301, 307) or request.method == 'HEAD':+ if response.status in (301, 307, 308) or request.method == 'HEAD':redirected = request.replace(url=redirected_url)return self._redirect(redirected, request, spider, response.status)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex e8c92affa..a2da4aa8f 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase):req2 = self.mw.process_response(req, rsp, self.spider)assert req2.priority > req.priority- def test_redirect_301(self):- def _test(method):- url = 'http://www.example.com/301'+ def test_redirect_3xx_permanent(self):+ def _test(method, status=301):+ url = 'http://www.example.com/{}'.format(status)url2 = 'http://www.example.com/redirected'req = Request(url, method=method)- rsp = Response(url, headers={'Location': url2}, status=301)+ rsp = Response(url, headers={'Location': url2}, status=status)req2 = self.mw.process_response(req, rsp, self.spider)assert isinstance(req2, Request)@@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase):_test('POST')_test('HEAD')+ _test('GET', status=308)+ _test('POST', status=308)+ _test('HEAD', status=308)+def test_dont_redirect(self):url = 'http://www.example.com/301'url2 = 'http://www.example.com/redirected'