Scrapy, a fast high-level web crawling & scraping framework for Python.
[MRG+1] redirect_reasons in Request.meta (#3687)
Maram Sumanth2fd8b7c28c25
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 0d976077b..f2f3ef466 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -733,6 +733,17 @@ RedirectMiddlewareThe urls which the request goes through (while being redirected) can be foundin the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.+.. reqmeta:: redirect_reasons++The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the+``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For+example: ``[301, 302, 307, 'meta refresh']``.++The format of a reason depends on the middleware that handled the corresponding+redirect. For example, :class:`RedirectMiddleware` indicates the triggering+response status code as an integer, while :class:`MetaRefreshMiddleware`+always uses the ``'meta refresh'`` string as reason.+The :class:`RedirectMiddleware` can be configured through the followingsettings (see the settings documentation for more info):@@ -796,8 +807,9 @@ settings (see the settings documentation for more info):* :setting:`METAREFRESH_ENABLED`* :setting:`METAREFRESH_MAXDELAY`-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described+for :class:`RedirectMiddleware`MetaRefreshMiddleware settingsdiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex ca59b46d8..ac6fe6e3f 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -299,6 +299,7 @@ Those are:* :reqmeta:`dont_merge_cookies`* :reqmeta:`cookiejar`* :reqmeta:`dont_cache`+* :reqmeta:`redirect_reasons`* :reqmeta:`redirect_urls`* :reqmeta:`bindaddress`* :reqmeta:`dont_obey_robotstxt`diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 30cae3fee..cb59d3fd2 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object):redirected.meta['redirect_ttl'] = ttl - 1redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \[request.url]+ redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \+ [reason]redirected.dont_filter = request.dont_filterredirected.priority = request.priority + self.priority_adjustlogger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 74137b4cd..6c81c94ca 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -139,6 +139,16 @@ class RedirectMiddlewareTest(unittest.TestCase):self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])+ def test_redirect_reasons(self):+ req1 = Request('http://scrapytest.org/first')+ rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301)+ req2 = self.mw.process_response(req1, rsp1, self.spider)+ rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301)+ req3 = self.mw.process_response(req2, rsp2, self.spider)++ self.assertEqual(req2.meta['redirect_reasons'], [301])+ self.assertEqual(req3.meta['redirect_reasons'], [301, 301])+def test_spider_handling(self):smartspider = self.crawler._create_spider('smarty')smartspider.handle_httpstatus_list = [404, 301, 302]@@ -259,6 +269,15 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])+ def test_redirect_reasons(self):+ req1 = Request('http://scrapytest.org/first')+ rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected'))+ req2 = self.mw.process_response(req1, rsp1, self.spider)+ rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1'))+ req3 = self.mw.process_response(req2, rsp2, self.spider)++ self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])+ self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])if __name__ == "__main__":unittest.main()