Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[MRG+1] redirect_reasons in Request.meta (#3687)

Maram Sumanth2fd8b7c28c25
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0d976077b..f2f3ef466 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -733,6 +733,17 @@ RedirectMiddleware
The urls which the request goes through (while being redirected) can be found
in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
+.. reqmeta:: redirect_reasons
+
+The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
+``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
+example: ``[301, 302, 307, 'meta refresh']``.
+
+The format of a reason depends on the middleware that handled the corresponding
+redirect. For example, :class:`RedirectMiddleware` indicates the triggering
+response status code as an integer, while :class:`MetaRefreshMiddleware`
+always uses the ``'meta refresh'`` string as reason.
+
The :class:`RedirectMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@@ -796,8 +807,9 @@ settings (see the settings documentation for more info):
* :setting:`METAREFRESH_ENABLED`
* :setting:`METAREFRESH_MAXDELAY`
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
MetaRefreshMiddleware settings
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ca59b46d8..ac6fe6e3f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -299,6 +299,7 @@ Those are:
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 30cae3fee..cb59d3fd2 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object):
redirected.meta['redirect_ttl'] = ttl - 1
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
[request.url]
+ redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
+ [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 74137b4cd..6c81c94ca 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -139,6 +139,16 @@ class RedirectMiddlewareTest(unittest.TestCase):
self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+ def test_redirect_reasons(self):
+ req1 = Request('http://scrapytest.org/first')
+ rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301)
+ req2 = self.mw.process_response(req1, rsp1, self.spider)
+ rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301)
+ req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+ self.assertEqual(req2.meta['redirect_reasons'], [301])
+ self.assertEqual(req3.meta['redirect_reasons'], [301, 301])
+
def test_spider_handling(self):
smartspider = self.crawler._create_spider('smarty')
smartspider.handle_httpstatus_list = [404, 301, 302]
@@ -259,6 +269,15 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+ def test_redirect_reasons(self):
+ req1 = Request('http://scrapytest.org/first')
+ rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected'))
+ req2 = self.mw.process_response(req1, rsp1, self.spider)
+ rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1'))
+ req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+ self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
+ self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
if __name__ == "__main__":
unittest.main()