Scrapy, a fast high-level web crawling & scraping framework for Python.
adjust variable wording and redirect logic
Bernardas3cef1cd451c8
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 976b5032b..b11e7095b 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -6,6 +6,7 @@ from collections import defaultdictfrom twisted.internet.defer import Deferred, DeferredListfrom twisted.python.failure import Failure+from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.settings import Settingsfrom scrapy.utils.defer import mustbe_deferred, defer_resultfrom scrapy.utils.request import request_fingerprint@@ -36,7 +37,7 @@ class MediaPipeline(object):self.allow_redirects = settings.getbool(resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS)- self.allow_httpstatus_list = settings.getlist(+ self.handle_httpstatus_list = settings.getlist(resolve('MEDIA_HTTPSTATUS_LIST'), [])@@ -107,17 +108,15 @@ class MediaPipeline(object):def _modify_media_request(self, request):httpstatus_list = []- if self.allow_httpstatus_list:- httpstatus_list = self.allow_httpstatus_list- elif self.allow_redirects:+ if self.handle_httpstatus_list:+ httpstatus_list = self.handle_httpstatus_list+ if self.allow_redirects:if not httpstatus_list:- httpstatus_list = list(range(0, 300)) + list(range(400, 1000))+ httpstatus_list = [i for i in range(1000)+ if i not in RedirectMiddleware.allowed_status]else:- for i in range(300, 400):- try:- httpstatus_list.remove(i)- except ValueError:- pass+ httpstatus_list = [i for i in httpstatus_list+ if i not in RedirectMiddleware.allowed_status]if httpstatus_list:request.meta['handle_httpstatus_list'] = httpstatus_listelse:diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 41ee9962f..66e98db29 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase):assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}request = Request('http://url')- self.pipe.allow_httpstatus_list = list(range(100))+ self.pipe.handle_httpstatus_list = list(range(100))assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}- self.pipe.allow_httpstatus_list = None+ self.pipe.handle_httpstatus_list = Nonerequest = Request('http://url')self.pipe.allow_redirects = True