Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

adjust variable wording and redirect logic

Bernardas3cef1cd451c8
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 976b5032b..b11e7095b 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -6,6 +6,7 @@ from collections import defaultdict
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.settings import Settings
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.request import request_fingerprint
@@ -36,7 +37,7 @@ class MediaPipeline(object):
self.allow_redirects = settings.getbool(
resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS
)
- self.allow_httpstatus_list = settings.getlist(
+ self.handle_httpstatus_list = settings.getlist(
resolve('MEDIA_HTTPSTATUS_LIST'), []
)
@@ -107,17 +108,15 @@ class MediaPipeline(object):
def _modify_media_request(self, request):
httpstatus_list = []
- if self.allow_httpstatus_list:
- httpstatus_list = self.allow_httpstatus_list
- elif self.allow_redirects:
+ if self.handle_httpstatus_list:
+ httpstatus_list = self.handle_httpstatus_list
+ if self.allow_redirects:
if not httpstatus_list:
- httpstatus_list = list(range(0, 300)) + list(range(400, 1000))
+ httpstatus_list = [i for i in range(1000)
+ if i not in RedirectMiddleware.allowed_status]
else:
- for i in range(300, 400):
- try:
- httpstatus_list.remove(i)
- except ValueError:
- pass
+ httpstatus_list = [i for i in httpstatus_list
+ if i not in RedirectMiddleware.allowed_status]
if httpstatus_list:
request.meta['handle_httpstatus_list'] = httpstatus_list
else:
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 41ee9962f..66e98db29 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
request = Request('http://url')
- self.pipe.allow_httpstatus_list = list(range(100))
+ self.pipe.handle_httpstatus_list = list(range(100))
assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}
- self.pipe.allow_httpstatus_list = None
+ self.pipe.handle_httpstatus_list = None
request = Request('http://url')
self.pipe.allow_redirects = True