Scrapy, a fast high-level web crawling & scraping framework for Python.
Add REDIRECT_HTTP_CODES for 201 redirect support and Response.follow_redirect() (#7801)
Adrianf0f6fd523a19
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex e2ee77755..d0db60ea3 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -908,6 +908,9 @@ RedirectMiddlewareThis middleware handles redirection of requests based on response status.+Requests are redirected to the URL from the ``Location`` header of responses+whose status is in :setting:`REDIRECT_HTTP_CODES`.+.. reqmeta:: redirect_urlsThe urls which the request goes through (while being redirected) can be found@@ -941,6 +944,7 @@ The :class:`RedirectMiddleware` can be configured through the followingsettings (see the settings documentation for more info):* :setting:`REDIRECT_ENABLED`+* :setting:`REDIRECT_HTTP_CODES`* :setting:`REDIRECT_MAX_TIMES`.. reqmeta:: dont_redirect@@ -948,6 +952,13 @@ settings (see the settings documentation for more info):If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``key set to True, the request will be ignored by this middleware.+.. reqmeta:: redirect_http_codes++.. versionadded:: VERSION++The ``redirect_http_codes`` key of :attr:`Request.meta <scrapy.Request.meta>`+overrides the :setting:`REDIRECT_HTTP_CODES` setting for a single request.+If you want to handle some redirect status codes in your spider, you canspecify these in the ``handle_httpstatus_list`` spider attribute.@@ -1013,6 +1024,53 @@ Default: ``True``Whether the Redirect middleware will be enabled.+.. setting:: REDIRECT_HTTP_CODES++REDIRECT_HTTP_CODES+^^^^^^^^^^^^^^^^^^^++.. versionadded:: VERSION++Default: ``[301, 302, 303, 307, 308]``++Response status codes whose ``Location`` header is followed.++The default value contains the status codes for which the HTTP standard defines+redirection handling. Add other status codes that may report a ``Location``+header to have it followed as well.++For example, an HTTP 201 (Created) response reports that the request created a+resource, and its ``Location`` header identifies that resource. Because that is+not a redirection target, it is not followed by default, matching the behavior+of web browsers. However, servers that create resources on the fly, e.g. to+serve an image that they generate on demand, sometimes send an empty 201+response and expect a separate request for the ``Location`` URL:++.. code-block:: python++ REDIRECT_HTTP_CODES = [201, 301, 302, 303, 307, 308]++Status codes other than the ones in the default value get conservative+handling, because their redirection semantics are unknown:++- Their ``Location`` header is only followed if the response has an empty+ body, as a response that carries a resource would otherwise be discarded.++- They are followed like 303 responses, i.e. with a GET request without a+ body, unless the original request used the GET or HEAD method, which is+ kept.++Use the :reqmeta:`redirect_http_codes` :attr:`Request.meta+<scrapy.Request.meta>` key to override this setting for a single request, and+:meth:`Response.follow_redirect() <scrapy.http.Response.follow_redirect>` to+follow a ``Location`` header from a spider callback instead.++To stop following a status code, you can also use the ``handle_httpstatus_list``+spider attribute or :attr:`Request.meta <scrapy.Request.meta>` key, which+additionally lets your spider callbacks receive the affected responses.++For media pipelines, see also :setting:`MEDIA_ALLOW_REDIRECTS`.+.. setting:: REDIRECT_MAX_TIMESREDIRECT_MAX_TIMESdiff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex a964ddfa9..e43b5e698 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -695,6 +695,28 @@ To handle media redirections, set this setting to ``True``:MEDIA_ALLOW_REDIRECTS = True+Media files created on the fly+------------------------------++.. versionchanged:: VERSION+ Added support for HTTP 201 responses.++Some servers create media files on the fly and report that with an HTTP 201+(Created) response instead of an HTTP 200 (OK) one. Media pipelines treat both+as successful downloads.++If such a response has an empty body, the media file must be requested+separately, from the URL in its ``Location`` header. To do that, add ``201`` to+:setting:`REDIRECT_HTTP_CODES`, so that+:class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` follows that+header, and enable :setting:`MEDIA_ALLOW_REDIRECTS`, so that media requests may+be redirected:++.. code-block:: python++ REDIRECT_HTTP_CODES = [201, 301, 302, 303, 307, 308]+ MEDIA_ALLOW_REDIRECTS = True+Limiting the result cache-------------------------diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 025bb3943..c23b87c84 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -838,6 +838,7 @@ Those are:* :reqmeta:`priority_adjust`* :reqmeta:`proxy`* :reqmeta:`proxy_headers`+* :reqmeta:`redirect_http_codes`* :reqmeta:`redirect_reasons`* :reqmeta:`redirect_times`* :reqmeta:`redirect_ttl`diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 488a3c0ec..80fe2da39 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -10,7 +10,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddlewarefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import global_object_name-from scrapy.utils.response import get_meta_refresh+from scrapy.utils.response import _body_size, get_meta_refreshfrom scrapy.utils.url import _redirect_urlif TYPE_CHECKING:@@ -22,6 +22,12 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+# Status codes for which the HTTP standard defines redirection handling. Other+# status codes from REDIRECT_HTTP_CODES get conservative handling, see+# RedirectMiddleware.+_STANDARD_HTTP_CODES = frozenset({301, 302, 303, 307, 308})++class BaseRedirectMiddleware:crawler: Crawlerenabled_setting: str = "REDIRECT_ENABLED"@@ -195,6 +201,28 @@ class BaseRedirectMiddleware:class RedirectMiddleware(BaseRedirectMiddleware):"""Handle redirection of requests based on response status."""+ def __init__(self, settings: BaseSettings):+ super().__init__(settings)+ self.redirect_http_codes: set[int] = {+ int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")+ }++ def _is_redirect(self, request: Request, response: Response) -> bool:+ meta_http_codes = request.meta.get("redirect_http_codes")+ http_codes = (+ {int(x) for x in meta_http_codes}+ if meta_http_codes is not None+ else self.redirect_http_codes+ )+ if response.status not in http_codes:+ return False+ if response.status in _STANDARD_HTTP_CODES:+ return True+ # For other status codes, e.g. 201, Location identifies a resource+ # rather than a redirection target, and the response body may be that+ # resource, which following Location would discard.+ return not _body_size(response)+@_warn_spider_argdef process_response(self, request: Request, response: Response, spider: Spider | None = None@@ -208,13 +236,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):):return response- if "Location" not in response.headers or response.status not in {- 301,- 302,- 303,- 307,- 308,- }:+ if "Location" not in response.headers or not self._is_redirect(+ request, response+ ):return responseassert response.headers["Location"] is not None@@ -224,8 +248,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):if urlparse_cached(redirected).scheme not in {"http", "https"}:return response+ # 307 and 308 responses keep the method and the body of the original+ # request as is, and so do 301 and 302 responses, except for POST+ # requests, for historical reasons. Any other status code gets the+ # handling of 303 responses, a bodyless GET request, which is the safest+ # choice for status codes with no defined redirection semantics.if (response.status in {301, 302} and request.method == "POST") or (- response.status == 303 and request.method not in {"GET", "HEAD"}+ response.status not in {301, 302, 307, 308}+ and request.method not in {"GET", "HEAD"}):redirected = self._redirect_request_using_get(request, response, redirected_urldiff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex d1c6ae0d3..d5d20e15f 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -321,10 +321,11 @@ class Response(object_ref):It raises :exc:`ValueError` if this response has no ``Location`` header.- The resulting request uses the GET method and has no body unless you- specify otherwise, even for 307 and 308 responses, for which+ Note that the resulting request uses the GET method and has no body,+ unless you specify otherwise. To reproduce instead how:class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`- keeps the method and body of the original request.+ follows the ``Location`` header of a given response status, see+ :setting:`REDIRECT_HTTP_CODES`."""location = self.headers.get("Location")if location is None:diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 484c6297c..e18dc5c2d 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -104,6 +104,9 @@ class MediaPipeline(ABC):resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline", settings=settings)+ self._redirect_http_codes: set[int] = {+ int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")+ }self.allow_redirects: bool = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)@@ -113,7 +116,11 @@ class MediaPipeline(ABC):def _handle_statuses(self, allow_redirects: bool) -> None:self.handle_httpstatus_list = Noneif allow_redirects:- self.handle_httpstatus_list = SequenceExclude(range(300, 400))+ # Statuses that RedirectMiddleware may redirect, i.e. those from+ # REDIRECT_HTTP_CODES, are left for it to handle.+ self.handle_httpstatus_list = SequenceExclude(+ {*range(300, 400), *self._redirect_http_codes}+ )def _key_for_pipe(self,diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex fd9d45f26..69560bc67 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -188,6 +188,7 @@ __all__ = ["RANDOMIZE_DOWNLOAD_DELAY","REACTOR_THREADPOOL_MAXSIZE","REDIRECT_ENABLED",+ "REDIRECT_HTTP_CODES","REDIRECT_MAX_TIMES","REDIRECT_PRIORITY_ADJUST","REFERER_ENABLED",@@ -555,6 +556,7 @@ RANDOMIZE_DOWNLOAD_DELAY = TrueREACTOR_THREADPOOL_MAXSIZE = 10REDIRECT_ENABLED = True+REDIRECT_HTTP_CODES = [301, 302, 303, 307, 308]REDIRECT_MAX_TIMES = 20 # uses Firefox default settingREDIRECT_PRIORITY_ADJUST = +2diff --git a/tests/mockserver/http.py b/tests/mockserver/http.pyindex e0a6f5981..e7944f6c2 100644--- a/tests/mockserver/http.py+++ b/tests/mockserver/http.py@@ -21,6 +21,7 @@ from .http_resources import (Compress,ConnectionId,ContentLengthHeaderResource,+ Created,Delay,Drop,DuplicateHeaderResource,@@ -68,6 +69,7 @@ class Root(BaseResource):put_child(self, b"alpayload", ArbitraryLengthPayloadResource())put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))put_child(self, b"redirect-to", RedirectTo())+ put_child(self, b"created", Created())put_child(self, b"text", Data(b"Works", "text/plain"))put_child(self, b"connection-id", ConnectionId())put_child(diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.pyindex 759e3765b..79ae43c35 100644--- a/tests/mockserver/http_resources.py+++ b/tests/mockserver/http_resources.py@@ -296,6 +296,16 @@ class RedirectTo(LeafResource):return b"redirecting..."+class Created(LeafResource):+ """Emulate a server that creates a resource on the fly, reporting it with an+ empty 201 response that points at the created resource through Location."""++ def render(self, request: Request) -> bytes:+ request.setResponseCode(201)+ request.setHeader(b"Location", getarg(request, b"goto", b"/"))+ return b""++class Partial(LeafResource):def render_GET(self, request: Request) -> int:request.setHeader(b"Content-Length", b"1024")diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 770d7fea1..d3cc1be0a 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -1,11 +1,14 @@+from __future__ import annotations+import logging+from abc import ABCfrom unittest.mock import MagicMockimport pytestfrom scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.exceptions import NotConfigured-from scrapy.http import Request, Response+from scrapy.http import BodyFileResponse, Request, Responsefrom scrapy.spidermiddlewares.referer import (POLICY_NO_REFERRER,POLICY_ORIGIN,@@ -368,6 +371,152 @@ class TestRedirectMiddleware(TestRedirectBase):response = Response(request.url, status=302)assert self.mw.process_response(request, response) is response+ @pytest.mark.parametrize("status", [201, 300])+ def test_status_not_in_redirect_http_codes(self, status):+ """Status codes outside REDIRECT_HTTP_CODES are not followed, even if they+ report a Location header, as 201 (Created) or 300 (Multiple Choices) may+ do."""+ url = f"https://example.com/{status}"+ request = Request(url)+ response = Response(url, status=status, headers={"Location": "/target"})+ assert self.mw.process_response(request, response) is response++ def test_standard_status_not_in_redirect_http_codes(self):+ crawler = get_crawler(DefaultSpider, {"REDIRECT_HTTP_CODES": [301]})+ crawler.spider = crawler._create_spider()+ mw = RedirectMiddleware.from_crawler(crawler)+ url = "https://example.com/302"+ request = Request(url)+ response = Response(url, status=302, headers={"Location": "/redirected"})+ assert mw.process_response(request, response) is response++ def test_meta_http_codes(self):+ """The redirect_http_codes request metadata key overrides the+ REDIRECT_HTTP_CODES setting."""+ url = "https://example.com/201"+ request = Request(url, meta={"redirect_http_codes": [201]})+ response = Response(url, status=201, headers={"Location": "/created"})+ redirect_request = self.mw.process_response(request, response)+ assert isinstance(redirect_request, Request)+ assert redirect_request.url == "https://example.com/created"++ def test_meta_http_codes_excluding_standard_status(self):+ url = "https://example.com/302"+ request = Request(url, meta={"redirect_http_codes": [201]})+ response = Response(url, status=302, headers={"Location": "/redirected"})+ assert self.mw.process_response(request, response) is response+++class NonStandardStatusMixin(ABC):+ """Tests of status codes added to REDIRECT_HTTP_CODES for which the HTTP+ standard does not define redirection handling, and which therefore get+ conservative handling."""++ status: int++ def setup_method(self):+ crawler = get_crawler(+ DefaultSpider,+ {"REDIRECT_HTTP_CODES": [self.status, 301, 302, 303, 307, 308]},+ )+ crawler.spider = crawler._create_spider()+ self.mw = RedirectMiddleware.from_crawler(crawler)+ self.url = f"https://example.com/{self.status}"++ def _response(+ self, body: bytes = b"", location: str | None = "/target"+ ) -> Response:+ headers = {"Location": location} if location else {}+ return Response(self.url, status=self.status, headers=headers, body=body)++ def test_empty_body(self):+ redirect_request = self.mw.process_response(Request(self.url), self._response())+ assert isinstance(redirect_request, Request)+ assert redirect_request.url == "https://example.com/target"+ assert redirect_request.method == "GET"+ assert redirect_request.meta["redirect_urls"] == [self.url]+ assert redirect_request.meta["redirect_reasons"] == [self.status]++ def test_non_empty_body(self):+ """A response that carries a resource in its body is not followed, doing so+ would discard that resource."""+ response = self._response(body=b"file content")+ assert self.mw.process_response(Request(self.url), response) is response++ @pytest.mark.parametrize("body", [b"", b"file content"])+ def test_body_file(self, body, tmp_path):+ body_file = tmp_path / "body"+ body_file.write_bytes(body)+ response = BodyFileResponse(+ self.url,+ status=self.status,+ headers={"Location": "/target"},+ body_file=body_file,+ )+ result = self.mw.process_response(Request(self.url), response)+ assert (result is response) is bool(body)++ def test_no_location(self):+ response = self._response(location=None)+ assert self.mw.process_response(Request(self.url), response) is response++ @pytest.mark.parametrize("method", ["POST", "PUT"])+ def test_method_becomes_get(self, method):+ request = Request(+ self.url,+ method=method,+ body=b"payload",+ headers={"Content-Type": "text/plain", "Content-Length": "7"},+ )+ redirect_request = self.mw.process_response(request, self._response())+ assert isinstance(redirect_request, Request)+ assert redirect_request.method == "GET"+ assert not redirect_request.body+ assert "Content-Type" not in redirect_request.headers+ assert "Content-Length" not in redirect_request.headers++ def test_head_method_preserved(self):+ request = Request(self.url, method="HEAD")+ redirect_request = self.mw.process_response(request, self._response())+ assert isinstance(redirect_request, Request)+ assert redirect_request.method == "HEAD"++ @pytest.mark.parametrize(+ "meta",+ [+ {"dont_redirect": True},+ {"handle_httpstatus_all": True},+ ],+ )+ def test_request_meta_handling(self, meta):+ request = Request(self.url, meta=meta)+ response = self._response()+ assert self.mw.process_response(request, response) is response++ def test_request_meta_status_handling(self):+ request = Request(self.url, meta={"handle_httpstatus_list": [self.status]})+ response = self._response()+ assert self.mw.process_response(request, response) is response++ def test_spider_handling(self):+ self.mw.crawler.spider.handle_httpstatus_list = [self.status] # type: ignore[union-attr]+ response = self._response()+ assert self.mw.process_response(Request(self.url), response) is response+++class TestRedirect201(NonStandardStatusMixin):+ """201 (Created) reports the created resource through Location."""++ status = 201+++class TestRedirect300(NonStandardStatusMixin):+ """300 (Multiple Choices) may report a preferred choice through Location. It+ is a 3xx status code, but the standard defines no redirection handling for+ it, so it gets the same conservative handling as 201."""++ status = 300+@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES)def test_redirect_schemes(url, location, target):diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex d0778e088..58cad18f3 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -61,6 +61,18 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):)+class CreatedMediaDownloadSpider(MediaDownloadSpider):+ """Requests media files from a server that creates them on the fly, reports+ that with an empty 201 response, and points at the created file through the+ Location header."""++ name = "createdmedia"++ def _process_url(self, url: str) -> str:+ assert self.mockserver+ return add_or_replace_parameter(self.mockserver.url("/created"), "goto", url)++class TestFileDownloadCrawl:pipeline_class = "scrapy.pipelines.files.FilesPipeline"store_setting_key = "FILES_STORE"@@ -200,6 +212,62 @@ class TestFileDownloadCrawl:self._assert_files_downloaded(self.items, caplog.text)assert crawler.stats.get_value("downloader/response_status_count/302") == 3+ @pytest.mark.parametrize(+ "settings",+ [+ {},+ # Following the Location header of a 201 response requires both+ # settings.+ {"REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308]},+ {"MEDIA_ALLOW_REDIRECTS": True},+ ],+ )+ @coroutine_test+ async def test_download_media_created_default_failure(+ self,+ settings: dict[str, Any],+ caplog: pytest.LogCaptureFixture,+ mockserver: MockServer,+ ) -> None:+ crawler = self._create_crawler(+ CreatedMediaDownloadSpider, {**self.settings, **settings}+ )+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ mockserver.url("/static/files/images/"),+ media_key=self.media_key,+ media_urls_key=self.media_urls_key,+ mockserver=mockserver,+ )+ assert len(self.items) == 1+ assert not self.items[0][self.media_key]+ assert crawler.stats+ assert crawler.stats.get_value("downloader/response_status_count/201") == 3+ assert caplog.text.count("File (empty-content): Empty file from") == 3+ assert not list(self.tmpmediastore.iterdir())++ @coroutine_test+ async def test_download_media_created_allowed(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ settings = {+ **self.settings,+ "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],+ "MEDIA_ALLOW_REDIRECTS": True,+ }+ crawler = self._create_crawler(CreatedMediaDownloadSpider, settings)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ mockserver.url("/static/files/images/"),+ media_key=self.media_key,+ media_urls_key=self.media_urls_key,+ mockserver=mockserver,+ )+ self._assert_files_downloaded(self.items, caplog.text)+ assert crawler.stats+ assert crawler.stats.get_value("downloader/response_status_count/201") == 3+ assert caplog.text.count("Redirecting (201)") == 3+@coroutine_testasync def test_download_media_file_path_error(self, caplog: pytest.LogCaptureFixture, mockserver: MockServerdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex a46cdee1b..e500ad8fa 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -472,6 +472,24 @@ class TestFilesPipeline:path = Path(self.tempdir) / result["files"][0]["path"]assert path.read_bytes() == b"data"+ @coroutine_test+ async def test_file_created_empty(self, caplog: pytest.LogCaptureFixture) -> None:+ """A 201 response without a body is an empty download, e.g. because its+ Location header was not followed, 201 not being in REDIRECT_HTTP_CODES."""+ item_url = "http://example.com/created-empty.pdf"+ item = _create_item_with_files(item_url)+ with (+ caplog.at_level(logging.WARNING),+ mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url, status=201, body=b"")],+ ),+ ):+ result = await self.pipeline.process_item(item)+ assert result["files"] == []+ assert "File (empty-content)" in caplog.text+@coroutine_testasync def test_checksum_algorithm(self) -> None:pipeline = self._create_pipeline(@@ -1731,12 +1749,13 @@ def _prepare_request_object(flags: list[str] | None = None,headers: dict[str, str] | None = None,status: int = 200,+ body: bytes = b"data",) -> Request:return Request(item_url,meta={"response": Response(- item_url, status=status, body=b"data", flags=flags, headers=headers+ item_url, status=status, body=body, flags=flags, headers=headers)},)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 09d470cc5..fcf23928e 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -511,7 +511,7 @@ class TestAsyncMediaDownloaded(TestMediaPipeline):class TestMediaPipelineAllowRedirectSettings:- def _assert_request_no3xx(+ def _assert_redirect_statuses_not_handled(self, pipeline_class: type[MediaPipeline], settings: dict[str, Any]) -> None:pipe = pipeline_class(crawler=get_crawler(None, settings))@@ -521,6 +521,8 @@ class TestMediaPipelineAllowRedirectSettings:assert "handle_httpstatus_list" in request.metafor status, check in [(200, True),+ # 201 is not in REDIRECT_HTTP_CODES by default+ (201, True),# These are the status codes we want# the downloader to handle itself(301, False),@@ -539,13 +541,32 @@ class TestMediaPipelineAllowRedirectSettings:assert status not in request.meta["handle_httpstatus_list"]def test_subclass_standard_setting(self):- self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})+ self._assert_redirect_statuses_not_handled(+ UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}+ )def test_subclass_specific_setting(self):- self._assert_request_no3xx(+ self._assert_redirect_statuses_not_handled(UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True})+ def test_custom_redirect_http_codes(self):+ """Statuses added to REDIRECT_HTTP_CODES are also left to+ RedirectMiddleware."""+ crawler = get_crawler(+ None,+ {+ "MEDIA_ALLOW_REDIRECTS": True,+ "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],+ },+ )+ request = Request("http://url")+ UserDefinedPipeline(crawler=crawler)._modify_media_request(request)++ assert 201 not in request.meta["handle_httpstatus_list"]+ assert 302 not in request.meta["handle_httpstatus_list"]+ assert 200 in request.meta["handle_httpstatus_list"]+class TestBuildFromCrawler:def setup_method(self):