Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add REDIRECT_HTTP_CODES for 201 redirect support and Response.follow_redirect() (#7801)

Adrianf0f6fd523a19
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e2ee77755..d0db60ea3 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -908,6 +908,9 @@ RedirectMiddleware
This middleware handles redirection of requests based on response status.
+Requests are redirected to the URL from the ``Location`` header of responses
+whose status is in :setting:`REDIRECT_HTTP_CODES`.
+
.. reqmeta:: redirect_urls
The urls which the request goes through (while being redirected) can be found
@@ -941,6 +944,7 @@ The :class:`RedirectMiddleware` can be configured through the following
settings (see the settings documentation for more info):
* :setting:`REDIRECT_ENABLED`
+* :setting:`REDIRECT_HTTP_CODES`
* :setting:`REDIRECT_MAX_TIMES`
.. reqmeta:: dont_redirect
@@ -948,6 +952,13 @@ settings (see the settings documentation for more info):
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``
key set to True, the request will be ignored by this middleware.
+.. reqmeta:: redirect_http_codes
+
+.. versionadded:: VERSION
+
+The ``redirect_http_codes`` key of :attr:`Request.meta <scrapy.Request.meta>`
+overrides the :setting:`REDIRECT_HTTP_CODES` setting for a single request.
+
If you want to handle some redirect status codes in your spider, you can
specify these in the ``handle_httpstatus_list`` spider attribute.
@@ -1013,6 +1024,53 @@ Default: ``True``
Whether the Redirect middleware will be enabled.
+.. setting:: REDIRECT_HTTP_CODES
+
+REDIRECT_HTTP_CODES
+^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: VERSION
+
+Default: ``[301, 302, 303, 307, 308]``
+
+Response status codes whose ``Location`` header is followed.
+
+The default value contains the status codes for which the HTTP standard defines
+redirection handling. Add other status codes that may report a ``Location``
+header to have it followed as well.
+
+For example, an HTTP 201 (Created) response reports that the request created a
+resource, and its ``Location`` header identifies that resource. Because that is
+not a redirection target, it is not followed by default, matching the behavior
+of web browsers. However, servers that create resources on the fly, e.g. to
+serve an image that they generate on demand, sometimes send an empty 201
+response and expect a separate request for the ``Location`` URL:
+
+.. code-block:: python
+
+ REDIRECT_HTTP_CODES = [201, 301, 302, 303, 307, 308]
+
+Status codes other than the ones in the default value get conservative
+handling, because their redirection semantics are unknown:
+
+- Their ``Location`` header is only followed if the response has an empty
+ body, as a response that carries a resource would otherwise be discarded.
+
+- They are followed like 303 responses, i.e. with a GET request without a
+ body, unless the original request used the GET or HEAD method, which is
+ kept.
+
+Use the :reqmeta:`redirect_http_codes` :attr:`Request.meta
+<scrapy.Request.meta>` key to override this setting for a single request, and
+:meth:`Response.follow_redirect() <scrapy.http.Response.follow_redirect>` to
+follow a ``Location`` header from a spider callback instead.
+
+To stop following a status code, you can also use the ``handle_httpstatus_list``
+spider attribute or :attr:`Request.meta <scrapy.Request.meta>` key, which
+additionally lets your spider callbacks receive the affected responses.
+
+For media pipelines, see also :setting:`MEDIA_ALLOW_REDIRECTS`.
+
.. setting:: REDIRECT_MAX_TIMES
REDIRECT_MAX_TIMES
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a964ddfa9..e43b5e698 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -695,6 +695,28 @@ To handle media redirections, set this setting to ``True``:
MEDIA_ALLOW_REDIRECTS = True
+Media files created on the fly
+------------------------------
+
+.. versionchanged:: VERSION
+ Added support for HTTP 201 responses.
+
+Some servers create media files on the fly and report that with an HTTP 201
+(Created) response instead of an HTTP 200 (OK) one. Media pipelines treat both
+as successful downloads.
+
+If such a response has an empty body, the media file must be requested
+separately, from the URL in its ``Location`` header. To do that, add ``201`` to
+:setting:`REDIRECT_HTTP_CODES`, so that
+:class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` follows that
+header, and enable :setting:`MEDIA_ALLOW_REDIRECTS`, so that media requests may
+be redirected:
+
+.. code-block:: python
+
+ REDIRECT_HTTP_CODES = [201, 301, 302, 303, 307, 308]
+ MEDIA_ALLOW_REDIRECTS = True
+
Limiting the result cache
-------------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 025bb3943..c23b87c84 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -838,6 +838,7 @@ Those are:
* :reqmeta:`priority_adjust`
* :reqmeta:`proxy`
* :reqmeta:`proxy_headers`
+* :reqmeta:`redirect_http_codes`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_times`
* :reqmeta:`redirect_ttl`
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 488a3c0ec..80fe2da39 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -10,7 +10,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import global_object_name
-from scrapy.utils.response import get_meta_refresh
+from scrapy.utils.response import _body_size, get_meta_refresh
from scrapy.utils.url import _redirect_url
if TYPE_CHECKING:
@@ -22,6 +22,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+# Status codes for which the HTTP standard defines redirection handling. Other
+# status codes from REDIRECT_HTTP_CODES get conservative handling, see
+# RedirectMiddleware.
+_STANDARD_HTTP_CODES = frozenset({301, 302, 303, 307, 308})
+
+
class BaseRedirectMiddleware:
crawler: Crawler
enabled_setting: str = "REDIRECT_ENABLED"
@@ -195,6 +201,28 @@ class BaseRedirectMiddleware:
class RedirectMiddleware(BaseRedirectMiddleware):
"""Handle redirection of requests based on response status."""
+ def __init__(self, settings: BaseSettings):
+ super().__init__(settings)
+ self.redirect_http_codes: set[int] = {
+ int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")
+ }
+
+ def _is_redirect(self, request: Request, response: Response) -> bool:
+ meta_http_codes = request.meta.get("redirect_http_codes")
+ http_codes = (
+ {int(x) for x in meta_http_codes}
+ if meta_http_codes is not None
+ else self.redirect_http_codes
+ )
+ if response.status not in http_codes:
+ return False
+ if response.status in _STANDARD_HTTP_CODES:
+ return True
+ # For other status codes, e.g. 201, Location identifies a resource
+ # rather than a redirection target, and the response body may be that
+ # resource, which following Location would discard.
+ return not _body_size(response)
+
@_warn_spider_arg
def process_response(
self, request: Request, response: Response, spider: Spider | None = None
@@ -208,13 +236,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
):
return response
- if "Location" not in response.headers or response.status not in {
- 301,
- 302,
- 303,
- 307,
- 308,
- }:
+ if "Location" not in response.headers or not self._is_redirect(
+ request, response
+ ):
return response
assert response.headers["Location"] is not None
@@ -224,8 +248,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
+ # 307 and 308 responses keep the method and the body of the original
+ # request as is, and so do 301 and 302 responses, except for POST
+ # requests, for historical reasons. Any other status code gets the
+ # handling of 303 responses, a bodyless GET request, which is the safest
+ # choice for status codes with no defined redirection semantics.
if (response.status in {301, 302} and request.method == "POST") or (
- response.status == 303 and request.method not in {"GET", "HEAD"}
+ response.status not in {301, 302, 307, 308}
+ and request.method not in {"GET", "HEAD"}
):
redirected = self._redirect_request_using_get(
request, response, redirected_url
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index d1c6ae0d3..d5d20e15f 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -321,10 +321,11 @@ class Response(object_ref):
It raises :exc:`ValueError` if this response has no ``Location`` header.
- The resulting request uses the GET method and has no body unless you
- specify otherwise, even for 307 and 308 responses, for which
+ Note that the resulting request uses the GET method and has no body,
+ unless you specify otherwise. To reproduce instead how
:class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
- keeps the method and body of the original request.
+ follows the ``Location`` header of a given response status, see
+ :setting:`REDIRECT_HTTP_CODES`.
"""
location = self.headers.get("Location")
if location is None:
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 484c6297c..e18dc5c2d 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -104,6 +104,9 @@ class MediaPipeline(ABC):
resolve = functools.partial(
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
)
+ self._redirect_http_codes: set[int] = {
+ int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")
+ }
self.allow_redirects: bool = settings.getbool(
resolve("MEDIA_ALLOW_REDIRECTS"), False
)
@@ -113,7 +116,11 @@ class MediaPipeline(ABC):
def _handle_statuses(self, allow_redirects: bool) -> None:
self.handle_httpstatus_list = None
if allow_redirects:
- self.handle_httpstatus_list = SequenceExclude(range(300, 400))
+ # Statuses that RedirectMiddleware may redirect, i.e. those from
+ # REDIRECT_HTTP_CODES, are left for it to handle.
+ self.handle_httpstatus_list = SequenceExclude(
+ {*range(300, 400), *self._redirect_http_codes}
+ )
def _key_for_pipe(
self,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index fd9d45f26..69560bc67 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -188,6 +188,7 @@ __all__ = [
"RANDOMIZE_DOWNLOAD_DELAY",
"REACTOR_THREADPOOL_MAXSIZE",
"REDIRECT_ENABLED",
+ "REDIRECT_HTTP_CODES",
"REDIRECT_MAX_TIMES",
"REDIRECT_PRIORITY_ADJUST",
"REFERER_ENABLED",
@@ -555,6 +556,7 @@ RANDOMIZE_DOWNLOAD_DELAY = True
REACTOR_THREADPOOL_MAXSIZE = 10
REDIRECT_ENABLED = True
+REDIRECT_HTTP_CODES = [301, 302, 303, 307, 308]
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
REDIRECT_PRIORITY_ADJUST = +2
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index e0a6f5981..e7944f6c2 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -21,6 +21,7 @@ from .http_resources import (
Compress,
ConnectionId,
ContentLengthHeaderResource,
+ Created,
Delay,
Drop,
DuplicateHeaderResource,
@@ -68,6 +69,7 @@ class Root(BaseResource):
put_child(self, b"alpayload", ArbitraryLengthPayloadResource())
put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
put_child(self, b"redirect-to", RedirectTo())
+ put_child(self, b"created", Created())
put_child(self, b"text", Data(b"Works", "text/plain"))
put_child(self, b"connection-id", ConnectionId())
put_child(
diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py
index 759e3765b..79ae43c35 100644
--- a/tests/mockserver/http_resources.py
+++ b/tests/mockserver/http_resources.py
@@ -296,6 +296,16 @@ class RedirectTo(LeafResource):
return b"redirecting..."
+class Created(LeafResource):
+ """Emulate a server that creates a resource on the fly, reporting it with an
+ empty 201 response that points at the created resource through Location."""
+
+ def render(self, request: Request) -> bytes:
+ request.setResponseCode(201)
+ request.setHeader(b"Location", getarg(request, b"goto", b"/"))
+ return b""
+
+
class Partial(LeafResource):
def render_GET(self, request: Request) -> int:
request.setHeader(b"Content-Length", b"1024")
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 770d7fea1..d3cc1be0a 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,11 +1,14 @@
+from __future__ import annotations
+
import logging
+from abc import ABC
from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured
-from scrapy.http import Request, Response
+from scrapy.http import BodyFileResponse, Request, Response
from scrapy.spidermiddlewares.referer import (
POLICY_NO_REFERRER,
POLICY_ORIGIN,
@@ -368,6 +371,152 @@ class TestRedirectMiddleware(TestRedirectBase):
response = Response(request.url, status=302)
assert self.mw.process_response(request, response) is response
+ @pytest.mark.parametrize("status", [201, 300])
+ def test_status_not_in_redirect_http_codes(self, status):
+ """Status codes outside REDIRECT_HTTP_CODES are not followed, even if they
+ report a Location header, as 201 (Created) or 300 (Multiple Choices) may
+ do."""
+ url = f"https://example.com/{status}"
+ request = Request(url)
+ response = Response(url, status=status, headers={"Location": "/target"})
+ assert self.mw.process_response(request, response) is response
+
+ def test_standard_status_not_in_redirect_http_codes(self):
+ crawler = get_crawler(DefaultSpider, {"REDIRECT_HTTP_CODES": [301]})
+ crawler.spider = crawler._create_spider()
+ mw = RedirectMiddleware.from_crawler(crawler)
+ url = "https://example.com/302"
+ request = Request(url)
+ response = Response(url, status=302, headers={"Location": "/redirected"})
+ assert mw.process_response(request, response) is response
+
+ def test_meta_http_codes(self):
+ """The redirect_http_codes request metadata key overrides the
+ REDIRECT_HTTP_CODES setting."""
+ url = "https://example.com/201"
+ request = Request(url, meta={"redirect_http_codes": [201]})
+ response = Response(url, status=201, headers={"Location": "/created"})
+ redirect_request = self.mw.process_response(request, response)
+ assert isinstance(redirect_request, Request)
+ assert redirect_request.url == "https://example.com/created"
+
+ def test_meta_http_codes_excluding_standard_status(self):
+ url = "https://example.com/302"
+ request = Request(url, meta={"redirect_http_codes": [201]})
+ response = Response(url, status=302, headers={"Location": "/redirected"})
+ assert self.mw.process_response(request, response) is response
+
+
+class NonStandardStatusMixin(ABC):
+ """Tests of status codes added to REDIRECT_HTTP_CODES for which the HTTP
+ standard does not define redirection handling, and which therefore get
+ conservative handling."""
+
+ status: int
+
+ def setup_method(self):
+ crawler = get_crawler(
+ DefaultSpider,
+ {"REDIRECT_HTTP_CODES": [self.status, 301, 302, 303, 307, 308]},
+ )
+ crawler.spider = crawler._create_spider()
+ self.mw = RedirectMiddleware.from_crawler(crawler)
+ self.url = f"https://example.com/{self.status}"
+
+ def _response(
+ self, body: bytes = b"", location: str | None = "/target"
+ ) -> Response:
+ headers = {"Location": location} if location else {}
+ return Response(self.url, status=self.status, headers=headers, body=body)
+
+ def test_empty_body(self):
+ redirect_request = self.mw.process_response(Request(self.url), self._response())
+ assert isinstance(redirect_request, Request)
+ assert redirect_request.url == "https://example.com/target"
+ assert redirect_request.method == "GET"
+ assert redirect_request.meta["redirect_urls"] == [self.url]
+ assert redirect_request.meta["redirect_reasons"] == [self.status]
+
+ def test_non_empty_body(self):
+ """A response that carries a resource in its body is not followed, doing so
+ would discard that resource."""
+ response = self._response(body=b"file content")
+ assert self.mw.process_response(Request(self.url), response) is response
+
+ @pytest.mark.parametrize("body", [b"", b"file content"])
+ def test_body_file(self, body, tmp_path):
+ body_file = tmp_path / "body"
+ body_file.write_bytes(body)
+ response = BodyFileResponse(
+ self.url,
+ status=self.status,
+ headers={"Location": "/target"},
+ body_file=body_file,
+ )
+ result = self.mw.process_response(Request(self.url), response)
+ assert (result is response) is bool(body)
+
+ def test_no_location(self):
+ response = self._response(location=None)
+ assert self.mw.process_response(Request(self.url), response) is response
+
+ @pytest.mark.parametrize("method", ["POST", "PUT"])
+ def test_method_becomes_get(self, method):
+ request = Request(
+ self.url,
+ method=method,
+ body=b"payload",
+ headers={"Content-Type": "text/plain", "Content-Length": "7"},
+ )
+ redirect_request = self.mw.process_response(request, self._response())
+ assert isinstance(redirect_request, Request)
+ assert redirect_request.method == "GET"
+ assert not redirect_request.body
+ assert "Content-Type" not in redirect_request.headers
+ assert "Content-Length" not in redirect_request.headers
+
+ def test_head_method_preserved(self):
+ request = Request(self.url, method="HEAD")
+ redirect_request = self.mw.process_response(request, self._response())
+ assert isinstance(redirect_request, Request)
+ assert redirect_request.method == "HEAD"
+
+ @pytest.mark.parametrize(
+ "meta",
+ [
+ {"dont_redirect": True},
+ {"handle_httpstatus_all": True},
+ ],
+ )
+ def test_request_meta_handling(self, meta):
+ request = Request(self.url, meta=meta)
+ response = self._response()
+ assert self.mw.process_response(request, response) is response
+
+ def test_request_meta_status_handling(self):
+ request = Request(self.url, meta={"handle_httpstatus_list": [self.status]})
+ response = self._response()
+ assert self.mw.process_response(request, response) is response
+
+ def test_spider_handling(self):
+ self.mw.crawler.spider.handle_httpstatus_list = [self.status] # type: ignore[union-attr]
+ response = self._response()
+ assert self.mw.process_response(Request(self.url), response) is response
+
+
+class TestRedirect201(NonStandardStatusMixin):
+ """201 (Created) reports the created resource through Location."""
+
+ status = 201
+
+
+class TestRedirect300(NonStandardStatusMixin):
+ """300 (Multiple Choices) may report a preferred choice through Location. It
+ is a 3xx status code, but the standard defines no redirection handling for
+ it, so it gets the same conservative handling as 201."""
+
+ status = 300
+
@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES)
def test_redirect_schemes(url, location, target):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index d0778e088..58cad18f3 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -61,6 +61,18 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
)
+class CreatedMediaDownloadSpider(MediaDownloadSpider):
+ """Requests media files from a server that creates them on the fly, reports
+ that with an empty 201 response, and points at the created file through the
+ Location header."""
+
+ name = "createdmedia"
+
+ def _process_url(self, url: str) -> str:
+ assert self.mockserver
+ return add_or_replace_parameter(self.mockserver.url("/created"), "goto", url)
+
+
class TestFileDownloadCrawl:
pipeline_class = "scrapy.pipelines.files.FilesPipeline"
store_setting_key = "FILES_STORE"
@@ -200,6 +212,62 @@ class TestFileDownloadCrawl:
self._assert_files_downloaded(self.items, caplog.text)
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
+ @pytest.mark.parametrize(
+ "settings",
+ [
+ {},
+ # Following the Location header of a 201 response requires both
+ # settings.
+ {"REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308]},
+ {"MEDIA_ALLOW_REDIRECTS": True},
+ ],
+ )
+ @coroutine_test
+ async def test_download_media_created_default_failure(
+ self,
+ settings: dict[str, Any],
+ caplog: pytest.LogCaptureFixture,
+ mockserver: MockServer,
+ ) -> None:
+ crawler = self._create_crawler(
+ CreatedMediaDownloadSpider, {**self.settings, **settings}
+ )
+ with caplog.at_level(logging.DEBUG):
+ await crawler.crawl_async(
+ mockserver.url("/static/files/images/"),
+ media_key=self.media_key,
+ media_urls_key=self.media_urls_key,
+ mockserver=mockserver,
+ )
+ assert len(self.items) == 1
+ assert not self.items[0][self.media_key]
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/response_status_count/201") == 3
+ assert caplog.text.count("File (empty-content): Empty file from") == 3
+ assert not list(self.tmpmediastore.iterdir())
+
+ @coroutine_test
+ async def test_download_media_created_allowed(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ settings = {
+ **self.settings,
+ "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],
+ "MEDIA_ALLOW_REDIRECTS": True,
+ }
+ crawler = self._create_crawler(CreatedMediaDownloadSpider, settings)
+ with caplog.at_level(logging.DEBUG):
+ await crawler.crawl_async(
+ mockserver.url("/static/files/images/"),
+ media_key=self.media_key,
+ media_urls_key=self.media_urls_key,
+ mockserver=mockserver,
+ )
+ self._assert_files_downloaded(self.items, caplog.text)
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/response_status_count/201") == 3
+ assert caplog.text.count("Redirecting (201)") == 3
+
@coroutine_test
async def test_download_media_file_path_error(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index a46cdee1b..e500ad8fa 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -472,6 +472,24 @@ class TestFilesPipeline:
path = Path(self.tempdir) / result["files"][0]["path"]
assert path.read_bytes() == b"data"
+ @coroutine_test
+ async def test_file_created_empty(self, caplog: pytest.LogCaptureFixture) -> None:
+ """A 201 response without a body is an empty download, e.g. because its
+ Location header was not followed, 201 not being in REDIRECT_HTTP_CODES."""
+ item_url = "http://example.com/created-empty.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ caplog.at_level(logging.WARNING),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url, status=201, body=b"")],
+ ),
+ ):
+ result = await self.pipeline.process_item(item)
+ assert result["files"] == []
+ assert "File (empty-content)" in caplog.text
+
@coroutine_test
async def test_checksum_algorithm(self) -> None:
pipeline = self._create_pipeline(
@@ -1731,12 +1749,13 @@ def _prepare_request_object(
flags: list[str] | None = None,
headers: dict[str, str] | None = None,
status: int = 200,
+ body: bytes = b"data",
) -> Request:
return Request(
item_url,
meta={
"response": Response(
- item_url, status=status, body=b"data", flags=flags, headers=headers
+ item_url, status=status, body=body, flags=flags, headers=headers
)
},
)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 09d470cc5..fcf23928e 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -511,7 +511,7 @@ class TestAsyncMediaDownloaded(TestMediaPipeline):
class TestMediaPipelineAllowRedirectSettings:
- def _assert_request_no3xx(
+ def _assert_redirect_statuses_not_handled(
self, pipeline_class: type[MediaPipeline], settings: dict[str, Any]
) -> None:
pipe = pipeline_class(crawler=get_crawler(None, settings))
@@ -521,6 +521,8 @@ class TestMediaPipelineAllowRedirectSettings:
assert "handle_httpstatus_list" in request.meta
for status, check in [
(200, True),
+ # 201 is not in REDIRECT_HTTP_CODES by default
+ (201, True),
# These are the status codes we want
# the downloader to handle itself
(301, False),
@@ -539,13 +541,32 @@ class TestMediaPipelineAllowRedirectSettings:
assert status not in request.meta["handle_httpstatus_list"]
def test_subclass_standard_setting(self):
- self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
+ self._assert_redirect_statuses_not_handled(
+ UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}
+ )
def test_subclass_specific_setting(self):
- self._assert_request_no3xx(
+ self._assert_redirect_statuses_not_handled(
UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}
)
+ def test_custom_redirect_http_codes(self):
+ """Statuses added to REDIRECT_HTTP_CODES are also left to
+ RedirectMiddleware."""
+ crawler = get_crawler(
+ None,
+ {
+ "MEDIA_ALLOW_REDIRECTS": True,
+ "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],
+ },
+ )
+ request = Request("http://url")
+ UserDefinedPipeline(crawler=crawler)._modify_media_request(request)
+
+ assert 201 not in request.meta["handle_httpstatus_list"]
+ assert 302 not in request.meta["handle_httpstatus_list"]
+ assert 200 in request.meta["handle_httpstatus_list"]
+
class TestBuildFromCrawler:
def setup_method(self):