Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Unify HTTP code handling configuration for the error and redirect middlewares (#7799)

Adrian0e77fdf6e4a6
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 537cc9243..a0a51e6ac 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -965,8 +965,9 @@ key set to True, the request will be ignored by this middleware.
The ``redirect_http_codes`` key of :attr:`Request.meta <scrapy.Request.meta>`
overrides the :setting:`REDIRECT_HTTP_CODES` setting for a single request.
-If you want to handle some redirect status codes in your spider, you can
-specify these in the ``handle_httpstatus_list`` spider attribute.
+If you want to handle some redirect status codes in your spider, declare them
+through the :setting:`HANDLE_HTTP_CODES` setting or the
+:reqmeta:`handle_http_codes` request meta key.
For example, if you want the redirect middleware to ignore 301 and 302
responses (and pass them through to your spider) you can do this:
@@ -974,13 +975,7 @@ responses (and pass them through to your spider) you can do this:
.. code-block:: python
class MySpider(CrawlSpider):
- handle_httpstatus_list = [301, 302]
-
-The ``handle_httpstatus_list`` key of :attr:`Request.meta
-<scrapy.Request.meta>` can also be used to specify which response codes to
-allow on a per-request basis. You can also set the meta key
-``handle_httpstatus_all`` to ``True`` if you want to allow any response code
-for a request.
+ custom_settings = {"HANDLE_HTTP_CODES": [301, 302]}
To decide whether to follow a redirect based on its target URL, subclass
:class:`RedirectMiddleware` and override
@@ -1071,9 +1066,10 @@ Use the :reqmeta:`redirect_http_codes` :attr:`Request.meta
:meth:`Response.follow_redirect() <scrapy.http.Response.follow_redirect>` to
follow a ``Location`` header from a spider callback instead.
-To stop following a status code, you can also use the ``handle_httpstatus_list``
-spider attribute or :attr:`Request.meta <scrapy.Request.meta>` key, which
-additionally lets your spider callbacks receive the affected responses.
+To stop following a status code, you can also use the
+:setting:`HANDLE_HTTP_CODES` setting or the :reqmeta:`handle_http_codes`
+request meta key, which additionally let your spider callbacks receive the
+affected responses.
For media pipelines, see also :setting:`MEDIA_ALLOW_REDIRECTS`.
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 68fcfe7b0..fa7762cee 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -698,6 +698,10 @@ To handle media redirections, set this setting to ``True``:
MEDIA_ALLOW_REDIRECTS = True
+To override this setting for a single media request, set the
+:reqmeta:`dont_redirect` request meta key in
+:meth:`~scrapy.pipelines.files.FilesPipeline.get_media_requests`.
+
Media files created on the fly
------------------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index eedbc4d90..1bc5329d3 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -829,8 +829,7 @@ Those are:
* :reqmeta:`ftp_password`
* :reqmeta:`ftp_user`
* :reqmeta:`give_up_log_level`
-* :reqmeta:`handle_httpstatus_all`
-* :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`handle_http_codes`
* :reqmeta:`http_auth_domain`
* :reqmeta:`http_pass`
* :reqmeta:`http_user`
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index bf663a05f..3fc091ad7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1639,6 +1639,55 @@ Default: ``None``
The Project ID that will be used when storing data on `Google Cloud Storage`_.
+.. setting:: HANDLE_HTTP_CODES
+
+HANDLE_HTTP_CODES
+-----------------
+
+.. versionadded:: VERSION
+
+Default: ``False``
+
+Response status codes outside the 2xx range that your spider handles itself,
+instead of letting Scrapy handle them.
+
+Supported values are:
+
+- ``False``, to let Scrapy handle all of them.
+
+- ``True``, to handle all of them yourself.
+
+- An integer, or a container of integers, to handle only those status codes
+ yourself, e.g. ``404``, ``[404, 410]`` or ``range(400, 500)``.
+
+Handling a status code yourself has 2 effects:
+
+- :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` sends
+ responses with that status code to your callback, instead of to your
+ errback as an
+ :exc:`~scrapy.spidermiddlewares.httperror.HttpError` exception.
+
+- :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` does not
+ follow responses with that status code, so that your callback gets the
+ redirect response itself. Note that
+ :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` is
+ not affected, because ``meta`` refresh redirects come in successful
+ responses; use :reqmeta:`dont_redirect` to stop those.
+
+Retries are a separate matter: handling a status code yourself does not
+prevent :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` from
+retrying it. A retried request either succeeds, in which case you get the new
+response, or runs out of retries, in which case you get the response with the
+status code that you handle. To also stop retries, remove the status code from
+:setting:`RETRY_HTTP_CODES`, or set the :reqmeta:`dont_retry` request meta key.
+
+.. reqmeta:: handle_http_codes
+
+Use the ``handle_http_codes`` key of :attr:`Request.meta <scrapy.Request.meta>`
+to set this on a per-request basis. It supports the same values as the setting,
+and it takes precedence over the setting, e.g. ``False`` makes Scrapy handle
+every status code outside the 2xx range even when the setting is ``True``.
+
.. setting:: HTTP2_MAX_FRAME_SIZE
HTTP2_MAX_FRAME_SIZE
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index bf87a9049..83a9380b1 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -247,21 +247,16 @@ DepthMiddleware
HttpErrorMiddleware
-------------------
-.. class:: scrapy.spidermiddlewares.httperror.HttpErrorMiddleware
-
- Filter out unsuccessful (erroneous) HTTP responses so that spiders don't
- have to deal with them, which (most of the time) imposes an overhead,
- consumes more resources, and makes the spider logic more complex.
+.. autoclass:: scrapy.spidermiddlewares.httperror.HttpErrorMiddleware
According to the `HTTP standard`_, successful responses are those whose
status codes are in the 200-300 range.
.. _HTTP standard: https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-If you still want to process response codes outside that range, you can
-specify which response codes the spider is able to handle using the
-``handle_httpstatus_list`` spider attribute or
-:setting:`HTTPERROR_ALLOWED_CODES` setting.
+If you still want to process response codes outside that range, use the
+:setting:`HANDLE_HTTP_CODES` setting or the :reqmeta:`handle_http_codes`
+request meta key to declare which status codes your spider handles itself.
For example, if you want your spider to handle 404 responses you can do
this:
@@ -272,24 +267,7 @@ this:
class MySpider(CrawlSpider):
- handle_httpstatus_list = [404]
-
-.. reqmeta:: handle_httpstatus_list
-
-.. reqmeta:: handle_httpstatus_all
-
-The ``handle_httpstatus_list`` key of :attr:`Request.meta
-<scrapy.Request.meta>` can also be used to specify which response codes to
-allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
-to ``True`` if you want to allow any response code for a request, and ``False`` to
-disable the effects of the ``handle_httpstatus_all`` key.
-
-Keep in mind, however, that it's usually a bad idea to handle non-200
-responses, unless you really know what you're doing.
-
-For more information see: `HTTP Status Code Definitions`_.
-
-.. _HTTP Status Code Definitions: https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+ custom_settings = {"HANDLE_HTTP_CODES": [404]}
Responses whose status code your spider does not handle reach your errback as
an :exc:`~scrapy.spidermiddlewares.httperror.HttpError` exception:
@@ -297,27 +275,6 @@ an :exc:`~scrapy.spidermiddlewares.httperror.HttpError` exception:
.. autoexception:: scrapy.spidermiddlewares.httperror.HttpError
:members:
-HttpErrorMiddleware settings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. setting:: HTTPERROR_ALLOWED_CODES
-
-HTTPERROR_ALLOWED_CODES
-^^^^^^^^^^^^^^^^^^^^^^^
-
-Default: ``[]``
-
-Pass all responses with non-200 status codes contained in this list.
-
-.. setting:: HTTPERROR_ALLOW_ALL
-
-HTTPERROR_ALLOW_ALL
-^^^^^^^^^^^^^^^^^^^
-
-Default: ``False``
-
-Pass all responses, regardless of its status code.
-
MetaCopyDetectionMiddleware
---------------------------
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 5502ab8d5..0e66f73ca 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -93,10 +93,9 @@ class Command(ScrapyCommand):
)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
- if not opts.no_redirect:
- request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400))
- else:
- request.meta["handle_httpstatus_all"] = True
+ request.meta["handle_http_codes"] = (
+ True if opts.no_redirect else SequenceExclude(range(300, 400))
+ )
spidercls: type[Spider] = DefaultSpider
assert self.crawler_process
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index c2eb206b4..a9d120740 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -84,12 +84,9 @@ class Command(ScrapyCommand):
if url:
raise UsageError("--curl cannot be combined with a URL argument")
request = _request_from_curl(opts.curl, dont_filter=True)
- if opts.no_redirect:
- request.meta["handle_httpstatus_all"] = True
- else:
- request.meta["handle_httpstatus_list"] = SequenceExclude(
- range(300, 400)
- )
+ request.meta["handle_http_codes"] = (
+ True if opts.no_redirect else SequenceExclude(range(300, 400))
+ )
url = request.url
elif url:
# first argument may be a local file
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index ef00aeb6b..b81c1de73 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -10,6 +10,7 @@ from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response
from scrapy.spidermiddlewares.referer import RefererMiddleware
+from scrapy.utils._httpstatus import StatusHandling
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import global_object_name
@@ -202,10 +203,20 @@ class RedirectMiddleware(BaseRedirectMiddleware):
def __init__(self, settings: BaseSettings):
super().__init__(settings)
+ self._status_handling = StatusHandling(settings, union_legacy_meta=True)
self.redirect_http_codes: set[int] = {
int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")
}
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ o = super().from_crawler(crawler)
+ crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+ return o
+
+ def spider_opened(self, spider: Spider) -> None:
+ self._status_handling.spider_opened(spider)
+
def _is_redirect(self, request: Request, response: Response) -> bool:
meta_http_codes = request.meta.get("redirect_http_codes")
http_codes = (
@@ -226,12 +237,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
def process_response(
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
- if (
- request.meta.get("dont_redirect", False)
- or response.status
- in getattr(self.crawler.spider, "handle_httpstatus_list", ())
- or response.status in request.meta.get("handle_httpstatus_list", ())
- or request.meta.get("handle_httpstatus_all", False)
+ if request.meta.get("dont_redirect", False) or self._status_handling.handles(
+ response.status, request.meta
):
return response
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 515c2ef8f..a6264670c 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -29,7 +29,7 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
- from collections.abc import Awaitable, Callable
+ from collections.abc import Awaitable, Callable, Container
from scrapy import Spider
from scrapy.crawler import Crawler
@@ -71,6 +71,16 @@ def _media_request_filtered(failure: Failure) -> bool:
return isinstance(failure.value, _MediaRequestFiltered)
+def _warn_deprecated_handle_httpstatus_list() -> None:
+ warnings.warn(
+ "MediaPipeline.handle_httpstatus_list is deprecated, use the "
+ "MEDIA_ALLOW_REDIRECTS setting or the handle_http_codes request meta "
+ "key instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=3,
+ )
+
+
class MediaPipeline(ABC):
LOG_FAILED_RESULTS: bool = True
@@ -105,23 +115,31 @@ class MediaPipeline(ABC):
resolve = functools.partial(
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
)
- self._redirect_http_codes: set[int] = {
- int(x) for x in settings.getlist("REDIRECT_HTTP_CODES")
- }
self.allow_redirects: bool = settings.getbool(
resolve("MEDIA_ALLOW_REDIRECTS"), False
)
- self._handle_statuses(self.allow_redirects)
+ self._legacy_handle_http_codes: Literal[True] | Container[int] | None = None
self._cache_size: int = settings.getint(resolve("MEDIA_CACHE_SIZE"))
- def _handle_statuses(self, allow_redirects: bool) -> None:
- self.handle_httpstatus_list = None
- if allow_redirects:
- # Statuses that RedirectMiddleware may redirect, i.e. those from
- # REDIRECT_HTTP_CODES, are left for it to handle.
- self.handle_httpstatus_list = SequenceExclude(
- {*range(300, 400), *self._redirect_http_codes}
- )
+ @property
+ def handle_httpstatus_list(self) -> Container[int] | None:
+ _warn_deprecated_handle_httpstatus_list()
+ if self._legacy_handle_http_codes is not None:
+ if self._legacy_handle_http_codes is True:
+ return None
+ return self._legacy_handle_http_codes
+ if not self.allow_redirects:
+ return None
+ redirect_http_codes = self.crawler.settings.getlist("REDIRECT_HTTP_CODES")
+ return SequenceExclude(
+ {*range(300, 400), *(int(code) for code in redirect_http_codes)}
+ )
+
+ @handle_httpstatus_list.setter
+ def handle_httpstatus_list(self, value: Container[int] | None) -> None:
+ _warn_deprecated_handle_httpstatus_list()
+ # A false value has always meant handling every status code.
+ self._legacy_handle_http_codes = value or True
def _key_for_pipe(
self,
@@ -224,10 +242,10 @@ class MediaPipeline(ABC):
return await maybe_deferred_to_future(wad) # it must return wad at last
def _modify_media_request(self, request: Request) -> None:
- if self.handle_httpstatus_list:
- request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
- else:
- request.meta["handle_httpstatus_all"] = True
+ if self._legacy_handle_http_codes is not None:
+ request.meta.setdefault("handle_http_codes", self._legacy_handle_http_codes)
+ elif not self.allow_redirects:
+ request.meta.setdefault("dont_redirect", True)
async def _check_media_to_download(
self, request: Request, info: SpiderInfo, item: Any
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d17fc8118..7e42347da 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -116,6 +116,7 @@ __all__ = [
"FTP_PASSWORD",
"FTP_USER",
"GCS_PROJECT_ID",
+ "HANDLE_HTTP_CODES",
"HTTP2_MAX_FRAME_SIZE",
"HTTPAUTH_DOMAIN",
"HTTPAUTH_PASS",
@@ -448,6 +449,8 @@ FTP_PASSWORD = "guest" # noqa: S105
GCS_PROJECT_ID = None
+HANDLE_HTTP_CODES = False
+
HTTP2_MAX_FRAME_SIZE = 16384
HTTPAUTH_USER = ""
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 0df0bbc7c..4c4a667ff 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -7,13 +7,16 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
+import warnings
from typing import TYPE_CHECKING, Any, Self
-from scrapy.exceptions import IgnoreRequest
+from scrapy import signals
+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
+from scrapy.utils._httpstatus import StatusHandling, normalize
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
- from collections.abc import Iterable
+ from collections.abc import Container, Iterable
from scrapy import Spider
from scrapy.crawler import Crawler
@@ -26,7 +29,8 @@ logger = logging.getLogger(__name__)
class HttpError(IgnoreRequest):
"""Raised by :class:`HttpErrorMiddleware` for a response whose status code
- is not successful and that the spider does not handle itself.
+ is not successful and that the spider does not handle itself. See
+ :setting:`HANDLE_HTTP_CODES`.
"""
def __init__(self, response: Response, *args: object, **kwargs: object):
@@ -35,41 +39,68 @@ class HttpError(IgnoreRequest):
super().__init__(*args, **kwargs)
+def _warn_deprecated_attribute(name: str) -> None:
+ warnings.warn(
+ f"HttpErrorMiddleware.{name} is deprecated, use the HANDLE_HTTP_CODES "
+ f"setting instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=3,
+ )
+
+
class HttpErrorMiddleware:
+ """Filter out unsuccessful (erroneous) HTTP responses so that spiders don't
+ have to deal with them, which (most of the time) imposes an overhead,
+ consumes more resources, and makes the spider logic more complex."""
+
crawler: Crawler
def __init__(self, settings: BaseSettings):
- self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")
- self.handle_httpstatus_list: list[int] = settings.getlist(
- "HTTPERROR_ALLOWED_CODES"
- )
+ self._status_handling = StatusHandling(settings, legacy_settings=True)
+
+ @property
+ def handle_httpstatus_all(self) -> bool:
+ _warn_deprecated_attribute("handle_httpstatus_all")
+ return self._status_handling._settings_value is True
+
+ @handle_httpstatus_all.setter
+ def handle_httpstatus_all(self, value: bool) -> None:
+ _warn_deprecated_attribute("handle_httpstatus_all")
+ if value:
+ self._status_handling._settings_value = True
+ elif self._status_handling._settings_value is True:
+ self._status_handling._settings_value = False
+
+ @property
+ def handle_httpstatus_list(self) -> Container[int]:
+ _warn_deprecated_attribute("handle_httpstatus_list")
+ value = self._status_handling._settings_value
+ return [] if isinstance(value, bool) else value
+
+ @handle_httpstatus_list.setter
+ def handle_httpstatus_list(self, value: Container[int]) -> None:
+ _warn_deprecated_attribute("handle_httpstatus_list")
+ # As before, handle_httpstatus_all takes precedence.
+ if self._status_handling._settings_value is not True:
+ self._status_handling._settings_value = normalize(value)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings)
o.crawler = crawler
+ crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
+ def spider_opened(self, spider: Spider) -> None:
+ self._status_handling.spider_opened(spider)
+
@_warn_spider_arg
def process_spider_input(
self, response: Response, spider: Spider | None = None
) -> None:
if 200 <= response.status < 300: # common case
return
- meta = response.meta
- if meta.get("handle_httpstatus_all", False):
- return
- if "handle_httpstatus_list" in meta:
- allowed_statuses = meta["handle_httpstatus_list"]
- elif self.handle_httpstatus_all:
- return
- else:
- allowed_statuses = getattr(
- self.crawler.spider,
- "handle_httpstatus_list",
- self.handle_httpstatus_list,
- )
- if response.status in allowed_statuses:
+ if self._status_handling.handles(response.status, response.meta):
return
raise HttpError(response, "Ignoring non-200 response")
diff --git a/scrapy/utils/_httpstatus.py b/scrapy/utils/_httpstatus.py
new file mode 100644
index 000000000..7d72ca0cc
--- /dev/null
+++ b/scrapy/utils/_httpstatus.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import warnings
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
+from weakref import WeakKeyDictionary
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
+
+if TYPE_CHECKING:
+ from collections.abc import Container, Mapping
+
+ from scrapy import Spider
+ from scrapy.settings import BaseSettings
+
+ # True means every status, False means no status, and a container is
+ # checked for membership.
+ HandledCodes: TypeAlias = bool | Container[int]
+
+
+SETTING = "HANDLE_HTTP_CODES"
+META_KEY = "handle_http_codes"
+
+_LEGACY_SETTING_ALL = "HTTPERROR_ALLOW_ALL"
+_LEGACY_SETTING_LIST = "HTTPERROR_ALLOWED_CODES"
+# Both a spider attribute and a request meta key.
+_LEGACY_LIST = "handle_httpstatus_list"
+_LEGACY_ALL = "handle_httpstatus_all"
+
+# Same string values that BaseSettings.getbool() accepts.
+_TRUE_STRINGS = frozenset({"1", "True", "true"})
+_FALSE_STRINGS = frozenset({"0", "False", "false"})
+
+_warned_meta_keys: WeakKeyDictionary[Any, set[str]] = WeakKeyDictionary()
+
+
+def normalize(value: Any) -> HandledCodes:
+ """Return *value* as either a boolean or a container of status codes.
+
+ Booleans are returned as they are, integers become single-code containers,
+ strings are parsed as they come from the command line or the environment,
+ and sequences have their items coerced to integers. Any other container is
+ returned untouched, so that objects such as
+ :class:`~scrapy.utils._datatypes.SequenceExclude` keep working.
+ """
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, int):
+ return frozenset({value})
+ if isinstance(value, str):
+ value = value.strip()
+ if not value:
+ return False
+ if value in _TRUE_STRINGS:
+ return True
+ if value in _FALSE_STRINGS:
+ return False
+ return frozenset(int(code) for code in value.split(","))
+ if isinstance(value, (list, tuple, set, frozenset)):
+ return frozenset(int(code) for code in value)
+ if not hasattr(value, "__contains__"):
+ raise ValueError(
+ f"Unsupported {SETTING} value: {value!r}. Expected a boolean, an "
+ f"integer, a string, or a container of integers."
+ )
+ return cast("Container[int]", value)
+
+
+def matches(value: HandledCodes | None, status: int) -> bool:
+ if value is True:
+ return True
+ if not value: # False, None or an empty container
+ return False
+ return status in value
+
+
+class StatusHandling:
+ """Tell whether the spider handles a response status code itself.
+
+ *legacy_settings* enables reading the deprecated
+ :setting:`HTTPERROR_ALLOWED_CODES` and :setting:`HTTPERROR_ALLOW_ALL`
+ settings, which only ever applied to
+ :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
+
+ *union_legacy_meta* combines a deprecated request meta key with the
+ deprecated spider attribute, instead of overriding it, as
+ :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` used to
+ do.
+
+ :meth:`spider_opened` must be called on the ``spider_opened`` signal, so
+ that the deprecated ``handle_httpstatus_list`` spider attribute is taken
+ into account.
+ """
+
+ def __init__(
+ self,
+ settings: BaseSettings,
+ *,
+ legacy_settings: bool = False,
+ union_legacy_meta: bool = False,
+ ):
+ self._settings_value: HandledCodes = self._from_settings(
+ settings, legacy_settings
+ )
+ self._union_legacy_meta = union_legacy_meta
+ self._spider_value: HandledCodes | None = None
+ self._warning_scope: Any = self
+
+ def spider_opened(self, spider: Spider) -> None:
+ # Deprecation warnings about request meta keys are emitted once per
+ # crawl, no matter how many components ask about the same key.
+ self._warning_scope = getattr(spider, "crawler", None) or self
+ value = getattr(spider, _LEGACY_LIST, None)
+ if value is None:
+ return
+ warn_on_deprecated_spider_attribute(_LEGACY_LIST, SETTING)
+ self._spider_value = normalize(value)
+
+ def handles(self, status: int, meta: Mapping[str, Any]) -> bool:
+ value, from_legacy_meta = self._from_meta(meta)
+ if value is None:
+ value = self._spider_value
+ if value is None:
+ value = self._settings_value
+ handled = matches(value, status)
+ if handled or not (from_legacy_meta and self._union_legacy_meta):
+ return handled
+ return matches(self._spider_value, status)
+
+ @staticmethod
+ def _from_settings(settings: BaseSettings, legacy: bool) -> HandledCodes:
+ if not legacy or (settings.getpriority(SETTING) or 0) > 0:
+ return normalize(settings.get(SETTING))
+ for name in (_LEGACY_SETTING_ALL, _LEGACY_SETTING_LIST):
+ if (settings.getpriority(name) or 0) > 0:
+ warnings.warn(
+ f"The {name} setting is deprecated, use {SETTING} instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if settings.getbool(_LEGACY_SETTING_ALL):
+ return True
+ return normalize(settings.getlist(_LEGACY_SETTING_LIST))
+
+ def _from_meta(self, meta: Mapping[str, Any]) -> tuple[HandledCodes | None, bool]:
+ """Return the value that *meta* defines, if any, and whether it comes
+ from a deprecated meta key."""
+ value = meta.get(META_KEY)
+ if value is not None:
+ return normalize(value), False
+ # The deprecated keys used to be checked in this order: a true
+ # handle_httpstatus_all took precedence over handle_httpstatus_list,
+ # and a false one was only taken into account on its own.
+ legacy_all = meta.get(_LEGACY_ALL)
+ if legacy_all:
+ self._warn_meta_key(_LEGACY_ALL)
+ return True, True
+ if _LEGACY_LIST in meta:
+ self._warn_meta_key(_LEGACY_LIST)
+ return normalize(meta[_LEGACY_LIST]), True
+ if legacy_all is not None:
+ self._warn_meta_key(_LEGACY_ALL)
+ return False, True
+ return None, False
+
+ def _warn_meta_key(self, key: str) -> None:
+ warned = _warned_meta_keys.setdefault(self._warning_scope, set())
+ if key in warned:
+ return
+ warned.add(key)
+ warnings.warn(
+ f"The {key!r} request meta key is deprecated, use {META_KEY!r} instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index ceb0b09f9..d34dd4bbb 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import BodyFileResponse, Request, Response
from scrapy.spidermiddlewares.referer import (
POLICY_NO_REFERRER,
@@ -319,28 +319,110 @@ class TestRedirectMiddleware(TestRedirectBase):
assert req2.url == url3
assert req2.method == "HEAD"
- def test_spider_handling(self):
- self.mw.crawler.spider.handle_httpstatus_list = [404, 301, 302]
- url = "http://www.example.com/301"
+ def _assert_passthrough(
+ self, req: Request, mw: RedirectMiddleware | None = None
+ ) -> None:
url2 = "http://www.example.com/redirected"
- req = Request(url)
- rsp = Response(url, headers={"Location": url2}, status=301)
- r = self.mw.process_response(req, rsp)
- assert r is rsp
+ rsp = Response(req.url, headers={"Location": url2}, status=301, request=req)
+ assert (mw or self.mw).process_response(req, rsp) is rsp
+
+ def _assert_redirected(
+ self, req: Request, mw: RedirectMiddleware | None = None
+ ) -> None:
+ url2 = "http://www.example.com/redirected"
+ rsp = Response(req.url, headers={"Location": url2}, status=301, request=req)
+ assert isinstance((mw or self.mw).process_response(req, rsp), Request)
+
+ def test_setting_handling(self):
+ url = "http://www.example.com/301"
+ crawler = get_crawler(DefaultSpider, {"HANDLE_HTTP_CODES": [404, 301, 302]})
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ self._assert_passthrough(Request(url), mw)
+
+ def test_setting_handling_all(self):
+ url = "http://www.example.com/301"
+ crawler = get_crawler(DefaultSpider, {"HANDLE_HTTP_CODES": True})
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ self._assert_passthrough(Request(url), mw)
+
+ def test_setting_handling_unrelated_code(self):
+ url = "http://www.example.com/301"
+ crawler = get_crawler(DefaultSpider, {"HANDLE_HTTP_CODES": [404]})
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ self._assert_redirected(Request(url), mw)
+
+ def test_httperror_settings_ignored(self):
+ """The deprecated HTTPERROR_* settings never applied to redirects."""
+ url = "http://www.example.com/301"
+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ self._assert_redirected(Request(url), mw)
+
+ def test_spider_handling(self):
+ mw = self._spider_attribute_mw([404, 301, 302])
+ self._assert_passthrough(Request("http://www.example.com/301"), mw)
def test_request_meta_handling(self):
url = "http://www.example.com/301"
- url2 = "http://www.example.com/redirected"
+ self._assert_passthrough(Request(url, meta={"handle_http_codes": [301]}))
+ self._assert_passthrough(Request(url, meta={"handle_http_codes": True}))
+ self._assert_redirected(Request(url, meta={"handle_http_codes": [404]}))
+ self._assert_redirected(Request(url, meta={"handle_http_codes": False}))
- def _test_passthrough(req: Request) -> None:
- rsp = Response(url, headers={"Location": url2}, status=301, request=req)
- r = self.mw.process_response(req, rsp)
- assert r is rsp
+ def test_request_meta_handling_deprecated(self):
+ url = "http://www.example.com/301"
+ with pytest.warns(ScrapyDeprecationWarning):
+ self._assert_passthrough(
+ Request(url, meta={"handle_httpstatus_list": [404, 301, 302]})
+ )
+ with pytest.warns(ScrapyDeprecationWarning):
+ self._assert_passthrough(Request(url, meta={"handle_httpstatus_all": True}))
+ # Already warned about above, hence no warning expected here.
+ self._assert_redirected(Request(url, meta={"handle_httpstatus_list": [404]}))
+
+ def _spider_attribute_mw(self, codes: list[int]) -> RedirectMiddleware:
+ class HandlingSpider(DefaultSpider):
+ handle_httpstatus_list = codes
+
+ crawler = get_crawler(HandlingSpider)
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw.spider_opened(crawler.spider)
+ return mw
- _test_passthrough(
- Request(url, meta={"handle_httpstatus_list": [404, 301, 302]})
+ @pytest.mark.parametrize(
+ "meta",
+ [
+ {"handle_httpstatus_list": [404]},
+ {"handle_httpstatus_all": False},
+ ],
+ )
+ def test_deprecated_meta_combines_with_spider_attribute(self, meta):
+ """This middleware used to combine the deprecated meta key with the
+ deprecated spider attribute, instead of overriding it."""
+ mw = self._spider_attribute_mw([301])
+ with pytest.warns(ScrapyDeprecationWarning):
+ self._assert_passthrough(
+ Request("http://www.example.com/301", meta=meta), mw
+ )
+
+ def test_meta_overrides_spider_attribute(self):
+ mw = self._spider_attribute_mw([301])
+ self._assert_redirected(
+ Request("http://www.example.com/301", meta={"handle_http_codes": [404]}), mw
)
- _test_passthrough(Request(url, meta={"handle_httpstatus_all": True}))
+
+ def test_request_meta_overrides_setting(self):
+ url = "http://www.example.com/301"
+ crawler = get_crawler(DefaultSpider, {"HANDLE_HTTP_CODES": True})
+ crawler.spider = crawler._create_spider()
+ mw = self.mwcls.from_crawler(crawler)
+ self._assert_redirected(Request(url, meta={"handle_http_codes": False}), mw)
def test_latin1_location(self):
req = Request("http://scrapytest.org/first")
@@ -485,7 +567,7 @@ class NonStandardStatusMixin(ABC):
"meta",
[
{"dont_redirect": True},
- {"handle_httpstatus_all": True},
+ {"handle_http_codes": True},
],
)
def test_request_meta_handling(self, meta):
@@ -494,14 +576,21 @@ class NonStandardStatusMixin(ABC):
assert self.mw.process_response(request, response) is response
def test_request_meta_status_handling(self):
- request = Request(self.url, meta={"handle_httpstatus_list": [self.status]})
+ request = Request(self.url, meta={"handle_http_codes": [self.status]})
response = self._response()
assert self.mw.process_response(request, response) is response
- def test_spider_handling(self):
- self.mw.crawler.spider.handle_httpstatus_list = [self.status] # type: ignore[union-attr]
+ def test_setting_handling(self):
+ crawler = get_crawler(
+ DefaultSpider,
+ {
+ "REDIRECT_HTTP_CODES": [self.status],
+ "HANDLE_HTTP_CODES": [self.status],
+ },
+ )
+ mw = RedirectMiddleware.from_crawler(crawler)
response = self._response()
- assert self.mw.process_response(Request(self.url), response) is response
+ assert mw.process_response(Request(self.url), response) is response
class TestRedirect201(NonStandardStatusMixin):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 6cae9d4df..e7b970157 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -779,9 +779,7 @@ class _RetryThenRedirectSpider(Spider):
url = self.mockserver.url(
"/redirect-to?" + urlencode({"goto": seen, "n": n})
)
- yield Request(
- url, meta={"dont_redirect": True, "handle_httpstatus_list": [302]}
- )
+ yield Request(url, meta={"dont_redirect": True, "handle_http_codes": [302]})
def parse(self, response: Response) -> Request | None:
if response.status == 200:
@@ -791,7 +789,7 @@ class _RetryThenRedirectSpider(Spider):
request = get_retry_request(response.request, spider=self)
assert request is not None
del request.meta["dont_redirect"]
- del request.meta["handle_httpstatus_list"]
+ del request.meta["handle_http_codes"]
return request
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 8abbb9eb3..ef0183eab 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -97,7 +97,12 @@ class TestBaseMediaPipeline:
def test_modify_media_request(self):
request = Request("http://url")
self.pipe._modify_media_request(request)
- assert request.meta == {"handle_httpstatus_all": True}
+ assert request.meta == {"dont_redirect": True}
+
+ def test_modify_media_request_keeps_meta(self):
+ request = Request("http://url", meta={"dont_redirect": False})
+ self.pipe._modify_media_request(request)
+ assert request.meta == {"dont_redirect": False}
def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:
item = {"name": "name"}
@@ -468,48 +473,23 @@ class TestAsyncMediaDownloaded(TestMediaPipeline):
class TestMediaPipelineAllowRedirectSettings:
- def _assert_redirect_statuses_not_handled(
- self, pipeline_class: type[MediaPipeline], settings: dict[str, Any]
- ) -> None:
- pipe = pipeline_class(crawler=get_crawler(None, settings))
+ @pytest.mark.parametrize(
+ "settings",
+ [
+ {"MEDIA_ALLOW_REDIRECTS": True},
+ {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True},
+ ],
+ )
+ def test_allow_redirects(self, settings: dict[str, Any]) -> None:
+ pipe = UserDefinedPipeline(crawler=get_crawler(None, settings))
request = Request("http://url")
pipe._modify_media_request(request)
+ assert request.meta == {}
- assert "handle_httpstatus_list" in request.meta
- for status, check in [
- (200, True),
- # 201 is not in REDIRECT_HTTP_CODES by default
- (201, True),
- # These are the status codes we want
- # the downloader to handle itself
- (301, False),
- (302, False),
- (303, False),
- (307, False),
- (308, False),
- # we still want to get 4xx and 5xx
- (400, True),
- (404, True),
- (500, True),
- ]:
- if check:
- assert status in request.meta["handle_httpstatus_list"]
- else:
- assert status not in request.meta["handle_httpstatus_list"]
-
- def test_subclass_standard_setting(self):
- self._assert_redirect_statuses_not_handled(
- UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}
- )
-
- def test_subclass_specific_setting(self):
- self._assert_redirect_statuses_not_handled(
- UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}
- )
-
- def test_custom_redirect_http_codes(self):
- """Statuses added to REDIRECT_HTTP_CODES are also left to
- RedirectMiddleware."""
+ def test_deprecated_attribute_get(self):
+ pipe = UserDefinedPipeline(crawler=get_crawler())
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ assert pipe.handle_httpstatus_list is None
crawler = get_crawler(
None,
{
@@ -517,12 +497,24 @@ class TestMediaPipelineAllowRedirectSettings:
"REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],
},
)
+ pipe = UserDefinedPipeline(crawler=crawler)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ codes = pipe.handle_httpstatus_list
+ assert codes is not None
+ assert 404 in codes
+ assert 201 not in codes
+ assert 302 not in codes
+
+ @pytest.mark.parametrize(("value", "expected"), [(None, True), ([404], [404])])
+ def test_deprecated_attribute_set(self, value: Any, expected: Any) -> None:
+ pipe = UserDefinedPipeline(crawler=get_crawler())
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ pipe.handle_httpstatus_list = value
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ assert pipe.handle_httpstatus_list == value
request = Request("http://url")
- UserDefinedPipeline(crawler=crawler)._modify_media_request(request)
-
- assert 201 not in request.meta["handle_httpstatus_list"]
- assert 302 not in request.meta["handle_httpstatus_list"]
- assert 200 in request.meta["handle_httpstatus_list"]
+ pipe._modify_media_request(request)
+ assert request.meta == {"handle_http_codes": expected}
class TestBuildFromCrawler:
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 43b41ce3f..012a3dfb7 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,12 +1,15 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING
+import warnings
+from typing import TYPE_CHECKING, Any
import pytest
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
+from scrapy.utils._datatypes import SequenceExclude
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -14,6 +17,7 @@ from tests.spiders import MockServerSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ from scrapy import Spider
from tests.mockserver.http import MockServer
@@ -75,12 +79,20 @@ def res404() -> Response:
return _response(req, 404)
+def _mw(
+ settings: dict[str, Any] | None = None, spidercls: type[Spider] = DefaultSpider
+) -> HttpErrorMiddleware:
+ crawler = get_crawler(spidercls, settings)
+ crawler.spider = crawler._create_spider()
+ mw = build_from_crawler(HttpErrorMiddleware, crawler)
+ mw.spider_opened(crawler.spider)
+ return mw
+
+
class TestHttpErrorMiddleware:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- crawler = get_crawler(DefaultSpider)
- crawler.spider = crawler._create_spider()
- return build_from_crawler(HttpErrorMiddleware, crawler)
+ return _mw()
def test_process_spider_input(
self, mw: HttpErrorMiddleware, res200: Response, res404: Response
@@ -95,18 +107,13 @@ class TestHttpErrorMiddleware:
assert mw.process_spider_exception(res404, HttpError(res404)) == ()
assert mw.process_spider_exception(res404, Exception()) is None
- def test_handle_httpstatus_list(
- self, mw: HttpErrorMiddleware, res404: Response
- ) -> None:
- request = Request(
- "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
- )
- res = _response(request, 404)
- mw.process_spider_input(res)
-
- assert mw.crawler.spider
- mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
- mw.process_spider_input(res404)
+ def test_meta(self, mw: HttpErrorMiddleware, res404: Response) -> None:
+ request = Request("http://example.com", meta={"handle_http_codes": [404]})
+ mw.process_spider_input(_response(request, 404))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 402))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res404)
class TestHttpErrorMiddlewareSettings:
@@ -114,9 +121,7 @@ class TestHttpErrorMiddlewareSettings:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
- crawler.spider = crawler._create_spider()
- return build_from_crawler(HttpErrorMiddleware, crawler)
+ return _mw({"HANDLE_HTTP_CODES": (402,)})
def test_process_spider_input(
self,
@@ -131,32 +136,25 @@ class TestHttpErrorMiddlewareSettings:
mw.process_spider_input(res402)
def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
- request = Request(
- "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
- )
- res404 = _response(request, 404)
- res402 = _response(request, 402)
-
- mw.process_spider_input(res404)
+ request = Request("http://example.com", meta={"handle_http_codes": [404]})
+ mw.process_spider_input(_response(request, 404))
with pytest.raises(HttpError):
- mw.process_spider_input(res402)
+ mw.process_spider_input(_response(request, 402))
- def test_spider_override_settings(
- self, mw: HttpErrorMiddleware, res402: Response, res404: Response
- ) -> None:
- assert mw.crawler.spider
- mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
- mw.process_spider_input(res404)
+ def test_meta_false_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
+ request = Request("http://example.com", meta={"handle_http_codes": False})
with pytest.raises(HttpError):
- mw.process_spider_input(res402)
+ mw.process_spider_input(_response(request, 402))
+
+ def test_meta_none_ignored(self, mw: HttpErrorMiddleware) -> None:
+ request = Request("http://example.com", meta={"handle_http_codes": None})
+ mw.process_spider_input(_response(request, 402))
class TestHttpErrorMiddlewareHandleAll:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
- crawler.spider = crawler._create_spider()
- return build_from_crawler(HttpErrorMiddleware, crawler)
+ return _mw({"HANDLE_HTTP_CODES": True})
def test_process_spider_input(
self,
@@ -168,31 +166,237 @@ class TestHttpErrorMiddlewareHandleAll:
mw.process_spider_input(res404)
def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
- request = Request(
- "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
- )
- res404 = _response(request, 404)
- res402 = _response(request, 402)
+ request = Request("http://example.com", meta={"handle_http_codes": [404]})
+ mw.process_spider_input(_response(request, 404))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 402))
- mw.process_spider_input(res404)
+ def test_meta_false_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
+ request = Request("http://example.com", meta={"handle_http_codes": False})
with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 402))
+
+
+class TestHttpErrorMiddlewareValues:
+ """Every supported way to express a HANDLE_HTTP_CODES value."""
+
+ @pytest.mark.parametrize(
+ ("value", "handled"),
+ [
+ (True, True),
+ (False, False),
+ (402, True),
+ ("402", True),
+ ("402,404", True),
+ ("True", True),
+ ("true", True),
+ ("1", True),
+ ("False", False),
+ ("false", False),
+ ("0", False),
+ ("", False),
+ ([402], True),
+ (["402"], True),
+ ((402,), True),
+ ({402}, True),
+ (frozenset({402}), True),
+ ([], False),
+ ([404], False),
+ (range(400, 500), True),
+ (SequenceExclude(range(300, 400)), True),
+ ],
+ )
+ def test_setting(self, value: Any, handled: bool) -> None:
+ mw = _mw({"HANDLE_HTTP_CODES": value})
+ res402 = _response(Request("http://example.com"), 402)
+ if handled:
mw.process_spider_input(res402)
+ else:
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res402)
+
+ @pytest.mark.parametrize(
+ ("value", "handled"),
+ [
+ (True, True),
+ (False, False),
+ (402, True),
+ ("402", True),
+ ([402], True),
+ ([404], False),
+ (SequenceExclude(range(300, 400)), True),
+ ],
+ )
+ def test_meta(self, value: Any, handled: bool) -> None:
+ mw = _mw()
+ request = Request("http://example.com", meta={"handle_http_codes": value})
+ if handled:
+ mw.process_spider_input(_response(request, 402))
+ else:
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 402))
+
+ @pytest.mark.parametrize("value", [None, object()])
+ def test_unsupported_value(self, value: Any) -> None:
+ with pytest.raises(ValueError, match="Unsupported HANDLE_HTTP_CODES value"):
+ _mw({"HANDLE_HTTP_CODES": value})
+
+
+class TestHttpErrorMiddlewareDeprecated:
+ def test_settings(self) -> None:
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="HTTPERROR_ALLOWED_CODES setting"
+ ):
+ mw = _mw({"HTTPERROR_ALLOWED_CODES": (402,)})
+ res402 = _response(Request("http://example.com"), 402)
+ res404 = _response(Request("http://example.com"), 404)
+ mw.process_spider_input(res402)
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res404)
- def test_httperror_allow_all_false(self) -> None:
- crawler = get_crawler(_HttpErrorSpider)
- mw = build_from_crawler(HttpErrorMiddleware, crawler)
- request_httpstatus_false = Request(
- "http://scrapytest.org", meta={"handle_httpstatus_all": False}
+ def test_allow_all_setting(self) -> None:
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="HTTPERROR_ALLOW_ALL setting"
+ ):
+ mw = _mw({"HTTPERROR_ALLOW_ALL": True})
+ mw.process_spider_input(_response(Request("http://example.com"), 404))
+
+ def test_new_setting_wins(self) -> None:
+ mw = _mw({"HANDLE_HTTP_CODES": False, "HTTPERROR_ALLOW_ALL": True})
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(Request("http://example.com"), 404))
+
+ def test_spider_attribute(self) -> None:
+ class HandlingSpider(DefaultSpider):
+ handle_httpstatus_list = [404]
+
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="'handle_httpstatus_list' spider attribute"
+ ):
+ mw = _mw(spidercls=HandlingSpider)
+ mw.process_spider_input(_response(Request("http://example.com"), 404))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(Request("http://example.com"), 402))
+
+ def test_spider_attribute_overrides_settings(self) -> None:
+ class HandlingSpider(DefaultSpider):
+ handle_httpstatus_list = [404]
+
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw = _mw({"HANDLE_HTTP_CODES": [402]}, spidercls=HandlingSpider)
+ mw.process_spider_input(_response(Request("http://example.com"), 404))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(Request("http://example.com"), 402))
+
+ def test_meta_overrides_spider_attribute(self) -> None:
+ """Unlike RedirectMiddleware, this middleware never combined the
+ deprecated meta key with the deprecated spider attribute."""
+
+ class HandlingSpider(DefaultSpider):
+ handle_httpstatus_list = [404]
+
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw = _mw(spidercls=HandlingSpider)
+ request = Request("http://example.com", meta={"handle_httpstatus_list": [402]})
+ with pytest.warns(ScrapyDeprecationWarning), pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 404))
+
+ def test_meta_list(self) -> None:
+ mw = _mw()
+ request = Request("http://example.com", meta={"handle_httpstatus_list": [404]})
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="'handle_httpstatus_list' request meta key"
+ ):
+ mw.process_spider_input(_response(request, 404))
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 402))
+
+ def test_meta_all(self) -> None:
+ mw = _mw()
+ request = Request("http://example.com", meta={"handle_httpstatus_all": True})
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="'handle_httpstatus_all' request meta key"
+ ):
+ mw.process_spider_input(_response(request, 404))
+
+ def test_meta_all_false(self) -> None:
+ mw = _mw({"HANDLE_HTTP_CODES": True})
+ request = Request("http://example.com", meta={"handle_httpstatus_all": False})
+ with pytest.warns(ScrapyDeprecationWarning), pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 404))
+
+ def test_meta_all_true_beats_meta_list(self) -> None:
+ mw = _mw()
+ request = Request(
+ "http://example.com",
+ meta={"handle_httpstatus_all": True, "handle_httpstatus_list": [404]},
)
- request_httpstatus_true = Request(
- "http://scrapytest.org", meta={"handle_httpstatus_all": True}
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw.process_spider_input(_response(request, 402))
+
+ def test_meta_all_false_defers_to_meta_list(self) -> None:
+ mw = _mw()
+ request = Request(
+ "http://example.com",
+ meta={"handle_httpstatus_all": False, "handle_httpstatus_list": [404]},
)
- res404 = _response(request_httpstatus_false, 404)
- res402 = _response(request_httpstatus_true, 402)
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw.process_spider_input(_response(request, 404))
+ def test_new_meta_key_wins(self) -> None:
+ mw = _mw()
+ request = Request(
+ "http://example.com",
+ meta={"handle_http_codes": False, "handle_httpstatus_all": True},
+ )
+ with pytest.raises(HttpError):
+ mw.process_spider_input(_response(request, 404))
+
+ def test_meta_key_warning_once_per_crawl(self) -> None:
+ mw = _mw()
+ request = Request("http://example.com", meta={"handle_httpstatus_all": True})
+ with pytest.warns(ScrapyDeprecationWarning):
+ mw.process_spider_input(_response(request, 404))
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ mw.process_spider_input(_response(request, 404))
+
+ @pytest.mark.parametrize(
+ ("setting", "handle_all", "handle_list"),
+ [(True, True, []), ([402], False, frozenset({402})), (False, False, [])],
+ )
+ def test_attributes_get(
+ self, setting: Any, handle_all: bool, handle_list: Any
+ ) -> None:
+ mw = _mw({"HANDLE_HTTP_CODES": setting})
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_all"):
+ assert mw.handle_httpstatus_all is handle_all
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ assert mw.handle_httpstatus_list == handle_list
+
+ def test_attributes_set(self) -> None:
+ mw = _mw()
+ res402 = _response(Request("http://example.com"), 402)
+ res404 = _response(Request("http://example.com"), 404)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ mw.handle_httpstatus_list = [402]
+ mw.process_spider_input(res402)
with pytest.raises(HttpError):
mw.process_spider_input(res404)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_all"):
+ mw.handle_httpstatus_all = True
+ mw.process_spider_input(res404)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
+ mw.handle_httpstatus_list = [404]
mw.process_spider_input(res402)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_all"):
+ mw.handle_httpstatus_all = False
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res402)
+ with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_all"):
+ mw.handle_httpstatus_all = False
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res402)
class TestHttpErrorMiddlewareIntegrational:
@@ -211,6 +415,22 @@ class TestHttpErrorMiddlewareIntegrational:
assert get_value("httperror/response_ignored_status_count/402") == 1
assert get_value("httperror/response_ignored_status_count/500") == 1
+ @coroutine_test
+ async def test_setting(self, mockserver: MockServer) -> None:
+ crawler = get_crawler(_HttpErrorSpider, {"HANDLE_HTTP_CODES": [402, 404]})
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, _HttpErrorSpider)
+ assert crawler.spider.parsed == {"200", "402", "404"}
+ assert crawler.spider.failed == {"500"}
+
+ @coroutine_test
+ async def test_setting_all(self, mockserver: MockServer) -> None:
+ crawler = get_crawler(_HttpErrorSpider, {"HANDLE_HTTP_CODES": True})
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, _HttpErrorSpider)
+ assert crawler.spider.parsed == {"200", "402", "404", "500"}
+ assert not crawler.spider.failed
+
@coroutine_test
async def test_logging(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer