Scrapy, a fast high-level web crawling & scraping framework for Python.
Use autodoc for exceptions and improve their docs (#7767)
Adrian67e52826845f
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rstindex 45acb1763..0aab90a43 100644--- a/docs/topics/exceptions.rst+++ b/docs/topics/exceptions.rst@@ -1,117 +1,25 @@.. _topics-exceptions:+.. _topics-exceptions-ref:==========Exceptions==========-.. module:: scrapy.exceptions- :synopsis: Scrapy exceptions--.. _topics-exceptions-ref:--Built-in Exceptions reference-=============================-Here's a list of all exceptions included in Scrapy and their usage, except forthe :ref:`download handler exceptions <download-handlers-exceptions>`.+.. module:: scrapy.exceptions-CloseSpider--------------.. exception:: CloseSpider(reason='cancelled')-- This exception can be raised from a spider callback to request the spider to be- closed/stopped. Supported arguments:-- :param reason: the reason for closing- :type reason: str--For example:--.. code-block:: python-- def parse_page(self, response):- if "Bandwidth exceeded" in response.text:- raise CloseSpider("bandwidth_exceeded")--DontCloseSpider------------------.. exception:: DontCloseSpider--This exception can be raised in a :signal:`spider_idle` signal handler to-prevent the spider from being closed.--DropItem-----------.. exception:: DropItem--The exception that must be raised by item pipeline stages to stop processing an-Item. For more information see :ref:`topics-item-pipeline`.--IgnoreRequest----------------.. exception:: IgnoreRequest--This exception can be raised by the Scheduler or any downloader middleware to-indicate that the request should be ignored.--NotConfigured----------------.. exception:: NotConfigured--This exception can be raised by some components to indicate that they will-remain disabled. Those components include:--- Extensions-- Item pipelines-- Downloader middlewares-- Spider middlewares--The exception must be raised in the component's ``__init__()`` or-``from_crawler()`` method.--NotSupported---------------.. exception:: NotSupported--This exception is raised to indicate an unsupported feature.--StopDownload----------------.. exception:: StopDownload(fail=True)--Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`-signal handler to indicate that no further bytes should be downloaded for a response.+.. autoexception:: CloseSpider-The ``fail`` boolean parameter controls which method will handle the resulting-response:+.. autoexception:: DontCloseSpider-* If ``fail=True`` (default), the request errback is called. The response object is- available as the ``response`` attribute of the ``StopDownload`` exception,- which is in turn stored as the ``value`` attribute of the received- :class:`~twisted.python.failure.Failure` object. This means that in an errback- defined as ``def errback(self, failure)``, the response can be accessed though- ``failure.value.response``.+.. autoexception:: DropItem-* If ``fail=False``, the request callback is called instead.+.. autoexception:: IgnoreRequest-In both cases, the response could have its body truncated: the body contains-all bytes received up until the exception is raised, including the bytes-received in the signal handler that raises the exception. Also, the response-object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags`-attribute.+.. autoexception:: NotConfigured-.. note:: ``fail`` is a keyword-only parameter, i.e. raising- ``StopDownload(False)`` or ``StopDownload(True)`` will raise- a :class:`TypeError`.+.. autoexception:: NotSupported-See the documentation for the :class:`~scrapy.signals.bytes_received` and-:class:`~scrapy.signals.headers_received` signals-and the :ref:`topics-stop-response-download` topic for additional information and examples.+.. autoexception:: StopDownloaddiff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 5330eab48..ccaccddf2 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -16,7 +16,15 @@ if TYPE_CHECKING:class NotConfigured(Exception):- """Indicates a missing configuration situation"""+ """Raised by a :ref:`component <topics-components>` from its ``__init__()``+ or :meth:`from_crawler` method to indicate that it will remain disabled.++ Only the following components can be disabled this way:++ - :ref:`Downloader middlewares <topics-downloader-middleware>`+ - :ref:`Extensions <topics-extensions>`+ - :ref:`Item pipelines <topics-item-pipeline>`+ - :ref:`Spider middlewares <topics-spider-middleware>`"""class _InvalidOutput(TypeError):@@ -30,15 +38,37 @@ class _InvalidOutput(TypeError):class IgnoreRequest(Exception):- """Indicates a decision was made not to process a request"""+ """Raised to indicate that a request should be ignored.++ A :ref:`downloader middleware <topics-downloader-middleware>` can raise it+ from its+ :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`+ or+ :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`+ method to drop a request, and a :signal:`request_scheduled` signal handler+ can raise it to drop a request before it reaches the+ :ref:`scheduler <topics-scheduler>`."""class DontCloseSpider(Exception):- """Request the spider not to be closed yet"""+ """Raised in a :signal:`spider_idle` signal handler to prevent the spider+ from being closed."""class CloseSpider(Exception):- """Raise this from callbacks to request the spider to be closed"""+ """Raised from a :ref:`spider callback <topics-spiders>` to request the+ spider to be closed/stopped.++ *reason* is a string with the reason for closing.++ For example:++ .. code-block:: python++ def parse_page(self, response):+ if "Bandwidth exceeded" in response.text:+ raise CloseSpider("bandwidth_exceeded")+ """def __init__(self, reason: str = "cancelled"):super().__init__()@@ -46,10 +76,27 @@ class CloseSpider(Exception):class StopDownload(Exception):- """- Stop the download of the body for a given response.- The 'fail' boolean parameter indicates whether or not the resulting partial response- should be handled by the request errback. Note that 'fail' is a keyword-only argument.+ """Raised from a :class:`~scrapy.signals.bytes_received` or+ :class:`~scrapy.signals.headers_received` signal handler to :ref:`stop the+ download <topics-stop-response-download>` of the response body.++ The ``fail`` boolean parameter controls which method will handle the+ resulting response:++ * If ``fail=True`` (default), the request errback is called. The response+ object is available as the ``response`` attribute of the ``StopDownload``+ exception, which is in turn stored as the ``value`` attribute of the+ received :class:`~twisted.python.failure.Failure` object. This means that+ in an errback defined as ``def errback(self, failure)``, the response can+ be accessed though ``failure.value.response``.++ * If ``fail=False``, the request callback is called instead.++ In both cases, the response could have its body truncated: the body contains+ all bytes received up until the exception is raised, including the bytes+ received in the signal handler that raises the exception. Also, the response+ object is marked with ``"download_stopped"`` in its+ :attr:`~scrapy.http.Response.flags` attribute."""response: Response | None@@ -91,7 +138,8 @@ class UnsupportedURLSchemeError(Exception):class DropItem(Exception):- """Drop item from the item pipeline"""+ """Raised from the :meth:`process_item` method of an :ref:`item pipeline+ <topics-item-pipeline>` to stop the processing of an item."""def __init__(self, message: str, log_level: str | None = None):super().__init__(message)@@ -99,7 +147,14 @@ class DropItem(Exception):class NotSupported(Exception):- """Indicates a feature or method is not supported"""+ """Raised to indicate that a requested feature is not supported.++ For example, Scrapy raises it when text-parsing shortcuts such as+ :meth:`response.css() <scrapy.http.TextResponse.css>` or+ :meth:`response.xpath() <scrapy.http.TextResponse.xpath>` are used on a+ :class:`~scrapy.http.Response` whose content is not text, or when sending a+ request whose URL scheme has no matching :ref:`download handler+ <topics-download-handlers>`."""# Commands