Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Ask custom download handlers not to use engine.download_async() (#7871)

Adrian6e2081ca4196
diff --git a/docs/conf.py b/docs/conf.py
index de722baac..1b41adaad 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -141,6 +141,8 @@ coverage_ignore_pyobjects = [
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
]
+# -- Options for the autodoc extension ----------------------------------------
+autodoc_member_order = "bysource"
# -- Options for the InterSphinx extension -----------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 34ab4f105..e0501c169 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -78,33 +78,15 @@ Writing your own download handler
A download handler is a :ref:`component <topics-components>` that defines
the following API:
-.. class:: SampleDownloadHandler
-
- .. attribute:: lazy
- :type: bool
-
- If ``False``, the handler will be instantiated when Scrapy is
- initialized.
-
- If ``True``, the handler will only be instantiated when the first
- request handled by it needs to be downloaded.
-
- .. method:: download_request(request: Request) -> Response
- :async:
-
- Download the given request and return a response.
-
- .. method:: close() -> None
- :async:
-
- Clean up any resources used by the handler.
+.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol
+ :members:
An optional base class for custom handlers is provided:
.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler
:members:
:undoc-members:
- :member-order: bysource
+ :exclude-members: close, download_request, lazy
.. _download-handlers-exceptions:
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index fb27cdb8b..84dc6216b 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -39,11 +39,23 @@ logger = logging.getLogger(__name__)
class DownloadHandlerProtocol(Protocol):
+ """Interface that :ref:`download handlers <topics-download-handlers>` must
+ implement.
+
+ Besides implementing this protocol, the contract of a download handler
+ includes **never** calling :meth:`crawler.engine.download_async()
+ <scrapy.core.engine.ExecutionEngine.download_async>`.
+ """
+
lazy: bool
+ """Whether to delay instantiation of the handler; see :ref:`lazy
+ <lazy-download-handlers>`."""
- async def download_request(self, request: Request) -> Response: ...
+ async def download_request(self, request: Request) -> Response:
+ """Download *request* and return a response."""
- async def close(self) -> None: ...
+ async def close(self) -> None:
+ """Clean up any resources used by the handler."""
class DownloadHandlers: