Scrapy, a fast high-level web crawling & scraping framework for Python.
Ask custom download handlers not to use engine.download_async() (#7871)
Adrian6e2081ca4196
diff --git a/docs/conf.py b/docs/conf.pyindex de722baac..1b41adaad 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -141,6 +141,8 @@ coverage_ignore_pyobjects = [r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",]+# -- Options for the autodoc extension ----------------------------------------+autodoc_member_order = "bysource"# -- Options for the InterSphinx extension -----------------------------------# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configurationdiff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 34ab4f105..e0501c169 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -78,33 +78,15 @@ Writing your own download handlerA download handler is a :ref:`component <topics-components>` that definesthe following API:-.. class:: SampleDownloadHandler-- .. attribute:: lazy- :type: bool-- If ``False``, the handler will be instantiated when Scrapy is- initialized.-- If ``True``, the handler will only be instantiated when the first- request handled by it needs to be downloaded.-- .. method:: download_request(request: Request) -> Response- :async:-- Download the given request and return a response.-- .. method:: close() -> None- :async:-- Clean up any resources used by the handler.+.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol+ :members:An optional base class for custom handlers is provided:.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler:members::undoc-members:- :member-order: bysource+ :exclude-members: close, download_request, lazy.. _download-handlers-exceptions:diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex fb27cdb8b..84dc6216b 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -39,11 +39,23 @@ logger = logging.getLogger(__name__)class DownloadHandlerProtocol(Protocol):+ """Interface that :ref:`download handlers <topics-download-handlers>` must+ implement.++ Besides implementing this protocol, the contract of a download handler+ includes **never** calling :meth:`crawler.engine.download_async()+ <scrapy.core.engine.ExecutionEngine.download_async>`.+ """+lazy: bool+ """Whether to delay instantiation of the handler; see :ref:`lazy+ <lazy-download-handlers>`."""- async def download_request(self, request: Request) -> Response: ...+ async def download_request(self, request: Request) -> Response:+ """Download *request* and return a response."""- async def close(self) -> None: ...+ async def close(self) -> None:+ """Clean up any resources used by the handler."""class DownloadHandlers: