Scrapy, a fast high-level web crawling & scraping framework for Python.
Async API for download handlers. (#7164)
Andrey Rakhmatullind1bd8eb49f7a
diff --git a/docs/index.rst b/docs/index.rstindex 71047f9ef..fe61f3513 100644--- a/docs/index.rst+++ b/docs/index.rst@@ -229,6 +229,7 @@ Extending Scrapytopics/signalstopics/schedulertopics/exporters+ topics/download-handlerstopics/componentstopics/api@@ -257,6 +258,9 @@ Extending Scrapy:doc:`topics/exporters`Quickly export your scraped items to a file (XML, CSV, etc).+:doc:`topics/download-handlers`+ Customize how requests are downloaded or add support for new URL schemes.+:doc:`topics/components`Learn the common API and some good practices when building custom Scrapycomponents.diff --git a/docs/news.rst b/docs/news.rstindex 19d990a50..7e15c2167 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -74,6 +74,12 @@ Backward-incompatible changes:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` orcalls its methods directly.+- The built-in :ref:`download handlers <download-handlers-ref>` were+ refactored, changing signatures of their methods. This change should only+ affect user code that subclasses any of these handlers or calls their+ methods directly.+ (:issue:`6778`, :issue:`7164`)+- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns acoroutine, previously it returned a:class:`~twisted.internet.defer.Deferred` object. This@@ -2616,7 +2622,7 @@ Modified requirements~~~~~~~~~~~~~~~~~~~~~- The h2_ dependency is now optional, only needed to- :ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)+ :ref:`enable HTTP/2 support <twisted-http2-handler>`. (:issue:`5113`).. _h2: https://pypi.org/project/h2/@@ -2998,7 +3004,7 @@ Highlights:- Official Python 3.9 support-- Experimental :ref:`HTTP/2 support <http2>`+- Experimental :ref:`HTTP/2 support <twisted-http2-handler>`- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` functionto retry requests from spider callbacks@@ -3029,7 +3035,7 @@ DeprecationsNew features~~~~~~~~~~~~-- Experimental :ref:`HTTP/2 support <http2>` through a new download handler+- Experimental :ref:`HTTP/2 support <twisted-http2-handler>` through a new download handlerthat can be assigned to the ``https`` protocol in the:setting:`DOWNLOAD_HANDLERS` setting.(:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)diff --git a/docs/requirements.txt b/docs/requirements.txtindex 81a3119e4..500db7a7e 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -1,3 +1,4 @@+h2==4.3.0pydantic==2.12.3scrapy-spider-metadata==0.2.0sphinx==8.1.3diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex dba14e749..64f00f622 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -175,16 +175,19 @@ Use a fallback component:class MyHandler:lazy = False- def __init__(self, settings, crawler):- dhcls = load_object(settings.get(FALLBACK_SETTING))+ def __init__(self, crawler):+ dhcls = load_object(crawler.settings.get(FALLBACK_SETTING))self._fallback_handler = build_from_crawler(dhcls, crawler)- def download_request(self, request, spider):+ async def download_request(self, request):if request.meta.get("my_params"):# handle the request...else:- return self._fallback_handler.download_request(request, spider)+ return await self._fallback_handler.download_request(request)++ async def close(self):+ passclass MyAddon:diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rstindex c82e6c2f3..a78e98828 100644--- a/docs/topics/coroutines.rst+++ b/docs/topics/coroutines.rst@@ -55,6 +55,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):- :ref:`Signal handlers that support deferreds <signal-deferred>`.+- Methods of :ref:`download handlers <topics-download-handlers>`.++ .. versionadded:: VERSION+.. _coroutine-deferred-apis:@@ -95,12 +99,6 @@ The following user-supplied methods can return:class:`~twisted.internet.defer.Deferred` objects (the methods that can alsoreturn coroutines are listed in :ref:`coroutine-support`):-- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`):-- - ``download_request()``-- - ``close()``-- Custom downloader implementations (see :setting:`DOWNLOADER`):- ``fetch()``@@ -146,11 +144,11 @@ For example:email is sent. You can use this object directly in Deferred-based code orconvert it into a :class:`~asyncio.Future` object with:func:`~scrapy.utils.defer.maybe_deferred_to_future`.-- A custom download handler needs to define a ``download_request()`` method- that returns a :class:`~twisted.internet.defer.Deferred` object. You can- write a method that works with Deferreds and returns one directly, or you- can write a coroutine and convert it into a function that returns a- Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`.+- A custom scheduler needs to define an ``open()`` method that can return a+ :class:`~twisted.internet.defer.Deferred` object. You can write a method+ that works with Deferreds and returns one directly, or you can write a+ coroutine and convert it into a function that returns a Deferred with+ :func:`~scrapy.utils.defer.deferred_f_from_coro_f`.General usagediff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstnew file mode 100644index 000000000..5f00d7131--- /dev/null+++ b/docs/topics/download-handlers.rst@@ -0,0 +1,219 @@+.. _topics-download-handlers:++=================+Download handlers+=================++Download handlers are Scrapy :ref:`components <topics-components>` used to+download :ref:`requests <topics-request-response>` and produce responses from+them.++Using download handlers+=======================++The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings+tell Scrapy which handler is responsible for a given URL scheme. Their values+are merged into a mapping from scheme names to handler classes. When Scrapy+initializes it creates instances of all configured download handlers (except+for :ref:`lazy ones <lazy-download-handlers>`) and stores them in a similar+mapping. When Scrapy needs to download a request it extracts the scheme from+its URL, finds the handler for this scheme, passes the request to it and gets a+response from it. If there is no handler for the scheme, the request is not+downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised.++The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of+handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers+for additional schemes and to replace or disable default ones:++.. code-block:: python++ DOWNLOAD_HANDLERS = {+ # disable support for ftp:// requests+ "ftp": None,+ # replace the default one for http://+ "http": "my.download_handlers.HttpHandler",+ # http:// and https:// are different schemes,+ # even though they may use the same handler+ "https": "my.download_handlers.HttpHandler",+ # support for any custom scheme can be added+ "sftp": "my.download_handlers.SftpHandler",+ }++Replacing HTTP(S) download handlers+-----------------------------------++While Scrapy provides a default handler for ``http`` and ``https`` schemes,+users may want to use a different handler, provided by Scrapy or by some+3rd-party package. There are several considerations to keep in mind related to+this.++First of all, as ``http`` and ``https`` are separate schemes, they need+separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's+likely that the same handler class will be used for both schemes.++Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`,+are honored by the default HTTP(S) handler but not necessarily by alternative+ones. The same may apply to other Scrapy features, e.g. the+:signal:`bytes_received` and :signal:`headers_received` signals.++.. _lazy-download-handlers:++Lazy instantiation of download handlers+---------------------------------------++A download handler can be marked as "lazy" by setting its ``lazy`` class+attribute to ``True``. Such handlers are only instantiated when they need to+download their first request. This may be useful when the instantiation is slow+or requires dependencies that are not always available, and the handler is not+needed on every spider run. For example, :class:`the built-in S3 handler+<.S3DownloadHandler>` is lazy.++Writing your own download handler+=================================++A download handler is a :ref:`component <topics-components>` that defines+the following methods:++.. class:: SampleDownloadHandler++ .. attribute:: lazy+ :type: bool++ If ``False``, the handler will be instantiated when Scrapy is+ initialized.++ If ``True``, the handler will only be instantiated when the first+ request handled by it needs to be downloaded.++ .. method:: download_request(request: Request) -> Response:+ :async:++ Download the given request and return a response.++ .. method:: close() -> None+ :async:++ Clean up any resources used by the handler.++An optional base class for custom handlers is provided:++.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler+ :members:+ :undoc-members:+ :member-order: bysource++.. _download-handlers-ref:++Built-in download handlers reference+====================================++DataURIDownloadHandler+----------------------++.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler++| Supported scheme: ``data``.+| Lazy: no.++This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.++FileDownloadHandler+-------------------++.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler++| Supported scheme: ``file``.+| Lazy: no.++This handler supports ``file:///path`` local file URIs. It doesn't+support remote files.++FTPDownloadHandler+------------------++.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler++| Supported scheme: ``ftp``.+| Lazy: no.++This handler supports ``ftp://host/path`` FTP URIs.++It's implemented using :mod:`twisted.protocols.ftp`.++.. _twisted-http2-handler:++H2DownloadHandler+-----------------++.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler++| Supported scheme: ``https``.+| Lazy: yes.++This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol+for them.++It's implemented using :mod:`twisted.web.client` and the ``h2`` library.++For this handler to work you need to install the ``Twisted[http2]`` extra+dependency.++If you want to use this handler you need to replace the default one for the+``https`` scheme:++.. code-block:: python++ DOWNLOAD_HANDLERS = {+ "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",+ }++.. warning::++ This handler is experimental, and not yet recommended for production+ environments. Future Scrapy versions may introduce related changes without+ a deprecation period or warning.++.. note::++ Known limitations of the HTTP/2 implementation in this handler include:++ - No support for HTTP/2 Cleartext (h2c), since no major browser supports+ HTTP/2 unencrypted (refer `http2 faq`_).++ - No setting to specify a maximum `frame size`_ larger than the default+ value, 16384. Connections to servers that send a larger frame will+ fail.++ - No support for `server pushes`_, which are ignored.++ - No support for the :signal:`bytes_received` and+ :signal:`headers_received` signals.++.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2+.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption+.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2++HTTP11DownloadHandler+---------------------++.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler++| Supported schemes: ``http``, ``https``.+| Lazy: no.++This handler supports ``http://host/path`` and ``https://host/path`` URLs and+uses the HTTP/1.1 protocol for them.++It's implemented using :mod:`twisted.web.client`.++S3DownloadHandler+-----------------++.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler++| Supported scheme: ``s3``.+| Lazy: yes.++This handler supports ``s3://bucket/path`` S3 URIs.++It's implemented using the ``botocore`` library and needs it to be installed.diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex c78045c4b..9be984f5d 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -129,9 +129,10 @@ defines one or more of these methods:.. method:: process_exception(request, exception)- Scrapy calls :meth:`process_exception` when a download handler- or a :meth:`process_request` (from a downloader middleware) raises an- exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)+ Scrapy calls :meth:`process_exception` when a :ref:`download handler+ <topics-download-handlers>` or a :meth:`process_request` (from a+ downloader middleware) raises an exception (including an+ :exc:`~scrapy.exceptions.IgnoreRequest` exception).:meth:`process_exception` should return: either ``None``,a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 27a8764f3..b446f197f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -691,24 +691,6 @@ Default: ``'scrapy.core.downloader.Downloader'``The downloader to use for crawling.-.. setting:: DOWNLOADER_HTTPCLIENTFACTORY--DOWNLOADER_HTTPCLIENTFACTORY-------------------------------Default: ``'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'``--Defines a Twisted ``protocol.ClientFactory`` class to use for HTTP/1.0-connections (for ``HTTP10DownloadHandler``).--.. note::-- HTTP/1.0 is rarely used nowadays and its Scrapy support is deprecated,- so you can safely ignore this setting,- unless you really want to use HTTP/1.0 and override- :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,- i.e. to ``'scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler'``.-.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORYDOWNLOADER_CLIENTCONTEXTFACTORY@@ -738,6 +720,12 @@ accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mappingparameter (``bool``) and a ``tls_ciphers`` parameter (see:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`).+.. note::++ This setting is specific to the built-in Twisted-based download handlers:+ :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and+ :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`.+.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERSDOWNLOADER_CLIENT_TLS_CIPHERS@@ -745,8 +733,8 @@ DOWNLOADER_CLIENT_TLS_CIPHERSDefault: ``'DEFAULT'``-Use this setting to customize the TLS/SSL ciphers used by the default-HTTP/1.1 downloader.+Use this setting to customize the TLS/SSL ciphers used by the HTTPS download+handler.The setting should contain a string in the `OpenSSL cipher list format`_,these ciphers will be used as client ciphers. Changing this setting may be@@ -756,6 +744,16 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it... _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers. Moreover, for the built-in Twisted-based+ download handlers+ (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and+ :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs+ to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class.+.. setting:: DOWNLOADER_CLIENT_TLS_METHODDOWNLOADER_CLIENT_TLS_METHOD@@ -763,8 +761,8 @@ DOWNLOADER_CLIENT_TLS_METHODDefault: ``'TLS'``-Use this setting to customize the TLS/SSL method used by the default-HTTP/1.1 downloader.+Use this setting to customize the TLS/SSL method used by the HTTPS download+handler.This setting must be one of these string values:@@ -776,6 +774,15 @@ This setting must be one of these string values:- ``'TLSv1.1'``: forces TLS version 1.1- ``'TLSv1.2'``: forces TLS version 1.2+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers. Moreover, for the built-in Twisted-based+ download handlers+ (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and+ :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs+ to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class... setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING@@ -786,10 +793,18 @@ Default: ``False``Setting this to ``True`` will enable DEBUG level messages about TLS connectionparameters after establishing HTTPS connections. The kind of information logged-depends on the versions of OpenSSL and pyOpenSSL.+depends on the implementation of the download handler and the versions of+the TLS-related libraries.++.. note::-This setting is only used for the default-:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.+ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers. Moreover, for the built-in Twisted-based+ download handlers+ (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and+ :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs+ to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class... setting:: DOWNLOADER_MIDDLEWARES@@ -882,7 +897,6 @@ It is also possible to change this setting per domain, although it requiresnon-trivial code. See the implementation of the :ref:`AutoThrottle<topics-autothrottle>` extension for an example.-.. setting:: DOWNLOAD_HANDLERSDOWNLOAD_HANDLERS@@ -890,7 +904,9 @@ DOWNLOAD_HANDLERSDefault: ``{}``-A dict containing the request downloader handlers enabled in your project.+A dict containing the :ref:`download handlers <topics-download-handlers>`+enabled in your project.+See :setting:`DOWNLOAD_HANDLERS_BASE` for example format... setting:: DOWNLOAD_HANDLERS_BASE@@ -912,9 +928,9 @@ Default:}-A dict containing the request download handlers enabled by default in Scrapy.-You should never modify this setting in your project, modify-:setting:`DOWNLOAD_HANDLERS` instead.+A dict containing the :ref:`download handlers <topics-download-handlers>`+enabled by default in Scrapy. You should never modify this setting in your+project, modify :setting:`DOWNLOAD_HANDLERS` instead.You can disable any of these download handlers by assigning ``None`` to theirURI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP@@ -926,46 +942,6 @@ handler (without replacement), place this in your ``settings.py``:"ftp": None,}-.. _http2:--The default HTTPS handler uses HTTP/1.1. To use HTTP/2:--#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to- enable HTTP/2 support in Twisted.--#. Update :setting:`DOWNLOAD_HANDLERS` as follows:-- .. code-block:: python-- DOWNLOAD_HANDLERS = {- "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",- }--.. warning::-- HTTP/2 support in Scrapy is experimental, and not yet recommended for- production environments. Future Scrapy versions may introduce related- changes without a deprecation period or warning.--.. note::-- Known limitations of the current HTTP/2 implementation of Scrapy include:-- - No support for HTTP/2 Cleartext (h2c), since no major browser supports- HTTP/2 unencrypted (refer `http2 faq`_).-- - No setting to specify a maximum `frame size`_ larger than the default- value, 16384. Connections to servers that send a larger frame will- fail.-- - No support for `server pushes`_, which are ignored.-- - No support for the :signal:`bytes_received` and- :signal:`headers_received` signals.--.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2-.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption-.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2.. setting:: DOWNLOAD_SLOTS@@ -1006,6 +982,12 @@ The amount of time (in secs) that the downloader will wait before timing out.This timeout can be per-request using the :reqmeta:`download_timeout`:attr:`.Request.meta` key.+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers.+.. setting:: DOWNLOAD_MAXSIZE.. reqmeta:: download_maxsize@@ -1028,6 +1010,12 @@ Use ``0`` to disable this limit.This limit can be set per-request using the :reqmeta:`download_maxsize`:attr:`.Request.meta` key.+.. note::++ Checking responses before decompressing them needs to be implemented inside+ the :ref:`download handler <topics-download-handlers>`, so it's not+ guaranteed to be supported by all 3rd-party handlers.+.. setting:: DOWNLOAD_WARNSIZE.. reqmeta:: download_warnsize@@ -1046,6 +1034,12 @@ Use ``0`` to disable this limit.This limit can be set per-request using the :reqmeta:`download_warnsize`:attr:`.Request.meta` key.+.. note::++ Checking responses before decompressing them needs to be implemented inside+ the :ref:`download handler <topics-download-handlers>`, so it's not+ guaranteed to be supported by all 3rd-party handlers.+.. setting:: DOWNLOAD_FAIL_ON_DATALOSSDOWNLOAD_FAIL_ON_DATALOSS@@ -1072,11 +1066,17 @@ Optionally, this can be set per-request basis by using theIf :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers.+.. warning::This setting is ignored by the:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`- download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss+ :ref:`download handler <topics-download-handlers>`. In case of a data losserror, the corresponding HTTP/2 connection may be corrupted, affecting otherrequests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``failure is always raised for every request that was using that connection.@@ -1269,6 +1269,12 @@ Default: ``True``Whether or not to use passive mode when initiating FTP transfers.+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers.+.. reqmeta:: ftp_password.. setting:: FTP_PASSWORD@@ -1288,6 +1294,12 @@ in ``Request`` meta... _RFC 1635: https://datatracker.ietf.org/doc/html/rfc1635+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers.+.. reqmeta:: ftp_user.. setting:: FTP_USER@@ -1299,6 +1311,12 @@ Default: ``"anonymous"``The username to use for FTP connections when there is no ``"ftp_user"``in ``Request`` meta.+.. note::++ Handling of this setting needs to be implemented inside the :ref:`download+ handler <topics-download-handlers>`, so it's not guaranteed to be supported+ by all 3rd-party handlers.+.. setting:: GCS_PROJECT_IDGCS_PROJECT_IDdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex db4e41c6a..85fa8f3c2 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -236,9 +236,7 @@ class Downloader:slot.transferring.add(request)try:# 1. Download the response- response: Response = await maybe_deferred_to_future(- self.handlers.download_request(request)- )+ response: Response = await self.handlers.download_request_async(request)# 2. Notify response_downloaded listeners about the recent download# before querying queue for next requestself.signals.send_catch_log(diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 2d3d40dae..fb27cdb8b 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -2,16 +2,21 @@from __future__ import annotations+import inspectimport logging+import warningsfrom typing import TYPE_CHECKING, Any, Protocol, castfrom scrapy import Request, Spider, signals-from scrapy.exceptions import NotConfigured, NotSupported-from scrapy.utils.decorators import _warn_spider_arg-from scrapy.utils.defer import ensure_awaitable+from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning+from scrapy.utils.defer import (+ deferred_from_coro,+ ensure_awaitable,+ maybe_deferred_to_future,+)from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_object-from scrapy.utils.python import without_none_values+from scrapy.utils.python import global_object_name, without_none_valuesif TYPE_CHECKING:from collections.abc import Callable@@ -25,10 +30,20 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+# This is the official API but we temporarily support the old deprecated one:+# * lazy is not mandatory (defaults to True).+# * download_request() can return a Deferred[Response] instead of a coroutine,+# and takes a spider argument in this case.+# * close() can return None or Deferred[None] instead of a coroutine.+# * close() is not mandatory.++class DownloadHandlerProtocol(Protocol):- def download_request(- self, request: Request, spider: Spider- ) -> Deferred[Response]: ...+ lazy: bool++ async def download_request(self, request: Request) -> Response: ...++ async def close(self) -> None: ...class DownloadHandlers:@@ -40,6 +55,8 @@ class DownloadHandlers:self._handlers: dict[str, DownloadHandlerProtocol] = {}# remembers failed handlersself._notconfigured: dict[str, str] = {}+ # remembers handlers with Deferred-based download_request()+ self._old_style_handlers: set[str] = set()handlers: dict[str, str | Callable[..., Any]] = without_none_values(cast("dict[str, str | Callable[..., Any]]",@@ -72,8 +89,17 @@ class DownloadHandlers:path = self._schemes[scheme]try:dhcls: type[DownloadHandlerProtocol] = load_object(path)- if skip_lazy and getattr(dhcls, "lazy", True):- return None+ if skip_lazy:+ if not hasattr(dhcls, "lazy"):+ warnings.warn(+ f"{global_object_name(dhcls)} doesn't define a 'lazy' attribute."+ f" This is deprecated, please add 'lazy = True' (which is the current"+ f" default value) to the class definition.",+ category=ScrapyDeprecationWarning,+ stacklevel=1,+ )+ if getattr(dhcls, "lazy", True):+ return Nonedh = build_from_crawler(dhcls,self._crawler,@@ -91,12 +117,28 @@ class DownloadHandlers:self._notconfigured[scheme] = str(ex)return Noneself._handlers[scheme] = dh+ if not inspect.iscoroutinefunction(dh.download_request): # pragma: no cover+ warnings.warn(+ f"{global_object_name(dh.download_request)} is not a coroutine function."+ f" This is deprecated, please rewrite it to return a coroutine and remove"+ f" the 'spider' argument.",+ category=ScrapyDeprecationWarning,+ stacklevel=1,+ )+ self._old_style_handlers.add(scheme)return dh- @_warn_spider_argdef download_request(self, request: Request, spider: Spider | None = None- ) -> Deferred[Response]:+ ) -> Deferred[Response]: # pragma: no cover+ warnings.warn(+ "DownloadHandlers.download_request() is deprecated, use download_request_async() instead",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return deferred_from_coro(self.download_request_async(request))++ async def download_request_async(self, request: Request) -> Response:scheme = urlparse_cached(request).schemehandler = self._get_handler(scheme)if not handler:@@ -104,11 +146,33 @@ class DownloadHandlers:f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")assert self._crawler.spider- return handler.download_request(request, self._crawler.spider)+ if scheme in self._old_style_handlers: # pragma: no cover+ return await maybe_deferred_to_future(+ cast(+ "Deferred[Response]",+ handler.download_request(request, self._crawler.spider), # type: ignore[call-arg]+ )+ )+ return await handler.download_request(request)async def _close(self) -> None:for dh in self._handlers.values():- if not hasattr(dh, "close"):+ if not hasattr(dh, "close"): # pragma: no cover+ warnings.warn(+ f"{global_object_name(dh)} doesn't define a close() method."+ f" This is deprecated, please add an empty 'async def close()' method.",+ category=ScrapyDeprecationWarning,+ stacklevel=1,+ )continue- await ensure_awaitable(dh.close())+ if inspect.iscoroutinefunction(dh.close):+ await dh.close()+ else: # pragma: no cover+ warnings.warn(+ f"{global_object_name(dh.close)} is not a coroutine function."+ f" This is deprecated, please rewrite it to return a coroutine.",+ category=ScrapyDeprecationWarning,+ stacklevel=1,+ )+ await ensure_awaitable(dh.close())diff --git a/scrapy/core/downloader/handlers/base.py b/scrapy/core/downloader/handlers/base.pynew file mode 100644index 000000000..999c003b4--- /dev/null+++ b/scrapy/core/downloader/handlers/base.py@@ -0,0 +1,32 @@+from __future__ import annotations++from abc import ABC, abstractmethod+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy import Request+ from scrapy.crawler import Crawler+ from scrapy.http import Response+++class BaseDownloadHandler(ABC):+ """Optional base class for download handlers."""++ lazy: bool = False++ def __init__(self, crawler: Crawler):+ self.crawler = crawler++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ return cls(crawler)++ @abstractmethod+ async def download_request(self, request: Request) -> Response:+ raise NotImplementedError++ async def close(self) -> None: # noqa: B027+ passdiff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.pyindex b3f286d87..84f241d88 100644--- a/scrapy/core/downloader/handlers/datauri.py+++ b/scrapy/core/downloader/handlers/datauri.py@@ -4,19 +4,16 @@ from typing import TYPE_CHECKING, Anyfrom w3lib.url import parse_data_uri+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes-from scrapy.utils.decorators import defersif TYPE_CHECKING:- from scrapy import Request, Spider+ from scrapy import Request-class DataURIDownloadHandler:- lazy = False-- @defers- def download_request(self, request: Request, spider: Spider) -> Response:+class DataURIDownloadHandler(BaseDownloadHandler):+ async def download_request(self, request: Request) -> Response:uri = parse_data_uri(request.url)respcls = responsetypes.from_mimetype(uri.media_type)diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.pyindex d55c516f0..21fd2c353 100644--- a/scrapy/core/downloader/handlers/file.py+++ b/scrapy/core/downloader/handlers/file.py@@ -5,19 +5,16 @@ from typing import TYPE_CHECKINGfrom w3lib.url import file_uri_to_path+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.responsetypes import responsetypes-from scrapy.utils.decorators import defersif TYPE_CHECKING:- from scrapy import Request, Spider+ from scrapy import Requestfrom scrapy.http import Response-class FileDownloadHandler:- lazy = False-- @defers- def download_request(self, request: Request, spider: Spider) -> Response:+class FileDownloadHandler(BaseDownloadHandler):+ async def download_request(self, request: Request) -> Response:filepath = file_uri_to_path(request.url)body = Path(filepath).read_bytes()respcls = responsetypes.from_args(filename=filepath, body=body)diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 1d947b1e3..f261e8e7f 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -33,27 +33,22 @@ from __future__ import annotationsimport refrom io import BytesIOfrom pathlib import Path-from typing import TYPE_CHECKING, Any, BinaryIO+from typing import TYPE_CHECKING, BinaryIOfrom urllib.parse import unquotefrom twisted.internet.protocol import ClientCreator, Protocol+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.http import Responsefrom scrapy.responsetypes import responsetypes+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.python import to_bytesif TYPE_CHECKING:- from twisted.internet.defer import Deferredfrom twisted.protocols.ftp import FTPClient- from twisted.python.failure import Failure- # typing.Self requires Python 3.11- from typing_extensions import Self-- from scrapy import Request, Spider+ from scrapy import Requestfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettingsclass ReceivedDataProtocol(Protocol):@@ -82,26 +77,21 @@ class ReceivedDataProtocol(Protocol):_CODE_RE = re.compile(r"\d+")-class FTPDownloadHandler:- lazy = False-+class FTPDownloadHandler(BaseDownloadHandler):CODE_MAPPING: dict[str, int] = {"550": 404,"default": 503,}- def __init__(self, settings: BaseSettings):- self.default_user = settings["FTP_USER"]- self.default_password = settings["FTP_PASSWORD"]- self.passive_mode = settings["FTP_PASSIVE_MODE"]-- @classmethod- def from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings)+ def __init__(self, crawler: Crawler):+ super().__init__(crawler)+ self.default_user = crawler.settings["FTP_USER"]+ self.default_password = crawler.settings["FTP_PASSWORD"]+ self.passive_mode = crawler.settings["FTP_PASSIVE_MODE"]- def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:+ async def download_request(self, request: Request) -> Response:from twisted.internet import reactor- from twisted.protocols.ftp import FTPClient+ from twisted.protocols.ftp import CommandFailed, FTPClientparsed_url = urlparse_cached(request)user = request.meta.get("ftp_user", self.default_user)@@ -112,43 +102,23 @@ class FTPDownloadHandler:creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)- dfd: Deferred[FTPClient] = creator.connectTCP(- parsed_url.hostname, parsed_url.port or 21+ client: FTPClient = await maybe_deferred_to_future(+ creator.connectTCP(parsed_url.hostname, parsed_url.port or 21))- return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))-- def gotClient(- self, client: FTPClient, request: Request, filepath: str- ) -> Deferred[Response]:- self.client = client+ filepath = unquote(parsed_url.path)protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))- d = client.retrieveFile(filepath, protocol)- d.addCallback(self._build_response, request, protocol)- d.addErrback(self._failed, request)- return d-- def _build_response(- self, result: Any, request: Request, protocol: ReceivedDataProtocol- ) -> Response:- self.result = result+ try:+ await maybe_deferred_to_future(client.retrieveFile(filepath, protocol))+ except CommandFailed as e:+ message = str(e)+ if m := _CODE_RE.search(message):+ ftpcode = m.group()+ httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])+ return Response(url=request.url, status=httpcode, body=message.encode())+ raiseprotocol.close()headers = {"local filename": protocol.filename or b"", "size": protocol.size}body = protocol.filename or protocol.body.read()respcls = responsetypes.from_args(url=request.url, body=body)# hints for Headers-related types may need to be fixed to not use AnyStrreturn respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]-- def _failed(self, result: Failure, request: Request) -> Response:- from twisted.protocols.ftp import CommandFailed-- message = result.getErrorMessage()- if result.type == CommandFailed:- m = _CODE_RE.search(message)- if m:- ftpcode = m.group()- httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])- return Response(- url=request.url, status=httpcode, body=to_bytes(message)- )- assert result.type- raise result.type(result.value)diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.pyindex 0fbe5fc23..af3588019 100644--- a/scrapy/core/downloader/handlers/http10.py+++ b/scrapy/core/downloader/handlers/http10.py@@ -6,17 +6,17 @@ import warningsfrom typing import TYPE_CHECKINGfrom scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:- from twisted.internet.defer import Deferredfrom twisted.internet.interfaces import IConnector# typing.Self requires Python 3.11from typing_extensions import Self- from scrapy import Request, Spider+ from scrapy import Requestfrom scrapy.core.downloader.contextfactory import ScrapyClientContextFactoryfrom scrapy.core.downloader.webclient import ScrapyHTTPClientFactoryfrom scrapy.crawler import Crawler@@ -46,11 +46,10 @@ class HTTP10DownloadHandler:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings, crawler)- def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:- """Return a deferred for the HTTP download"""+ async def download_request(self, request: Request) -> Response:factory = self.HTTPClientFactory(request)self._connect(factory)- return factory.deferred+ return await maybe_deferred_to_future(factory.deferred)def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:from twisted.internet import reactor@@ -63,3 +62,6 @@ class HTTP10DownloadHandler:)return reactor.connectSSL(host, port, factory, client_context_factory)return reactor.connectTCP(host, port, factory)++ async def close(self) -> None:+ passdiff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 02aaf7c5c..cec73566a 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -30,11 +30,13 @@ from twisted.web.http_headers import Headers as TxHeadersfrom twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponsefrom zope.interface import implementer-from scrapy import Request, Spider, signals+from scrapy import Request, signalsfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settings+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import StopDownloadfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypes+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode@@ -44,11 +46,10 @@ if TYPE_CHECKING:from twisted.internet.base import ReactorBasefrom twisted.internet.interfaces import IConsumer- # typing.NotRequired and typing.Self require Python 3.11- from typing_extensions import NotRequired, Self+ # typing.NotRequired requires Python 3.11+ from typing_extensions import NotRequiredfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettingslogger = logging.getLogger(__name__)@@ -65,37 +66,34 @@ class _ResultT(TypedDict):failure: NotRequired[Failure | None]-class HTTP11DownloadHandler:- lazy = False-- def __init__(self, settings: BaseSettings, crawler: Crawler):+class HTTP11DownloadHandler(BaseDownloadHandler):+ def __init__(self, crawler: Crawler):+ super().__init__(crawler)self._crawler = crawlerfrom twisted.internet import reactorself._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)- self._pool.maxPersistentPerHost = settings.getint(+ self._pool.maxPersistentPerHost = crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")self._pool._factory.noisy = Falseself._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings(- settings, crawler+ crawler.settings, crawler+ )+ self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")+ self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")+ self._fail_on_dataloss: bool = crawler.settings.getbool(+ "DOWNLOAD_FAIL_ON_DATALOSS")- self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE")- self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE")- self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")self._disconnect_timeout: int = 1- @classmethod- def from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings, crawler)-- def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:+ async def download_request(self, request: Request) -> Response:"""Return a deferred for the HTTP download"""- if hasattr(spider, "download_maxsize"): # pragma: no cover+ if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no coverwarn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")- if hasattr(spider, "download_warnsize"): # pragma: no cover+ if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no coverwarn_on_deprecated_spider_attribute("download_warnsize", "DOWNLOAD_WARNSIZE")@@ -103,14 +101,18 @@ class HTTP11DownloadHandler:agent = ScrapyAgent(contextFactory=self._contextFactory,pool=self._pool,- maxsize=getattr(spider, "download_maxsize", self._default_maxsize),- warnsize=getattr(spider, "download_warnsize", self._default_warnsize),+ maxsize=getattr(+ self._crawler.spider, "download_maxsize", self._default_maxsize+ ),+ warnsize=getattr(+ self._crawler.spider, "download_warnsize", self._default_warnsize+ ),fail_on_dataloss=self._fail_on_dataloss,crawler=self._crawler,)- return agent.download_request(request)+ return await maybe_deferred_to_future(agent.download_request(request))- def close(self) -> Deferred[None]:+ async def close(self) -> None:from twisted.internet import reactord: Deferred[None] = self._pool.closeCachedConnections()@@ -118,19 +120,19 @@ class HTTP11DownloadHandler:# we'll manually timeout the deferred.## Twisted issue addressing this problem can be found here:- # https://twistedmatrix.com/trac/ticket/7738.+ # https://github.com/twisted/twisted/issues/7738## closeCachedConnections doesn't handle external errbacks, so we'll# issue a callback after `_disconnect_timeout` seconds.+ #+ # See also https://github.com/scrapy/scrapy/issues/2653delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])- def cancel_delayed_call(result: _T) -> _T:+ try:+ await maybe_deferred_to_future(d)+ finally:if delayed_call.active():delayed_call.cancel()- return result-- d.addBoth(cancel_delayed_call)- return dclass TunnelError(Exception):diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 8e6236094..ff6e8ab98 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -8,7 +8,9 @@ from twisted.internet.error import TimeoutError as TxTimeoutErrorfrom twisted.web.client import URIfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settings+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes@@ -17,37 +19,37 @@ if TYPE_CHECKING:from twisted.internet.defer import Deferredfrom twisted.web.iweb import IPolicyForHTTPS- # typing.Self requires Python 3.11- from typing_extensions import Self-from scrapy.crawler import Crawlerfrom scrapy.http import Request, Response- from scrapy.settings import Settingsfrom scrapy.spiders import Spider-class H2DownloadHandler:- def __init__(self, settings: Settings, crawler: Crawler):+class H2DownloadHandler(BaseDownloadHandler):+ lazy = True++ def __init__(self, crawler: Crawler):+ super().__init__(crawler)self._crawler = crawlerfrom twisted.internet import reactor- self._pool = H2ConnectionPool(reactor, settings)- self._context_factory = load_context_factory_from_settings(settings, crawler)-- @classmethod- def from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings, crawler)+ self._pool = H2ConnectionPool(reactor, crawler.settings)+ self._context_factory = load_context_factory_from_settings(+ crawler.settings, crawler+ )- def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:+ async def download_request(self, request: Request) -> Response:agent = ScrapyH2Agent(context_factory=self._context_factory,pool=self._pool,crawler=self._crawler,)- return agent.download_request(request, spider)+ assert self._crawler.spider+ return await maybe_deferred_to_future(+ agent.download_request(request, self._crawler.spider)+ )- def close(self) -> None:+ async def close(self) -> None:self._pool.close_connections()diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 05a71b742..a601bc2aa 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,7 +1,8 @@from __future__ import annotations-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.boto import is_botocore_available@@ -9,57 +10,28 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawlerif TYPE_CHECKING:- from twisted.internet.defer import Deferred-- # typing.Self requires Python 3.11- from typing_extensions import Self-- from scrapy import Request, Spider+ from scrapy import Requestfrom scrapy.crawler import Crawlerfrom scrapy.http import Response- from scrapy.settings import BaseSettings-class S3DownloadHandler:- def __init__(- self,- settings: BaseSettings,- *,- crawler: Crawler,- aws_access_key_id: str | None = None,- aws_secret_access_key: str | None = None,- aws_session_token: str | None = None,- httpdownloadhandler: type[HTTP11DownloadHandler] = HTTP11DownloadHandler,- **kw: Any,- ):+class S3DownloadHandler(BaseDownloadHandler):+ lazy = True++ def __init__(self, crawler: Crawler):if not is_botocore_available():raise NotConfigured("missing botocore library")- if not aws_access_key_id:- aws_access_key_id = settings["AWS_ACCESS_KEY_ID"]- if not aws_secret_access_key:- aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"]- if not aws_session_token:- aws_session_token = settings["AWS_SESSION_TOKEN"]-- # If no credentials could be found anywhere,- # consider this an anonymous connection request by default;- # unless 'anon' was set explicitly (True/False).- anon = kw.get("anon")- if anon is None and not aws_access_key_id and not aws_secret_access_key:- kw["anon"] = True- self.anon = kw.get("anon")-+ super().__init__(crawler)+ aws_access_key_id = crawler.settings["AWS_ACCESS_KEY_ID"]+ aws_secret_access_key = crawler.settings["AWS_SECRET_ACCESS_KEY"]+ aws_session_token = crawler.settings["AWS_SESSION_TOKEN"]+ self.anon = not aws_access_key_id and not aws_secret_access_keyself._signer = None- import botocore.auth # noqa: PLC0415- import botocore.credentials # noqa: PLC0415-- kw.pop("anon", None)- if kw:- raise TypeError(f"Unexpected keyword arguments: {kw}")if not self.anon:- assert aws_access_key_id is not None- assert aws_secret_access_key is not None+ import botocore.auth # noqa: PLC0415+ import botocore.credentials # noqa: PLC0415+SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]# botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses doself._signer = SignerCls( # type: ignore[call-arg]@@ -68,17 +40,10 @@ class S3DownloadHandler:))- _http_handler = build_from_crawler(- httpdownloadhandler,- crawler,- )+ _http_handler = build_from_crawler(HTTP11DownloadHandler, crawler)self._download_http = _http_handler.download_request- @classmethod- def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:- return cls(crawler.settings, crawler=crawler, **kwargs)-- def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:+ async def download_request(self, request: Request) -> Response:p = urlparse_cached(request)scheme = "https" if request.meta.get("is_secure") else "http"bucket = p.hostname@@ -98,4 +63,4 @@ class S3DownloadHandler:assert self._signerself._signer.add_auth(awsrequest)request = request.replace(url=url, headers=awsrequest.headers.items())- return self._download_http(request, spider)+ return await self._download_http(request)diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 094a0195e..d24c1b6c4 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -15,10 +15,10 @@ from typing import TYPE_CHECKING, Anyfrom twisted.conch import telnetfrom twisted.conch.insults import insultsfrom twisted.internet import protocol+from twisted.internet.defer import fail, succeedfrom scrapy import signalsfrom scrapy.exceptions import NotConfigured-from scrapy.utils.decorators import defersfrom scrapy.utils.engine import print_engine_statusfrom scrapy.utils.reactor import listen_tcpfrom scrapy.utils.trackref import print_live_refs@@ -80,20 +80,19 @@ class TelnetConsole(protocol.ServerFactory):class Portal:"""An implementation of IPortal"""- @defersdef login(self_, credentials, mind, *interfaces): # pylint: disable=no-self-argumentif not (credentials.username == self.username.encode("utf8")and credentials.checkPassword(self.password.encode("utf8"))):- raise ValueError("Invalid credentials")+ return fail(ValueError("Invalid credentials"))from twisted.conch import manholeprotocol = telnet.TelnetBootstrapProtocol(insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars())- return (interfaces[0], protocol, lambda: None)+ return succeed((interfaces[0], protocol, lambda: None))return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal())diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex 2dde0aa55..044d1d4c7 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -42,8 +42,13 @@ def deprecated(return deco-def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:+def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: # pragma: no cover"""Decorator to make sure a function always returns a deferred"""+ warnings.warn(+ "@defers is deprecated, you can use maybeDeferred() directly if needed.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )@wraps(func)def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 699ac3911..9fd5a40df 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -420,6 +420,12 @@ with multiples linesself._assert_retried(caplog.text)assert "Got response 200" in caplog.text+ @deferred_f_from_coro_f+ async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:+ crawler = get_crawler(SimpleSpider)+ await maybe_deferred_to_future(crawler.crawl("foo://bar"))+ assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text+class TestCrawlSpider:mockserver: MockServerdiff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pyindex a43fe318d..6cda607cc 100644--- a/tests/test_downloader_handler_twisted_ftp.py+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport osimport sys+from abc import ABC, abstractmethodfrom pathlib import Pathfrom tempfile import mkstempfrom typing import TYPE_CHECKING, Any@@ -13,17 +14,18 @@ from twisted.cred import checkers, credentials, portalfrom scrapy.core.downloader.handlers.ftp import FTPDownloadHandlerfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.http.response.text import TextResponse-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes-from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generator+ from twisted.protocols.ftp import FTPFactory-class TestFTPBase:++class TestFTPBase(ABC):username = "scrapy"password = "passwd"req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}@@ -34,21 +36,13 @@ class TestFTPBase:("html-file-without-extension", b"<!DOCTYPE html>\n<title>.</title>"),)+ @abstractmethoddef _create_files(self, root: Path) -> None:- userdir = root / self.username- userdir.mkdir()- for filename, content in self.test_files:- (userdir / filename).write_bytes(content)+ raise NotImplementedError- def _get_factory(self, root):- from twisted.protocols.ftp import FTPFactory, FTPRealm-- realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))- p = portal.Portal(realm)- users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()- users_checker.addUser(self.username, self.password)- p.registerChecker(users_checker, credentials.IUsernamePassword)- return FTPFactory(portal=p)+ @abstractmethod+ def _get_factory(self, tmp_path: Path) -> FTPFactory:+ raise NotImplementedError@async_yield_fixtureasync def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:@@ -76,18 +70,12 @@ class TestFTPBase:assert dh.client.transportdh.client.transport.loseConnection()- @staticmethod- async def download_request(dh: FTPDownloadHandler, request: Request) -> Response:- return await maybe_deferred_to_future(- dh.download_request(request, DefaultSpider())- )-@deferred_f_from_coro_fasync def test_ftp_download_success(self, server_url: str, dh: FTPDownloadHandler) -> None:request = Request(url=server_url + "file.txt", meta=self.req_meta)- r = await self.download_request(dh, request)+ r = await dh.download_request(request)assert r.status == 200assert r.body == b"I have the power!"assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}@@ -101,7 +89,7 @@ class TestFTPBase:url=server_url + "file with spaces.txt",meta=self.req_meta,)- r = await self.download_request(dh, request)+ r = await dh.download_request(request)assert r.status == 200assert r.body == b"Moooooooooo power!"assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}@@ -111,8 +99,9 @@ class TestFTPBase:self, server_url: str, dh: FTPDownloadHandler) -> None:request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)- r = await self.download_request(dh, request)+ r = await dh.download_request(request)assert r.status == 404+ assert r.body == b"['550 nonexistent.txt: No such file or directory.']"@deferred_f_from_coro_fasync def test_ftp_local_filename(@@ -125,7 +114,7 @@ class TestFTPBase:meta = {"ftp_local_filename": fname_bytes}meta.update(self.req_meta)request = Request(url=server_url + "file.txt", meta=meta)- r = await self.download_request(dh, request)+ r = await dh.download_request(request)assert r.body == fname_bytesassert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}assert local_path.exists()@@ -153,12 +142,28 @@ class TestFTPBase:meta = {}meta.update(self.req_meta)request = Request(url=server_url + filename, meta=meta)- r = await self.download_request(dh, request)+ r = await dh.download_request(request)assert type(r) is response_class # pylint: disable=unidiomatic-typechecklocal_fname_path.unlink()class TestFTP(TestFTPBase):+ def _create_files(self, root: Path) -> None:+ userdir = root / self.username+ userdir.mkdir()+ for filename, content in self.test_files:+ (userdir / filename).write_bytes(content)++ def _get_factory(self, root):+ from twisted.protocols.ftp import FTPFactory, FTPRealm++ realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))+ p = portal.Portal(realm)+ users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()+ users_checker.addUser(self.username, self.password)+ p.registerChecker(users_checker, credentials.IUsernamePassword)+ return FTPFactory(portal=p)+@deferred_f_from_coro_fasync def test_invalid_credentials(self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str@@ -174,7 +179,7 @@ class TestFTP(TestFTPBase):meta.update({"ftp_password": "invalid"})request = Request(url=server_url + "file.txt", meta=meta)with pytest.raises(ConnectionLost):- await self.download_request(dh, request)+ await dh.download_request(request)class TestAnonymousFTP(TestFTPBase):diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.pyindex c3e0aa7c4..4745869e4 100644--- a/tests/test_downloader_handler_twisted_http10.py+++ b/tests/test_downloader_handler_twisted_http10.py@@ -9,11 +9,7 @@ import pytestfrom scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandlerfrom scrapy.http import Requestfrom scrapy.utils.defer import deferred_f_from_coro_f-from tests.test_downloader_handlers_http_base import (- TestHttpBase,- TestHttpProxyBase,- download_request,-)+from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBaseif TYPE_CHECKING:from scrapy.core.downloader.handlers import DownloadHandlerProtocol@@ -31,13 +27,12 @@ class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):"""HTTP 1.0 test case"""@deferred_f_from_coro_f- async def test_protocol(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_protocol(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/host", is_secure=self.is_secure), method="GET")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.protocol == "HTTP/1.0"diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 1e638392f..f60d5e7b5 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -13,10 +13,7 @@ from twisted.web.error import SchemeNotSupportedfrom twisted.web.http import H2_ENABLEDfrom scrapy.http import Request-from scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future-from scrapy.utils.misc import build_from_crawler-from scrapy.utils.test import get_crawlerfrom tests.test_downloader_handlers_http_base import (TestHttpProxyBase,TestHttps11Base,@@ -25,7 +22,6 @@ from tests.test_downloader_handlers_http_base import (TestHttpsInvalidDNSPatternBase,TestHttpsWrongHostnameBase,TestHttpWithCrawlerBase,- download_request,)if TYPE_CHECKING:@@ -53,13 +49,12 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"@deferred_f_from_coro_f- async def test_protocol(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_protocol(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/host", is_secure=self.is_secure), method="GET")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.protocol == "h2"@deferred_f_from_coro_f@@ -68,9 +63,6 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):) -> None:from twisted.internet import reactor- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 1_500})- download_handler = build_from_crawler(self.download_handler_cls, crawler)-with mock.patch("scrapy.core.http2.stream.logger") as logger:request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure)@@ -79,8 +71,9 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):def check(logger: mock.Mock) -> None:logger.error.assert_called_once_with(mock.ANY)- with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await download_handler.download_request(request)# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queue@@ -91,12 +84,11 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):await maybe_deferred_to_future(d)@deferred_f_from_coro_f- async def test_unsupported_scheme(- self, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_unsupported_scheme(self) -> None:request = Request("ftp://unsupported.scheme")- with pytest.raises(SchemeNotSupported):- await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ with pytest.raises(SchemeNotSupported):+ await download_handler.download_request(request)def test_download_cause_data_loss(self) -> None: # type: ignore[override]pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)@@ -109,49 +101,46 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):@deferred_f_from_coro_fasync def test_concurrent_requests_same_domain(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:request1 = Request(mockserver.url("/text", is_secure=self.is_secure))- response1 = await download_request(download_handler, request1)- assert response1.body == b"Works"-request2 = Request(mockserver.url("/echo", is_secure=self.is_secure), method="POST")- response2 = await download_request(download_handler, request2)- assert response2.headers["Content-Length"] == b"79"+ async with self.get_dh() as download_handler:+ response1 = await download_handler.download_request(request1)+ assert response1.body == b"Works"+ response2 = await download_handler.download_request(request2)+ assert response2.headers["Content-Length"] == b"79"@pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")@deferred_f_from_coro_f- async def test_connect_request(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_connect_request(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/file", is_secure=self.is_secure), method="CONNECT")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b""@deferred_f_from_coro_f- async def test_custom_content_length_good(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_custom_content_length_good(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))custom_content_length = str(len(request.body))request.headers["Content-Length"] = custom_content_length- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.text == custom_content_length@deferred_f_from_coro_f- async def test_custom_content_length_bad(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_custom_content_length_bad(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))actual_content_length = str(len(request.body))bad_content_length = str(len(request.body) + 1)request.headers["Content-Length"] = bad_content_length- with LogCapture() as log:- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ with LogCapture() as log:+ response = await download_handler.download_request(request)assert response.text == actual_content_lengthlog.check_present((@@ -164,14 +153,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):)@deferred_f_from_coro_f- async def test_duplicate_header(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_duplicate_header(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/echo", is_secure=self.is_secure))header, value1, value2 = "Custom-Header", "foo", "bar"request.headers.appendlist(header, value1)request.headers.appendlist(header, value2)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert json.loads(response.text)["headers"][header] == [value1, value2]@@ -213,26 +201,18 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):@deferred_f_from_coro_fasync def test_download_with_proxy_https_timeout(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:with pytest.raises(NotImplementedError):await maybe_deferred_to_future(- super().test_download_with_proxy_https_timeout(- proxy_mockserver, download_handler- )+ super().test_download_with_proxy_https_timeout(proxy_mockserver))@deferred_f_from_coro_fasync def test_download_with_proxy_without_http_scheme(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:with pytest.raises(SchemeNotSupported):await maybe_deferred_to_future(- super().test_download_with_proxy_without_http_scheme(- proxy_mockserver, download_handler- )+ super().test_download_with_proxy_without_http_scheme(proxy_mockserver))diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex e37932fbb..602b2186f 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -15,22 +15,26 @@ from scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.handlers.datauri import DataURIDownloadHandlerfrom scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandler-from scrapy.exceptions import NotConfigured-from scrapy.http import Request, Response+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.http import Requestfrom scrapy.responsetypes import responsetypes-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.boto import is_botocore_available+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.misc import build_from_crawler-from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerclass DummyDH:lazy = False+ async def download_request(self, request):+ pass+class DummyLazyDH:- # Default is lazy for backward compatibility- pass+ # Default (but deprecated) is lazy for backward compatibility+ async def download_request(self, request):+ passclass OffDH:@@ -44,6 +48,17 @@ class OffDH:return cls(crawler)+class BuggyDH:+ lazy = False++ def __init__(self, crawler):+ raise ValueError++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++class TestLoad:def test_enabled_handler(self):handlers = {"scheme": DummyDH}@@ -61,6 +76,18 @@ class TestLoad:assert "scheme" not in dh._handlersassert "scheme" in dh._notconfigured+ def test_buggy_handler(self, caplog: pytest.LogCaptureFixture) -> None:+ handlers = {"scheme": BuggyDH}+ crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})+ dh = DownloadHandlers(crawler)+ assert "scheme" in dh._schemes+ assert "scheme" not in dh._handlers+ assert "scheme" in dh._notconfigured+ assert (+ 'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for scheme "scheme"'+ in caplog.text+ )+def test_disabled_handler(self):handlers = {"scheme": None}crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})@@ -74,7 +101,11 @@ class TestLoad:def test_lazy_handlers(self):handlers = {"scheme": DummyLazyDH}crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})- dh = DownloadHandlers(crawler)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="DummyLazyDH doesn't define a 'lazy' attribute",+ ):+ dh = DownloadHandlers(crawler)assert "scheme" in dh._schemesassert "scheme" not in dh._handlersfor scheme in handlers: # force load lazy handler@@ -88,17 +119,13 @@ class TestFile:# add a special char to check that they are handled correctlyself.fd, self.tmpname = mkstemp(suffix="^")Path(self.tmpname).write_text("0123456789", encoding="utf-8")- self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler())+ download_handler = build_from_crawler(FileDownloadHandler, get_crawler())+ self.download_request = download_handler.download_requestdef teardown_method(self):os.close(self.fd)Path(self.tmpname).unlink()- async def download_request(self, request: Request) -> Response:- return await maybe_deferred_to_future(- self.download_handler.download_request(request, DefaultSpider())- )-@deferred_f_from_coro_fasync def test_download(self):request = Request(path_to_file_uri(self.tmpname))@@ -121,7 +148,7 @@ class HttpDownloadHandlerMock:def __init__(self, *args, **kwargs):pass- def download_request(self, request, spider):+ async def download_request(self, request):return request@@ -129,18 +156,17 @@ class HttpDownloadHandlerMock:class TestS3Anon:def setup_method(self):crawler = get_crawler()- self.s3reqh = build_from_crawler(- S3DownloadHandler,- crawler,- httpdownloadhandler=HttpDownloadHandlerMock,- # anon=True, # implicit- )+ with mock.patch(+ "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler",+ HttpDownloadHandlerMock,+ ):+ self.s3reqh = build_from_crawler(S3DownloadHandler, crawler)self.download_request = self.s3reqh.download_request- self.spider = DefaultSpider()- def test_anon_request(self):+ @deferred_f_from_coro_f+ async def test_anon_request(self):req = Request("s3://aws-publicdatasets/")- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert hasattr(self.s3reqh, "anon")assert self.s3reqh.anonassert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"@@ -148,26 +174,22 @@ class TestS3Anon:@pytest.mark.requires_botocoreclass TestS3:- download_handler_cls: type = S3DownloadHandler-- # test use same example keys than amazon developer guide- # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf- # and the tests described here are the examples from that manual-- AWS_ACCESS_KEY_ID = "0PN5J17HBGZHT7JJ3X82"- AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"-def setup_method(self):- crawler = get_crawler()- s3reqh = build_from_crawler(- S3DownloadHandler,- crawler,- aws_access_key_id=self.AWS_ACCESS_KEY_ID,- aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,- httpdownloadhandler=HttpDownloadHandlerMock,+ # test use same example keys than amazon developer guide+ # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf+ # and the tests described here are the examples from that manual+ crawler = get_crawler(+ settings_dict={+ "AWS_ACCESS_KEY_ID": "0PN5J17HBGZHT7JJ3X82",+ "AWS_SECRET_ACCESS_KEY": "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o",+ })+ with mock.patch(+ "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler",+ HttpDownloadHandlerMock,+ ):+ s3reqh = build_from_crawler(S3DownloadHandler, crawler)self.download_request = s3reqh.download_request- self.spider = DefaultSpider()@contextlib.contextmanagerdef _mocked_date(self, date):@@ -183,27 +205,20 @@ class TestS3:mock_formatdate.return_value = dateyield- def test_extra_kw(self):- crawler = get_crawler()- with pytest.raises((TypeError, NotConfigured)):- build_from_crawler(- S3DownloadHandler,- crawler,- extra_kw=True,- )-- def test_request_signing1(self):+ @deferred_f_from_coro_f+ async def test_request_signing1(self):# gets an object from the johnsmith bucket.date = "Tue, 27 Mar 2007 19:36:42 +0000"req = Request("s3://johnsmith/photos/puppy.jpg", headers={"Date": date})with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=")- def test_request_signing2(self):+ @deferred_f_from_coro_f+ async def test_request_signing2(self):# puts an object into the johnsmith bucket.date = "Tue, 27 Mar 2007 21:15:45 +0000"req = Request(@@ -216,13 +231,14 @@ class TestS3:},)with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=")- def test_request_signing3(self):+ @deferred_f_from_coro_f+ async def test_request_signing3(self):# lists the content of the johnsmith bucket.date = "Tue, 27 Mar 2007 19:42:41 +0000"req = Request(@@ -234,24 +250,26 @@ class TestS3:},)with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=")- def test_request_signing4(self):+ @deferred_f_from_coro_f+ async def test_request_signing4(self):# fetches the access control policy sub-resource for the 'johnsmith' bucket.date = "Tue, 27 Mar 2007 19:44:46 +0000"req = Request("s3://johnsmith/?acl", method="GET", headers={"Date": date})with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=")- def test_request_signing6(self):+ @deferred_f_from_coro_f+ async def test_request_signing6(self):# uploads an object to a CNAME style virtual hosted bucket with metadata.date = "Tue, 27 Mar 2007 21:06:08 +0000"req = Request(@@ -273,13 +291,14 @@ class TestS3:},)with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=")- def test_request_signing7(self):+ @deferred_f_from_coro_f+ async def test_request_signing7(self):# ensure that spaces are quoted properly before signingdate = "Tue, 27 Mar 2007 19:42:41 +0000"req = Request(@@ -288,22 +307,25 @@ class TestS3:headers={"Date": date},)with self._mocked_date(date):- httpreq = self.download_request(req, self.spider)+ httpreq = await self.download_request(req)assert (httpreq.headers["Authorization"]== b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=")[email protected](is_botocore_available(), reason="Requires not having botocore")+def test_s3_no_botocore() -> None:+ crawler = get_crawler()+ with pytest.raises(NotConfigured, match="missing botocore library"):+ build_from_crawler(S3DownloadHandler, crawler)++class TestDataURI:def setup_method(self):crawler = get_crawler()- self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler)-- async def download_request(self, request: Request) -> Response:- return await maybe_deferred_to_future(- self.download_handler.download_request(request, DefaultSpider())- )+ download_handler = build_from_crawler(DataURIDownloadHandler, crawler)+ self.download_request = download_handler.download_request@deferred_f_from_coro_fasync def test_response_attrs(self):diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex a2459911f..e325c0463 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -6,19 +6,18 @@ import gzipimport jsonimport sysfrom abc import ABC, abstractmethod+from contextlib import asynccontextmanagerfrom http import HTTPStatusfrom typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytest-from pytest_twisted import async_yield_fixturefrom testfixtures import LogCapturefrom twisted.internet import defer, errorfrom twisted.web._newclient import ResponseFailedfrom twisted.web.http import _DataLossfrom scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse-from scrapy.spiders import Spiderfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import (deferred_f_from_coro_f,@@ -40,27 +39,6 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer-async def download_request(- download_handler: DownloadHandlerProtocol,- request: Request,- spider: Spider = DefaultSpider(),-) -> Response:- return await maybe_deferred_to_future(- download_handler.download_request(request, spider)- )---async def close_dh(dh: DownloadHandlerProtocol) -> None:- # needed because the interface of close() is not clearly defined- if not hasattr(dh, "close"):- return- c = dh.close()- if c is None:- return- # covers coroutines and Deferreds; won't work if close() uses Futures inside- await c--class TestHttpBase(ABC):is_secure = False@@ -69,30 +47,32 @@ class TestHttpBase(ABC):def download_handler_cls(self) -> type[DownloadHandlerProtocol]:raise NotImplementedError- @async_yield_fixture- async def download_handler(self) -> AsyncGenerator[DownloadHandlerProtocol]:- dh = build_from_crawler(self.download_handler_cls, get_crawler())-- yield dh-- await close_dh(dh)+ @asynccontextmanager+ async def get_dh(+ self, settings_dict: dict[str, Any] | None = None+ ) -> AsyncGenerator[DownloadHandlerProtocol]:+ crawler = get_crawler(DefaultSpider, settings_dict)+ crawler.spider = crawler._create_spider()+ dh = build_from_crawler(self.download_handler_cls, crawler)+ try:+ yield dh+ finally:+ await dh.close()@deferred_f_from_coro_f- async def test_download(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"Works"@deferred_f_from_coro_f- async def test_download_head(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download_head(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure), method="HEAD")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b""@pytest.mark.parametrize(@@ -105,22 +85,18 @@ class TestHttpBase(ABC):)@deferred_f_from_coro_fasync def test_download_has_correct_http_status_code(- self,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,- http_status: HTTPStatus,+ self, mockserver: MockServer, http_status: HTTPStatus) -> None:request = Request(mockserver.url(f"/status?n={http_status.value}", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == http_status.value@deferred_f_from_coro_fasync def test_server_receives_correct_request_headers(- self,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,+ self, mockserver: MockServer) -> None:request_headers = {# common request headers@@ -136,7 +112,8 @@ class TestHttpBase(ABC):mockserver.url("/echo", is_secure=self.is_secure),headers=request_headers,)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == HTTPStatus.OKbody = json.loads(response.body.decode("utf-8"))assert "headers" in body@@ -146,9 +123,7 @@ class TestHttpBase(ABC):@deferred_f_from_coro_fasync def test_server_receives_correct_request_body(- self,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,+ self, mockserver: MockServer) -> None:request_body = {"message": "It works!",@@ -157,16 +132,15 @@ class TestHttpBase(ABC):mockserver.url("/echo", is_secure=self.is_secure),body=json.dumps(request_body),)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == HTTPStatus.OKbody = json.loads(response.body.decode("utf-8"))assert json.loads(body["body"]) == request_body@deferred_f_from_coro_fasync def test_download_has_correct_response_headers(- self,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,+ self, mockserver: MockServer) -> None:# these headers will be set on the response in the resource and returnedresponse_headers = {@@ -192,7 +166,8 @@ class TestHttpBase(ABC):headers={"content-type": "application/json"},body=json.dumps(response_headers),)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 200for header_name, header_value in response_headers.items():assert header_name in response.headers, (@@ -203,29 +178,24 @@ class TestHttpBase(ABC):)@deferred_f_from_coro_f- async def test_redirect_status(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_redirect_status(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/redirect", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 302@deferred_f_from_coro_f- async def test_redirect_status_head(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_redirect_status_head(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 302@deferred_f_from_coro_fasync def test_timeout_download_from_spider_nodata_rcvd(- self,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,- reactor_pytest: str,+ self, mockserver: MockServer, reactor_pytest: str) -> None:if reactor_pytest == "asyncio" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279@@ -236,15 +206,15 @@ class TestHttpBase(ABC):# client connects but no data is receivedmeta = {"download_timeout": 0.5}request = Request(mockserver.url("/wait", is_secure=self.is_secure), meta=meta)- d = deferred_from_coro(download_request(download_handler, request))- with pytest.raises((defer.TimeoutError, error.TimeoutError)):- await maybe_deferred_to_future(d)+ async with self.get_dh() as download_handler:+ d = deferred_from_coro(download_handler.download_request(request))+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):+ await maybe_deferred_to_future(d)@deferred_f_from_coro_fasync def test_timeout_download_from_spider_server_hangs(self,mockserver: MockServer,- download_handler: DownloadHandlerProtocol,reactor_pytest: str,) -> None:if reactor_pytest == "asyncio" and sys.platform == "win32":@@ -257,24 +227,21 @@ class TestHttpBase(ABC):request = Request(mockserver.url("/hang-after-headers", is_secure=self.is_secure), meta=meta)- d = deferred_from_coro(download_request(download_handler, request))- with pytest.raises((defer.TimeoutError, error.TimeoutError)):- await maybe_deferred_to_future(d)+ async with self.get_dh() as download_handler:+ d = deferred_from_coro(download_handler.download_request(request))+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):+ await maybe_deferred_to_future(d)@pytest.mark.parametrize("send_header", [True, False])@deferred_f_from_coro_f- async def test_host_header(- self,- send_header: bool,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,- ) -> None:+ async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None:host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}"request = Request(mockserver.url("/host", is_secure=self.is_secure),headers={"Host": host_port} if send_header else {},)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == host_port.encode()if send_header:assert request.headers.get("Host") == host_port.encode()@@ -283,7 +250,7 @@ class TestHttpBase(ABC):@deferred_f_from_coro_fasync def test_content_length_zero_bodyless_post_request_headers(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:"""Tests if "Content-Length: 0" is sent for bodyless POST requests.@@ -298,43 +265,43 @@ class TestHttpBase(ABC):request = Request(mockserver.url("/contentlength", is_secure=self.is_secure), method="POST")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"0"@deferred_f_from_coro_fasync def test_content_length_zero_bodyless_post_only_one(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:request = Request(mockserver.url("/echo", is_secure=self.is_secure), method="POST")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)headers = Headers(json.loads(response.text)["headers"])contentlengths = headers.getlist("Content-Length")assert len(contentlengths) == 1assert contentlengths == [b"0"]@deferred_f_from_coro_f- async def test_payload(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_payload(self, mockserver: MockServer) -> None:body = b"1" * 100 # PayloadResource requires body length to be 100request = Request(mockserver.url("/payload", is_secure=self.is_secure),method="POST",body=body,)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == body@deferred_f_from_coro_f- async def test_response_header_content_length(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_response_header_content_length(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure), method="GET")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.headers[b"content-length"] == b"5"@pytest.mark.parametrize(@@ -351,25 +318,24 @@ class TestHttpBase(ABC):body: bytes,response_class: type[Response],mockserver: MockServer,- download_handler: DownloadHandlerProtocol,) -> None:request = Request(mockserver.url(f"/{filename}", is_secure=self.is_secure), body=body)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert type(response) is response_class # pylint: disable=unidiomatic-typecheck@deferred_f_from_coro_f- async def test_get_duplicate_header(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_get_duplicate_header(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]@deferred_f_from_coro_fasync def test_download_is_not_automatically_gzip_decoded(- self, download_handler: DownloadHandlerProtocol, mockserver: MockServer+ self, mockserver: MockServer) -> None:"""Test download handler does not automatically decode content using the scheme provided in Content-Encoding header"""@@ -382,7 +348,8 @@ class TestHttpBase(ABC):"accept-encoding": "gzip",},)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 200@@ -402,7 +369,7 @@ class TestHttpBase(ABC):@deferred_f_from_coro_fasync def test_no_cookie_processing_or_persistence(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:cookie_name = "foo"cookie_value = "bar"@@ -413,34 +380,34 @@ class TestHttpBase(ABC):f"/set-cookie?{cookie_name}={cookie_value}", is_secure=self.is_secure))- response = await download_request(download_handler, request)- assert response.status == 200- set_cookie = response.headers.get(b"Set-Cookie")- assert set_cookie == f"{cookie_name}={cookie_value}".encode()+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.status == 200+ set_cookie = response.headers.get(b"Set-Cookie")+ assert set_cookie == f"{cookie_name}={cookie_value}".encode()- # check that cookies are not sent in the next request- request = Request(mockserver.url("/echo", is_secure=self.is_secure))- response = await download_request(download_handler, request)- assert response.status == 200- headers = Headers(json.loads(response.text)["headers"])- assert "Cookie" not in headers- assert "cookie" not in headers+ # check that cookies are not sent in the next request+ request = Request(mockserver.url("/echo", is_secure=self.is_secure))+ response = await download_handler.download_request(request)+ assert response.status == 200+ headers = Headers(json.loads(response.text)["headers"])+ assert "Cookie" not in headers+ assert "cookie" not in headersclass TestHttp11Base(TestHttpBase):"""HTTP 1.1 test case"""@deferred_f_from_coro_f- async def test_download_without_maxsize_limit(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"Works"@deferred_f_from_coro_fasync def test_response_class_choosing_request(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:"""Tests choosing of correct response typein case of Content-Type is empty but body contains text.@@ -449,7 +416,8 @@ class TestHttp11Base(TestHttpBase):request = Request(mockserver.url("/nocontenttype", is_secure=self.is_secure), body=body)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck@deferred_f_from_coro_f@@ -458,24 +426,19 @@ class TestHttp11Base(TestHttpBase):# 10 is minimal size for this request and the limit is only counted on# response body. (regardless of headers)- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 5})- download_handler = build_from_crawler(self.download_handler_cls, crawler)- response = await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"Works"- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 4})- download_handler = build_from_crawler(self.download_handler_cls, crawler)-- with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 4}) as download_handler:+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await download_handler.download_request(request)@deferred_f_from_coro_fasync def test_download_with_maxsize_very_large_file(self, mockserver: MockServer) -> None:# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 1_500})- download_handler = build_from_crawler(self.download_handler_cls, crawler)with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure)@@ -484,8 +447,9 @@ class TestHttp11Base(TestHttpBase):def check(logger: mock.Mock) -> None:logger.warning.assert_called_once_with(mock.ANY, mock.ANY)- with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await download_handler.download_request(request)# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queue@@ -496,69 +460,61 @@ class TestHttp11Base(TestHttpBase):await maybe_deferred_to_future(d)@deferred_f_from_coro_f- async def test_download_with_maxsize_per_req(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None:meta = {"download_maxsize": 2}request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta)- with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await download_handler.download_request(request)@deferred_f_from_coro_fasync def test_download_with_small_maxsize_via_setting(self, mockserver: MockServer) -> None:- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 2})- download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(mockserver.url("/text", is_secure=self.is_secure))- with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 2}) as download_handler:+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await download_handler.download_request(request)@deferred_f_from_coro_fasync def test_download_with_large_maxsize_via_setting(self, mockserver: MockServer) -> None:- crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 5})- download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(mockserver.url("/text", is_secure=self.is_secure))- response = await download_request(download_handler, request, Spider("foo"))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 100}) as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"Works"@deferred_f_from_coro_f- async def test_download_chunked_content(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download_chunked_content(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/chunked", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"chunked content\n"@pytest.mark.parametrize("url", ["broken", "broken-chunked"])@deferred_f_from_coro_fasync def test_download_cause_data_loss(- self,- url: str,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,+ self, url: str, mockserver: MockServer) -> None:# TODO: this one checks for Twisted-specific exceptionsrequest = Request(mockserver.url(f"/{url}", is_secure=self.is_secure))- with pytest.raises(ResponseFailed) as exc_info:- await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ with pytest.raises(ResponseFailed) as exc_info:+ await download_handler.download_request(request)assert any(r.check(_DataLoss) for r in exc_info.value.reasons)@pytest.mark.parametrize("url", ["broken", "broken-chunked"])@deferred_f_from_coro_fasync def test_download_allow_data_loss(- self,- url: str,- mockserver: MockServer,- download_handler: DownloadHandlerProtocol,+ self, url: str, mockserver: MockServer) -> None:request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure),meta={"download_fail_on_dataloss": False},)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.flags == ["dataloss"]@pytest.mark.parametrize("url", ["broken", "broken-chunked"])@@ -566,27 +522,20 @@ class TestHttp11Base(TestHttpBase):async def test_download_allow_data_loss_via_setting(self, url: str, mockserver: MockServer) -> None:- crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})- download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure))- try:- response = await maybe_deferred_to_future(- download_handler.download_request(request, DefaultSpider())- )- finally:- d = download_handler.close() # type: ignore[attr-defined]- if d is not None:- await maybe_deferred_to_future(d)+ async with self.get_dh(+ {"DOWNLOAD_FAIL_ON_DATALOSS": False}+ ) as download_handler:+ response = await download_handler.download_request(request)assert response.flags == ["dataloss"]@deferred_f_from_coro_f- async def test_protocol(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_protocol(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/host", is_secure=self.is_secure), method="GET")- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.protocol == "HTTP/1.1"@@ -600,24 +549,16 @@ class TestHttps11Base(TestHttp11Base):@deferred_f_from_coro_fasync def test_tls_logging(self, mockserver: MockServer) -> None:- crawler = get_crawler(- settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}- )- download_handler = build_from_crawler(self.download_handler_cls, crawler)- try:+ request = Request(mockserver.url("/text", is_secure=self.is_secure))+ async with self.get_dh(+ {"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}+ ) as download_handler:with LogCapture() as log_capture:- request = Request(mockserver.url("/text", is_secure=self.is_secure))- response = await maybe_deferred_to_future(- download_handler.download_request(request, DefaultSpider())- )- assert response.body == b"Works"- log_capture.check_present(- ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)- )- finally:- d = download_handler.close() # type: ignore[attr-defined]- if d is not None:- await maybe_deferred_to_future(d)+ response = await download_handler.download_request(request)+ assert response.body == b"Works"+ log_capture.check_present(+ ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)+ )class TestSimpleHttpsBase(ABC):@@ -645,25 +586,25 @@ class TestSimpleHttpsBase(ABC):def download_handler_cls(self) -> type[DownloadHandlerProtocol]:raise NotImplementedError- @async_yield_fixture- async def download_handler(self) -> AsyncGenerator[DownloadHandlerProtocol]:+ @asynccontextmanager+ async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]:if self.cipher_string is not None:settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string}else:settings_dict = None- crawler = get_crawler(settings_dict=settings_dict)+ crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)+ crawler.spider = crawler._create_spider()dh = build_from_crawler(self.download_handler_cls, crawler)-- yield dh-- await close_dh(dh)+ try:+ yield dh+ finally:+ await dh.close()@deferred_f_from_coro_f- async def test_download(- self, url: str, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download(self, url: str) -> None:request = Request(url)- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.body == b"0123456789"@@ -748,65 +689,63 @@ class TestHttpProxyBase(ABC):with ProxyEchoMockServer() as proxy:yield proxy- @async_yield_fixture- async def download_handler(self) -> AsyncGenerator[DownloadHandlerProtocol]:- dh = build_from_crawler(self.download_handler_cls, get_crawler())-- yield dh-- await close_dh(dh)+ @asynccontextmanager+ async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ dh = build_from_crawler(self.download_handler_cls, crawler)+ try:+ yield dh+ finally:+ await dh.close()@deferred_f_from_coro_fasync def test_download_with_proxy(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:http_proxy = proxy_mockserver.url("", is_secure=self.is_secure)request = Request("http://example.com", meta={"proxy": http_proxy})- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 200assert response.url == request.urlassert response.body == self.expected_http_proxy_request_body@deferred_f_from_coro_fasync def test_download_without_proxy(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:request = Request(proxy_mockserver.url("/path/to/resource", is_secure=self.is_secure))- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 200assert response.url == request.urlassert response.body == b"/path/to/resource"@deferred_f_from_coro_fasync def test_download_with_proxy_https_timeout(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:if NON_EXISTING_RESOLVABLE:pytest.skip("Non-existing hosts are resolvable")http_proxy = proxy_mockserver.url("", is_secure=self.is_secure)domain = "https://no-such-domain.nosuch"request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})- with pytest.raises(error.TimeoutError) as exc_info:- await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ with pytest.raises(error.TimeoutError) as exc_info:+ await download_handler.download_request(request)assert domain in exc_info.value.osError@deferred_f_from_coro_fasync def test_download_with_proxy_without_http_scheme(- self,- proxy_mockserver: ProxyEchoMockServer,- download_handler: DownloadHandlerProtocol,+ self, proxy_mockserver: ProxyEchoMockServer) -> None:http_proxy = f"{proxy_mockserver.host}:{proxy_mockserver.port()}"request = Request("http://example.com", meta={"proxy": http_proxy})- response = await download_request(download_handler, request)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)assert response.status == 200assert response.url == request.urlassert response.body == self.expected_http_proxy_request_body