Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow downloading response bodies into files (#7987)

Adrian832e37c8d41a
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 5ffeb8d03..7e7cdaf9b 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -288,6 +288,7 @@ HTTP/2 No (not supported by the library)
Bad header handling Fail (not supported by the library)
``response.certificate`` DER bytes
Per-request ``bindaddress`` No (not supported by the library)
+:reqmeta:`body_file` Yes
TLS implementation Standard library ``ssl``
=========================== =======================================
@@ -329,6 +330,7 @@ HTTP/2 Yes
Bad header handling Not applicable (HTTP/2 only)
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
+:reqmeta:`body_file` No (not implemented)
TLS implementation ``pyOpenSSL``/``cryptography``
=========================== ================================================
@@ -374,6 +376,7 @@ HTTP/2 No (implemented as a separate handler)
Bad header handling Skip bad, like web browsers do
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
+:reqmeta:`body_file` Yes
TLS implementation ``pyOpenSSL``/``cryptography``
=========================== ================================================
@@ -440,6 +443,7 @@ HTTP/2 Yes
Bad header handling Fail (not supported by the library)
``response.certificate`` DER bytes
Per-request ``bindaddress`` No (not supported by the library)
+:reqmeta:`body_file` Yes
TLS implementation Standard library ``ssl``
=========================== =======================================
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index bd302a42a..f75d03541 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -805,6 +805,7 @@ Those are:
* :reqmeta:`allow_offsite`
* :reqmeta:`autothrottle_dont_adjust_delay`
* :reqmeta:`bindaddress`
+* :reqmeta:`body_file`
* :reqmeta:`cache_timestamp`
* :reqmeta:`cookiejar`
* :reqmeta:`depth`
@@ -888,6 +889,39 @@ This meta key is not supported by
:class:`~scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler`, but
the :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported by them.
+.. reqmeta:: body_file
+
+body_file
+---------
+
+.. versionadded:: VERSION
+
+Path of a file where the response body is written as it is downloaded, or
+``True`` to write it to a new temporary file, so that responses of any size can
+be downloaded without fitting them in memory:
+
+.. code-block:: python
+
+ Request("https://example.com/big.zip", meta={"body_file": "big.zip"})
+
+The response is then a :class:`BodyFileResponse`.
+
+Each download attempt, e.g. a retry or a redirect, writes the file from the
+start. If the download fails, e.g. because the response exceeds
+:setting:`DOWNLOAD_MAXSIZE`, the file may hold part of the body, except for
+temporary files, which Scrapy deletes. Deleting temporary files of successful
+downloads is up to you.
+
+Scrapy does not ask for a compressed response, and does not decompress the file
+if the server sends one anyway, as indicated by the ``Content-Encoding``
+response header.
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` skips
+these requests.
+
+Download handlers that do not support this key, such as :ref:`some built-in
+ones <download-handlers-ref>`, make the request fail with
+:exc:`~scrapy.exceptions.NotSupported`.
+
.. reqmeta:: download_handler
download_handler
@@ -1509,6 +1543,12 @@ JsonResponse objects
<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
header.
+BodyFileResponse objects
+------------------------
+
+.. autoclass:: BodyFileResponse
+ :members: body_file
+
Other functions related to responses
====================================
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 68db7ed49..1bc90f6e4 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -4,11 +4,20 @@ from __future__ import annotations
import inspect
import logging
+import os
import warnings
+from pathlib import Path
+from tempfile import mkstemp
from typing import TYPE_CHECKING, Any, Protocol, cast
from scrapy import Request, Spider, signals
-from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning
+from scrapy.exceptions import (
+ NotConfigured,
+ NotSupported,
+ ScrapyDeprecationWarning,
+ StopDownload,
+)
+from scrapy.http import BodyFileResponse
from scrapy.utils.defer import (
deferred_from_coro,
ensure_awaitable,
@@ -190,6 +199,13 @@ class DownloadHandlers:
f"Unusable download handler {handler_id!r}: "
f"{self._notconfigured[handler_id]}"
)
+ if not request.meta.get("body_file"):
+ return await self._download(handler_id, handler, request)
+ return await self._download_to_file(handler_id, handler, request)
+
+ async def _download(
+ self, handler_id: str, handler: DownloadHandlerProtocol, request: Request
+ ) -> Response:
assert self._crawler.spider
if handler_id in self._old_style_handlers: # pragma: no cover
return await maybe_deferred_to_future(
@@ -200,6 +216,31 @@ class DownloadHandlers:
)
return await handler.download_request(request)
+ async def _download_to_file(
+ self, handler_id: str, handler: DownloadHandlerProtocol, request: Request
+ ) -> Response:
+ temporary = request.meta["body_file"] is True
+ if temporary:
+ fd, request.meta["body_file"] = mkstemp()
+ os.close(fd)
+ try:
+ response = await self._download(handler_id, handler, request)
+ if not isinstance(response, BodyFileResponse):
+ raise NotSupported(
+ f"{global_object_name(type(handler))} does not support the "
+ f"body_file request meta key."
+ )
+ except StopDownload:
+ # Its response, which points to the body file, reaches errbacks.
+ raise
+ except BaseException:
+ if temporary:
+ Path(request.meta["body_file"]).unlink() # noqa: ASYNC240
+ # Retries of the request get a new temporary file.
+ request.meta["body_file"] = True
+ raise
+ return response
+
async def _close(self) -> None:
for dh in self._handlers.values():
if not hasattr(dh, "close"): # pragma: no cover
diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.py
index 8d5c19931..ea4b661bb 100644
--- a/scrapy/core/downloader/handlers/_aioftp.py
+++ b/scrapy/core/downloader/handlers/_aioftp.py
@@ -20,7 +20,7 @@ from scrapy.exceptions import (
DownloadTimeoutError,
NotConfigured,
)
-from scrapy.http import Headers
+from scrapy.http import BodyFileResponse, Headers
from scrapy.utils._ssl import _log_sslobj_debug_info, _make_ssl_context
from scrapy.utils.httpobj import urlparse_cached
@@ -199,7 +199,7 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):
@staticmethod
def _local_filename(response: _AioftpResponse, request: Request) -> bytes | None:
- if response.data is None:
+ if response.data is None or request.meta.get("body_file"):
return None
return request.meta.get("ftp_local_filename")
@@ -230,6 +230,8 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):
) -> None:
if local_filename := self._local_filename(response, request):
size = Path(local_filename.decode()).stat().st_size
+ elif isinstance(result, BodyFileResponse):
+ size = Path(result.body_file).stat().st_size
else:
size = len(result.body)
result.headers.update({"Local Filename": local_filename or b"", "Size": size})
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 08eed5d25..050bf4df8 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -5,6 +5,7 @@ import logging
import time
from abc import abstractmethod
from io import BytesIO
+from pathlib import Path
from typing import (
TYPE_CHECKING,
Any,
@@ -39,6 +40,7 @@ from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
+ import os
from collections.abc import AsyncIterable
from contextlib import AbstractAsyncContextManager
from ipaddress import IPv4Address, IPv6Address
@@ -61,6 +63,7 @@ class _BaseResponseArgs(TypedDict):
certificate: NotRequired[Any]
ip_address: NotRequired[IPv4Address | IPv6Address | None]
protocol: str | None
+ body_file: NotRequired[str | os.PathLike[str] | None]
class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT]):
@@ -151,12 +154,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
def _open_body_buffer(self, response: _ResponseT, request: Request) -> BinaryIO:
"""Return a file object to write the response body into."""
+ if body_file := request.meta.get("body_file"):
+ return Path(body_file).open("wb")
return BytesIO()
def _get_body(
self, buffer: BinaryIO, response: _ResponseT, request: Request
) -> bytes:
"""Return the response body given the *buffer* it was written into."""
+ if request.meta.get("body_file"):
+ return b""
assert isinstance(buffer, BytesIO)
return buffer.getvalue()
@@ -198,23 +205,24 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
make_response_base_args = self._build_base_response_args(
response, request, headers
)
+ make_response_base_args["body_file"] = request.meta.get("body_file")
if self._tls_verbose_logging:
self._log_tls_info(response, request)
- if stop_download := check_stop_download(
- signals.headers_received,
- self.crawler,
- request,
- headers=headers,
- body_length=expected_size,
- ):
- return make_response(
- **make_response_base_args,
- stop_download=stop_download,
- )
-
with self._open_body_buffer(response, request) as response_body:
+ if stop_download := check_stop_download(
+ signals.headers_received,
+ self.crawler,
+ request,
+ headers=headers,
+ body_length=expected_size,
+ ):
+ return make_response(
+ **make_response_base_args,
+ stop_download=stop_download,
+ )
+
bytes_received = 0
try:
async for chunk in self._iter_body_chunks(response):
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index e3b8f6331..25a29ec30 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -41,7 +41,7 @@ from twisted.internet.protocol import ClientCreator, Protocol
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import DownloadCancelledError, DownloadFailedError, NotConfigured
-from scrapy.http import Response
+from scrapy.http import BodyFileResponse, Response
from scrapy.responsetypes import responsetypes
from scrapy.utils._download_handlers import (
get_maxsize_msg,
@@ -52,6 +52,8 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
+ import os
+
from twisted.protocols.ftp import FTPClient
from scrapy import Request
@@ -69,11 +71,16 @@ class ReceivedDataProtocol(Protocol):
request: Request | None = None,
maxsize: int = 0,
warnsize: int = 0,
+ body_file: str | os.PathLike[str] | None = None,
):
self.__filename: bytes | None = filename
- self.body: BinaryIO = (
- Path(filename.decode()).open("wb") if filename else BytesIO()
- )
+ self.body: BinaryIO
+ if filename:
+ self.body = Path(filename.decode()).open("wb")
+ elif body_file:
+ self.body = Path(body_file).open("wb")
+ else:
+ self.body = BytesIO()
self.size: int = 0
self._request: Request | None = request
self._maxsize: int = maxsize
@@ -105,10 +112,10 @@ class ReceivedDataProtocol(Protocol):
return self.__filename
def close(self) -> None:
- if self.filename:
- self.body.close()
- else:
+ if isinstance(self.body, BytesIO):
self.body.seek(0)
+ else:
+ self.body.close()
_CODE_RE = re.compile(r"\d+")
@@ -149,11 +156,13 @@ class FTPDownloadHandler(BaseDownloadHandler):
filepath = unquote(parsed_url.path)
is_listing = filepath == "" or filepath.endswith("/")
maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
+ body_file = request.meta.get("body_file")
protocol = ReceivedDataProtocol(
- request.meta.get("ftp_local_filename"),
+ None if body_file else request.meta.get("ftp_local_filename"),
request=request,
maxsize=maxsize,
warnsize=get_warnsize(request.meta, self._default_warnsize),
+ body_file=body_file,
)
try:
if is_listing:
@@ -170,6 +179,11 @@ class FTPDownloadHandler(BaseDownloadHandler):
httpcode = self.CODE_MAPPING.get(
m.group(), self.CODE_MAPPING["default"]
)
+ if body_file:
+ protocol.body.write(message.encode())
+ return BodyFileResponse(
+ url=request.url, status=httpcode, body_file=body_file
+ )
return Response(url=request.url, status=httpcode, body=message.encode())
except ConnectionLost as e:
raise DownloadFailedError(str(e)) from e
@@ -184,6 +198,10 @@ class FTPDownloadHandler(BaseDownloadHandler):
logger.warning(warning_msg)
raise DownloadCancelledError(warning_msg)
headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size}
+ if body_file:
+ return BodyFileResponse(
+ url=request.url, status=200, headers=headers, body_file=body_file
+ )
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 10a40e29f..d39d11f38 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,8 +8,9 @@ import re
from contextlib import suppress
from functools import partial
from io import BytesIO
+from pathlib import Path
from time import monotonic
-from typing import TYPE_CHECKING, Any, NotRequired, TypedDict, TypeVar, cast
+from typing import TYPE_CHECKING, Any, BinaryIO, NotRequired, TypedDict, TypeVar, cast
from urllib.parse import urldefrag
from twisted.internet import ssl
@@ -68,6 +69,8 @@ from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
+ import os
+
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IAddress, IConsumer
from twisted.web._newclient import Request as TxRequest
@@ -559,6 +562,8 @@ class _ScrapyAgent:
if isinstance(agent, _TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
+ body_file = request.meta.get("body_file")
+ bodybuf: BinaryIO = Path(body_file).open("wb") if body_file else BytesIO()
start_time = monotonic()
d: Deferred[IResponse] = agent.request(
method,
@@ -569,13 +574,19 @@ class _ScrapyAgent:
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
- d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
- d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)
+ d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request, bodybuf)
+ d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url, body_file)
# check download timeout
self._timeout_cl = reactor.callLater(timeout, self._cb_timed_out, d3, request)
d3.addBoth(self._cb_timeout, request, url, timeout)
+ d3.addBoth(self._cb_close, bodybuf)
return d3
+ @staticmethod
+ def _cb_close(result: _T, bodybuf: BinaryIO) -> _T:
+ bodybuf.close()
+ return result
+
def _cb_timed_out(self, d3: Deferred[Response], request: Request) -> None:
fail_on_dataloss = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
@@ -614,7 +625,7 @@ class _ScrapyAgent:
return headers
def _cb_bodyready(
- self, txresponse: TxResponse, request: Request
+ self, txresponse: TxResponse, request: Request, bodybuf: BinaryIO
) -> _ResultT | Deferred[_ResultT]:
if stop_download := check_stop_download(
signals.headers_received,
@@ -670,6 +681,7 @@ class _ScrapyAgent:
finished=d,
txresponse=txresponse,
request=request,
+ bodybuf=bodybuf,
maxsize=maxsize,
warnsize=warnsize,
reached_warnsize=reached_warnsize,
@@ -689,7 +701,12 @@ class _ScrapyAgent:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
- def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
+ def _cb_bodydone(
+ self,
+ result: _ResultT,
+ url: str,
+ body_file: str | os.PathLike[str] | None,
+ ) -> Response:
headers = self._headers_from_twisted_response(result["txresponse"])
version = result["txresponse"].version
protocol = f"{version[0].decode()}/{version[1]}.{version[2]}"
@@ -703,6 +720,7 @@ class _ScrapyAgent:
ip_address=result.get("ip_address"),
protocol=protocol,
stop_download=result.get("stop_download"),
+ body_file=body_file,
)
@@ -729,6 +747,7 @@ class _ResponseReader(Protocol):
finished: Deferred[_ResultT],
txresponse: TxResponse,
request: Request,
+ bodybuf: BinaryIO,
maxsize: int,
warnsize: int,
fail_on_dataloss: bool,
@@ -740,7 +759,7 @@ class _ResponseReader(Protocol):
self._finished: Deferred[_ResultT] = finished
self._txresponse: TxResponse = txresponse
self._request: Request = request
- self._bodybuf: BytesIO = BytesIO()
+ self._bodybuf: BinaryIO = bodybuf
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
@@ -757,7 +776,11 @@ class _ResponseReader(Protocol):
self._finished.callback(
{
"txresponse": self._txresponse,
- "body": self._bodybuf.getvalue(),
+ "body": (
+ self._bodybuf.getvalue()
+ if isinstance(self._bodybuf, BytesIO)
+ else b""
+ ),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e86117ab1..b192750c4 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -20,7 +20,7 @@ from scrapy.exceptions import (
IgnoreRequest,
ScrapyDeprecationWarning,
)
-from scrapy.http import Request, Response
+from scrapy.http import BodyFileResponse, Request, Response
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
@@ -94,7 +94,7 @@ class Slot:
# this Deferred will be awaited in enqueue_scrape()
deferred: Deferred[None] = Deferred()
self.queue.append((result, request, deferred))
- if isinstance(result, Response):
+ if isinstance(result, Response) and not isinstance(result, BodyFileResponse):
self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
else:
self.active_size += self.MIN_RESPONSE_SIZE
@@ -107,7 +107,7 @@ class Slot:
def finish_response(self, result: Response | Failure, request: Request) -> None:
self.active.remove(request)
- if isinstance(result, Response):
+ if isinstance(result, Response) and not isinstance(result, BodyFileResponse):
self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
else:
self.active_size -= self.MIN_RESPONSE_SIZE
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 998c6b812..abbd94a6e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -72,7 +72,9 @@ class HttpCacheMiddleware:
return None
# Skip uncacheable requests
- if not self.policy.should_cache_request(request):
+ if request.meta.get("body_file") or not self.policy.should_cache_request(
+ request
+ ):
request.meta["_dont_cache"] = True # flag as uncacheable
return None
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 8beee247e..3fa7db59e 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -12,7 +12,7 @@ from scrapy.exceptions import (
NotConfigured,
ScrapyDeprecationWarning,
)
-from scrapy.http import Response, TextResponse
+from scrapy.http import BodyFileResponse, Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._compression import (
_DECOMPRESSION_ERRORS,
@@ -90,6 +90,9 @@ class HttpCompressionMiddleware:
def process_request(
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
+ if request.meta.get("body_file"):
+ # Their body is written to a file, never decompressed.
+ return None
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
return None
@@ -97,7 +100,7 @@ class HttpCompressionMiddleware:
async def process_response(
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
- if request.method == "HEAD":
+ if request.method == "HEAD" or isinstance(response, BodyFileResponse):
return response
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index ba108be48..1a7fc89da 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -8,6 +8,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import global_object_name
from scrapy.utils.request import request_httprepr
+from scrapy.utils.response import _body_size
if TYPE_CHECKING:
from scrapy import Request, Spider
@@ -59,7 +60,7 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_count")
self.stats.inc_value(f"downloader/response_status_count/{response.status}")
reslen = (
- len(response.body)
+ _body_size(response)
+ get_header_size(response.headers)
+ get_status_size(response.status)
+ len(b"\r\n") # the empty line after the headers
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index e6532457e..44876ebcc 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Self
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured
+from scrapy.utils.response import _body_size
if TYPE_CHECKING:
from scrapy.core.downloader import Slot
@@ -71,7 +72,7 @@ class AutoThrottle:
self._adjust_delay(slot, latency, response)
if self.debug:
diff = slot.delay - olddelay
- size = len(response.body)
+ size = _body_size(response)
conc = len(slot.transferring)
logger.info(
"slot: %(slot)s | conc:%(concurrency)2d | "
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index 0e5c2b53b..ae6486c4e 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -11,12 +11,14 @@ from scrapy.http.request.form import FormRequest
from scrapy.http.request.json_request import JsonRequest
from scrapy.http.request.rpc import XmlRpcRequest
from scrapy.http.response import Response
+from scrapy.http.response.body_file import BodyFileResponse
from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
from scrapy.http.response.text import TextResponse
from scrapy.http.response.xml import XmlResponse
__all__ = [
+ "BodyFileResponse",
"FormRequest",
"Headers",
"HtmlResponse",
diff --git a/scrapy/http/response/body_file.py b/scrapy/http/response/body_file.py
new file mode 100644
index 000000000..0001c7bf0
--- /dev/null
+++ b/scrapy/http/response/body_file.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy.http.response import Response
+
+if TYPE_CHECKING:
+ import os
+
+
+class BodyFileResponse(Response):
+ """A :class:`~scrapy.http.Response` whose body was written to a file, as
+ requested with the :reqmeta:`body_file` request meta key.
+
+ .. versionadded:: VERSION
+
+ Reading :attr:`~scrapy.http.Response.body` raises :exc:`AttributeError`.
+ """
+
+ attributes: tuple[str, ...] = (
+ *(attr for attr in Response.attributes if attr != "body"),
+ "body_file",
+ )
+
+ __slots__ = ("body_file",)
+
+ def __init__(self, *args: Any, body_file: str | os.PathLike[str], **kwargs: Any):
+ if "body" in kwargs:
+ raise TypeError(f"{type(self).__name__} does not take a body.")
+ super().__init__(*args, **kwargs)
+ self.body_file: str | os.PathLike[str] = body_file
+ """Path of the file that holds the response body."""
+
+ @property
+ def body(self) -> bytes:
+ raise AttributeError(
+ f"The body of {self} is not in memory, it was written to "
+ f"{self.body_file!r}, as requested with the body_file request meta "
+ f"key."
+ )
diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py
index 57330bd2b..4c2819088 100644
--- a/scrapy/utils/_download_handlers.py
+++ b/scrapy/utils/_download_handlers.py
@@ -24,9 +24,11 @@ from scrapy.exceptions import (
StopDownload,
UnsupportedURLSchemeError,
)
+from scrapy.http import BodyFileResponse
from scrapy.utils.log import logger
if TYPE_CHECKING:
+ import os
from collections.abc import Iterator
from http.client import HTTPResponse
from http.cookiejar import Cookie
@@ -101,18 +103,26 @@ def make_response(
ip_address: IPv4Address | IPv6Address | None = None,
protocol: str | None = None,
stop_download: StopDownload | None = None,
+ body_file: str | os.PathLike[str] | None = None,
) -> Response:
- respcls = responsetypes.responsetypes.from_args(headers=headers, url=url, body=body)
- response = respcls(
- url=url,
- status=status,
- headers=headers,
- body=body,
- flags=flags,
- certificate=certificate,
- ip_address=ip_address,
- protocol=protocol,
- )
+ kwargs: dict[str, Any] = {
+ "url": url,
+ "status": status,
+ "headers": headers,
+ "flags": flags,
+ "certificate": certificate,
+ "ip_address": ip_address,
+ "protocol": protocol,
+ }
+ response: Response
+ if body_file is not None:
+ assert not body
+ response = BodyFileResponse(**kwargs, body_file=body_file)
+ else:
+ respcls = responsetypes.responsetypes.from_args(
+ headers=headers, url=url, body=body
+ )
+ response = respcls(**kwargs, body=body)
if stop_download:
response.flags.append("download_stopped")
if stop_download.fail:
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 984dc7064..fa8911aeb 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -10,6 +10,7 @@ import os
import re
import tempfile
import webbrowser
+from pathlib import Path
from typing import TYPE_CHECKING, Any
from weakref import WeakKeyDictionary
@@ -17,6 +18,7 @@ from twisted.web import http
from w3lib import html
from scrapy.http.headers import Headers
+from scrapy.http.response.body_file import BodyFileResponse
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
@@ -24,6 +26,13 @@ if TYPE_CHECKING:
from scrapy.http import Response, TextResponse
+
+def _body_size(response: Response) -> int:
+ if isinstance(response, BodyFileResponse):
+ return Path(response.body_file).stat().st_size
+ return len(response.body)
+
+
_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index bc32f7a1f..239ee4b3b 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,6 +4,7 @@ import json
import logging
import re
from ipaddress import IPv4Address
+from pathlib import Path
from socket import gethostbyname
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlparse
@@ -16,7 +17,7 @@ from twisted.python.failure import Failure
from scrapy import Spider, signals
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
-from scrapy.http import Request
+from scrapy.http import BodyFileResponse, Request
from scrapy.http.response import Response
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.engine import (
@@ -62,6 +63,8 @@ from tests.spiders import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
from scrapy.statscollectors import StatsCollector
from tests.mockserver.http import MockServer
@@ -76,6 +79,24 @@ class CloseSpiderProcessResponseMiddleware:
raise CloseSpider("my_reason", error=request.meta["close_error"])
+class BodyFileSpider(Spider):
+ name = "body_file"
+
+ def __init__(self, url: str, *args: Any, **kwargs: Any):
+ super().__init__(*args, **kwargs)
+ self.url = url
+ self.body: bytes | None = None
+
+ async def start(self) -> AsyncIterator[Any]:
+ yield Request(self.url, meta={"body_file": True})
+
+ def parse(self, response: Response) -> None:
+ assert isinstance(response, BodyFileResponse)
+ body_file = Path(response.body_file)
+ self.body = body_file.read_bytes()
+ body_file.unlink()
+
+
class TestCrawl:
@coroutine_test
async def test_follow_all(self, mockserver: MockServer) -> None:
@@ -463,6 +484,15 @@ with multiples lines
)
assert not crawler.crawling
+ @coroutine_test
+ async def test_body_file(self, mockserver: MockServer) -> None:
+ crawler = get_crawler(BodyFileSpider)
+ await crawler.crawl_async(mockserver.url("/text"))
+ assert isinstance(crawler.spider, BodyFileSpider)
+ assert crawler.spider.body == b"Works"
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/response_bytes")
+
@coroutine_test
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(SimpleSpider)
diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.py
index 610c76964..a3fd74fb5 100644
--- a/tests/test_downloader_handler_aioftp.py
+++ b/tests/test_downloader_handler_aioftp.py
@@ -19,7 +19,7 @@ from scrapy.exceptions import (
NotConfigured,
ResponseDataLossError,
)
-from scrapy.http import HtmlResponse, Request, TextResponse
+from scrapy.http import BodyFileResponse, HtmlResponse, Request, TextResponse
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
@@ -349,6 +349,19 @@ async def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None
assert (tmp_path / "file.txt").read_bytes() == FILES["file.txt"]
+@coroutine_test
+async def test_body_file(ftp_server: MockFTPServer, tmp_path: Path) -> None:
+ body_file = tmp_path / "file.txt"
+ request = Request(
+ ftp_server.url("file.txt"),
+ meta={"body_file": body_file, "ftp_local_filename": b"ignored"},
+ )
+ response = await _download(request)
+ assert isinstance(response, BodyFileResponse)
+ assert response.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
+ assert body_file.read_bytes() == FILES["file.txt"]
+
+
@asynccontextmanager
async def _socks_proxy(**kwargs: Any) -> AsyncGenerator[str]:
server = await asyncio.start_server(
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
index 0d4d42901..6c9895e2d 100644
--- a/tests/test_downloader_handler_twisted_ftp.py
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -15,7 +15,7 @@ from scrapy import Spider
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import DownloadCancelledError, DownloadFailedError, NotConfigured
-from scrapy.http import HtmlResponse, Request, Response
+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Response
from scrapy.http.response.text import TextResponse
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
@@ -145,6 +145,33 @@ class TestFTPBase(ABC):
assert local_path.read_bytes() == b"I have the power!"
local_path.unlink()
+ @deferred_f_from_coro_f
+ async def test_body_file(
+ self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path
+ ) -> None:
+ body_file = tmp_path / "body"
+ meta = {**self.req_meta, "body_file": body_file}
+ request = Request(url=server_url + "file.txt", meta=meta)
+ r = await dh.download_request(request)
+ assert isinstance(r, BodyFileResponse)
+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
+ assert body_file.read_bytes() == b"I have the power!"
+
+ @deferred_f_from_coro_f
+ async def test_body_file_nonexistent(
+ self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path
+ ) -> None:
+ body_file = tmp_path / "body"
+ meta = {**self.req_meta, "body_file": body_file}
+ request = Request(url=server_url + "nonexistent.txt", meta=meta)
+ r = await dh.download_request(request)
+ assert isinstance(r, BodyFileResponse)
+ assert r.status == 404
+ assert (
+ body_file.read_bytes()
+ == b"['550 nonexistent.txt: No such file or directory.']"
+ )
+
@pytest.mark.parametrize("filename", ["file.txt", "big.bin"])
@pytest.mark.parametrize("local", [False, True])
@deferred_f_from_coro_f
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 7f0b04b23..a88158a8d 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -91,6 +91,7 @@ def test_pool_keeps_newer_connection() -> None:
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
http2 = True
+ handler_supports_body_file = False
@coroutine_test
async def test_protocol(self, mockserver: MockServer) -> None:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ca9860271..95a6c524b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -30,9 +30,10 @@ from scrapy.exceptions import (
NotConfigured,
NotSupported,
ScrapyDeprecationWarning,
+ StopDownload,
UnsupportedURLSchemeError,
)
-from scrapy.http import Request, Response, TextResponse
+from scrapy.http import BodyFileResponse, Request, Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._download_handlers import wrap_twisted_exceptions
from scrapy.utils.boto import is_botocore_available
@@ -62,6 +63,18 @@ class SchemeDH:
return Response(request.url, body=b"scheme")
+class BodyFileDH:
+ lazy = False
+
+ async def download_request(self, request):
+ Path(request.meta["body_file"]).write_bytes(b"body")
+ if request.url.endswith("/fail"):
+ raise DownloadFailedError
+ if request.url.endswith("/stop"):
+ raise StopDownload(fail=True)
+ return BodyFileResponse(request.url, body_file=request.meta["body_file"])
+
+
class DummyLazyDH:
# Default (but deprecated) is lazy for backward compatibility
async def download_request(self, request):
@@ -198,6 +211,90 @@ class TestHandlerID:
await dh.download_request_async(request)
+class TestBodyFile:
+ @staticmethod
+ def _get_dh() -> DownloadHandlers:
+ crawler = get_crawler(
+ settings_dict={
+ "DOWNLOAD_HANDLERS": {"https": BodyFileDH},
+ "DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH},
+ }
+ )
+ crawler.spider = crawler._create_spider()
+ return DownloadHandlers(crawler)
+
+ @coroutine_test
+ async def test_path(self, tmp_path: Path) -> None:
+ body_file = tmp_path / "body"
+ request = Request("https://example.com", meta={"body_file": body_file})
+ response = await self._get_dh().download_request_async(request)
+ assert isinstance(response, BodyFileResponse)
+ assert response.body_file == body_file
+ assert body_file.read_bytes() == b"body"
+
+ @coroutine_test
+ async def test_temporary(self) -> None:
+ request = Request("https://example.com", meta={"body_file": True})
+ response = await self._get_dh().download_request_async(request)
+ assert isinstance(response, BodyFileResponse)
+ body_file = Path(response.body_file)
+ assert request.meta["body_file"] == response.body_file
+ assert body_file.read_bytes() == b"body"
+ body_file.unlink()
+
+ @pytest.mark.parametrize(
+ ("url", "meta", "exception"),
+ [
+ ("https://example.com/fail", {}, DownloadFailedError),
+ ("https://example.com", {"download_handler": "named"}, NotSupported),
+ ],
+ )
+ @coroutine_test
+ async def test_temporary_failure(
+ self, url: str, meta: dict[str, str], exception: type[Exception]
+ ) -> None:
+ request = Request(url, meta={**meta, "body_file": True})
+ created: list[str] = []
+ real_mkstemp = mkstemp
+
+ def mock_mkstemp() -> tuple[int, str]:
+ fd, path = real_mkstemp()
+ created.append(path)
+ return fd, path
+
+ with (
+ mock.patch("scrapy.core.downloader.handlers.mkstemp", mock_mkstemp),
+ pytest.raises(exception),
+ ):
+ await self._get_dh().download_request_async(request)
+ assert request.meta["body_file"] is True
+ assert len(created) == 1
+ assert not Path(created[0]).exists()
+
+ @coroutine_test
+ async def test_temporary_stop_download(self) -> None:
+ request = Request("https://example.com/stop", meta={"body_file": True})
+ with pytest.raises(StopDownload):
+ await self._get_dh().download_request_async(request)
+ body_file = Path(request.meta["body_file"])
+ assert body_file.read_bytes() == b"body"
+ body_file.unlink()
+
+ @coroutine_test
+ async def test_unsupported(self, tmp_path: Path) -> None:
+ body_file = tmp_path / "body"
+ request = Request(
+ "https://example.com",
+ meta={"body_file": body_file, "download_handler": "named"},
+ )
+ with pytest.raises(
+ NotSupported,
+ match="NamedDH does not support the body_file request meta key",
+ ):
+ await self._get_dh().download_request_async(request)
+ assert not body_file.exists()
+
+
class TestFile:
def setup_method(self):
# add a special char to check that they are handled correctly
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index d80dd945c..e90342515 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -17,7 +17,7 @@ import pytest
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.extensions.httpcache import DummyPolicy, rfc1123_to_epoch
-from scrapy.http import HtmlResponse, Request, Response
+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
@@ -281,6 +281,17 @@ class DummyPolicyTestMixin(PolicyTestMixin):
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
+ def test_body_file(self):
+ with self._middleware() as mw:
+ mw.process_response(self.request, self.response)
+ request = self.request.replace(meta={"body_file": "body"})
+ assert mw.process_request(request) is None
+ response = BodyFileResponse(request.url, body_file="body")
+ assert mw.process_response(request, response) is response
+ cached = mw.process_request(self.request)
+ assert isinstance(cached, HtmlResponse)
+ self.assertEqualResponse(self.response, cached)
+
def test_different_request_response_urls(self):
with self._middleware() as mw:
req = Request("http://host.com/path")
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index bf5a64099..40b5fc3d3 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -20,7 +20,7 @@ from scrapy.exceptions import (
NotConfigured,
ScrapyDeprecationWarning,
)
-from scrapy.http import HtmlResponse, Request, Response
+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded, gunzip
@@ -138,6 +138,22 @@ class TestHttpCompression:
self.mw.process_request(request)
assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)
+ def test_process_request_body_file(self):
+ request = Request("https://example.com", meta={"body_file": "body"})
+ self.mw.process_request(request)
+ assert "Accept-Encoding" not in request.headers
+
+ @coroutine_test
+ async def test_process_response_body_file(self):
+ request = Request("https://example.com", meta={"body_file": "body"})
+ response = BodyFileResponse(
+ "https://example.com",
+ headers={"Content-Encoding": "gzip"},
+ body_file="body",
+ request=request,
+ )
+ assert await self.mw.process_response(request, response) is response
+
@coroutine_test
async def test_process_response_gzip(self):
response = self._getresponse("gzip")
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 918f6b28b..f0588e056 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,14 +1,19 @@
from __future__ import annotations
+from typing import TYPE_CHECKING
+
import pytest
from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
from scrapy.exceptions import NotConfigured
-from scrapy.http import Request, Response
+from scrapy.http import BodyFileResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from pathlib import Path
+
class MyException(Exception):
pass
@@ -61,6 +66,15 @@ class TestDownloaderStats:
self.mw.process_response(self.req, res)
self.assertStatsEqual("downloader/response_bytes", len(raw_response))
+ def test_response_bytes_body_file(self, tmp_path: Path) -> None:
+ body_file = tmp_path / "body"
+ body_file.write_bytes(b"body")
+ res = BodyFileResponse("https://example.com", body_file=body_file)
+ self.mw.process_response(self.req, res)
+ self.assertStatsEqual(
+ "downloader/response_bytes", len(b"HTTP/1.1 200 OK\r\n\r\nbody")
+ )
+
def test_process_exception(self) -> None:
self.mw.process_exception(self.req, MyException())
self.assertStatsEqual("downloader/exception_count", 1)
diff --git a/tests/test_http_response_body_file.py b/tests/test_http_response_body_file.py
new file mode 100644
index 000000000..f4302d102
--- /dev/null
+++ b/tests/test_http_response_body_file.py
@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import pytest
+
+from scrapy.http import BodyFileResponse
+
+
+def test_body() -> None:
+ response = BodyFileResponse("https://example.com", body_file="body")
+ with pytest.raises(AttributeError, match="written to 'body'"):
+ response.body
+
+
+def test_replace() -> None:
+ response = BodyFileResponse("https://example.com", body_file="body")
+ new_response = response.replace(status=404)
+ assert isinstance(new_response, BodyFileResponse)
+ assert new_response.body_file == "body"
+ assert new_response.status == 404
+ with pytest.raises(TypeError, match="does not take a body"):
+ response.replace(body=b"body")
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index 8739436a0..5c73dbe43 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -31,7 +31,14 @@ from scrapy.exceptions import (
StopDownload,
UnsupportedURLSchemeError,
)
-from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
+from scrapy.http import (
+ BodyFileResponse,
+ Headers,
+ HtmlResponse,
+ Request,
+ Response,
+ TextResponse,
+)
from scrapy.utils._deps_compat import (
TWISTED_TLS_LIMITS_OFFBY1,
W3LIB_STRIPS_IPV6_BRACKETS,
@@ -74,6 +81,7 @@ class TestHttpBase(ABC):
http2: bool = False
# whether the handler supports per-request bindaddress
handler_supports_bindaddress_meta: bool = True
+ handler_supports_body_file: bool = True
# whether the handler can return the body received so far, instead of
# erroring out, when a request times out mid-response
handler_supports_partial_response_on_timeout: bool = False
@@ -1164,6 +1172,54 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request)
assert response.body == path.encode()
+ @coroutine_test
+ async def test_body_file(self, mockserver: MockServer, tmp_path: Path) -> None:
+ if not self.handler_supports_body_file:
+ pytest.skip("The handler does not support body_file")
+ body_file = tmp_path / "body"
+ request = Request(
+ mockserver.url("/text", is_secure=self.is_secure),
+ meta={"body_file": body_file},
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert isinstance(response, BodyFileResponse)
+ assert response.body_file == body_file
+ assert body_file.read_bytes() == b"Works"
+
+ @coroutine_test
+ async def test_body_file_empty_response(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
+ if not self.handler_supports_body_file:
+ pytest.skip("The handler does not support body_file")
+ body_file = tmp_path / "body"
+ request = Request(
+ mockserver.url("/text", is_secure=self.is_secure),
+ method="HEAD",
+ meta={"body_file": body_file},
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert isinstance(response, BodyFileResponse)
+ assert body_file.read_bytes() == b""
+
+ @coroutine_test
+ async def test_body_file_maxsize(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
+ if not self.handler_supports_body_file:
+ pytest.skip("The handler does not support body_file")
+ body_file = tmp_path / "body"
+ request = Request(
+ mockserver.url("/largechunkedfile", is_secure=self.is_secure),
+ meta={"body_file": body_file},
+ )
+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:
+ with pytest.raises(DownloadCancelledError):
+ await download_handler.download_request(request)
+ assert body_file.read_bytes() == b""
+
class TestHttpsBase(TestHttpBase, ABC):
is_secure = True