Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow downloading response bodies into files (#7987)
Adrian832e37c8d41a
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 5ffeb8d03..7e7cdaf9b 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -288,6 +288,7 @@ HTTP/2 No (not supported by the library)Bad header handling Fail (not supported by the library)``response.certificate`` DER bytesPer-request ``bindaddress`` No (not supported by the library)+:reqmeta:`body_file` YesTLS implementation Standard library ``ssl``=========================== =======================================@@ -329,6 +330,7 @@ HTTP/2 YesBad header handling Not applicable (HTTP/2 only)``response.certificate`` :class:`twisted.internet.ssl.Certificate` objectPer-request ``bindaddress`` Yes+:reqmeta:`body_file` No (not implemented)TLS implementation ``pyOpenSSL``/``cryptography``=========================== ================================================@@ -374,6 +376,7 @@ HTTP/2 No (implemented as a separate handler)Bad header handling Skip bad, like web browsers do``response.certificate`` :class:`twisted.internet.ssl.Certificate` objectPer-request ``bindaddress`` Yes+:reqmeta:`body_file` YesTLS implementation ``pyOpenSSL``/``cryptography``=========================== ================================================@@ -440,6 +443,7 @@ HTTP/2 YesBad header handling Fail (not supported by the library)``response.certificate`` DER bytesPer-request ``bindaddress`` No (not supported by the library)+:reqmeta:`body_file` YesTLS implementation Standard library ``ssl``=========================== =======================================diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex bd302a42a..f75d03541 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -805,6 +805,7 @@ Those are:* :reqmeta:`allow_offsite`* :reqmeta:`autothrottle_dont_adjust_delay`* :reqmeta:`bindaddress`+* :reqmeta:`body_file`* :reqmeta:`cache_timestamp`* :reqmeta:`cookiejar`* :reqmeta:`depth`@@ -888,6 +889,39 @@ This meta key is not supported by:class:`~scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler`, butthe :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported by them.+.. reqmeta:: body_file++body_file+---------++.. versionadded:: VERSION++Path of a file where the response body is written as it is downloaded, or+``True`` to write it to a new temporary file, so that responses of any size can+be downloaded without fitting them in memory:++.. code-block:: python++ Request("https://example.com/big.zip", meta={"body_file": "big.zip"})++The response is then a :class:`BodyFileResponse`.++Each download attempt, e.g. a retry or a redirect, writes the file from the+start. If the download fails, e.g. because the response exceeds+:setting:`DOWNLOAD_MAXSIZE`, the file may hold part of the body, except for+temporary files, which Scrapy deletes. Deleting temporary files of successful+downloads is up to you.++Scrapy does not ask for a compressed response, and does not decompress the file+if the server sends one anyway, as indicated by the ``Content-Encoding``+response header.+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` skips+these requests.++Download handlers that do not support this key, such as :ref:`some built-in+ones <download-handlers-ref>`, make the request fail with+:exc:`~scrapy.exceptions.NotSupported`.+.. reqmeta:: download_handlerdownload_handler@@ -1509,6 +1543,12 @@ JsonResponse objects<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`header.+BodyFileResponse objects+------------------------++.. autoclass:: BodyFileResponse+ :members: body_file+Other functions related to responses====================================diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 68db7ed49..1bc90f6e4 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -4,11 +4,20 @@ from __future__ import annotationsimport inspectimport logging+import osimport warnings+from pathlib import Path+from tempfile import mkstempfrom typing import TYPE_CHECKING, Any, Protocol, castfrom scrapy import Request, Spider, signals-from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning+from scrapy.exceptions import (+ NotConfigured,+ NotSupported,+ ScrapyDeprecationWarning,+ StopDownload,+)+from scrapy.http import BodyFileResponsefrom scrapy.utils.defer import (deferred_from_coro,ensure_awaitable,@@ -190,6 +199,13 @@ class DownloadHandlers:f"Unusable download handler {handler_id!r}: "f"{self._notconfigured[handler_id]}")+ if not request.meta.get("body_file"):+ return await self._download(handler_id, handler, request)+ return await self._download_to_file(handler_id, handler, request)++ async def _download(+ self, handler_id: str, handler: DownloadHandlerProtocol, request: Request+ ) -> Response:assert self._crawler.spiderif handler_id in self._old_style_handlers: # pragma: no coverreturn await maybe_deferred_to_future(@@ -200,6 +216,31 @@ class DownloadHandlers:)return await handler.download_request(request)+ async def _download_to_file(+ self, handler_id: str, handler: DownloadHandlerProtocol, request: Request+ ) -> Response:+ temporary = request.meta["body_file"] is True+ if temporary:+ fd, request.meta["body_file"] = mkstemp()+ os.close(fd)+ try:+ response = await self._download(handler_id, handler, request)+ if not isinstance(response, BodyFileResponse):+ raise NotSupported(+ f"{global_object_name(type(handler))} does not support the "+ f"body_file request meta key."+ )+ except StopDownload:+ # Its response, which points to the body file, reaches errbacks.+ raise+ except BaseException:+ if temporary:+ Path(request.meta["body_file"]).unlink() # noqa: ASYNC240+ # Retries of the request get a new temporary file.+ request.meta["body_file"] = True+ raise+ return response+async def _close(self) -> None:for dh in self._handlers.values():if not hasattr(dh, "close"): # pragma: no coverdiff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.pyindex 8d5c19931..ea4b661bb 100644--- a/scrapy/core/downloader/handlers/_aioftp.py+++ b/scrapy/core/downloader/handlers/_aioftp.py@@ -20,7 +20,7 @@ from scrapy.exceptions import (DownloadTimeoutError,NotConfigured,)-from scrapy.http import Headers+from scrapy.http import BodyFileResponse, Headersfrom scrapy.utils._ssl import _log_sslobj_debug_info, _make_ssl_contextfrom scrapy.utils.httpobj import urlparse_cached@@ -199,7 +199,7 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):@staticmethoddef _local_filename(response: _AioftpResponse, request: Request) -> bytes | None:- if response.data is None:+ if response.data is None or request.meta.get("body_file"):return Nonereturn request.meta.get("ftp_local_filename")@@ -230,6 +230,8 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):) -> None:if local_filename := self._local_filename(response, request):size = Path(local_filename.decode()).stat().st_size+ elif isinstance(result, BodyFileResponse):+ size = Path(result.body_file).stat().st_sizeelse:size = len(result.body)result.headers.update({"Local Filename": local_filename or b"", "Size": size})diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.pyindex 08eed5d25..050bf4df8 100644--- a/scrapy/core/downloader/handlers/_base_streaming.py+++ b/scrapy/core/downloader/handlers/_base_streaming.py@@ -5,6 +5,7 @@ import loggingimport timefrom abc import abstractmethodfrom io import BytesIO+from pathlib import Pathfrom typing import (TYPE_CHECKING,Any,@@ -39,6 +40,7 @@ from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandlerif TYPE_CHECKING:+ import osfrom collections.abc import AsyncIterablefrom contextlib import AbstractAsyncContextManagerfrom ipaddress import IPv4Address, IPv6Address@@ -61,6 +63,7 @@ class _BaseResponseArgs(TypedDict):certificate: NotRequired[Any]ip_address: NotRequired[IPv4Address | IPv6Address | None]protocol: str | None+ body_file: NotRequired[str | os.PathLike[str] | None]class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT]):@@ -151,12 +154,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])def _open_body_buffer(self, response: _ResponseT, request: Request) -> BinaryIO:"""Return a file object to write the response body into."""+ if body_file := request.meta.get("body_file"):+ return Path(body_file).open("wb")return BytesIO()def _get_body(self, buffer: BinaryIO, response: _ResponseT, request: Request) -> bytes:"""Return the response body given the *buffer* it was written into."""+ if request.meta.get("body_file"):+ return b""assert isinstance(buffer, BytesIO)return buffer.getvalue()@@ -198,23 +205,24 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])make_response_base_args = self._build_base_response_args(response, request, headers)+ make_response_base_args["body_file"] = request.meta.get("body_file")if self._tls_verbose_logging:self._log_tls_info(response, request)- if stop_download := check_stop_download(- signals.headers_received,- self.crawler,- request,- headers=headers,- body_length=expected_size,- ):- return make_response(- **make_response_base_args,- stop_download=stop_download,- )-with self._open_body_buffer(response, request) as response_body:+ if stop_download := check_stop_download(+ signals.headers_received,+ self.crawler,+ request,+ headers=headers,+ body_length=expected_size,+ ):+ return make_response(+ **make_response_base_args,+ stop_download=stop_download,+ )+bytes_received = 0try:async for chunk in self._iter_body_chunks(response):diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex e3b8f6331..25a29ec30 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -41,7 +41,7 @@ from twisted.internet.protocol import ClientCreator, Protocolfrom scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import DownloadCancelledError, DownloadFailedError, NotConfigured-from scrapy.http import Response+from scrapy.http import BodyFileResponse, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._download_handlers import (get_maxsize_msg,@@ -52,6 +52,8 @@ from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:+ import os+from twisted.protocols.ftp import FTPClientfrom scrapy import Request@@ -69,11 +71,16 @@ class ReceivedDataProtocol(Protocol):request: Request | None = None,maxsize: int = 0,warnsize: int = 0,+ body_file: str | os.PathLike[str] | None = None,):self.__filename: bytes | None = filename- self.body: BinaryIO = (- Path(filename.decode()).open("wb") if filename else BytesIO()- )+ self.body: BinaryIO+ if filename:+ self.body = Path(filename.decode()).open("wb")+ elif body_file:+ self.body = Path(body_file).open("wb")+ else:+ self.body = BytesIO()self.size: int = 0self._request: Request | None = requestself._maxsize: int = maxsize@@ -105,10 +112,10 @@ class ReceivedDataProtocol(Protocol):return self.__filenamedef close(self) -> None:- if self.filename:- self.body.close()- else:+ if isinstance(self.body, BytesIO):self.body.seek(0)+ else:+ self.body.close()_CODE_RE = re.compile(r"\d+")@@ -149,11 +156,13 @@ class FTPDownloadHandler(BaseDownloadHandler):filepath = unquote(parsed_url.path)is_listing = filepath == "" or filepath.endswith("/")maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)+ body_file = request.meta.get("body_file")protocol = ReceivedDataProtocol(- request.meta.get("ftp_local_filename"),+ None if body_file else request.meta.get("ftp_local_filename"),request=request,maxsize=maxsize,warnsize=get_warnsize(request.meta, self._default_warnsize),+ body_file=body_file,)try:if is_listing:@@ -170,6 +179,11 @@ class FTPDownloadHandler(BaseDownloadHandler):httpcode = self.CODE_MAPPING.get(m.group(), self.CODE_MAPPING["default"])+ if body_file:+ protocol.body.write(message.encode())+ return BodyFileResponse(+ url=request.url, status=httpcode, body_file=body_file+ )return Response(url=request.url, status=httpcode, body=message.encode())except ConnectionLost as e:raise DownloadFailedError(str(e)) from e@@ -184,6 +198,10 @@ class FTPDownloadHandler(BaseDownloadHandler):logger.warning(warning_msg)raise DownloadCancelledError(warning_msg)headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size}+ if body_file:+ return BodyFileResponse(+ url=request.url, status=200, headers=headers, body_file=body_file+ )body = protocol.filename or protocol.body.read()respcls = responsetypes.from_args(url=request.url, body=body)return respcls(url=request.url, status=200, body=body, headers=headers)diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 10a40e29f..d39d11f38 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -8,8 +8,9 @@ import refrom contextlib import suppressfrom functools import partialfrom io import BytesIO+from pathlib import Pathfrom time import monotonic-from typing import TYPE_CHECKING, Any, NotRequired, TypedDict, TypeVar, cast+from typing import TYPE_CHECKING, Any, BinaryIO, NotRequired, TypedDict, TypeVar, castfrom urllib.parse import urldefragfrom twisted.internet import ssl@@ -68,6 +69,8 @@ from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandlerif TYPE_CHECKING:+ import os+from twisted.internet.base import ReactorBasefrom twisted.internet.interfaces import IAddress, IConsumerfrom twisted.web._newclient import Request as TxRequest@@ -559,6 +562,8 @@ class _ScrapyAgent:if isinstance(agent, _TunnelingAgent):headers.removeHeader(b"Proxy-Authorization")bodyproducer = _RequestBodyProducer(request.body) if request.body else None+ body_file = request.meta.get("body_file")+ bodybuf: BinaryIO = Path(body_file).open("wb") if body_file else BytesIO()start_time = monotonic()d: Deferred[IResponse] = agent.request(method,@@ -569,13 +574,19 @@ class _ScrapyAgent:# set download latencyd.addCallback(self._cb_latency, request, start_time)# response body is ready to be consumed- d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)- d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)+ d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request, bodybuf)+ d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url, body_file)# check download timeoutself._timeout_cl = reactor.callLater(timeout, self._cb_timed_out, d3, request)d3.addBoth(self._cb_timeout, request, url, timeout)+ d3.addBoth(self._cb_close, bodybuf)return d3+ @staticmethod+ def _cb_close(result: _T, bodybuf: BinaryIO) -> _T:+ bodybuf.close()+ return result+def _cb_timed_out(self, d3: Deferred[Response], request: Request) -> None:fail_on_dataloss = request.meta.get("download_fail_on_dataloss", self._fail_on_dataloss@@ -614,7 +625,7 @@ class _ScrapyAgent:return headersdef _cb_bodyready(- self, txresponse: TxResponse, request: Request+ self, txresponse: TxResponse, request: Request, bodybuf: BinaryIO) -> _ResultT | Deferred[_ResultT]:if stop_download := check_stop_download(signals.headers_received,@@ -670,6 +681,7 @@ class _ScrapyAgent:finished=d,txresponse=txresponse,request=request,+ bodybuf=bodybuf,maxsize=maxsize,warnsize=warnsize,reached_warnsize=reached_warnsize,@@ -689,7 +701,12 @@ class _ScrapyAgent:# Abort connection immediately.txresponse._transport._producer.abortConnection()- def _cb_bodydone(self, result: _ResultT, url: str) -> Response:+ def _cb_bodydone(+ self,+ result: _ResultT,+ url: str,+ body_file: str | os.PathLike[str] | None,+ ) -> Response:headers = self._headers_from_twisted_response(result["txresponse"])version = result["txresponse"].versionprotocol = f"{version[0].decode()}/{version[1]}.{version[2]}"@@ -703,6 +720,7 @@ class _ScrapyAgent:ip_address=result.get("ip_address"),protocol=protocol,stop_download=result.get("stop_download"),+ body_file=body_file,)@@ -729,6 +747,7 @@ class _ResponseReader(Protocol):finished: Deferred[_ResultT],txresponse: TxResponse,request: Request,+ bodybuf: BinaryIO,maxsize: int,warnsize: int,fail_on_dataloss: bool,@@ -740,7 +759,7 @@ class _ResponseReader(Protocol):self._finished: Deferred[_ResultT] = finishedself._txresponse: TxResponse = txresponseself._request: Request = request- self._bodybuf: BytesIO = BytesIO()+ self._bodybuf: BinaryIO = bodybufself._maxsize: int = maxsizeself._warnsize: int = warnsizeself._fail_on_dataloss: bool = fail_on_dataloss@@ -757,7 +776,11 @@ class _ResponseReader(Protocol):self._finished.callback({"txresponse": self._txresponse,- "body": self._bodybuf.getvalue(),+ "body": (+ self._bodybuf.getvalue()+ if isinstance(self._bodybuf, BytesIO)+ else b""+ ),"flags": flags,"certificate": self._certificate,"ip_address": self._ip_address,diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex e86117ab1..b192750c4 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -20,7 +20,7 @@ from scrapy.exceptions import (IgnoreRequest,ScrapyDeprecationWarning,)-from scrapy.http import Request, Response+from scrapy.http import BodyFileResponse, Request, Responsefrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available@@ -94,7 +94,7 @@ class Slot:# this Deferred will be awaited in enqueue_scrape()deferred: Deferred[None] = Deferred()self.queue.append((result, request, deferred))- if isinstance(result, Response):+ if isinstance(result, Response) and not isinstance(result, BodyFileResponse):self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)else:self.active_size += self.MIN_RESPONSE_SIZE@@ -107,7 +107,7 @@ class Slot:def finish_response(self, result: Response | Failure, request: Request) -> None:self.active.remove(request)- if isinstance(result, Response):+ if isinstance(result, Response) and not isinstance(result, BodyFileResponse):self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)else:self.active_size -= self.MIN_RESPONSE_SIZEdiff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex 998c6b812..abbd94a6e 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -72,7 +72,9 @@ class HttpCacheMiddleware:return None# Skip uncacheable requests- if not self.policy.should_cache_request(request):+ if request.meta.get("body_file") or not self.policy.should_cache_request(+ request+ ):request.meta["_dont_cache"] = True # flag as uncacheablereturn Nonediff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 8beee247e..3fa7db59e 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -12,7 +12,7 @@ from scrapy.exceptions import (NotConfigured,ScrapyDeprecationWarning,)-from scrapy.http import Response, TextResponse+from scrapy.http import BodyFileResponse, Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._compression import (_DECOMPRESSION_ERRORS,@@ -90,6 +90,9 @@ class HttpCompressionMiddleware:def process_request(self, request: Request, spider: Spider | None = None) -> Request | Response | None:+ if request.meta.get("body_file"):+ # Their body is written to a file, never decompressed.+ return Nonerequest.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))return None@@ -97,7 +100,7 @@ class HttpCompressionMiddleware:async def process_response(self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:- if request.method == "HEAD":+ if request.method == "HEAD" or isinstance(response, BodyFileResponse):return responsecontent_encoding = response.headers.getlist("Content-Encoding")if content_encoding:diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex ba108be48..1a7fc89da 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -8,6 +8,7 @@ from scrapy.exceptions import NotConfiguredfrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.python import global_object_namefrom scrapy.utils.request import request_httprepr+from scrapy.utils.response import _body_sizeif TYPE_CHECKING:from scrapy import Request, Spider@@ -59,7 +60,7 @@ class DownloaderStats:self.stats.inc_value("downloader/response_count")self.stats.inc_value(f"downloader/response_status_count/{response.status}")reslen = (- len(response.body)+ _body_size(response)+ get_header_size(response.headers)+ get_status_size(response.status)+ len(b"\r\n") # the empty line after the headersdiff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex e6532457e..44876ebcc 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Selffrom scrapy import Request, Spider, signalsfrom scrapy.exceptions import NotConfigured+from scrapy.utils.response import _body_sizeif TYPE_CHECKING:from scrapy.core.downloader import Slot@@ -71,7 +72,7 @@ class AutoThrottle:self._adjust_delay(slot, latency, response)if self.debug:diff = slot.delay - olddelay- size = len(response.body)+ size = _body_size(response)conc = len(slot.transferring)logger.info("slot: %(slot)s | conc:%(concurrency)2d | "diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.pyindex 0e5c2b53b..ae6486c4e 100644--- a/scrapy/http/__init__.py+++ b/scrapy/http/__init__.py@@ -11,12 +11,14 @@ from scrapy.http.request.form import FormRequestfrom scrapy.http.request.json_request import JsonRequestfrom scrapy.http.request.rpc import XmlRpcRequestfrom scrapy.http.response import Response+from scrapy.http.response.body_file import BodyFileResponsefrom scrapy.http.response.html import HtmlResponsefrom scrapy.http.response.json import JsonResponsefrom scrapy.http.response.text import TextResponsefrom scrapy.http.response.xml import XmlResponse__all__ = [+ "BodyFileResponse","FormRequest","Headers","HtmlResponse",diff --git a/scrapy/http/response/body_file.py b/scrapy/http/response/body_file.pynew file mode 100644index 000000000..0001c7bf0--- /dev/null+++ b/scrapy/http/response/body_file.py@@ -0,0 +1,40 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++from scrapy.http.response import Response++if TYPE_CHECKING:+ import os+++class BodyFileResponse(Response):+ """A :class:`~scrapy.http.Response` whose body was written to a file, as+ requested with the :reqmeta:`body_file` request meta key.++ .. versionadded:: VERSION++ Reading :attr:`~scrapy.http.Response.body` raises :exc:`AttributeError`.+ """++ attributes: tuple[str, ...] = (+ *(attr for attr in Response.attributes if attr != "body"),+ "body_file",+ )++ __slots__ = ("body_file",)++ def __init__(self, *args: Any, body_file: str | os.PathLike[str], **kwargs: Any):+ if "body" in kwargs:+ raise TypeError(f"{type(self).__name__} does not take a body.")+ super().__init__(*args, **kwargs)+ self.body_file: str | os.PathLike[str] = body_file+ """Path of the file that holds the response body."""++ @property+ def body(self) -> bytes:+ raise AttributeError(+ f"The body of {self} is not in memory, it was written to "+ f"{self.body_file!r}, as requested with the body_file request meta "+ f"key."+ )diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.pyindex 57330bd2b..4c2819088 100644--- a/scrapy/utils/_download_handlers.py+++ b/scrapy/utils/_download_handlers.py@@ -24,9 +24,11 @@ from scrapy.exceptions import (StopDownload,UnsupportedURLSchemeError,)+from scrapy.http import BodyFileResponsefrom scrapy.utils.log import loggerif TYPE_CHECKING:+ import osfrom collections.abc import Iteratorfrom http.client import HTTPResponsefrom http.cookiejar import Cookie@@ -101,18 +103,26 @@ def make_response(ip_address: IPv4Address | IPv6Address | None = None,protocol: str | None = None,stop_download: StopDownload | None = None,+ body_file: str | os.PathLike[str] | None = None,) -> Response:- respcls = responsetypes.responsetypes.from_args(headers=headers, url=url, body=body)- response = respcls(- url=url,- status=status,- headers=headers,- body=body,- flags=flags,- certificate=certificate,- ip_address=ip_address,- protocol=protocol,- )+ kwargs: dict[str, Any] = {+ "url": url,+ "status": status,+ "headers": headers,+ "flags": flags,+ "certificate": certificate,+ "ip_address": ip_address,+ "protocol": protocol,+ }+ response: Response+ if body_file is not None:+ assert not body+ response = BodyFileResponse(**kwargs, body_file=body_file)+ else:+ respcls = responsetypes.responsetypes.from_args(+ headers=headers, url=url, body=body+ )+ response = respcls(**kwargs, body=body)if stop_download:response.flags.append("download_stopped")if stop_download.fail:diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 984dc7064..fa8911aeb 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -10,6 +10,7 @@ import osimport reimport tempfileimport webbrowser+from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom weakref import WeakKeyDictionary@@ -17,6 +18,7 @@ from twisted.web import httpfrom w3lib import htmlfrom scrapy.http.headers import Headers+from scrapy.http.response.body_file import BodyFileResponsefrom scrapy.utils.misc import load_objectif TYPE_CHECKING:@@ -24,6 +26,13 @@ if TYPE_CHECKING:from scrapy.http import Response, TextResponse++def _body_size(response: Response) -> int:+ if isinstance(response, BodyFileResponse):+ return Path(response.body_file).stat().st_size+ return len(response.body)++_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex bc32f7a1f..239ee4b3b 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -4,6 +4,7 @@ import jsonimport loggingimport refrom ipaddress import IPv4Address+from pathlib import Pathfrom socket import gethostbynamefrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urlencode, urlparse@@ -16,7 +17,7 @@ from twisted.python.failure import Failurefrom scrapy import Spider, signalsfrom scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload-from scrapy.http import Request+from scrapy.http import BodyFileResponse, Requestfrom scrapy.http.response import Responsefrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.engine import (@@ -62,6 +63,8 @@ from tests.spiders import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ from collections.abc import AsyncIterator+from scrapy.statscollectors import StatsCollectorfrom tests.mockserver.http import MockServer@@ -76,6 +79,24 @@ class CloseSpiderProcessResponseMiddleware:raise CloseSpider("my_reason", error=request.meta["close_error"])+class BodyFileSpider(Spider):+ name = "body_file"++ def __init__(self, url: str, *args: Any, **kwargs: Any):+ super().__init__(*args, **kwargs)+ self.url = url+ self.body: bytes | None = None++ async def start(self) -> AsyncIterator[Any]:+ yield Request(self.url, meta={"body_file": True})++ def parse(self, response: Response) -> None:+ assert isinstance(response, BodyFileResponse)+ body_file = Path(response.body_file)+ self.body = body_file.read_bytes()+ body_file.unlink()++class TestCrawl:@coroutine_testasync def test_follow_all(self, mockserver: MockServer) -> None:@@ -463,6 +484,15 @@ with multiples lines)assert not crawler.crawling+ @coroutine_test+ async def test_body_file(self, mockserver: MockServer) -> None:+ crawler = get_crawler(BodyFileSpider)+ await crawler.crawl_async(mockserver.url("/text"))+ assert isinstance(crawler.spider, BodyFileSpider)+ assert crawler.spider.body == b"Works"+ assert crawler.stats+ assert crawler.stats.get_value("downloader/response_bytes")+@coroutine_testasync def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:crawler = get_crawler(SimpleSpider)diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.pyindex 610c76964..a3fd74fb5 100644--- a/tests/test_downloader_handler_aioftp.py+++ b/tests/test_downloader_handler_aioftp.py@@ -19,7 +19,7 @@ from scrapy.exceptions import (NotConfigured,ResponseDataLossError,)-from scrapy.http import HtmlResponse, Request, TextResponse+from scrapy.http import BodyFileResponse, HtmlResponse, Request, TextResponsefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServer@@ -349,6 +349,19 @@ async def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> Noneassert (tmp_path / "file.txt").read_bytes() == FILES["file.txt"]+@coroutine_test+async def test_body_file(ftp_server: MockFTPServer, tmp_path: Path) -> None:+ body_file = tmp_path / "file.txt"+ request = Request(+ ftp_server.url("file.txt"),+ meta={"body_file": body_file, "ftp_local_filename": b"ignored"},+ )+ response = await _download(request)+ assert isinstance(response, BodyFileResponse)+ assert response.headers == {b"Local Filename": [b""], b"Size": [b"17"]}+ assert body_file.read_bytes() == FILES["file.txt"]++@asynccontextmanagerasync def _socks_proxy(**kwargs: Any) -> AsyncGenerator[str]:server = await asyncio.start_server(diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pyindex 0d4d42901..6c9895e2d 100644--- a/tests/test_downloader_handler_twisted_ftp.py+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -15,7 +15,7 @@ from scrapy import Spiderfrom scrapy.core.downloader.handlers.ftp import FTPDownloadHandlerfrom scrapy.crawler import Crawlerfrom scrapy.exceptions import DownloadCancelledError, DownloadFailedError, NotConfigured-from scrapy.http import HtmlResponse, Request, Response+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Responsefrom scrapy.http.response.text import TextResponsefrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawler@@ -145,6 +145,33 @@ class TestFTPBase(ABC):assert local_path.read_bytes() == b"I have the power!"local_path.unlink()+ @deferred_f_from_coro_f+ async def test_body_file(+ self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path+ ) -> None:+ body_file = tmp_path / "body"+ meta = {**self.req_meta, "body_file": body_file}+ request = Request(url=server_url + "file.txt", meta=meta)+ r = await dh.download_request(request)+ assert isinstance(r, BodyFileResponse)+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}+ assert body_file.read_bytes() == b"I have the power!"++ @deferred_f_from_coro_f+ async def test_body_file_nonexistent(+ self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path+ ) -> None:+ body_file = tmp_path / "body"+ meta = {**self.req_meta, "body_file": body_file}+ request = Request(url=server_url + "nonexistent.txt", meta=meta)+ r = await dh.download_request(request)+ assert isinstance(r, BodyFileResponse)+ assert r.status == 404+ assert (+ body_file.read_bytes()+ == b"['550 nonexistent.txt: No such file or directory.']"+ )+@pytest.mark.parametrize("filename", ["file.txt", "big.bin"])@pytest.mark.parametrize("local", [False, True])@deferred_f_from_coro_fdiff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 7f0b04b23..a88158a8d 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -91,6 +91,7 @@ def test_pool_keeps_newer_connection() -> None:class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):http2 = True+ handler_supports_body_file = False@coroutine_testasync def test_protocol(self, mockserver: MockServer) -> None:diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex ca9860271..95a6c524b 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -30,9 +30,10 @@ from scrapy.exceptions import (NotConfigured,NotSupported,ScrapyDeprecationWarning,+ StopDownload,UnsupportedURLSchemeError,)-from scrapy.http import Request, Response, TextResponse+from scrapy.http import BodyFileResponse, Request, Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._download_handlers import wrap_twisted_exceptionsfrom scrapy.utils.boto import is_botocore_available@@ -62,6 +63,18 @@ class SchemeDH:return Response(request.url, body=b"scheme")+class BodyFileDH:+ lazy = False++ async def download_request(self, request):+ Path(request.meta["body_file"]).write_bytes(b"body")+ if request.url.endswith("/fail"):+ raise DownloadFailedError+ if request.url.endswith("/stop"):+ raise StopDownload(fail=True)+ return BodyFileResponse(request.url, body_file=request.meta["body_file"])++class DummyLazyDH:# Default (but deprecated) is lazy for backward compatibilityasync def download_request(self, request):@@ -198,6 +211,90 @@ class TestHandlerID:await dh.download_request_async(request)+class TestBodyFile:+ @staticmethod+ def _get_dh() -> DownloadHandlers:+ crawler = get_crawler(+ settings_dict={+ "DOWNLOAD_HANDLERS": {"https": BodyFileDH},+ "DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH},+ }+ )+ crawler.spider = crawler._create_spider()+ return DownloadHandlers(crawler)++ @coroutine_test+ async def test_path(self, tmp_path: Path) -> None:+ body_file = tmp_path / "body"+ request = Request("https://example.com", meta={"body_file": body_file})+ response = await self._get_dh().download_request_async(request)+ assert isinstance(response, BodyFileResponse)+ assert response.body_file == body_file+ assert body_file.read_bytes() == b"body"++ @coroutine_test+ async def test_temporary(self) -> None:+ request = Request("https://example.com", meta={"body_file": True})+ response = await self._get_dh().download_request_async(request)+ assert isinstance(response, BodyFileResponse)+ body_file = Path(response.body_file)+ assert request.meta["body_file"] == response.body_file+ assert body_file.read_bytes() == b"body"+ body_file.unlink()++ @pytest.mark.parametrize(+ ("url", "meta", "exception"),+ [+ ("https://example.com/fail", {}, DownloadFailedError),+ ("https://example.com", {"download_handler": "named"}, NotSupported),+ ],+ )+ @coroutine_test+ async def test_temporary_failure(+ self, url: str, meta: dict[str, str], exception: type[Exception]+ ) -> None:+ request = Request(url, meta={**meta, "body_file": True})+ created: list[str] = []+ real_mkstemp = mkstemp++ def mock_mkstemp() -> tuple[int, str]:+ fd, path = real_mkstemp()+ created.append(path)+ return fd, path++ with (+ mock.patch("scrapy.core.downloader.handlers.mkstemp", mock_mkstemp),+ pytest.raises(exception),+ ):+ await self._get_dh().download_request_async(request)+ assert request.meta["body_file"] is True+ assert len(created) == 1+ assert not Path(created[0]).exists()++ @coroutine_test+ async def test_temporary_stop_download(self) -> None:+ request = Request("https://example.com/stop", meta={"body_file": True})+ with pytest.raises(StopDownload):+ await self._get_dh().download_request_async(request)+ body_file = Path(request.meta["body_file"])+ assert body_file.read_bytes() == b"body"+ body_file.unlink()++ @coroutine_test+ async def test_unsupported(self, tmp_path: Path) -> None:+ body_file = tmp_path / "body"+ request = Request(+ "https://example.com",+ meta={"body_file": body_file, "download_handler": "named"},+ )+ with pytest.raises(+ NotSupported,+ match="NamedDH does not support the body_file request meta key",+ ):+ await self._get_dh().download_request_async(request)+ assert not body_file.exists()++class TestFile:def setup_method(self):# add a special char to check that they are handled correctlydiff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex d80dd945c..e90342515 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -17,7 +17,7 @@ import pytestfrom scrapy.downloadermiddlewares.httpcache import HttpCacheMiddlewarefrom scrapy.exceptions import IgnoreRequestfrom scrapy.extensions.httpcache import DummyPolicy, rfc1123_to_epoch-from scrapy.http import HtmlResponse, Request, Response+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.test import get_crawler@@ -281,6 +281,17 @@ class DummyPolicyTestMixin(PolicyTestMixin):self.assertEqualResponse(self.response, response)assert "cached" in response.flags+ def test_body_file(self):+ with self._middleware() as mw:+ mw.process_response(self.request, self.response)+ request = self.request.replace(meta={"body_file": "body"})+ assert mw.process_request(request) is None+ response = BodyFileResponse(request.url, body_file="body")+ assert mw.process_response(request, response) is response+ cached = mw.process_request(self.request)+ assert isinstance(cached, HtmlResponse)+ self.assertEqualResponse(self.response, cached)+def test_different_request_response_urls(self):with self._middleware() as mw:req = Request("http://host.com/path")diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex bf5a64099..40b5fc3d3 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -20,7 +20,7 @@ from scrapy.exceptions import (NotConfigured,ScrapyDeprecationWarning,)-from scrapy.http import HtmlResponse, Request, Response+from scrapy.http import BodyFileResponse, HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spiderfrom scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded, gunzip@@ -138,6 +138,22 @@ class TestHttpCompression:self.mw.process_request(request)assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)+ def test_process_request_body_file(self):+ request = Request("https://example.com", meta={"body_file": "body"})+ self.mw.process_request(request)+ assert "Accept-Encoding" not in request.headers++ @coroutine_test+ async def test_process_response_body_file(self):+ request = Request("https://example.com", meta={"body_file": "body"})+ response = BodyFileResponse(+ "https://example.com",+ headers={"Content-Encoding": "gzip"},+ body_file="body",+ request=request,+ )+ assert await self.mw.process_response(request, response) is response+@coroutine_testasync def test_process_response_gzip(self):response = self._getresponse("gzip")diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 918f6b28b..f0588e056 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -1,14 +1,19 @@from __future__ import annotations+from typing import TYPE_CHECKING+import pytestfrom scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_sizefrom scrapy.exceptions import NotConfigured-from scrapy.http import Request, Response+from scrapy.http import BodyFileResponse, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler+if TYPE_CHECKING:+ from pathlib import Path+class MyException(Exception):pass@@ -61,6 +66,15 @@ class TestDownloaderStats:self.mw.process_response(self.req, res)self.assertStatsEqual("downloader/response_bytes", len(raw_response))+ def test_response_bytes_body_file(self, tmp_path: Path) -> None:+ body_file = tmp_path / "body"+ body_file.write_bytes(b"body")+ res = BodyFileResponse("https://example.com", body_file=body_file)+ self.mw.process_response(self.req, res)+ self.assertStatsEqual(+ "downloader/response_bytes", len(b"HTTP/1.1 200 OK\r\n\r\nbody")+ )+def test_process_exception(self) -> None:self.mw.process_exception(self.req, MyException())self.assertStatsEqual("downloader/exception_count", 1)diff --git a/tests/test_http_response_body_file.py b/tests/test_http_response_body_file.pynew file mode 100644index 000000000..f4302d102--- /dev/null+++ b/tests/test_http_response_body_file.py@@ -0,0 +1,21 @@+from __future__ import annotations++import pytest++from scrapy.http import BodyFileResponse+++def test_body() -> None:+ response = BodyFileResponse("https://example.com", body_file="body")+ with pytest.raises(AttributeError, match="written to 'body'"):+ response.body+++def test_replace() -> None:+ response = BodyFileResponse("https://example.com", body_file="body")+ new_response = response.replace(status=404)+ assert isinstance(new_response, BodyFileResponse)+ assert new_response.body_file == "body"+ assert new_response.status == 404+ with pytest.raises(TypeError, match="does not take a body"):+ response.replace(body=b"body")diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex 8739436a0..5c73dbe43 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -31,7 +31,14 @@ from scrapy.exceptions import (StopDownload,UnsupportedURLSchemeError,)-from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse+from scrapy.http import (+ BodyFileResponse,+ Headers,+ HtmlResponse,+ Request,+ Response,+ TextResponse,+)from scrapy.utils._deps_compat import (TWISTED_TLS_LIMITS_OFFBY1,W3LIB_STRIPS_IPV6_BRACKETS,@@ -74,6 +81,7 @@ class TestHttpBase(ABC):http2: bool = False# whether the handler supports per-request bindaddresshandler_supports_bindaddress_meta: bool = True+ handler_supports_body_file: bool = True# whether the handler can return the body received so far, instead of# erroring out, when a request times out mid-responsehandler_supports_partial_response_on_timeout: bool = False@@ -1164,6 +1172,54 @@ class TestHttpBase(ABC):response = await download_handler.download_request(request)assert response.body == path.encode()+ @coroutine_test+ async def test_body_file(self, mockserver: MockServer, tmp_path: Path) -> None:+ if not self.handler_supports_body_file:+ pytest.skip("The handler does not support body_file")+ body_file = tmp_path / "body"+ request = Request(+ mockserver.url("/text", is_secure=self.is_secure),+ meta={"body_file": body_file},+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert isinstance(response, BodyFileResponse)+ assert response.body_file == body_file+ assert body_file.read_bytes() == b"Works"++ @coroutine_test+ async def test_body_file_empty_response(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:+ if not self.handler_supports_body_file:+ pytest.skip("The handler does not support body_file")+ body_file = tmp_path / "body"+ request = Request(+ mockserver.url("/text", is_secure=self.is_secure),+ method="HEAD",+ meta={"body_file": body_file},+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert isinstance(response, BodyFileResponse)+ assert body_file.read_bytes() == b""++ @coroutine_test+ async def test_body_file_maxsize(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:+ if not self.handler_supports_body_file:+ pytest.skip("The handler does not support body_file")+ body_file = tmp_path / "body"+ request = Request(+ mockserver.url("/largechunkedfile", is_secure=self.is_secure),+ meta={"body_file": body_file},+ )+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:+ with pytest.raises(DownloadCancelledError):+ await download_handler.download_request(request)+ assert body_file.read_bytes() == b""+class TestHttpsBase(TestHttpBase, ABC):is_secure = True