Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not count body_file requests toward RESPONSE_MAX_ACTIVE_SIZE (#8466)
Adrian9ddd3a70c76d
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex d099cfb8e..41fa28609 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2049,12 +2049,14 @@ the limit.It counts the body size of responses that have been downloaded or returned by:ref:`downloader middlewares <topics-downloader-middleware>` and remain inmemory, and the :setting:`rough size <RESPONSE_ROUGH_SIZE>` of requests-currently being downloaded. While that total is above this value, Scrapy stops-taking requests from the scheduler. While the part of it that comes from-requests sent with :meth:`~scrapy.core.engine.ExecutionEngine.download_async`,-which include :ref:`awaited requests <inline-requests>` and those of-:ref:`media pipelines <topics-media-pipeline>`, is above this value, new-requests of that kind wait before being downloaded.+currently being downloaded. Requests that use :reqmeta:`body_file` do not+count, and neither do their responses. While that total is above this value,+Scrapy stops taking requests from the scheduler. While the part of it that+comes from requests sent with+:meth:`~scrapy.core.engine.ExecutionEngine.download_async`, which include+:ref:`awaited requests <inline-requests>` and those of :ref:`media pipelines+<topics-media-pipeline>`, is above this value, new requests of that kind wait+before being downloaded.Requests already being downloaded when the limit is reached still finish: up to:setting:`CONCURRENT_REQUESTS` from the scheduler, and up to this value divideddiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 762ed2ab5..b0a4be58e 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -264,6 +264,7 @@ class Downloader:# has resumed yet.self._size_blocked = Falseself._unscheduled_transfers = 0+ self._unscheduled_rough_size = 0self._size_bypass_call: CallLaterResult | None = Noneself._size_bypass_due = Falseself._size_bypass_warned = False@@ -534,6 +535,7 @@ class Downloader:slot.transferring.add(request)if request in self.middleware._unscheduled_requests:self._unscheduled_transfers += 1+ self._unscheduled_rough_size += self.middleware._rough_size(request)self._cancel_size_bypass()else:slot._scheduled_queued -= 1@@ -613,8 +615,7 @@ class Downloader:if not middleware._max_active_size:return Falsesize = (- middleware._unscheduled_response_active_size- + self._unscheduled_transfers * middleware._response_rough_size+ middleware._unscheduled_response_active_size + self._unscheduled_rough_size)return size >= middleware._max_active_size@@ -722,6 +723,7 @@ class Downloader:unscheduled = request in self.middleware._unscheduled_requestsif unscheduled:self._unscheduled_transfers -= 1+ self._unscheduled_rough_size -= self.middleware._rough_size(request)self._process_queue(slot)if unscheduled:self._resume_size_blocked()diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex d5d527c27..d33d87ade 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -121,14 +121,20 @@ class DownloaderMiddlewareManager(MiddlewareManager): # pylint: disable=abstracdef _total_active_size(self) -> int:return self._response_active_size + self._rough_active_size+ def _rough_size(self, request: Request) -> int:+ # The body of these responses is written to a file.+ if request.meta.get("body_file"):+ return 0+ return self._response_rough_size+def _count_rough_size(self, request: Request) -> None:# Counting the same request twice, which concurrent downloads of the# same request object would do, would leak its rough size, since only# one discount call can find it.if request in self._rough_sizes:return- self._rough_sizes[request] = self._response_rough_size- self._rough_active_size += self._response_rough_size+ size = self._rough_sizes[request] = self._rough_size(request)+ self._rough_active_size += sizedef _discount_rough_size(self, request: Request) -> None:self._rough_active_size -= self._rough_sizes.pop(request, 0)diff --git a/tests/test_downloader.py b/tests/test_downloader.pyindex 71e86a71f..857538f80 100644--- a/tests/test_downloader.py+++ b/tests/test_downloader.py@@ -2,6 +2,7 @@ import asyncioimport gcimport loggingimport warnings+from pathlib import Pathfrom typing import Anyimport pytest@@ -14,7 +15,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandlerfrom scrapy.core.engine import ExecutionEnginefrom scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning-from scrapy.http import HtmlResponse, Response+from scrapy.http import BodyFileResponse, HtmlResponse, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FilesPipelinefrom scrapy.utils.asyncio import call_later, sleep@@ -100,6 +101,16 @@ class _IgnoreResponse:raise IgnoreRequest+class _BodyFileHandler:+ lazy = False++ async def download_request(self, request: Request) -> BodyFileResponse:+ await sleep(0.01)+ body_file = request.meta["body_file"]+ Path(body_file).write_bytes(b"a" * 2048)+ return BodyFileResponse(request.url, body_file=body_file)++async def _crawl_without_gc(crawler: Crawler) -> None:# The log capture of pytest keeps the tracebacks of logged errors, and with# them the responses they refer to.@@ -366,6 +377,88 @@ class TestResponseRoughSize:}assert _backout_stats(crawler) == expected_stats+ @coroutine_test+ async def test_body_file(self, tmp_path):+ """Requests and responses whose body is written to a file do not count+ toward the limit."""++ class TestSpider(Spider):+ name = "test"+ custom_settings = {+ "DOWNLOAD_HANDLERS": {"https": _BodyFileHandler},+ "RESPONSE_MAX_ACTIVE_SIZE": 512,+ "RESPONSE_ROUGH_SIZE": 1024,+ }++ async def start(self):+ for n in range(2):+ yield Request(+ f"https://example.com/{n}",+ meta={"body_file": tmp_path / str(n)},+ )++ def parse(self, response):+ pass++ crawler = get_crawler(TestSpider)+ self.caplog.clear()+ with self.caplog.at_level("INFO"):+ await maybe_deferred_to_future(crawler.crawl())++ assert crawler.stats+ assert crawler.stats.get_value("response_received_count") == 2+ assert _count_backout_logs(self.caplog) == 0+ assert _backout_stats(crawler) == {}++ @pytest.mark.only_asyncio+ @coroutine_test+ async def test_body_file_download_async(self, tmp_path):+ """download_async() requests whose response body is written to a file+ are not held back by the limit."""+ active = 0+ max_active = 0++ class Handler(_BodyFileHandler):+ async def download_request(self, request: Request) -> BodyFileResponse:+ nonlocal active, max_active+ active += 1+ max_active = max(max_active, active)+ try:+ return await super().download_request(request)+ finally:+ active -= 1++ class TestSpider(Spider):+ name = "test"+ start_urls = ["data:,"]+ custom_settings = {+ "DOWNLOAD_HANDLERS": {"https": Handler},+ "RESPONSE_MAX_ACTIVE_SIZE": 512,+ "RESPONSE_ROUGH_SIZE": 1024,+ }++ async def parse(self, response):+ assert self.crawler.engine+ await asyncio.gather(+ *(+ self.crawler.engine.download_async(+ Request(+ f"https://example.com/{n}",+ meta={"body_file": tmp_path / str(n)},+ )+ )+ for n in range(2)+ )+ )+ yield {}++ crawler = get_crawler(TestSpider)+ await maybe_deferred_to_future(crawler.crawl())++ assert crawler.stats+ assert crawler.stats.get_value("item_scraped_count") == 1+ assert max_active == 2+class TestRequestBackout:@pytest.fixture(autouse=True)