Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Do not count body_file requests toward RESPONSE_MAX_ACTIVE_SIZE (#8466)

Adrian9ddd3a70c76d
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d099cfb8e..41fa28609 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -2049,12 +2049,14 @@ the limit.
It counts the body size of responses that have been downloaded or returned by
:ref:`downloader middlewares <topics-downloader-middleware>` and remain in
memory, and the :setting:`rough size <RESPONSE_ROUGH_SIZE>` of requests
-currently being downloaded. While that total is above this value, Scrapy stops
-taking requests from the scheduler. While the part of it that comes from
-requests sent with :meth:`~scrapy.core.engine.ExecutionEngine.download_async`,
-which include :ref:`awaited requests <inline-requests>` and those of
-:ref:`media pipelines <topics-media-pipeline>`, is above this value, new
-requests of that kind wait before being downloaded.
+currently being downloaded. Requests that use :reqmeta:`body_file` do not
+count, and neither do their responses. While that total is above this value,
+Scrapy stops taking requests from the scheduler. While the part of it that
+comes from requests sent with
+:meth:`~scrapy.core.engine.ExecutionEngine.download_async`, which include
+:ref:`awaited requests <inline-requests>` and those of :ref:`media pipelines
+<topics-media-pipeline>`, is above this value, new requests of that kind wait
+before being downloaded.
Requests already being downloaded when the limit is reached still finish: up to
:setting:`CONCURRENT_REQUESTS` from the scheduler, and up to this value divided
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 762ed2ab5..b0a4be58e 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -264,6 +264,7 @@ class Downloader:
# has resumed yet.
self._size_blocked = False
self._unscheduled_transfers = 0
+ self._unscheduled_rough_size = 0
self._size_bypass_call: CallLaterResult | None = None
self._size_bypass_due = False
self._size_bypass_warned = False
@@ -534,6 +535,7 @@ class Downloader:
slot.transferring.add(request)
if request in self.middleware._unscheduled_requests:
self._unscheduled_transfers += 1
+ self._unscheduled_rough_size += self.middleware._rough_size(request)
self._cancel_size_bypass()
else:
slot._scheduled_queued -= 1
@@ -613,8 +615,7 @@ class Downloader:
if not middleware._max_active_size:
return False
size = (
- middleware._unscheduled_response_active_size
- + self._unscheduled_transfers * middleware._response_rough_size
+ middleware._unscheduled_response_active_size + self._unscheduled_rough_size
)
return size >= middleware._max_active_size
@@ -722,6 +723,7 @@ class Downloader:
unscheduled = request in self.middleware._unscheduled_requests
if unscheduled:
self._unscheduled_transfers -= 1
+ self._unscheduled_rough_size -= self.middleware._rough_size(request)
self._process_queue(slot)
if unscheduled:
self._resume_size_blocked()
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index d5d527c27..d33d87ade 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -121,14 +121,20 @@ class DownloaderMiddlewareManager(MiddlewareManager): # pylint: disable=abstrac
def _total_active_size(self) -> int:
return self._response_active_size + self._rough_active_size
+ def _rough_size(self, request: Request) -> int:
+ # The body of these responses is written to a file.
+ if request.meta.get("body_file"):
+ return 0
+ return self._response_rough_size
+
def _count_rough_size(self, request: Request) -> None:
# Counting the same request twice, which concurrent downloads of the
# same request object would do, would leak its rough size, since only
# one discount call can find it.
if request in self._rough_sizes:
return
- self._rough_sizes[request] = self._response_rough_size
- self._rough_active_size += self._response_rough_size
+ size = self._rough_sizes[request] = self._rough_size(request)
+ self._rough_active_size += size
def _discount_rough_size(self, request: Request) -> None:
self._rough_active_size -= self._rough_sizes.pop(request, 0)
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index 71e86a71f..857538f80 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -2,6 +2,7 @@ import asyncio
import gc
import logging
import warnings
+from pathlib import Path
from typing import Any
import pytest
@@ -14,7 +15,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.engine import ExecutionEngine
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
-from scrapy.http import HtmlResponse, Response
+from scrapy.http import BodyFileResponse, HtmlResponse, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FilesPipeline
from scrapy.utils.asyncio import call_later, sleep
@@ -100,6 +101,16 @@ class _IgnoreResponse:
raise IgnoreRequest
+class _BodyFileHandler:
+ lazy = False
+
+ async def download_request(self, request: Request) -> BodyFileResponse:
+ await sleep(0.01)
+ body_file = request.meta["body_file"]
+ Path(body_file).write_bytes(b"a" * 2048)
+ return BodyFileResponse(request.url, body_file=body_file)
+
+
async def _crawl_without_gc(crawler: Crawler) -> None:
# The log capture of pytest keeps the tracebacks of logged errors, and with
# them the responses they refer to.
@@ -366,6 +377,88 @@ class TestResponseRoughSize:
}
assert _backout_stats(crawler) == expected_stats
+ @coroutine_test
+ async def test_body_file(self, tmp_path):
+ """Requests and responses whose body is written to a file do not count
+ toward the limit."""
+
+ class TestSpider(Spider):
+ name = "test"
+ custom_settings = {
+ "DOWNLOAD_HANDLERS": {"https": _BodyFileHandler},
+ "RESPONSE_MAX_ACTIVE_SIZE": 512,
+ "RESPONSE_ROUGH_SIZE": 1024,
+ }
+
+ async def start(self):
+ for n in range(2):
+ yield Request(
+ f"https://example.com/{n}",
+ meta={"body_file": tmp_path / str(n)},
+ )
+
+ def parse(self, response):
+ pass
+
+ crawler = get_crawler(TestSpider)
+ self.caplog.clear()
+ with self.caplog.at_level("INFO"):
+ await maybe_deferred_to_future(crawler.crawl())
+
+ assert crawler.stats
+ assert crawler.stats.get_value("response_received_count") == 2
+ assert _count_backout_logs(self.caplog) == 0
+ assert _backout_stats(crawler) == {}
+
+ @pytest.mark.only_asyncio
+ @coroutine_test
+ async def test_body_file_download_async(self, tmp_path):
+ """download_async() requests whose response body is written to a file
+ are not held back by the limit."""
+ active = 0
+ max_active = 0
+
+ class Handler(_BodyFileHandler):
+ async def download_request(self, request: Request) -> BodyFileResponse:
+ nonlocal active, max_active
+ active += 1
+ max_active = max(max_active, active)
+ try:
+ return await super().download_request(request)
+ finally:
+ active -= 1
+
+ class TestSpider(Spider):
+ name = "test"
+ start_urls = ["data:,"]
+ custom_settings = {
+ "DOWNLOAD_HANDLERS": {"https": Handler},
+ "RESPONSE_MAX_ACTIVE_SIZE": 512,
+ "RESPONSE_ROUGH_SIZE": 1024,
+ }
+
+ async def parse(self, response):
+ assert self.crawler.engine
+ await asyncio.gather(
+ *(
+ self.crawler.engine.download_async(
+ Request(
+ f"https://example.com/{n}",
+ meta={"body_file": tmp_path / str(n)},
+ )
+ )
+ for n in range(2)
+ )
+ )
+ yield {}
+
+ crawler = get_crawler(TestSpider)
+ await maybe_deferred_to_future(crawler.crawl())
+
+ assert crawler.stats
+ assert crawler.stats.get_value("item_scraped_count") == 1
+ assert max_active == 2
+
class TestRequestBackout:
@pytest.fixture(autouse=True)