Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix counting free downloader slots on asyncio. (#8257)

Andrey Rakhmatullinbb6dc3eee9a4
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 51ce5b1e8..041f876a2 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -304,6 +304,9 @@ class Downloader:
while slot.queue and slot.free_transfer_slots() > 0:
slot.lastseen = now
request, queue_dfd = slot.queue.popleft()
+ # Do this immediately so that following free_transfer_slots() calls
+ # account for this request.
+ slot.transferring.add(request)
download_dfd = deferred_from_coro(
self._wait_for_download(slot, request, queue_dfd)
)
@@ -325,7 +328,6 @@ class Downloader:
async def _download(self, slot: Slot, request: Request) -> Response:
# The order is very important for the following logic. Do not change!
- slot.transferring.add(request)
try:
# 1. Download the response
response: Response = await self.handlers.download_request_async(request)
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index dccb63bfd..4de25c15d 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -34,10 +34,13 @@ from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
_ScrapyClientContextFactory,
)
+from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer
from scrapy.exceptions import DownloadCancelledError, ScrapyDeprecationWarning
+from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN
+from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
@@ -56,7 +59,7 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
from twisted.web.iweb import IBodyProducer
- from scrapy.http import Response
+ from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
@@ -716,6 +719,47 @@ async def test_stop_cancels_pending_download_tasks() -> None:
assert failures[0].check(CancelledError)
+class ConcurrencyRecordingHandler(BaseDownloadHandler):
+ """Download handler that keeps each download in progress for a moment and
+ records the most downloads it had in progress at once."""
+
+ def __init__(self, crawler: Crawler):
+ super().__init__(crawler)
+ self._in_progress = 0
+
+ async def download_request(self, request: Request) -> Response:
+ self._in_progress += 1
+ self.crawler.stats.max_value("test/max_in_progress", self._in_progress)
+ await sleep(0.05)
+ self._in_progress -= 1
+ return Response(request.url)
+
+
+class ManyRequestsSpider(Spider):
+ name = "many_requests"
+
+ async def start(self):
+ for i in range(10):
+ yield Request(f"https://example.com/{i}")
+
+ def parse(self, response):
+ pass
+
+
+@coroutine_test
+async def test_slot_concurrency() -> None:
+ crawler = get_crawler(
+ ManyRequestsSpider,
+ {
+ "CONCURRENT_REQUESTS_PER_DOMAIN": 2,
+ "DOWNLOAD_HANDLERS": {"https": ConcurrencyRecordingHandler},
+ },
+ )
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("test/max_in_progress") == 2
+ assert crawler.stats.get_value("response_received_count") == 10
+
+
def test_deprecated_tls_module_names() -> None:
with pytest.warns(
ScrapyDeprecationWarning,