Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix counting free downloader slots on asyncio. (#8257)
Andrey Rakhmatullinbb6dc3eee9a4
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 51ce5b1e8..041f876a2 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -304,6 +304,9 @@ class Downloader:while slot.queue and slot.free_transfer_slots() > 0:slot.lastseen = nowrequest, queue_dfd = slot.queue.popleft()+ # Do this immediately so that following free_transfer_slots() calls+ # account for this request.+ slot.transferring.add(request)download_dfd = deferred_from_coro(self._wait_for_download(slot, request, queue_dfd))@@ -325,7 +328,6 @@ class Downloader:async def _download(self, slot: Slot, request: Request) -> Response:# The order is very important for the following logic. Do not change!- slot.transferring.add(request)try:# 1. Download the responseresponse: Response = await self.handlers.download_request_async(request)diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex dccb63bfd..4de25c15d 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -34,10 +34,13 @@ from scrapy.core.downloader.contextfactory import (_load_context_factory_from_settings,_ScrapyClientContextFactory,)+from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.core.downloader.handlers.http11 import _RequestBodyProducerfrom scrapy.exceptions import DownloadCancelledError, ScrapyDeprecationWarning+from scrapy.http import Responsefrom scrapy.resolver import dnscachefrom scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN+from scrapy.utils.asyncio import sleepfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes@@ -56,7 +59,7 @@ if TYPE_CHECKING:from twisted.python.failure import Failurefrom twisted.web.iweb import IBodyProducer- from scrapy.http import Response+ from scrapy.crawler import Crawlerfrom tests.mockserver.http import MockServer@@ -716,6 +719,47 @@ async def test_stop_cancels_pending_download_tasks() -> None:assert failures[0].check(CancelledError)+class ConcurrencyRecordingHandler(BaseDownloadHandler):+ """Download handler that keeps each download in progress for a moment and+ records the most downloads it had in progress at once."""++ def __init__(self, crawler: Crawler):+ super().__init__(crawler)+ self._in_progress = 0++ async def download_request(self, request: Request) -> Response:+ self._in_progress += 1+ self.crawler.stats.max_value("test/max_in_progress", self._in_progress)+ await sleep(0.05)+ self._in_progress -= 1+ return Response(request.url)+++class ManyRequestsSpider(Spider):+ name = "many_requests"++ async def start(self):+ for i in range(10):+ yield Request(f"https://example.com/{i}")++ def parse(self, response):+ pass+++@coroutine_test+async def test_slot_concurrency() -> None:+ crawler = get_crawler(+ ManyRequestsSpider,+ {+ "CONCURRENT_REQUESTS_PER_DOMAIN": 2,+ "DOWNLOAD_HANDLERS": {"https": ConcurrencyRecordingHandler},+ },+ )+ await crawler.crawl_async()+ assert crawler.stats.get_value("test/max_in_progress") == 2+ assert crawler.stats.get_value("response_received_count") == 10++def test_deprecated_tls_module_names() -> None:with pytest.warns(ScrapyDeprecationWarning,