Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix the fast stop hanging when it cancels a download before it starts. (#8312)
Andrey Rakhmatullin15a3cf427b2c
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 38e04e2eb..b784b4efe 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -314,6 +314,7 @@ class Downloader:assert isinstance(download_dfd, Deferred)self._download_tasks[request] = download_dfddownload_dfd.addBoth(self._download_task_done, request)+ download_dfd.addBoth(self._release_unstarted, slot, request, queue_dfd)# prevent burst if inter-request delays were configuredif delay:self._process_queue(slot)@@ -323,6 +324,20 @@ class Downloader:self._download_tasks.pop(request, None)return result+ def _release_unstarted(+ self, result: Any, slot: Slot, request: Request, queue_dfd: Deferred[Response]+ ) -> Any:+ # Runs when the download task finishes. The task itself normally ends+ # the transfer (in _download()) and fires queue_dfd (in+ # _wait_for_download()). But a task that is cancelled before it starts+ # running, e.g. by stop(), never runs any of its code, so this does+ # both instead.+ if request in slot.transferring:+ self._end_transfer(slot, request)+ if not queue_dfd.called:+ queue_dfd.errback(result)+ return result+def _latercall(self, slot: Slot) -> None:slot.latercall = Noneself._process_queue(slot)@@ -349,13 +364,16 @@ class Downloader:# state to free up the transferring slot so it can be used by the# following requests (perhaps those which came from the downloader# middleware itself)- slot.transferring.remove(request)- self._process_queue(slot)- self.signals.send_catch_log(- signal=signals.request_left_downloader,- request=request,- spider=self.crawler.spider,- )+ self._end_transfer(slot, request)++ def _end_transfer(self, slot: Slot, request: Request) -> None:+ slot.transferring.remove(request)+ self._process_queue(slot)+ self.signals.send_catch_log(+ signal=signals.request_left_downloader,+ request=request,+ spider=self.crawler.spider,+ )async def _wait_for_download(self, slot: Slot, request: Request, queue_dfd: Deferred[Response]diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 7cad5f537..7b9d0dc93 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -41,7 +41,11 @@ from scrapy.http import Responsefrom scrapy.resolver import dnscachefrom scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONNfrom scrapy.utils.asyncio import sleep-from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future+from scrapy.utils.defer import (+ _schedule_coro,+ deferred_from_coro,+ maybe_deferred_to_future,+)from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpider@@ -736,6 +740,24 @@ async def test_fast_stop_with_download_in_flight(mockserver: MockServer) -> Noneassert crawler.stats.get_value("response_received_count") is None+@coroutine_test+async def test_fast_stop_before_download_starts() -> None:+ """A fast stop that cancels a download before the download starts ends+ the crawl and frees the slot."""+ crawler = get_crawler(SimpleSpider)++ def stop(request: Request, spider: Spider) -> None:+ # With asyncio, the stop runs before the download task that the+ # downloader creates right after this signal.+ _schedule_coro(crawler.stop_async(mode="fast"))++ crawler.signals.connect(stop, signals.request_reached_downloader)+ await crawler.crawl_async(url="data:,")+ (slot,) = crawler.engine.downloader.slots.values()+ assert not slot.active+ assert not slot.transferring++class ConcurrencyRecordingHandler(BaseDownloadHandler):"""Download handler that keeps each download in progress for a moment andrecords the most downloads it had in progress at once."""