Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix the fast stop hanging when it cancels a download before it starts. (#8312)

Andrey Rakhmatullin15a3cf427b2c
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 38e04e2eb..b784b4efe 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -314,6 +314,7 @@ class Downloader:
assert isinstance(download_dfd, Deferred)
self._download_tasks[request] = download_dfd
download_dfd.addBoth(self._download_task_done, request)
+ download_dfd.addBoth(self._release_unstarted, slot, request, queue_dfd)
# prevent burst if inter-request delays were configured
if delay:
self._process_queue(slot)
@@ -323,6 +324,20 @@ class Downloader:
self._download_tasks.pop(request, None)
return result
+ def _release_unstarted(
+ self, result: Any, slot: Slot, request: Request, queue_dfd: Deferred[Response]
+ ) -> Any:
+ # Runs when the download task finishes. The task itself normally ends
+ # the transfer (in _download()) and fires queue_dfd (in
+ # _wait_for_download()). But a task that is cancelled before it starts
+ # running, e.g. by stop(), never runs any of its code, so this does
+ # both instead.
+ if request in slot.transferring:
+ self._end_transfer(slot, request)
+ if not queue_dfd.called:
+ queue_dfd.errback(result)
+ return result
+
def _latercall(self, slot: Slot) -> None:
slot.latercall = None
self._process_queue(slot)
@@ -349,13 +364,16 @@ class Downloader:
# state to free up the transferring slot so it can be used by the
# following requests (perhaps those which came from the downloader
# middleware itself)
- slot.transferring.remove(request)
- self._process_queue(slot)
- self.signals.send_catch_log(
- signal=signals.request_left_downloader,
- request=request,
- spider=self.crawler.spider,
- )
+ self._end_transfer(slot, request)
+
+ def _end_transfer(self, slot: Slot, request: Request) -> None:
+ slot.transferring.remove(request)
+ self._process_queue(slot)
+ self.signals.send_catch_log(
+ signal=signals.request_left_downloader,
+ request=request,
+ spider=self.crawler.spider,
+ )
async def _wait_for_download(
self, slot: Slot, request: Request, queue_dfd: Deferred[Response]
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 7cad5f537..7b9d0dc93 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -41,7 +41,11 @@ from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN
from scrapy.utils.asyncio import sleep
-from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
+from scrapy.utils.defer import (
+ _schedule_coro,
+ deferred_from_coro,
+ maybe_deferred_to_future,
+)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
@@ -736,6 +740,24 @@ async def test_fast_stop_with_download_in_flight(mockserver: MockServer) -> None
assert crawler.stats.get_value("response_received_count") is None
+@coroutine_test
+async def test_fast_stop_before_download_starts() -> None:
+ """A fast stop that cancels a download before the download starts ends
+ the crawl and frees the slot."""
+ crawler = get_crawler(SimpleSpider)
+
+ def stop(request: Request, spider: Spider) -> None:
+ # With asyncio, the stop runs before the download task that the
+ # downloader creates right after this signal.
+ _schedule_coro(crawler.stop_async(mode="fast"))
+
+ crawler.signals.connect(stop, signals.request_reached_downloader)
+ await crawler.crawl_async(url="data:,")
+ (slot,) = crawler.engine.downloader.slots.values()
+ assert not slot.active
+ assert not slot.transferring
+
+
class ConcurrencyRecordingHandler(BaseDownloadHandler):
"""Download handler that keeps each download in progress for a moment and
records the most downloads it had in progress at once."""