Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix the fast stop hanging with a download in flight on asyncio. (#8296)
Andrey Rakhmatullin03b2b7b25a8d
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 041f876a2..38e04e2eb 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport randomimport warningsfrom collections import deque@@ -361,7 +362,7 @@ class Downloader:) -> None:try:response = await self._download(slot, request)- except Exception:+ except (Exception, asyncio.CancelledError):if not queue_dfd.called:queue_dfd.errback(Failure())else:diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 4de25c15d..7cad5f537 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -23,7 +23,7 @@ from twisted.web.client import ()from twisted.web.client import Response as TxResponse-from scrapy import Request, Spider+from scrapy import Request, Spider, signalsfrom scrapy.core.downloader import Downloader, Slot, tlsfrom scrapy.core.downloader._idna_patch import (_install_twisted_idna_fallbacks,@@ -41,7 +41,7 @@ from scrapy.http import Responsefrom scrapy.resolver import dnscachefrom scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONNfrom scrapy.utils.asyncio import sleep-from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpider@@ -49,6 +49,7 @@ from scrapy.utils.test import get_crawlerfrom tests import IDNA_REJECTED_HOSTNAMESfrom tests.mockserver.http_resources import PayloadResource, put_childfrom tests.mockserver.utils import ssl_context_factory+from tests.spiders import SimpleSpiderfrom tests.utils.cmdline import procfrom tests.utils.decorators import coroutine_test@@ -719,6 +720,22 @@ async def test_stop_cancels_pending_download_tasks() -> None:assert failures[0].check(CancelledError)+@coroutine_test+async def test_fast_stop_with_download_in_flight(mockserver: MockServer) -> None:+ crawler = get_crawler(SimpleSpider)+ crawl_dfd = deferred_from_coro(+ crawler.crawl_async(url=mockserver.url("/delay?n=10"))+ )+ await crawler.signals.wait_for(signals.headers_received)+ # Let the download handler start waiting for the response body.+ await sleep(0)+ await crawler.stop_async(mode="fast")+ await maybe_deferred_to_future(crawl_dfd)+ assert crawler.stats+ assert crawler.stats.get_value("downloader/request_dropped_count") == 1+ assert crawler.stats.get_value("response_received_count") is None++class ConcurrencyRecordingHandler(BaseDownloadHandler):"""Download handler that keeps each download in progress for a moment andrecords the most downloads it had in progress at once."""