Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix the fast stop hanging with a download in flight on asyncio. (#8296)

Andrey Rakhmatullin03b2b7b25a8d
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 041f876a2..38e04e2eb 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import random
import warnings
from collections import deque
@@ -361,7 +362,7 @@ class Downloader:
) -> None:
try:
response = await self._download(slot, request)
- except Exception:
+ except (Exception, asyncio.CancelledError):
if not queue_dfd.called:
queue_dfd.errback(Failure())
else:
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 4de25c15d..7cad5f537 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -23,7 +23,7 @@ from twisted.web.client import (
)
from twisted.web.client import Response as TxResponse
-from scrapy import Request, Spider
+from scrapy import Request, Spider, signals
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader._idna_patch import (
_install_twisted_idna_fallbacks,
@@ -41,7 +41,7 @@ from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN
from scrapy.utils.asyncio import sleep
-from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
@@ -49,6 +49,7 @@ from scrapy.utils.test import get_crawler
from tests import IDNA_REJECTED_HOSTNAMES
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
+from tests.spiders import SimpleSpider
from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test
@@ -719,6 +720,22 @@ async def test_stop_cancels_pending_download_tasks() -> None:
assert failures[0].check(CancelledError)
+@coroutine_test
+async def test_fast_stop_with_download_in_flight(mockserver: MockServer) -> None:
+ crawler = get_crawler(SimpleSpider)
+ crawl_dfd = deferred_from_coro(
+ crawler.crawl_async(url=mockserver.url("/delay?n=10"))
+ )
+ await crawler.signals.wait_for(signals.headers_received)
+ # Let the download handler start waiting for the response body.
+ await sleep(0)
+ await crawler.stop_async(mode="fast")
+ await maybe_deferred_to_future(crawl_dfd)
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/request_dropped_count") == 1
+ assert crawler.stats.get_value("response_received_count") is None
+
+
class ConcurrencyRecordingHandler(BaseDownloadHandler):
"""Download handler that keeps each download in progress for a moment and
records the most downloads it had in progress at once."""