Scrapy, a fast high-level web crawling & scraping framework for Python.
Don't consider the spider idle with self._slot.inprogress. (#8355)
Andrey Rakhmatullinbc824d2f467c
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 0fd6a22cb..0b5dce8cf 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -586,6 +586,8 @@ class ExecutionEngine:raise RuntimeError("Engine slot not assigned")if not self.scraper.slot.is_idle(): # type: ignore[union-attr]return False+ if self._slot.inprogress: # the downloader may not have these requests yet+ return Falseif self.downloader.active: # downloader has pending requestsreturn Falseif self._start is not None: # not all start requests are handleddiff --git a/tests/test_engine.py b/tests/test_engine.pyindex 7088c17d4..8b39e9d0e 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -20,7 +20,7 @@ from scrapy.exceptions import (DownloadCancelledError,IgnoreRequest,)-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.asyncio import sleepfrom scrapy.utils.defer import (@@ -31,6 +31,7 @@ from scrapy.utils.defer import (from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler+from tests.spiders import SingleRequestSpiderfrom tests.utils.bases.engine import TestEngineBasefrom tests.utils.decorators import coroutine_test, inline_callbacks_testfrom tests.utils.engine import (@@ -531,6 +532,33 @@ class TestSpiderIdle:await crawler.crawl_async()assert crawler.stats.get_value("downloader/request_count") == 1+ @coroutine_test+ async def test_request_not_in_downloader(self) -> None:+ """The spider is not idle while a request taken from the scheduler is+ not in the downloader yet."""++ class SlowFetchDownloader(Downloader):+ def fetch(self, request: Request) -> Deferred[Response | Request]: # type: ignore[override]+ return deferred_from_coro(self._slow_fetch(request))++ async def _slow_fetch(self, request: Request) -> Response | Request:+ await sleep(0.1)+ return await maybe_deferred_to_future(super().fetch(request))++ events: list[str] = []++ def response_received() -> None:+ events.append("response_received")++ def spider_idle() -> None:+ events.append("spider_idle")++ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER": SlowFetchDownloader})+ crawler.signals.connect(response_received, signals.response_received)+ crawler.signals.connect(spider_idle, signals.spider_idle)+ await crawler.crawl_async(seed="data:,")+ assert events == ["response_received", "spider_idle"]+@coroutine_testasync def test_download_wrong_type(caplog: pytest.LogCaptureFixture) -> None: