Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't consider the spider idle with self._slot.inprogress. (#8355)

Andrey Rakhmatullinbc824d2f467c
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 0fd6a22cb..0b5dce8cf 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -586,6 +586,8 @@ class ExecutionEngine:
raise RuntimeError("Engine slot not assigned")
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
return False
+ if self._slot.inprogress: # the downloader may not have these requests yet
+ return False
if self.downloader.active: # downloader has pending requests
return False
if self._start is not None: # not all start requests are handled
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 7088c17d4..8b39e9d0e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -20,7 +20,7 @@ from scrapy.exceptions import (
DownloadCancelledError,
IgnoreRequest,
)
-from scrapy.http import Request
+from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
@@ -31,6 +31,7 @@ from scrapy.utils.defer import (
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
+from tests.spiders import SingleRequestSpider
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.engine import (
@@ -531,6 +532,33 @@ class TestSpiderIdle:
await crawler.crawl_async()
assert crawler.stats.get_value("downloader/request_count") == 1
+ @coroutine_test
+ async def test_request_not_in_downloader(self) -> None:
+ """The spider is not idle while a request taken from the scheduler is
+ not in the downloader yet."""
+
+ class SlowFetchDownloader(Downloader):
+ def fetch(self, request: Request) -> Deferred[Response | Request]: # type: ignore[override]
+ return deferred_from_coro(self._slow_fetch(request))
+
+ async def _slow_fetch(self, request: Request) -> Response | Request:
+ await sleep(0.1)
+ return await maybe_deferred_to_future(super().fetch(request))
+
+ events: list[str] = []
+
+ def response_received() -> None:
+ events.append("response_received")
+
+ def spider_idle() -> None:
+ events.append("spider_idle")
+
+ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER": SlowFetchDownloader})
+ crawler.signals.connect(response_received, signals.response_received)
+ crawler.signals.connect(spider_idle, signals.spider_idle)
+ await crawler.crawl_async(seed="data:,")
+ assert events == ["response_received", "spider_idle"]
+
@coroutine_test
async def test_download_wrong_type(caplog: pytest.LogCaptureFixture) -> None: