Scrapy, a fast high-level web crawling & scraping framework for Python.
Make ExecutionEngine._slot request changes symmetrical. (#8315)
Andrey Rakhmatullin60e85959ff08
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 46b759e7e..848e23348 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -522,6 +522,7 @@ class ExecutionEngine:self.signals.send_catch_log(signals.scheduler_empty)return False+ self._slot.add_request(request)d: Deferred[Response | Request] = self._download(request)d.addBoth(self._handle_downloader_output, request)d.addErrback(@@ -648,12 +649,13 @@ class ExecutionEngine:if self.spider is None:raise RuntimeError(f"No open spider to crawl: {request}")while True:+ assert self._slot is not None+ self._slot.add_request(request)try:response_or_request = await maybe_deferred_to_future(self._download(request))finally:- assert self._slot is not Noneself._slot.remove_request(request)if not isinstance(response_or_request, Request):return response_or_request@@ -666,7 +668,6 @@ class ExecutionEngine:assert self._slot is not None # typingassert self.spider is not None- self._slot.add_request(request)try:result: Response | Requestif self._downloader_fetch_needs_spider: