Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix setting ExecutionEngine._slot to None on close. (#6910)
Andrey Rakhmatullinff7795b15936
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex d6c171247..0631f7feb 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -266,10 +266,10 @@ class ExecutionEngine:See :ref:`start-requests-lazy` for an example."""- assert self._slot is not None # typingassert self.scraper.slot is not None # typingreturn (not self.running+ or not self._slotor bool(self._slot.closing)or self.downloader.needs_backout()or self.scraper.slot.needs_backout()@@ -530,10 +530,16 @@ class ExecutionEngine:))- dfd.addBoth(lambda _: setattr(self, "slot", None))+ def unassign_slot(_: Any) -> None:+ self._slot = None++ dfd.addBoth(unassign_slot)dfd.addErrback(log_failure("Error while unassigning slot"))- dfd.addBoth(lambda _: setattr(self, "spider", None))+ def unassign_spider(_: Any) -> None:+ self.spider = None++ dfd.addBoth(unassign_spider)dfd.addErrback(log_failure("Error while unassigning spider"))dfd.addBoth(lambda _: self._spider_closed_callback(spider))diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex bfb8eeced..4e82ad4ba 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -112,7 +112,7 @@ class TestMain(TestCase):with LogCapture(level=ERROR) as log:await maybe_deferred_to_future(crawler.crawl())- assert not log.records, f"{log.records=}"+ assert not log.recordsfinish_reason = crawler.stats.get_value("finish_reason")assert finish_reason == "shutdown", f"{finish_reason=}"expected_urls = []