Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix setting ExecutionEngine._slot to None on close. (#6910)

Andrey Rakhmatullinef01a953b108
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 6c6d24d5c..b3a5e1906 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -262,10 +262,10 @@ class ExecutionEngine:
See :ref:`start-requests-lazy` for an example.
"""
- assert self._slot is not None # typing
assert self.scraper.slot is not None # typing
return (
not self.running
+ or not self._slot
or bool(self._slot.closing)
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout()
@@ -528,10 +528,16 @@ class ExecutionEngine:
)
)
- dfd.addBoth(lambda _: setattr(self, "slot", None))
+ def unassign_slot(_: Any) -> None:
+ self._slot = None
+
+ dfd.addBoth(unassign_slot)
dfd.addErrback(log_failure("Error while unassigning slot"))
- dfd.addBoth(lambda _: setattr(self, "spider", None))
+ def unassign_spider(_: Any) -> None:
+ self.spider = None
+
+ dfd.addBoth(unassign_spider)
dfd.addErrback(log_failure("Error while unassigning spider"))
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index c7dbc82d4..8211667fb 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -112,7 +112,7 @@ class MainTestCase(TestCase):
with LogCapture(level=ERROR) as log:
await maybe_deferred_to_future(crawler.crawl())
- assert not log.records, f"{log.records=}"
+ assert not log.records
finish_reason = crawler.stats.get_value("finish_reason")
assert finish_reason == "shutdown", f"{finish_reason=}"
expected_urls = []