Scrapy, a fast high-level web crawling & scraping framework for Python.
Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867)
Andrey Rakhmatullin657e6cb2b57d
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex d9361a674..fe635dc82 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -174,7 +174,8 @@ class ExecutionEngine:return self.close_spider(self.spider, reason="shutdown") # will also close downloader- self.downloader.close()+ if hasattr(self, "downloader"):+ self.downloader.close()return succeed(None)def pause(self) -> None:diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 1f79a081d..9f618437c 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -22,6 +22,7 @@ from unittest.mock import Mockfrom urllib.parse import urlparseimport attr+import pytestfrom itemadapter import ItemAdapterfrom pydispatch import dispatcherfrom twisted.internet import defer@@ -433,6 +434,19 @@ class TestEngine(TestEngineBase):e = ExecutionEngine(get_crawler(MySpider), lambda _: None)yield e.close()+ def test_close_without_downloader(self):+ class CustomException(Exception):+ pass++ class BadDownloader:+ def __init__(self, crawler):+ raise CustomException++ with pytest.raises(CustomException):+ ExecutionEngine(+ get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None+ )+@defer.inlineCallbacksdef test_start_already_running_exception(self):e = ExecutionEngine(get_crawler(MySpider), lambda _: None)