Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867)

Andrey Rakhmatullin657e6cb2b57d
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index d9361a674..fe635dc82 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -174,7 +174,8 @@ class ExecutionEngine:
return self.close_spider(
self.spider, reason="shutdown"
) # will also close downloader
- self.downloader.close()
+ if hasattr(self, "downloader"):
+ self.downloader.close()
return succeed(None)
def pause(self) -> None:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 1f79a081d..9f618437c 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -22,6 +22,7 @@ from unittest.mock import Mock
from urllib.parse import urlparse
import attr
+import pytest
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer
@@ -433,6 +434,19 @@ class TestEngine(TestEngineBase):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.close()
+ def test_close_without_downloader(self):
+ class CustomException(Exception):
+ pass
+
+ class BadDownloader:
+ def __init__(self, crawler):
+ raise CustomException
+
+ with pytest.raises(CustomException):
+ ExecutionEngine(
+ get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None
+ )
+
@defer.inlineCallbacks
def test_start_already_running_exception(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)