Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867)

Andrey Rakhmatullind329eedfefd9
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 7f5dd0405..0df9ad2b2 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -170,7 +170,8 @@ class ExecutionEngine:
return self.close_spider(
self.spider, reason="shutdown"
) # will also close downloader
- self.downloader.close()
+ if hasattr(self, "downloader"):
+ self.downloader.close()
return succeed(None)
def pause(self) -> None:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index b60b510b2..b2e436425 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -22,6 +22,7 @@ from unittest.mock import Mock
from urllib.parse import urlparse
import attr
+import pytest
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer, reactor
@@ -431,6 +432,19 @@ class TestEngine(TestEngineBase):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.close()
+ def test_close_without_downloader(self):
+ class CustomException(Exception):
+ pass
+
+ class BadDownloader:
+ def __init__(self, crawler):
+ raise CustomException
+
+ with pytest.raises(CustomException):
+ ExecutionEngine(
+ get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None
+ )
+
@defer.inlineCallbacks
def test_start_already_running_exception(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)