Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add missed CloseSpider.error handling for downloader middlewares. (#8352)

Andrey Rakhmatullinb678e91edfac
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7ead932df..e86117ab1 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -283,7 +283,9 @@ class Scraper:
exc = result.value
assert isinstance(exc, CloseSpider) # typing
_schedule_coro(
- self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
+ self.crawler.engine.close_spider_async(
+ reason=exc.reason or "cancelled", error=exc.error
+ )
)
return
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index dc809df43..bc32f7a1f 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -68,12 +68,12 @@ if TYPE_CHECKING:
class CloseSpiderProcessRequestMiddleware:
def process_request(self, request):
- raise CloseSpider("my_reason")
+ raise CloseSpider("my_reason", error=request.meta["close_error"])
class CloseSpiderProcessResponseMiddleware:
def process_response(self, request, response):
- raise CloseSpider("my_reason")
+ raise CloseSpider("my_reason", error=request.meta["close_error"])
class TestCrawl:
@@ -1039,17 +1039,21 @@ class TestCrawlSpider:
"middleware",
[CloseSpiderProcessRequestMiddleware, CloseSpiderProcessResponseMiddleware],
)
+ @pytest.mark.parametrize("error", [False, True])
@coroutine_test
async def test_raise_closespider_downloader_middleware(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
middleware: type,
+ error: bool,
) -> None:
crawler = get_crawler(
SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {middleware: 1}}
)
+ seed = Request(mockserver.url("/"), meta={"close_error": error})
with caplog.at_level(logging.INFO):
- await crawler.crawl_async(seed=mockserver.url("/"))
+ await crawler.crawl_async(seed=seed)
assert "Closing spider (my_reason)" in caplog.text
assert "Error downloading" not in caplog.text
+ assert crawler.stats.get_value("finish_reason_error") is error