Scrapy, a fast high-level web crawling & scraping framework for Python.
Add missed CloseSpider.error handling for downloader middlewares. (#8352)
Andrey Rakhmatullinb678e91edfac
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 7ead932df..e86117ab1 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -283,7 +283,9 @@ class Scraper:exc = result.valueassert isinstance(exc, CloseSpider) # typing_schedule_coro(- self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")+ self.crawler.engine.close_spider_async(+ reason=exc.reason or "cancelled", error=exc.error+ ))returndiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex dc809df43..bc32f7a1f 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -68,12 +68,12 @@ if TYPE_CHECKING:class CloseSpiderProcessRequestMiddleware:def process_request(self, request):- raise CloseSpider("my_reason")+ raise CloseSpider("my_reason", error=request.meta["close_error"])class CloseSpiderProcessResponseMiddleware:def process_response(self, request, response):- raise CloseSpider("my_reason")+ raise CloseSpider("my_reason", error=request.meta["close_error"])class TestCrawl:@@ -1039,17 +1039,21 @@ class TestCrawlSpider:"middleware",[CloseSpiderProcessRequestMiddleware, CloseSpiderProcessResponseMiddleware],)+ @pytest.mark.parametrize("error", [False, True])@coroutine_testasync def test_raise_closespider_downloader_middleware(self,caplog: pytest.LogCaptureFixture,mockserver: MockServer,middleware: type,+ error: bool,) -> None:crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {middleware: 1}})+ seed = Request(mockserver.url("/"), meta={"close_error": error})with caplog.at_level(logging.INFO):- await crawler.crawl_async(seed=mockserver.url("/"))+ await crawler.crawl_async(seed=seed)assert "Closing spider (my_reason)" in caplog.textassert "Error downloading" not in caplog.text+ assert crawler.stats.get_value("finish_reason_error") is error