Scrapy, a fast high-level web crawling & scraping framework for Python.
CrawlSpider: extract links even when a rule's callback raises (#8200)
Adrian6651567bb115
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 28b2c2f63..27f3bcae6 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -194,19 +194,20 @@ class CrawlSpider(Spider):cb_kwargs: dict[str, Any],follow: bool = True,) -> AsyncIterator[Any]:- if callback:- cb_res = callback(response, **cb_kwargs) or ()- if isinstance(cb_res, AsyncIterator):- cb_res = await collect_asyncgen(cb_res)- elif isinstance(cb_res, Awaitable):- cb_res = await cb_res- cb_res = self.process_results(response, cb_res)- for request_or_item in iterate_spider_output(cb_res):- yield request_or_item-- if follow and self._follow_links:- for request_or_item in self._requests_to_follow(response):- yield request_or_item+ try:+ if callback:+ cb_res = callback(response, **cb_kwargs) or ()+ if isinstance(cb_res, AsyncIterator):+ cb_res = await collect_asyncgen(cb_res)+ elif isinstance(cb_res, Awaitable):+ cb_res = await cb_res+ cb_res = self.process_results(response, cb_res)+ for request_or_item in iterate_spider_output(cb_res):+ yield request_or_item+ finally:+ if follow and self._follow_links:+ for request_or_item in self._requests_to_follow(response):+ yield request_or_itemdef _parse_response(self,diff --git a/tests/spiders.py b/tests/spiders.pyindex 3cf0a0304..86329066b 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -536,6 +536,26 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):)+class CrawlSpiderWithCallbackException(CrawlSpiderWithParseMethod):+ name = "crawl_spider_with_callback_exception"++ async def start(self) -> AsyncIterator[Any]:+ test_body = b"""+ <html>+ <head><title>Page title</title></head>+ <body>+ <p><a href="/status?n=200">Item 200</a></p> <!-- still followed -->+ </body>+ </html>+ """+ assert self.mockserver+ url = self.mockserver.url("/alpayload")+ yield Request(url, method="POST", body=test_body)++ def parse_start_url(self, response: Response, **kwargs: Any) -> None:+ raise ValueError("callback exception")++class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):name = "crawl_spider_with_errback"rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 26c2bc529..018b82f23 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -45,6 +45,7 @@ from tests.spiders import (BytesReceivedErrbackSpider,CrawlSpiderWithAsyncCallback,CrawlSpiderWithAsyncGeneratorCallback,+ CrawlSpiderWithCallbackException,CrawlSpiderWithErrback,CrawlSpiderWithoutErrback,CrawlSpiderWithParseMethod,@@ -542,6 +543,19 @@ class TestCrawlSpider:assert "[errback]" not in caplog.textassert crawler.stats.get_value("downloader/response_status_count/404") == 1+ @coroutine_test+ async def test_crawlspider_with_callback_exception(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ crawler = get_crawler(CrawlSpiderWithCallbackException)+ with caplog.at_level(logging.INFO):+ await crawler.crawl_async(mockserver=mockserver)++ # The link is followed even though parse_start_url() raised, and the+ # exception is still logged as a spider error.+ assert "[parse] status 200 (foo: None)" in caplog.text+ assert crawler.stats.get_value("spider_exceptions/ValueError") == 1+@coroutine_testasync def test_crawlspider_process_request_cb_kwargs(self, caplog: pytest.LogCaptureFixture, mockserver: MockServer