Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CrawlSpider: extract links even when a rule's callback raises (#8200)

Adrian6651567bb115
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 28b2c2f63..27f3bcae6 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -194,19 +194,20 @@ class CrawlSpider(Spider):
cb_kwargs: dict[str, Any],
follow: bool = True,
) -> AsyncIterator[Any]:
- if callback:
- cb_res = callback(response, **cb_kwargs) or ()
- if isinstance(cb_res, AsyncIterator):
- cb_res = await collect_asyncgen(cb_res)
- elif isinstance(cb_res, Awaitable):
- cb_res = await cb_res
- cb_res = self.process_results(response, cb_res)
- for request_or_item in iterate_spider_output(cb_res):
- yield request_or_item
-
- if follow and self._follow_links:
- for request_or_item in self._requests_to_follow(response):
- yield request_or_item
+ try:
+ if callback:
+ cb_res = callback(response, **cb_kwargs) or ()
+ if isinstance(cb_res, AsyncIterator):
+ cb_res = await collect_asyncgen(cb_res)
+ elif isinstance(cb_res, Awaitable):
+ cb_res = await cb_res
+ cb_res = self.process_results(response, cb_res)
+ for request_or_item in iterate_spider_output(cb_res):
+ yield request_or_item
+ finally:
+ if follow and self._follow_links:
+ for request_or_item in self._requests_to_follow(response):
+ yield request_or_item
def _parse_response(
self,
diff --git a/tests/spiders.py b/tests/spiders.py
index 3cf0a0304..86329066b 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -536,6 +536,26 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
)
+class CrawlSpiderWithCallbackException(CrawlSpiderWithParseMethod):
+ name = "crawl_spider_with_callback_exception"
+
+ async def start(self) -> AsyncIterator[Any]:
+ test_body = b"""
+ <html>
+ <head><title>Page title</title></head>
+ <body>
+ <p><a href="/status?n=200">Item 200</a></p> <!-- still followed -->
+ </body>
+ </html>
+ """
+ assert self.mockserver
+ url = self.mockserver.url("/alpayload")
+ yield Request(url, method="POST", body=test_body)
+
+ def parse_start_url(self, response: Response, **kwargs: Any) -> None:
+ raise ValueError("callback exception")
+
+
class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_errback"
rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 26c2bc529..018b82f23 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -45,6 +45,7 @@ from tests.spiders import (
BytesReceivedErrbackSpider,
CrawlSpiderWithAsyncCallback,
CrawlSpiderWithAsyncGeneratorCallback,
+ CrawlSpiderWithCallbackException,
CrawlSpiderWithErrback,
CrawlSpiderWithoutErrback,
CrawlSpiderWithParseMethod,
@@ -542,6 +543,19 @@ class TestCrawlSpider:
assert "[errback]" not in caplog.text
assert crawler.stats.get_value("downloader/response_status_count/404") == 1
+ @coroutine_test
+ async def test_crawlspider_with_callback_exception(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ crawler = get_crawler(CrawlSpiderWithCallbackException)
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async(mockserver=mockserver)
+
+ # The link is followed even though parse_start_url() raised, and the
+ # exception is still logged as a spider error.
+ assert "[parse] status 200 (foo: None)" in caplog.text
+ assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
+
@coroutine_test
async def test_crawlspider_process_request_cb_kwargs(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer