Scrapy, a fast high-level web crawling & scraping framework for Python.
Report exceptions from Spider.start() (#7884)
Adrianbee31890a23f
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex f060710a4..acbf7ddef 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -338,15 +338,22 @@ spider_error.. signal:: spider_error.. function:: spider_error(failure, response, spider)- Sent when a spider callback generates an error (i.e. raises an exception).+ Sent when a spider callback or the :meth:`~scrapy.Spider.start` method of a+ spider generates an error (i.e. raises an exception).++ .. versionchanged:: VERSION+ Exceptions from :meth:`~scrapy.Spider.start` are also reported, see+ :ref:`start-error`.This signal does not support :ref:`asynchronous handlers <signal-deferred>`.:param failure: the exception raised:type failure: twisted.python.failure.Failure- :param response: the response being processed when the exception was raised- :type response: :class:`~scrapy.http.Response` object+ :param response: the response being processed when the exception was+ raised, or ``None`` if the exception came from+ :meth:`~scrapy.Spider.start`.+ :type response: :class:`~scrapy.http.Response` | ``None``:param spider: the spider which raised the exception:type spider: :class:`~scrapy.Spider` objectdiff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex e68c208b7..95c80d5dc 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -411,6 +411,38 @@ scheduled requests:await self.crawler.signals.wait_for(signals.scheduler_empty)yield item_or_request+.. _start-error:++Handling start errors+---------------------++An exception raised by :meth:`~scrapy.Spider.start` ends its iteration, so any+remaining start items and requests are never sent. Scrapy logs the exception,+sends the :signal:`spider_error` signal, and, once the already scheduled+requests are done, closes the spider with the ``start_error``+:stat:`finish_reason`.++.. versionchanged:: VERSION+ The close reason used to be ``finished``, and neither the+ :signal:`spider_error` signal nor the :stat:`spider_exceptions/count` stat+ reported the exception.++To keep the iteration going, catch the exception yourself:++.. code-block:: python++ async def start(self):+ for url in self.start_urls:+ try:+ request = Request(url)+ except ValueError:+ self.logger.exception(f"Skipping start URL {url}")+ else:+ yield request++To stop the crawl instead, and choose your own :stat:`finish_reason`, raise+:exc:`~scrapy.exceptions.CloseSpider`.+.. _builtin-spiders:Generic Spidersdiff --git a/docs/topics/stats.rst b/docs/topics/stats.rstindex b558c1cf2..6fb7783d1 100644--- a/docs/topics/stats.rst+++ b/docs/topics/stats.rst@@ -301,6 +301,10 @@ one per actual value of the placeholder.- ``shutdown``: the crawl was interrupted, e.g. by a system signal suchas ``SIGINT`` (:kbd:`Ctrl-C`).+ - ``start_error``: :meth:`~scrapy.Spider.start` raised an exception, so+ some :ref:`start requests <start-requests>` may never have been sent,+ see :ref:`start-error`.+Third-party components and your own code may use any other reason, e.g. byraising :exc:`~scrapy.exceptions.CloseSpider` with it.@@ -721,18 +725,21 @@ one per actual value of the placeholder... stat:: spider_exceptions/count``spider_exceptions/count``- Number of unhandled exceptions raised by spider callbacks.+ Number of unhandled exceptions raised by spider callbacks or by+ :meth:`~scrapy.Spider.start`.- Set by the :ref:`scraper <topics-architecture>`.+ Set by the :ref:`engine <topics-architecture>` and the :ref:`scraper+ <topics-architecture>`... stat:: spider_exceptions/{exception}``spider_exceptions/{exception}``- Number of unhandled exceptions raised by spider callbacks, per exception,- where ``{exception}`` is the class name of the exception, e.g.+ Same as :stat:`spider_exceptions/count`, per exception, where+ ``{exception}`` is the class name of the exception, e.g.``spider_exceptions/ValueError``.- Set by the :ref:`scraper <topics-architecture>`.+ Set by the :ref:`engine <topics-architecture>` and the :ref:`scraper+ <topics-architecture>`... stat:: start_timediff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex efac5f71c..b7a1e0fcd 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -125,6 +125,9 @@ class ExecutionEngine:] = spider_closed_callbackself.start_time: float | None = Noneself._start: AsyncIterator[Any] | None = None+ # Whether Spider.start() raised, i.e. some start items or requests may+ # never have reached the engine.+ self._start_error: bool = Falseself._closewait: Deferred[None] | None = Noneself._start_request_processing_awaitable: (asyncio.Future[None] | Deferred[None] | None@@ -277,13 +280,30 @@ class ExecutionEngine:item_or_request = await anext(self._start)except StopAsyncIteration:self._start = None+ except CloseSpider as exception:+ self._start = None+ _schedule_coro(+ self.close_spider_async(reason=exception.reason or "cancelled")+ )except Exception as exception:self._start = None+ self._start_error = Trueexception_traceback = format_exc()logger.error(f"Error while reading start items and requests: {exception}.\n{exception_traceback}",exc_info=True,)+ self.signals.send_catch_log(+ signal=signals.spider_error,+ failure=Failure(),+ response=None,+ spider=self.spider,+ )+ assert self.crawler.stats+ self.crawler.stats.inc_value("spider_exceptions/count")+ self.crawler.stats.inc_value(+ f"spider_exceptions/{type(exception).__name__}"+ )else:if not self.spider:return # spider already closed@@ -579,7 +599,8 @@ class ExecutionEngine:if DontCloseSpider in detected_ex:returnif self.spider_is_idle():- ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished"))+ default_reason = "start_error" if self._start_error else "finished"+ ex = detected_ex.get(CloseSpider, CloseSpider(reason=default_reason))assert isinstance(ex, CloseSpider) # typing_schedule_coro(self.close_spider_async(reason=ex.reason))diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex ccaccddf2..fa5bc1c1e 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -56,8 +56,12 @@ class DontCloseSpider(Exception):class CloseSpider(Exception):- """Raised from a :ref:`spider callback <topics-spiders>` to request the- spider to be closed/stopped.+ """Raised from a :ref:`spider callback <topics-spiders>` or from+ :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped.++ .. versionchanged:: VERSION+ Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead+ of being reported as a start error.*reason* is a string with the reason for closing.diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 240482586..defee1041 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -400,7 +400,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):def test_reactorless_import_hook(self) -> None:log = self.run_script("reactorless_import_hook.py")assert "Not using a Twisted reactor" in log- assert "Spider closed (finished)" in log+ assert "Spider closed (start_error)" in logassert "ImportError: Import of twisted.internet.reactor is forbidden" in logdef test_reactorless_import_hook_uninstall(self) -> None:diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 14ec3d184..1ecf8b8de 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signalsfrom scrapy.core.scheduler import BaseScheduler+from scrapy.exceptions import CloseSpiderfrom scrapy.utils.asyncio import call_later, sleepfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer@@ -140,6 +141,74 @@ class TestMain:assert crawler.stats.get_value("finish_reason") == "shutdown"assert not actual_urls+ @coroutine_test+ async def test_start_error(self, caplog: pytest.LogCaptureFixture) -> None:+ class TestSpider(Spider):+ name = "test"++ async def start(self):+ yield Request("data:,a")+ raise ValueError++ def parse(self, response):+ pass++ actual_urls = []+ errors = []++ def track_url(request, spider):+ actual_urls.append(request.url)++ def track_error(failure, response, spider):+ errors.append((failure, response))++ settings = {"SCHEDULER": MemoryScheduler}+ crawler = get_crawler(TestSpider, settings_dict=settings)+ crawler.signals.connect(track_url, signals.request_reached_downloader)+ crawler.signals.connect(track_error, signals.spider_error)++ caplog.clear()+ with caplog.at_level(ERROR):+ await crawler.crawl_async()++ # The requests yielded before the exception are still crawled.+ assert actual_urls == ["data:,a"]+ assert len(caplog.records) == 1+ assert len(errors) == 1+ failure, response = errors[0]+ assert isinstance(failure.value, ValueError)+ assert response is None+ assert crawler.stats+ assert crawler.stats.get_value("finish_reason") == "start_error"+ assert crawler.stats.get_value("spider_exceptions/count") == 1+ assert crawler.stats.get_value("spider_exceptions/ValueError") == 1++ @coroutine_test+ async def test_close_spider_from_start(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ class TestSpider(Spider):+ name = "test"++ async def start(self):+ yield Request("data:,a")+ raise CloseSpider("my_reason")++ def parse(self, response):+ pass++ settings = {"SCHEDULER": MemoryScheduler}+ crawler = get_crawler(TestSpider, settings_dict=settings)++ caplog.clear()+ with caplog.at_level(ERROR):+ await crawler.crawl_async()++ assert not caplog.records+ assert crawler.stats+ assert crawler.stats.get_value("finish_reason") == "my_reason"+ assert crawler.stats.get_value("spider_exceptions/count") is None+class TestRequestSendOrder:seconds = 0.1 # increase if flaky