Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix a regression in errback result handling. (#6863)
Andrey Rakhmatullin54474ceb0d14
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 9378f2651..975344103 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -188,6 +188,7 @@ class Scraper:)assert self.crawler.spider+ output: Iterable[Any] | AsyncIterator[Any]if isinstance(result, Response):try:# call the spider middlewares and the request callback with the response@@ -204,7 +205,7 @@ class Scraper:try:# call the request errback with the downloader error- await self.call_spider_async(result, request)+ output = await self.call_spider_async(result, request)except Exception as spider_exc:# the errback didn't silence the exceptionif not result.check(IgnoreRequest):@@ -219,6 +220,8 @@ class Scraper:if spider_exc is not result.value:# the errback raised a different exception, handle itself.handle_spider_error(Failure(), request, result)+ else:+ await self.handle_spider_output_async(output, request, result)def call_spider(self, result: Response | Failure, request: Request, spider: Spider | None = None@@ -309,7 +312,7 @@ class Scraper:self,result: Iterable[_T] | AsyncIterator[_T],request: Request,- response: Response,+ response: Response | Failure,spider: Spider | None = None,) -> Deferred[None]:"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""@@ -327,7 +330,7 @@ class Scraper:self,result: Iterable[_T] | AsyncIterator[_T],request: Request,- response: Response,+ response: Response | Failure,) -> None:"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""if isinstance(result, AsyncIterator):@@ -352,7 +355,9 @@ class Scraper:)@deferred_f_from_coro_f- async def _process_spidermw_output(self, output: Any, response: Response) -> None:+ async def _process_spidermw_output(+ self, output: Any, response: Response | Failure+ ) -> None:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider.@@ -368,7 +373,9 @@ class Scraper:)@deferred_f_from_coro_f- async def start_itemproc(self, item: Any, *, response: Response | None) -> None:+ async def start_itemproc(+ self, item: Any, *, response: Response | Failure | None+ ) -> None:"""Send *item* to the item pipelines for processing.*response* is the source of the item data. If the item does not comediff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 4f08918ae..e81a9ec93 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -116,7 +116,7 @@ class LogFormatter:self,item: Any,exception: BaseException,- response: Response | None,+ response: Response | Failure | None,spider: Spider,) -> LogFormatterResult:"""Logs a message when an item is dropped while it is passing through the item pipeline."""@@ -137,7 +137,7 @@ class LogFormatter:self,item: Any,exception: BaseException,- response: Response | None,+ response: Response | Failure | None,spider: Spider,) -> LogFormatterResult:"""Logs a message when an item causes an error while it is passingdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex b90706027..8289b2243 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -5,6 +5,7 @@ import loggingimport unittestfrom ipaddress import IPv4Addressfrom socket import gethostbyname+from typing import Anyfrom urllib.parse import urlparseimport pytest@@ -419,6 +420,8 @@ with multiples linesclass TestCrawlSpider(TestCase):+ mockserver: MockServer+@classmethoddef setUpClass(cls):cls.mockserver = MockServer()@@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase):)assert "Spider error processing" in str(log)+ @defer.inlineCallbacks+ def test_spider_errback_item(self):+ def eb(failure: Failure) -> Any:+ return {"foo": "bar"}++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "'item_scraped_count': 1" in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_request(self):+ def eb(failure: Failure) -> Request:+ return Request(self.mockserver.url("/"))++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "Crawled (200)" in str(log)+@defer.inlineCallbacksdef test_spider_errback_downloader_error(self):failures = []@@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase):assert "Spider error processing" not in str(log)@defer.inlineCallbacks- def test_spider_errback_exception_downloader_error(self):+ def test_spider_errback_downloader_error_exception(self):def eb(failure: Failure) -> None:raise ValueError("foo")@@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase):assert "Error downloading" in str(log)assert "Spider error processing" in str(log)+ @defer.inlineCallbacks+ def test_spider_errback_downloader_error_item(self):+ def eb(failure: Failure) -> Any:+ return {"foo": "bar"}++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "'item_scraped_count': 1" in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_downloader_error_request(self):+ def eb(failure: Failure) -> Request:+ return Request(self.mockserver.url("/"))++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "Crawled (200)" in str(log)+@defer.inlineCallbacksdef test_raise_closespider(self):def cb(response):