Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix a regression in errback result handling. (#6863)
Andrey Rakhmatullind602f13e8cd2
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 9fd68bce5..1f0d57c63 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -189,6 +189,7 @@ class Scraper:)assert self.crawler.spider+ output: Iterable[Any] | AsyncIterator[Any]if isinstance(result, Response):try:# call the spider middlewares and the request callback with the response@@ -203,7 +204,7 @@ class Scraper:try:# call the request errback with the downloader error- await self.call_spider_async(result, request)+ output = await self.call_spider_async(result, request)except Exception as spider_exc:# the errback didn't silence the exceptionif not result.check(IgnoreRequest):@@ -218,6 +219,8 @@ class Scraper:if spider_exc is not result.value:# the errback raised a different exception, handle itself.handle_spider_error(Failure(), request, result)+ else:+ await self.handle_spider_output_async(output, request, result)def call_spider(self, result: Response | Failure, request: Request, spider: Spider | None = None@@ -308,7 +311,7 @@ class Scraper:self,result: Iterable[_T] | AsyncIterator[_T],request: Request,- response: Response,+ response: Response | Failure,spider: Spider | None = None,) -> Deferred[None]:"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""@@ -326,7 +329,7 @@ class Scraper:self,result: Iterable[_T] | AsyncIterator[_T],request: Request,- response: Response,+ response: Response | Failure,) -> None:"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""it: Iterable[_T] | AsyncIterator[_T]@@ -361,7 +364,7 @@ class Scraper:)def _process_spidermw_output(- self, output: Any, response: Response+ self, output: Any, response: Response | Failure) -> Deferred[None]:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider.@@ -371,7 +374,7 @@ class Scraper:return deferred_from_coro(self._process_spidermw_output_async(output, response))async def _process_spidermw_output_async(- self, output: Any, response: Response+ self, output: Any, response: Response | Failure) -> None:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider.@@ -385,7 +388,9 @@ class Scraper:if output is not None:await self.start_itemproc_async(output, response=response)- def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]:+ def start_itemproc(+ self, item: Any, *, response: Response | Failure | None+ ) -> Deferred[None]:"""Send *item* to the item pipelines for processing.*response* is the source of the item data. If the item does not come@@ -394,7 +399,7 @@ class Scraper:return deferred_from_coro(self.start_itemproc_async(item, response=response))async def start_itemproc_async(- self, item: Any, *, response: Response | None+ self, item: Any, *, response: Response | Failure | None) -> None:"""Send *item* to the item pipelines for processing.diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 4f08918ae..e81a9ec93 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -116,7 +116,7 @@ class LogFormatter:self,item: Any,exception: BaseException,- response: Response | None,+ response: Response | Failure | None,spider: Spider,) -> LogFormatterResult:"""Logs a message when an item is dropped while it is passing through the item pipeline."""@@ -137,7 +137,7 @@ class LogFormatter:self,item: Any,exception: BaseException,- response: Response | None,+ response: Response | Failure | None,spider: Spider,) -> LogFormatterResult:"""Logs a message when an item causes an error while it is passingdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex b90706027..8289b2243 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -5,6 +5,7 @@ import loggingimport unittestfrom ipaddress import IPv4Addressfrom socket import gethostbyname+from typing import Anyfrom urllib.parse import urlparseimport pytest@@ -419,6 +420,8 @@ with multiples linesclass TestCrawlSpider(TestCase):+ mockserver: MockServer+@classmethoddef setUpClass(cls):cls.mockserver = MockServer()@@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase):)assert "Spider error processing" in str(log)+ @defer.inlineCallbacks+ def test_spider_errback_item(self):+ def eb(failure: Failure) -> Any:+ return {"foo": "bar"}++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "'item_scraped_count': 1" in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_request(self):+ def eb(failure: Failure) -> Request:+ return Request(self.mockserver.url("/"))++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "Crawled (200)" in str(log)+@defer.inlineCallbacksdef test_spider_errback_downloader_error(self):failures = []@@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase):assert "Spider error processing" not in str(log)@defer.inlineCallbacks- def test_spider_errback_exception_downloader_error(self):+ def test_spider_errback_downloader_error_exception(self):def eb(failure: Failure) -> None:raise ValueError("foo")@@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase):assert "Error downloading" in str(log)assert "Spider error processing" in str(log)+ @defer.inlineCallbacks+ def test_spider_errback_downloader_error_item(self):+ def eb(failure: Failure) -> Any:+ return {"foo": "bar"}++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "'item_scraped_count': 1" in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_downloader_error_request(self):+ def eb(failure: Failure) -> Request:+ return Request(self.mockserver.url("/"))++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)+ assert "Crawled (200)" in str(log)+@defer.inlineCallbacksdef test_raise_closespider(self):def cb(response):