Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix a regression in errback result handling. (#6863)

Andrey Rakhmatullin54474ceb0d14
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 9378f2651..975344103 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -188,6 +188,7 @@ class Scraper:
)
assert self.crawler.spider
+ output: Iterable[Any] | AsyncIterator[Any]
if isinstance(result, Response):
try:
# call the spider middlewares and the request callback with the response
@@ -204,7 +205,7 @@ class Scraper:
try:
# call the request errback with the downloader error
- await self.call_spider_async(result, request)
+ output = await self.call_spider_async(result, request)
except Exception as spider_exc:
# the errback didn't silence the exception
if not result.check(IgnoreRequest):
@@ -219,6 +220,8 @@ class Scraper:
if spider_exc is not result.value:
# the errback raised a different exception, handle it
self.handle_spider_error(Failure(), request, result)
+ else:
+ await self.handle_spider_output_async(output, request, result)
def call_spider(
self, result: Response | Failure, request: Request, spider: Spider | None = None
@@ -309,7 +312,7 @@ class Scraper:
self,
result: Iterable[_T] | AsyncIterator[_T],
request: Request,
- response: Response,
+ response: Response | Failure,
spider: Spider | None = None,
) -> Deferred[None]:
"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
@@ -327,7 +330,7 @@ class Scraper:
self,
result: Iterable[_T] | AsyncIterator[_T],
request: Request,
- response: Response,
+ response: Response | Failure,
) -> None:
"""Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
if isinstance(result, AsyncIterator):
@@ -352,7 +355,9 @@ class Scraper:
)
@deferred_f_from_coro_f
- async def _process_spidermw_output(self, output: Any, response: Response) -> None:
+ async def _process_spidermw_output(
+ self, output: Any, response: Response | Failure
+ ) -> None:
"""Process each Request/Item (given in the output parameter) returned
from the given spider.
@@ -368,7 +373,9 @@ class Scraper:
)
@deferred_f_from_coro_f
- async def start_itemproc(self, item: Any, *, response: Response | None) -> None:
+ async def start_itemproc(
+ self, item: Any, *, response: Response | Failure | None
+ ) -> None:
"""Send *item* to the item pipelines for processing.
*response* is the source of the item data. If the item does not come
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 4f08918ae..e81a9ec93 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -116,7 +116,7 @@ class LogFormatter:
self,
item: Any,
exception: BaseException,
- response: Response | None,
+ response: Response | Failure | None,
spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
@@ -137,7 +137,7 @@ class LogFormatter:
self,
item: Any,
exception: BaseException,
- response: Response | None,
+ response: Response | Failure | None,
spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item causes an error while it is passing
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index b90706027..8289b2243 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,6 +5,7 @@ import logging
import unittest
from ipaddress import IPv4Address
from socket import gethostbyname
+from typing import Any
from urllib.parse import urlparse
import pytest
@@ -419,6 +420,8 @@ with multiples lines
class TestCrawlSpider(TestCase):
+ mockserver: MockServer
+
@classmethod
def setUpClass(cls):
cls.mockserver = MockServer()
@@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase):
)
assert "Spider error processing" in str(log)
+ @defer.inlineCallbacks
+ def test_spider_errback_item(self):
+ def eb(failure: Failure) -> Any:
+ return {"foo": "bar"}
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+ assert "HTTP status code is not handled or not allowed" not in str(log)
+ assert "Spider error processing" not in str(log)
+ assert "'item_scraped_count': 1" in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_request(self):
+ def eb(failure: Failure) -> Request:
+ return Request(self.mockserver.url("/"))
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+ assert "HTTP status code is not handled or not allowed" not in str(log)
+ assert "Spider error processing" not in str(log)
+ assert "Crawled (200)" in str(log)
+
@defer.inlineCallbacks
def test_spider_errback_downloader_error(self):
failures = []
@@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase):
assert "Spider error processing" not in str(log)
@defer.inlineCallbacks
- def test_spider_errback_exception_downloader_error(self):
+ def test_spider_errback_downloader_error_exception(self):
def eb(failure: Failure) -> None:
raise ValueError("foo")
@@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase):
assert "Error downloading" in str(log)
assert "Spider error processing" in str(log)
+ @defer.inlineCallbacks
+ def test_spider_errback_downloader_error_item(self):
+ def eb(failure: Failure) -> Any:
+ return {"foo": "bar"}
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+ )
+ assert "HTTP status code is not handled or not allowed" not in str(log)
+ assert "Spider error processing" not in str(log)
+ assert "'item_scraped_count': 1" in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_downloader_error_request(self):
+ def eb(failure: Failure) -> Request:
+ return Request(self.mockserver.url("/"))
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+ )
+ assert "HTTP status code is not handled or not allowed" not in str(log)
+ assert "Spider error processing" not in str(log)
+ assert "Crawled (200)" in str(log)
+
@defer.inlineCallbacks
def test_raise_closespider(self):
def cb(response):