Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate returning Deferreds from spider callbacks and errbacks. (#7151)

Andrey Rakhmatullin483e059d5953
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index d29dc2c06..455326a0a 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -318,6 +318,13 @@ class Scraper:
callback = result.request.callback or self.crawler.spider._parse
warn_on_generator_with_return_value(self.crawler.spider, callback)
output = callback(result, **result.request.cb_kwargs)
+ if isinstance(output, Deferred):
+ warnings.warn(
+ f"{callback} returned a Deferred."
+ f" Returning Deferreds from spider callbacks is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
else: # result is a Failure
# TODO: properly type adding this attribute to a Failure
result.request = request # type: ignore[attr-defined]
@@ -329,6 +336,13 @@ class Scraper:
output.raiseException()
# else the errback returned actual output (like a callback),
# which needs to be passed to iterate_spider_output()
+ if isinstance(output, Deferred):
+ warnings.warn(
+ f"{request.errback} returned a Deferred."
+ f" Returning Deferreds from spider errbacks is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return await ensure_awaitable(iterate_spider_output(output))
@_warn_spider_arg
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index dc85d9d68..b0060a76a 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,13 +9,13 @@ from urllib.parse import urlencode, urlparse
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, succeed
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from scrapy import Spider, signals
from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import CloseSpider, StopDownload
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
@@ -720,6 +720,18 @@ class TestCrawlSpider:
"failure"
].value.response.headers == crawler.spider.meta.get("headers_received")
+ @inlineCallbacks
+ def test_spider_callback_deferred_deprecated(self):
+ def cb(response: Response) -> Any:
+ return succeed(None)
+
+ crawler = get_crawler(SingleRequestSpider)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="Returning Deferreds from spider callbacks is deprecated",
+ ):
+ yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
+
@inlineCallbacks
def test_spider_errback(self):
failures = []
@@ -851,6 +863,20 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log)
+ @inlineCallbacks
+ def test_spider_errback_deferred_deprecated(self):
+ def eb(failure: Failure) -> Any:
+ return succeed(None)
+
+ crawler = get_crawler(SingleRequestSpider)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="Returning Deferreds from spider errbacks is deprecated",
+ ):
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+
@inlineCallbacks
def test_raise_closespider(self):
def cb(response):