Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate returning Deferreds from spider callbacks and errbacks. (#7151)
Andrey Rakhmatullin483e059d5953
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex d29dc2c06..455326a0a 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -318,6 +318,13 @@ class Scraper:callback = result.request.callback or self.crawler.spider._parsewarn_on_generator_with_return_value(self.crawler.spider, callback)output = callback(result, **result.request.cb_kwargs)+ if isinstance(output, Deferred):+ warnings.warn(+ f"{callback} returned a Deferred."+ f" Returning Deferreds from spider callbacks is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )else: # result is a Failure# TODO: properly type adding this attribute to a Failureresult.request = request # type: ignore[attr-defined]@@ -329,6 +336,13 @@ class Scraper:output.raiseException()# else the errback returned actual output (like a callback),# which needs to be passed to iterate_spider_output()+ if isinstance(output, Deferred):+ warnings.warn(+ f"{request.errback} returned a Deferred."+ f" Returning Deferreds from spider errbacks is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return await ensure_awaitable(iterate_spider_output(output))@_warn_spider_argdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex dc85d9d68..b0060a76a 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -9,13 +9,13 @@ from urllib.parse import urlencode, urlparseimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacks+from twisted.internet.defer import inlineCallbacks, succeedfrom twisted.internet.ssl import Certificatefrom twisted.python.failure import Failurefrom scrapy import Spider, signalsfrom scrapy.crawler import CrawlerRunner-from scrapy.exceptions import CloseSpider, StopDownload+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future@@ -720,6 +720,18 @@ class TestCrawlSpider:"failure"].value.response.headers == crawler.spider.meta.get("headers_received")+ @inlineCallbacks+ def test_spider_callback_deferred_deprecated(self):+ def cb(response: Response) -> Any:+ return succeed(None)++ crawler = get_crawler(SingleRequestSpider)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Returning Deferreds from spider callbacks is deprecated",+ ):+ yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)+@inlineCallbacksdef test_spider_errback(self):failures = []@@ -851,6 +863,20 @@ class TestCrawlSpider:assert "Spider error processing" not in str(log)assert "Crawled (200)" in str(log)+ @inlineCallbacks+ def test_spider_errback_deferred_deprecated(self):+ def eb(failure: Failure) -> Any:+ return succeed(None)++ crawler = get_crawler(SingleRequestSpider)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Returning Deferreds from spider errbacks is deprecated",+ ):+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+@inlineCallbacksdef test_raise_closespider(self):def cb(response):