Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate _parse_response and implement parse_with_rules (#6804)
Rodrigosnrocha9cc23641ccc9
diff --git a/docs/conf.py b/docs/conf.pyindex 1167ce050..493a62976 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -3,7 +3,6 @@# For the full list of built-in configuration values, see the documentation:# https://www.sphinx-doc.org/en/master/usage/configuration.html-# pylint: disable=import-errorimport osimport sysfrom collections.abc import Sequencediff --git a/pyproject.toml b/pyproject.tomlindex 871da8020..02ab7858d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -161,6 +161,9 @@ extension-pkg-allow-list=[enable = ["useless-suppression",]+# Make INFO checks like useless-suppression also cause pylint to return a+# non-zero exit code.+fail-on = "I"disable = [# Ones we want to ignore"attribute-defined-outside-init",diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 171d8479c..f44f70e40 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rstfrom __future__ import annotationsimport copy+import warningsfrom collections.abc import AsyncIterator, Awaitable, Callablefrom typing import TYPE_CHECKING, Any, Optional, TypeVar, cast@@ -18,6 +19,8 @@ from scrapy.link import Linkfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgen+from scrapy.utils.deprecate import method_is_overridden+from scrapy.utils.python import global_object_namefrom scrapy.utils.spider import iterate_spider_outputif TYPE_CHECKING:@@ -95,9 +98,17 @@ class CrawlSpider(Spider):def __init__(self, *a: Any, **kw: Any):super().__init__(*a, **kw)self._compile_rules()+ if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"):+ warnings.warn(+ f"The CrawlSpider._parse_response method, which the "+ f"{global_object_name(self.__class__)} class overrides, is "+ f"deprecated: it will be removed in future Scrapy releases. "+ f"Please override the CrawlSpider.parse_with_rules method "+ f"instead."+ )def _parse(self, response: Response, **kwargs: Any) -> Any:- return self._parse_response(+ return self.parse_with_rules(response=response,callback=self.parse_start_url,cb_kwargs=kwargs,@@ -137,7 +148,7 @@ class CrawlSpider(Spider):def _callback(self, response: Response, **cb_kwargs: Any) -> Any:rule = self._rules[cast(int, response.meta["rule"])]- return self._parse_response(+ return self.parse_with_rules(response,cast("CallbackT", rule.callback),{**rule.cb_kwargs, **cb_kwargs},@@ -150,7 +161,7 @@ class CrawlSpider(Spider):failure, cast(Callable[[Failure], Any], rule.errback))- async def _parse_response(+ async def parse_with_rules(self,response: Response,callback: CallbackT | None,@@ -171,6 +182,21 @@ class CrawlSpider(Spider):for request_or_item in self._requests_to_follow(response):yield request_or_item+ def _parse_response(+ self,+ response: Response,+ callback: CallbackT | None,+ cb_kwargs: dict[str, Any],+ follow: bool = True,+ ) -> AsyncIterator[Any]:+ warnings.warn(+ "The CrawlSpider._parse_response method is deprecated: "+ "it will be removed in future Scrapy releases. "+ "Please use the CrawlSpider.parse_with_rules method instead.",+ stacklevel=2,+ )+ return self.parse_with_rules(response, callback, cb_kwargs, follow)+def _handle_failure(self, failure: Failure, errback: Callable[[Failure], Any] | None) -> Iterable[Any]:diff --git a/tests/test_spider.py b/tests/test_spider.pyindex b4aa649a3..4e4a99638 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider):assert "Error while reading start items and requests" in str(log)assert "did you miss an 's'?" in str(log)+ def test_parse_response_use(self):+ class _CrawlSpider(CrawlSpider):+ name = "test"+ start_urls = "https://www.example.com"+ _follow_links = False++ with warnings.catch_warnings(record=True) as w:+ spider = _CrawlSpider()+ assert len(w) == 0+ spider._parse_response(+ TextResponse(spider.start_urls, body=b""), None, None+ )+ assert len(w) == 1++ def test_parse_response_override(self):+ class _CrawlSpider(CrawlSpider):+ def _parse_response(self, response, callback, cb_kwargs, follow=True):+ pass++ name = "test"+ start_urls = "https://www.example.com"+ _follow_links = False++ with warnings.catch_warnings(record=True) as w:+ assert len(w) == 0+ spider = _CrawlSpider()+ assert len(w) == 1+ spider._parse_response(+ TextResponse(spider.start_urls, body=b""), None, None+ )+ assert len(w) == 1++ def test_parse_with_rules(self):+ class _CrawlSpider(CrawlSpider):+ name = "test"+ start_urls = "https://www.example.com"++ with warnings.catch_warnings(record=True) as w:+ spider = _CrawlSpider()+ spider.parse_with_rules(+ TextResponse(spider.start_urls, body=b""), None, None+ )+ assert len(w) == 0+class TestSitemapSpider(TestSpider):spider_class = SitemapSpider