Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate _parse_response and implement parse_with_rules (#6804)

Rodrigosnrocha9cc23641ccc9
diff --git a/docs/conf.py b/docs/conf.py
index 1167ce050..493a62976 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -3,7 +3,6 @@
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
-# pylint: disable=import-error
import os
import sys
from collections.abc import Sequence
diff --git a/pyproject.toml b/pyproject.toml
index 871da8020..02ab7858d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -161,6 +161,9 @@ extension-pkg-allow-list=[
enable = [
"useless-suppression",
]
+# Make INFO checks like useless-suppression also cause pylint to return a
+# non-zero exit code.
+fail-on = "I"
disable = [
# Ones we want to ignore
"attribute-defined-outside-init",
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 171d8479c..f44f70e40 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst
from __future__ import annotations
import copy
+import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
@@ -18,6 +19,8 @@ from scrapy.link import Link
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.deprecate import method_is_overridden
+from scrapy.utils.python import global_object_name
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
@@ -95,9 +98,17 @@ class CrawlSpider(Spider):
def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
self._compile_rules()
+ if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"):
+ warnings.warn(
+ f"The CrawlSpider._parse_response method, which the "
+ f"{global_object_name(self.__class__)} class overrides, is "
+ f"deprecated: it will be removed in future Scrapy releases. "
+ f"Please override the CrawlSpider.parse_with_rules method "
+ f"instead."
+ )
def _parse(self, response: Response, **kwargs: Any) -> Any:
- return self._parse_response(
+ return self.parse_with_rules(
response=response,
callback=self.parse_start_url,
cb_kwargs=kwargs,
@@ -137,7 +148,7 @@ class CrawlSpider(Spider):
def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
rule = self._rules[cast(int, response.meta["rule"])]
- return self._parse_response(
+ return self.parse_with_rules(
response,
cast("CallbackT", rule.callback),
{**rule.cb_kwargs, **cb_kwargs},
@@ -150,7 +161,7 @@ class CrawlSpider(Spider):
failure, cast(Callable[[Failure], Any], rule.errback)
)
- async def _parse_response(
+ async def parse_with_rules(
self,
response: Response,
callback: CallbackT | None,
@@ -171,6 +182,21 @@ class CrawlSpider(Spider):
for request_or_item in self._requests_to_follow(response):
yield request_or_item
+ def _parse_response(
+ self,
+ response: Response,
+ callback: CallbackT | None,
+ cb_kwargs: dict[str, Any],
+ follow: bool = True,
+ ) -> AsyncIterator[Any]:
+ warnings.warn(
+ "The CrawlSpider._parse_response method is deprecated: "
+ "it will be removed in future Scrapy releases. "
+ "Please use the CrawlSpider.parse_with_rules method instead.",
+ stacklevel=2,
+ )
+ return self.parse_with_rules(response, callback, cb_kwargs, follow)
+
def _handle_failure(
self, failure: Failure, errback: Callable[[Failure], Any] | None
) -> Iterable[Any]:
diff --git a/tests/test_spider.py b/tests/test_spider.py
index b4aa649a3..4e4a99638 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider):
assert "Error while reading start items and requests" in str(log)
assert "did you miss an 's'?" in str(log)
+ def test_parse_response_use(self):
+ class _CrawlSpider(CrawlSpider):
+ name = "test"
+ start_urls = "https://www.example.com"
+ _follow_links = False
+
+ with warnings.catch_warnings(record=True) as w:
+ spider = _CrawlSpider()
+ assert len(w) == 0
+ spider._parse_response(
+ TextResponse(spider.start_urls, body=b""), None, None
+ )
+ assert len(w) == 1
+
+ def test_parse_response_override(self):
+ class _CrawlSpider(CrawlSpider):
+ def _parse_response(self, response, callback, cb_kwargs, follow=True):
+ pass
+
+ name = "test"
+ start_urls = "https://www.example.com"
+ _follow_links = False
+
+ with warnings.catch_warnings(record=True) as w:
+ assert len(w) == 0
+ spider = _CrawlSpider()
+ assert len(w) == 1
+ spider._parse_response(
+ TextResponse(spider.start_urls, body=b""), None, None
+ )
+ assert len(w) == 1
+
+ def test_parse_with_rules(self):
+ class _CrawlSpider(CrawlSpider):
+ name = "test"
+ start_urls = "https://www.example.com"
+
+ with warnings.catch_warnings(record=True) as w:
+ spider = _CrawlSpider()
+ spider.parse_with_rules(
+ TextResponse(spider.start_urls, body=b""), None, None
+ )
+ assert len(w) == 0
+
class TestSitemapSpider(TestSpider):
spider_class = SitemapSpider