Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate spider attributes that can be replaced by settings, round 2 (#7039)
Laerte Pereira11073c86802d
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 77b17287d..0d23aa85c 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -1,7 +1,6 @@from __future__ import annotationsimport random-import warningsfrom collections import dequefrom datetime import datetimefrom time import time@@ -13,7 +12,6 @@ from twisted.python.failure import Failurefrom scrapy import Request, Spider, signalsfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManager-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.resolver import dnscachefrom scrapy.utils.asyncio import (AsyncioLoopingCall,@@ -27,6 +25,7 @@ from scrapy.utils.defer import (_schedule_coro,maybe_deferred_to_future,)+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:@@ -97,13 +96,9 @@ def _get_concurrency_delay(if hasattr(spider, "download_delay"):delay = spider.download_delay- if hasattr(spider, "max_concurrent_requests"):- warnings.warn(- "The 'max_concurrent_requests' spider attribute is deprecated. "- "Use Spider.custom_settings or Spider.update_settings() instead. "- "The corresponding setting name is 'CONCURRENT_REQUESTS'.",- category=ScrapyDeprecationWarning,- stacklevel=2,+ if hasattr(spider, "max_concurrent_requests"): # pragma: no cover+ warn_on_deprecated_spider_attribute(+ "max_concurrent_requests", "CONCURRENT_REQUESTS")concurrency = spider.max_concurrent_requestsdiff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex d8965c130..02aaf7c5c 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -35,6 +35,7 @@ from scrapy.core.downloader.contextfactory import load_context_factory_from_settfrom scrapy.exceptions import StopDownloadfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypes+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicodefrom scrapy.utils.url import add_http_if_no_scheme@@ -92,6 +93,13 @@ class HTTP11DownloadHandler:def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:"""Return a deferred for the HTTP download"""+ if hasattr(spider, "download_maxsize"): # pragma: no cover+ warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")+ if hasattr(spider, "download_warnsize"): # pragma: no cover+ warn_on_deprecated_spider_attribute(+ "download_warnsize", "DOWNLOAD_WARNSIZE"+ )+agent = ScrapyAgent(contextFactory=self._contextFactory,pool=self._pool,diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex cf2742de6..ee9211efc 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -34,6 +34,7 @@ from zope.interface import implementerfrom scrapy.core.http2.stream import Stream, StreamCloseReasonfrom scrapy.http import Request, Response+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributeif TYPE_CHECKING:from ipaddress import IPv4Address, IPv6Address@@ -191,6 +192,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):def _new_stream(self, request: Request, spider: Spider) -> Stream:"""Instantiates a new Stream object"""+ if hasattr(spider, "download_maxsize"): # pragma: no cover+ warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")+ if hasattr(spider, "download_warnsize"): # pragma: no cover+ warn_on_deprecated_spider_attribute(+ "download_warnsize", "DOWNLOAD_WARNSIZE"+ )+stream = Stream(stream_id=next(self._stream_id_generator),request=request,diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.pyindex b57d5c2a9..bccfb230c 100644--- a/scrapy/downloadermiddlewares/downloadtimeout.py+++ b/scrapy/downloadermiddlewares/downloadtimeout.py@@ -10,6 +10,7 @@ from typing import TYPE_CHECKINGfrom scrapy import Request, Spider, signalsfrom scrapy.utils.decorators import _warn_spider_arg+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributeif TYPE_CHECKING:# typing.Self requires Python 3.11@@ -30,6 +31,8 @@ class DownloadTimeoutMiddleware:return odef spider_opened(self, spider: Spider) -> None:+ if hasattr(spider, "download_timeout"): # pragma: no cover+ warn_on_deprecated_spider_attribute("download_timeout", "DOWNLOAD_TIMEOUT")self._timeout = getattr(spider, "download_timeout", self._timeout)@_warn_spider_argdiff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex e81888d9b..d4fa2d4d7 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -6,7 +6,7 @@ from logging import getLoggerfrom typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signals-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._compression import (@@ -16,6 +16,7 @@ from scrapy.utils._compression import (_unzstd,)from scrapy.utils.decorators import _warn_spider_arg+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.gz import gunzipif TYPE_CHECKING:@@ -85,21 +86,11 @@ class HttpCompressionMiddleware:def open_spider(self, spider: Spider) -> None:if hasattr(spider, "download_maxsize"):- warnings.warn(- "The 'download_maxsize' spider attribute is deprecated. "- "Use Spider.custom_settings or Spider.update_settings() instead. "- "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )+ warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")self._max_size = spider.download_maxsizeif hasattr(spider, "download_warnsize"):- warnings.warn(- "The 'download_warnsize' spider attribute is deprecated. "- "Use Spider.custom_settings or Spider.update_settings() instead. "- "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.",- category=ScrapyDeprecationWarning,- stacklevel=2,+ warn_on_deprecated_spider_attribute(+ "download_warnsize", "DOWNLOAD_WARNSIZE")self._warn_size = spider.download_warnsizediff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.pyindex c43a0195c..61f84b518 100644--- a/scrapy/downloadermiddlewares/useragent.py+++ b/scrapy/downloadermiddlewares/useragent.py@@ -6,6 +6,7 @@ from typing import TYPE_CHECKINGfrom scrapy import Request, Spider, signalsfrom scrapy.utils.decorators import _warn_spider_arg+from scrapy.utils.deprecate import warn_on_deprecated_spider_attributeif TYPE_CHECKING:# typing.Self requires Python 3.11@@ -28,6 +29,9 @@ class UserAgentMiddleware:return odef spider_opened(self, spider: Spider) -> None:+ if hasattr(spider, "user_agent"): # pragma: no cover+ warn_on_deprecated_spider_attribute("user_agent", "USER_AGENT")+self.user_agent = getattr(spider, "user_agent", self.user_agent)@_warn_spider_argdiff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 1f529b2cb..e6e6becd6 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -219,3 +219,13 @@ def argument_is_required(func: Callable[..., Any], arg_name: str) -> bool:args = get_func_args_dict(func)param = args.get(arg_name)return param is not None and param.default is inspect.Parameter.empty+++def warn_on_deprecated_spider_attribute(attribute_name: str, setting_name: str) -> None:+ warnings.warn(+ f"The '{attribute_name}' spider attribute is deprecated. "+ "Use Spider.custom_settings or Spider.update_settings() instead. "+ f"The corresponding setting name is '{setting_name}'.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 3e3e677a2..1e638392f 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -15,6 +15,8 @@ from twisted.web.http import H2_ENABLEDfrom scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawlerfrom tests.test_downloader_handlers_http_base import (TestHttpProxyBase,TestHttps11Base,@@ -31,7 +33,6 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServerfrom tests.mockserver.proxy_echo import ProxyEchoMockServer-pytestmark = pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled")@@ -63,10 +64,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):@deferred_f_from_coro_fasync def test_download_with_maxsize_very_large_file(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:from twisted.internet import reactor+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 1_500})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)+with mock.patch("scrapy.core.http2.stream.logger") as logger:request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure)@@ -76,9 +80,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):logger.error.assert_called_once_with(mock.ANY)with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(- download_handler, request, Spider("foo", download_maxsize=1500)- )+ await download_request(download_handler, request, Spider("foo"))# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queuediff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 8fefc0dd7..a2459911f 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -453,28 +453,29 @@ class TestHttp11Base(TestHttpBase):assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck@deferred_f_from_coro_f- async def test_download_with_maxsize(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol- ) -> None:+ async def test_download_with_maxsize(self, mockserver: MockServer) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure))# 10 is minimal size for this request and the limit is only counted on# response body. (regardless of headers)- response = await download_request(- download_handler, request, Spider("foo", download_maxsize=5)- )+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 5})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)+ response = await download_request(download_handler, request, Spider("foo"))assert response.body == b"Works"+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 4})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)+with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(- download_handler, request, Spider("foo", download_maxsize=4)- )+ await download_request(download_handler, request, Spider("foo"))@deferred_f_from_coro_fasync def test_download_with_maxsize_very_large_file(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ self, mockserver: MockServer) -> None:# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 1_500})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure)@@ -484,9 +485,7 @@ class TestHttp11Base(TestHttpBase):logger.warning.assert_called_once_with(mock.ANY, mock.ANY)with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(- download_handler, request, Spider("foo", download_maxsize=1500)- )+ await download_request(download_handler, request, Spider("foo"))# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queue@@ -506,23 +505,23 @@ class TestHttp11Base(TestHttpBase):await download_request(download_handler, request)@deferred_f_from_coro_f- async def test_download_with_small_maxsize_per_spider(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ async def test_download_with_small_maxsize_via_setting(+ self, mockserver: MockServer) -> None:+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 2})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(mockserver.url("/text", is_secure=self.is_secure))with pytest.raises((defer.CancelledError, error.ConnectionAborted)):- await download_request(- download_handler, request, Spider("foo", download_maxsize=2)- )+ await download_request(download_handler, request, Spider("foo"))@deferred_f_from_coro_f- async def test_download_with_large_maxsize_per_spider(- self, mockserver: MockServer, download_handler: DownloadHandlerProtocol+ async def test_download_with_large_maxsize_via_setting(+ self, mockserver: MockServer) -> None:+ crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 5})+ download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(mockserver.url("/text", is_secure=self.is_secure))- response = await download_request(- download_handler, request, Spider("foo", download_maxsize=100)- )+ response = await download_request(download_handler, request, Spider("foo"))assert response.body == b"Works"@deferred_f_from_coro_fdiff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.pyindex 3707cee18..c744d259c 100644--- a/tests/test_downloadermiddleware_downloadtimeout.py+++ b/tests/test_downloadermiddleware_downloadtimeout.py@@ -23,16 +23,14 @@ class TestDownloadTimeoutMiddleware:assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 20.1- def test_spider_has_download_timeout(self):- req, spider, mw = self.get_request_spider_mw()- spider.download_timeout = 2+ def test_setting_has_download_timeout(self):+ req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})mw.spider_opened(spider)assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 2def test_request_has_download_timeout(self):- req, spider, mw = self.get_request_spider_mw()- spider.download_timeout = 2+ req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})mw.spider_opened(spider)req.meta["download_timeout"] = 1assert mw.process_request(req) is Nonediff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.pyindex 60dc2ae7a..539183cd6 100644--- a/tests/test_downloadermiddleware_useragent.py+++ b/tests/test_downloadermiddleware_useragent.py@@ -16,26 +16,8 @@ class TestUserAgentMiddleware:assert mw.process_request(req) is Noneassert req.headers["User-Agent"] == b"default_useragent"- def test_remove_agent(self):- # settings USER_AGENT to None should remove the user agent- spider, mw = self.get_spider_and_mw("default_useragent")- spider.user_agent = None- mw.spider_opened(spider)- req = Request("http://scrapytest.org/")- assert mw.process_request(req) is None- assert req.headers.get("User-Agent") is None-- def test_spider_agent(self):- spider, mw = self.get_spider_and_mw("default_useragent")- spider.user_agent = "spider_useragent"- mw.spider_opened(spider)- req = Request("http://scrapytest.org/")- assert mw.process_request(req) is None- assert req.headers["User-Agent"] == b"spider_useragent"-def test_header_agent(self):spider, mw = self.get_spider_and_mw("default_useragent")- spider.user_agent = "spider_useragent"mw.spider_opened(spider)req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"}@@ -45,7 +27,6 @@ class TestUserAgentMiddleware:def test_no_agent(self):spider, mw = self.get_spider_and_mw(None)- spider.user_agent = Nonemw.spider_opened(spider)req = Request("http://scrapytest.org/")assert mw.process_request(req) is None