Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate setting overrides through spider attributes (#6994)
NavpreetSSidhud27d0a4ed9f9
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 627e16331..d368d7f92 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -93,9 +93,23 @@ def _get_concurrency_delay() -> tuple[int, float]:delay: float = settings.getfloat("DOWNLOAD_DELAY")if hasattr(spider, "download_delay"):+ warnings.warn(+ "The 'download_delay' spider attribute is deprecated. "+ "Use Spider.custom_settings or Spider.update_settings() instead. "+ "The corresponding setting name is 'DOWNLOAD_DELAY'.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )delay = spider.download_delayif hasattr(spider, "max_concurrent_requests"):+ warnings.warn(+ "The 'max_concurrent_requests' spider attribute is deprecated. "+ "Use Spider.custom_settings or Spider.update_settings() instead. "+ "The corresponding setting name is 'CONCURRENT_REQUESTS'.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )concurrency = spider.max_concurrent_requestsreturn concurrency, delaydiff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 58891b952..314baf1dc 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,11 +1,12 @@from __future__ import annotations+import warningsfrom itertools import chainfrom logging import getLoggerfrom typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signals-from scrapy.exceptions import IgnoreRequest, NotConfigured+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._compression import (@@ -74,8 +75,22 @@ class HttpCompressionMiddleware:def open_spider(self, spider: Spider) -> None:if hasattr(spider, "download_maxsize"):+ warnings.warn(+ "The 'download_maxsize' spider attribute is deprecated. "+ "Use Spider.custom_settings or Spider.update_settings() instead. "+ "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )self._max_size = spider.download_maxsizeif hasattr(spider, "download_warnsize"):+ warnings.warn(+ "The 'download_warnsize' spider attribute is deprecated. "+ "Use Spider.custom_settings or Spider.update_settings() instead. "+ "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )self._warn_size = spider.download_warnsizedef process_request(diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 0b3941c09..4389f044c 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -551,17 +551,21 @@ class TestHttpCompression:with pytest.raises(IgnoreRequest):mw.process_response(response.request, response, spider)+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_br(self):_skip_if_no_br()self._test_compression_bomb_spider_attr("br")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_deflate(self):self._test_compression_bomb_spider_attr("deflate")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_gzip(self):self._test_compression_bomb_spider_attr("gzip")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_zstd(self):_skip_if_no_zstd()@@ -664,17 +668,21 @@ class TestHttpCompression:),)+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_download_warnsize_spider_attr_br(self):_skip_if_no_br()self._test_download_warnsize_spider_attr("br")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_download_warnsize_spider_attr_deflate(self):self._test_download_warnsize_spider_attr("deflate")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_download_warnsize_spider_attr_gzip(self):self._test_download_warnsize_spider_attr("gzip")+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_download_warnsize_spider_attr_zstd(self):_skip_if_no_zstd()diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 4a532ed78..dcc9d1e82 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -726,6 +726,7 @@ Sitemap: /sitemap-relative-url.xmlresponse = Response(url="https://example.com", body=body, request=request)assert spider._get_sitemap_body(response) is None+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr(self):class DownloadMaxSizeSpider(self.spider_class):download_maxsize = 10_000_000@@ -773,6 +774,7 @@ Sitemap: /sitemap-relative-url.xml),)+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_download_warnsize_spider_attr(self):class DownloadWarnSizeSpider(self.spider_class):download_warnsize = 10_000_000