Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate setting overrides through spider attributes (#6994)

NavpreetSSidhud27d0a4ed9f9
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 627e16331..d368d7f92 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -93,9 +93,23 @@ def _get_concurrency_delay(
) -> tuple[int, float]:
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
+ warnings.warn(
+ "The 'download_delay' spider attribute is deprecated. "
+ "Use Spider.custom_settings or Spider.update_settings() instead. "
+ "The corresponding setting name is 'DOWNLOAD_DELAY'.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
delay = spider.download_delay
if hasattr(spider, "max_concurrent_requests"):
+ warnings.warn(
+ "The 'max_concurrent_requests' spider attribute is deprecated. "
+ "Use Spider.custom_settings or Spider.update_settings() instead. "
+ "The corresponding setting name is 'CONCURRENT_REQUESTS'.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
concurrency = spider.max_concurrent_requests
return concurrency, delay
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 58891b952..314baf1dc 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,11 +1,12 @@
from __future__ import annotations
+import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._compression import (
@@ -74,8 +75,22 @@ class HttpCompressionMiddleware:
def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"):
+ warnings.warn(
+ "The 'download_maxsize' spider attribute is deprecated. "
+ "Use Spider.custom_settings or Spider.update_settings() instead. "
+ "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):
+ warnings.warn(
+ "The 'download_warnsize' spider attribute is deprecated. "
+ "Use Spider.custom_settings or Spider.update_settings() instead. "
+ "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self._warn_size = spider.download_warnsize
def process_request(
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 0b3941c09..4389f044c 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -551,17 +551,21 @@ class TestHttpCompression:
with pytest.raises(IgnoreRequest):
mw.process_response(response.request, response, spider)
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
_skip_if_no_br()
self._test_compression_bomb_spider_attr("br")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_deflate(self):
self._test_compression_bomb_spider_attr("deflate")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_gzip(self):
self._test_compression_bomb_spider_attr("gzip")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_zstd(self):
_skip_if_no_zstd()
@@ -664,17 +668,21 @@ class TestHttpCompression:
),
)
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr_br(self):
_skip_if_no_br()
self._test_download_warnsize_spider_attr("br")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr_deflate(self):
self._test_download_warnsize_spider_attr("deflate")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr_gzip(self):
self._test_download_warnsize_spider_attr("gzip")
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr_zstd(self):
_skip_if_no_zstd()
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 4a532ed78..dcc9d1e82 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -726,6 +726,7 @@ Sitemap: /sitemap-relative-url.xml
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
download_maxsize = 10_000_000
@@ -773,6 +774,7 @@ Sitemap: /sitemap-relative-url.xml
),
)
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000