Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update default_settings, deprecate CRAWLSPIDER_FOLLOW_LINKS. (#7592)

Andrey Rakhmatullinbeb6c51c173e
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 037fe87fa..b542c6c05 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format.
Enabling your Media Pipeline
============================
-.. setting:: IMAGES_STORE
-.. setting:: FILES_STORE
-
To enable your media pipeline you must first add it to your project
:setting:`ITEM_PIPELINES` setting.
@@ -102,6 +99,8 @@ For Files Pipeline, use:
.. note::
You can also use both the Files and Images Pipeline at the same time.
+.. setting:: IMAGES_STORE
+.. setting:: FILES_STORE
Then, configure the target storage setting to a valid value that will be used
for storing the downloaded images. Otherwise the pipeline will remain disabled,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7d5612026..e8a9400d7 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -20,19 +20,26 @@ from typing import Any
__all__ = [
"ADDONS",
- "AJAXCRAWL_ENABLED",
- "AJAXCRAWL_MAXSIZE",
"ASYNCIO_EVENT_LOOP",
"AUTOTHROTTLE_DEBUG",
"AUTOTHROTTLE_ENABLED",
"AUTOTHROTTLE_MAX_DELAY",
"AUTOTHROTTLE_START_DELAY",
"AUTOTHROTTLE_TARGET_CONCURRENCY",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_ENDPOINT_URL",
+ "AWS_REGION_NAME",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_SESSION_TOKEN",
+ "AWS_USE_SSL",
+ "AWS_VERIFY",
"BOT_NAME",
"CLOSESPIDER_ERRORCOUNT",
"CLOSESPIDER_ITEMCOUNT",
"CLOSESPIDER_PAGECOUNT",
+ "CLOSESPIDER_PAGECOUNT_NO_ITEM",
"CLOSESPIDER_TIMEOUT",
+ "CLOSESPIDER_TIMEOUT_NO_ITEM",
"COMMANDS_MODULE",
"COMPRESSION_ENABLED",
"CONCURRENT_ITEMS",
@@ -64,11 +71,14 @@ __all__ = [
"DOWNLOAD_HANDLERS",
"DOWNLOAD_HANDLERS_BASE",
"DOWNLOAD_MAXSIZE",
+ "DOWNLOAD_SLOTS",
"DOWNLOAD_TIMEOUT",
"DOWNLOAD_TLS_MAX_VERSION",
"DOWNLOAD_TLS_MIN_VERSION",
+ "DOWNLOAD_VERIFY_CERTIFICATES",
"DOWNLOAD_WARNSIZE",
"DUPEFILTER_CLASS",
+ "DUPEFILTER_DEBUG",
"EDITOR",
"EXTENSIONS",
"EXTENSIONS_BASE",
@@ -87,7 +97,9 @@ __all__ = [
"FEED_STORAGE_S3_ACL",
"FEED_STORE_EMPTY",
"FEED_TEMPDIR",
+ "FEED_URI",
"FEED_URI_PARAMS",
+ "FILES_STORE",
"FILES_STORE_GCS_ACL",
"FILES_STORE_S3_ACL",
"FORCE_CRAWLER_PROCESS",
@@ -107,9 +119,12 @@ __all__ = [
"HTTPCACHE_IGNORE_SCHEMES",
"HTTPCACHE_POLICY",
"HTTPCACHE_STORAGE",
+ "HTTPERROR_ALLOWED_CODES",
+ "HTTPERROR_ALLOW_ALL",
"HTTPPROXY_AUTH_ENCODING",
"HTTPPROXY_ENABLED",
"HTTPX_HTTP2_ENABLED",
+ "IMAGES_STORE",
"IMAGES_STORE_GCS_ACL",
"IMAGES_STORE_S3_ACL",
"ITEM_PIPELINES",
@@ -132,6 +147,8 @@ __all__ = [
"MAIL_HOST",
"MAIL_PASS",
"MAIL_PORT",
+ "MAIL_SSL",
+ "MAIL_TLS",
"MAIL_USER",
"MEMDEBUG_ENABLED",
"MEMDEBUG_NOTIFY",
@@ -153,6 +170,7 @@ __all__ = [
"REDIRECT_MAX_TIMES",
"REDIRECT_PRIORITY_ADJUST",
"REFERER_ENABLED",
+ "REFERRER_POLICIES",
"REFERRER_POLICY",
"REQUEST_FINGERPRINTER_CLASS",
"RETRY_ENABLED",
@@ -198,9 +216,6 @@ __all__ = [
ADDONS = {}
-AJAXCRAWL_ENABLED = False
-AJAXCRAWL_MAXSIZE = 32768
-
ASYNCIO_EVENT_LOOP = None
AUTOTHROTTLE_ENABLED = False
@@ -209,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0
AUTOTHROTTLE_START_DELAY = 5.0
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+AWS_ACCESS_KEY_ID = None
+AWS_SECRET_ACCESS_KEY = None
+AWS_ENDPOINT_URL = None
+AWS_REGION_NAME = None
+AWS_SESSION_TOKEN = None
+AWS_USE_SSL = None
+AWS_VERIFY = None
+
BOT_NAME = "scrapybot"
CLOSESPIDER_ERRORCOUNT = 0
CLOSESPIDER_ITEMCOUNT = 0
CLOSESPIDER_PAGECOUNT = 0
CLOSESPIDER_TIMEOUT = 0
+CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
+CLOSESPIDER_TIMEOUT_NO_ITEM = 0
COMMANDS_MODULE = ""
@@ -267,6 +292,8 @@ DOWNLOAD_HANDLERS_BASE = {
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
+DOWNLOAD_SLOTS = {}
+
DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOAD_TLS_MAX_VERSION = None
@@ -304,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
DOWNLOADER_STATS = True
DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
+DUPEFILTER_DEBUG = False
EDITOR = "vi"
if sys.platform == "win32":
@@ -354,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False
FEED_STORAGE_GCS_ACL = ""
FEED_STORAGE_S3_ACL = ""
FEED_TEMPDIR = None
+FEED_URI = None
FEED_URI_PARAMS = None # a function to extend uri arguments
+FILES_STORE = None
FILES_STORE_GCS_ACL = ""
FILES_STORE_S3_ACL = "private"
@@ -380,11 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"]
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+HTTPERROR_ALLOW_ALL = False
+HTTPERROR_ALLOWED_CODES = []
+
HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = "latin-1"
HTTPX_HTTP2_ENABLED = False
+IMAGES_STORE = None
IMAGES_STORE_GCS_ACL = ""
IMAGES_STORE_S3_ACL = "private"
@@ -425,6 +459,8 @@ MAIL_HOST = "localhost"
MAIL_PORT = 25
MAIL_USER = None
MAIL_PASS = None
+MAIL_SSL = False
+MAIL_TLS = False
MEMDEBUG_ENABLED = False # enable memory debugging
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
@@ -455,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
+REFERRER_POLICIES = {}
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e14279b64..d2da31f35 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -12,6 +12,7 @@ import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.link import Link
from scrapy.linkextractors import LinkExtractor
@@ -220,4 +221,11 @@ class CrawlSpider(Spider):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
+ if not spider._follow_links:
+ warnings.warn(
+ "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated."
+ " You can set follow=False in your rules to achieve the same effect.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return spider
diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py
index 05d77912f..c97934b74 100644
--- a/tests/test_spider_crawl.py
+++ b/tests/test_spider_crawl.py
@@ -4,6 +4,7 @@ import re
import warnings
from logging import ERROR
+import pytest
from testfixtures import LogCapture
from w3lib.url import safe_url_string
@@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider):
"HtmlResponse",
]
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")