Scrapy, a fast high-level web crawling & scraping framework for Python.
Update default_settings, deprecate CRAWLSPIDER_FOLLOW_LINKS. (#7592)
Andrey Rakhmatullinbeb6c51c173e
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 037fe87fa..b542c6c05 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format.Enabling your Media Pipeline============================-.. setting:: IMAGES_STORE-.. setting:: FILES_STORE-To enable your media pipeline you must first add it to your project:setting:`ITEM_PIPELINES` setting.@@ -102,6 +99,8 @@ For Files Pipeline, use:.. note::You can also use both the Files and Images Pipeline at the same time.+.. setting:: IMAGES_STORE+.. setting:: FILES_STOREThen, configure the target storage setting to a valid value that will be usedfor storing the downloaded images. Otherwise the pipeline will remain disabled,diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 7d5612026..e8a9400d7 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -20,19 +20,26 @@ from typing import Any__all__ = ["ADDONS",- "AJAXCRAWL_ENABLED",- "AJAXCRAWL_MAXSIZE","ASYNCIO_EVENT_LOOP","AUTOTHROTTLE_DEBUG","AUTOTHROTTLE_ENABLED","AUTOTHROTTLE_MAX_DELAY","AUTOTHROTTLE_START_DELAY","AUTOTHROTTLE_TARGET_CONCURRENCY",+ "AWS_ACCESS_KEY_ID",+ "AWS_ENDPOINT_URL",+ "AWS_REGION_NAME",+ "AWS_SECRET_ACCESS_KEY",+ "AWS_SESSION_TOKEN",+ "AWS_USE_SSL",+ "AWS_VERIFY","BOT_NAME","CLOSESPIDER_ERRORCOUNT","CLOSESPIDER_ITEMCOUNT","CLOSESPIDER_PAGECOUNT",+ "CLOSESPIDER_PAGECOUNT_NO_ITEM","CLOSESPIDER_TIMEOUT",+ "CLOSESPIDER_TIMEOUT_NO_ITEM","COMMANDS_MODULE","COMPRESSION_ENABLED","CONCURRENT_ITEMS",@@ -64,11 +71,14 @@ __all__ = ["DOWNLOAD_HANDLERS","DOWNLOAD_HANDLERS_BASE","DOWNLOAD_MAXSIZE",+ "DOWNLOAD_SLOTS","DOWNLOAD_TIMEOUT","DOWNLOAD_TLS_MAX_VERSION","DOWNLOAD_TLS_MIN_VERSION",+ "DOWNLOAD_VERIFY_CERTIFICATES","DOWNLOAD_WARNSIZE","DUPEFILTER_CLASS",+ "DUPEFILTER_DEBUG","EDITOR","EXTENSIONS","EXTENSIONS_BASE",@@ -87,7 +97,9 @@ __all__ = ["FEED_STORAGE_S3_ACL","FEED_STORE_EMPTY","FEED_TEMPDIR",+ "FEED_URI","FEED_URI_PARAMS",+ "FILES_STORE","FILES_STORE_GCS_ACL","FILES_STORE_S3_ACL","FORCE_CRAWLER_PROCESS",@@ -107,9 +119,12 @@ __all__ = ["HTTPCACHE_IGNORE_SCHEMES","HTTPCACHE_POLICY","HTTPCACHE_STORAGE",+ "HTTPERROR_ALLOWED_CODES",+ "HTTPERROR_ALLOW_ALL","HTTPPROXY_AUTH_ENCODING","HTTPPROXY_ENABLED","HTTPX_HTTP2_ENABLED",+ "IMAGES_STORE","IMAGES_STORE_GCS_ACL","IMAGES_STORE_S3_ACL","ITEM_PIPELINES",@@ -132,6 +147,8 @@ __all__ = ["MAIL_HOST","MAIL_PASS","MAIL_PORT",+ "MAIL_SSL",+ "MAIL_TLS","MAIL_USER","MEMDEBUG_ENABLED","MEMDEBUG_NOTIFY",@@ -153,6 +170,7 @@ __all__ = ["REDIRECT_MAX_TIMES","REDIRECT_PRIORITY_ADJUST","REFERER_ENABLED",+ "REFERRER_POLICIES","REFERRER_POLICY","REQUEST_FINGERPRINTER_CLASS","RETRY_ENABLED",@@ -198,9 +216,6 @@ __all__ = [ADDONS = {}-AJAXCRAWL_ENABLED = False-AJAXCRAWL_MAXSIZE = 32768-ASYNCIO_EVENT_LOOP = NoneAUTOTHROTTLE_ENABLED = False@@ -209,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0AUTOTHROTTLE_START_DELAY = 5.0AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0+AWS_ACCESS_KEY_ID = None+AWS_SECRET_ACCESS_KEY = None+AWS_ENDPOINT_URL = None+AWS_REGION_NAME = None+AWS_SESSION_TOKEN = None+AWS_USE_SSL = None+AWS_VERIFY = None+BOT_NAME = "scrapybot"CLOSESPIDER_ERRORCOUNT = 0CLOSESPIDER_ITEMCOUNT = 0CLOSESPIDER_PAGECOUNT = 0CLOSESPIDER_TIMEOUT = 0+CLOSESPIDER_PAGECOUNT_NO_ITEM = 0+CLOSESPIDER_TIMEOUT_NO_ITEM = 0COMMANDS_MODULE = ""@@ -267,6 +292,8 @@ DOWNLOAD_HANDLERS_BASE = {DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024mDOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m+DOWNLOAD_SLOTS = {}+DOWNLOAD_TIMEOUT = 180 # 3minsDOWNLOAD_TLS_MAX_VERSION = None@@ -304,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {DOWNLOADER_STATS = TrueDUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"+DUPEFILTER_DEBUG = FalseEDITOR = "vi"if sys.platform == "win32":@@ -354,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = FalseFEED_STORAGE_GCS_ACL = ""FEED_STORAGE_S3_ACL = ""FEED_TEMPDIR = None+FEED_URI = NoneFEED_URI_PARAMS = None # a function to extend uri arguments+FILES_STORE = NoneFILES_STORE_GCS_ACL = ""FILES_STORE_S3_ACL = "private"@@ -380,11 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"]HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"+HTTPERROR_ALLOW_ALL = False+HTTPERROR_ALLOWED_CODES = []+HTTPPROXY_ENABLED = TrueHTTPPROXY_AUTH_ENCODING = "latin-1"HTTPX_HTTP2_ENABLED = False+IMAGES_STORE = NoneIMAGES_STORE_GCS_ACL = ""IMAGES_STORE_S3_ACL = "private"@@ -425,6 +459,8 @@ MAIL_HOST = "localhost"MAIL_PORT = 25MAIL_USER = NoneMAIL_PASS = None+MAIL_SSL = False+MAIL_TLS = FalseMEMDEBUG_ENABLED = False # enable memory debuggingMEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown@@ -455,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2REFERER_ENABLED = TrueREFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"+REFERRER_POLICIES = {}REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex e14279b64..d2da31f35 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -12,6 +12,7 @@ import warningsfrom collections.abc import AsyncIterator, Awaitable, Callablefrom typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.link import Linkfrom scrapy.linkextractors import LinkExtractor@@ -220,4 +221,11 @@ class CrawlSpider(Spider):def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:spider = super().from_crawler(crawler, *args, **kwargs)spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")+ if not spider._follow_links:+ warnings.warn(+ "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated."+ " You can set follow=False in your rules to achieve the same effect.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )return spiderdiff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.pyindex 05d77912f..c97934b74 100644--- a/tests/test_spider_crawl.py+++ b/tests/test_spider_crawl.py@@ -4,6 +4,7 @@ import reimport warningsfrom logging import ERROR+import pytestfrom testfixtures import LogCapturefrom w3lib.url import safe_url_string@@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider):"HtmlResponse",]+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_follow_links_attribute_population(self):crawler = get_crawler()spider = self.spider_class.from_crawler(crawler, "example.com")