Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fixing settings order (#6849)

Mehraz Hossain Rummanc6698b9fe8d3
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 4a27017a6..7cd470f11 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -1,16 +1,16 @@
-"""
-This module contains the default values for all settings used by Scrapy.
+"""This module contains the default values for all settings used by Scrapy.
For more information about these settings you can read the settings
documentation in docs/topics/settings.rst
Scrapy developers, if you add a setting here remember to:
-* add it in alphabetical order
+* add it in alphabetical order, with the exception that enabling flags and
+ other high-level settings for a group should come first in their group
+ and pairs like host/port and user/password should be in the usual order
* group similar settings without leaving blank lines
* add its documentation to the available settings documentation
(docs/topics/settings.rst)
-
"""
import sys
@@ -31,10 +31,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
BOT_NAME = "scrapybot"
-CLOSESPIDER_TIMEOUT = 0
-CLOSESPIDER_PAGECOUNT = 0
-CLOSESPIDER_ITEMCOUNT = 0
CLOSESPIDER_ERRORCOUNT = 0
+CLOSESPIDER_ITEMCOUNT = 0
+CLOSESPIDER_PAGECOUNT = 0
+CLOSESPIDER_TIMEOUT = 0
COMMANDS_MODULE = ""
@@ -59,8 +59,8 @@ DEFAULT_REQUEST_HEADERS = {
}
DEPTH_LIMIT = 0
-DEPTH_STATS_VERBOSE = False
DEPTH_PRIORITY = 0
+DEPTH_STATS_VERBOSE = False
DNSCACHE_ENABLED = True
DNSCACHE_SIZE = 10000
@@ -69,6 +69,8 @@ DNS_TIMEOUT = 60
DOWNLOAD_DELAY = 0
+DOWNLOAD_FAIL_ON_DATALOSS = True
+
DOWNLOAD_HANDLERS = {}
DOWNLOAD_HANDLERS_BASE = {
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
@@ -79,18 +81,13 @@ DOWNLOAD_HANDLERS_BASE = {
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
-DOWNLOAD_TIMEOUT = 180 # 3mins
-
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
-DOWNLOAD_FAIL_ON_DATALOSS = True
+DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOADER = "scrapy.core.downloader.Downloader"
-DOWNLOADER_HTTPCLIENTFACTORY = (
- "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
-)
DOWNLOADER_CLIENTCONTEXTFACTORY = (
"scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
)
@@ -99,8 +96,11 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
-DOWNLOADER_MIDDLEWARES = {}
+DOWNLOADER_HTTPCLIENTFACTORY = (
+ "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
+)
+DOWNLOADER_MIDDLEWARES = {}
DOWNLOADER_MIDDLEWARES_BASE = {
# Engine side
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
@@ -130,7 +130,6 @@ if sys.platform == "win32":
EDITOR = "%s -m idlelib.idle"
EXTENSIONS = {}
-
EXTENSIONS_BASE = {
"scrapy.extensions.corestats.CoreStats": 0,
"scrapy.extensions.telnet.TelnetConsole": 0,
@@ -143,22 +142,11 @@ EXTENSIONS_BASE = {
"scrapy.extensions.throttle.AutoThrottle": 0,
}
-FEED_TEMPDIR = None
FEEDS = {}
-FEED_URI_PARAMS = None # a function to extend uri arguments
-FEED_STORE_EMPTY = True
+FEED_EXPORT_BATCH_ITEM_COUNT = 0
FEED_EXPORT_ENCODING = None
FEED_EXPORT_FIELDS = None
-FEED_STORAGES = {}
-FEED_STORAGES_BASE = {
- "": "scrapy.extensions.feedexport.FileFeedStorage",
- "file": "scrapy.extensions.feedexport.FileFeedStorage",
- "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
- "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
- "s3": "scrapy.extensions.feedexport.S3FeedStorage",
- "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
-}
-FEED_EXPORT_BATCH_ITEM_COUNT = 0
+FEED_EXPORT_INDENT = 0
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
"json": "scrapy.exporters.JsonItemExporter",
@@ -170,59 +158,69 @@ FEED_EXPORTERS_BASE = {
"marshal": "scrapy.exporters.MarshalItemExporter",
"pickle": "scrapy.exporters.PickleItemExporter",
}
-FEED_EXPORT_INDENT = 0
-
+FEED_STORE_EMPTY = True
+FEED_STORAGES = {}
+FEED_STORAGES_BASE = {
+ "": "scrapy.extensions.feedexport.FileFeedStorage",
+ "file": "scrapy.extensions.feedexport.FileFeedStorage",
+ "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
+ "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
+ "s3": "scrapy.extensions.feedexport.S3FeedStorage",
+ "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
+}
FEED_STORAGE_FTP_ACTIVE = False
FEED_STORAGE_GCS_ACL = ""
FEED_STORAGE_S3_ACL = ""
+FEED_TEMPDIR = None
+FEED_URI_PARAMS = None # a function to extend uri arguments
-FILES_STORE_S3_ACL = "private"
FILES_STORE_GCS_ACL = ""
+FILES_STORE_S3_ACL = "private"
FORCE_CRAWLER_PROCESS = False
+FTP_PASSIVE_MODE = True
FTP_USER = "anonymous"
FTP_PASSWORD = "guest" # noqa: S105
-FTP_PASSIVE_MODE = True
GCS_PROJECT_ID = None
HTTPCACHE_ENABLED = False
+HTTPCACHE_ALWAYS_STORE = False
+HTTPCACHE_DBM_MODULE = "dbm"
HTTPCACHE_DIR = "httpcache"
-HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
HTTPCACHE_EXPIRATION_SECS = 0
-HTTPCACHE_ALWAYS_STORE = False
+HTTPCACHE_GZIP = False
HTTPCACHE_IGNORE_HTTP_CODES = []
-HTTPCACHE_IGNORE_SCHEMES = ["file"]
+HTTPCACHE_IGNORE_MISSING = False
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = "dbm"
+HTTPCACHE_IGNORE_SCHEMES = ["file"]
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
-HTTPCACHE_GZIP = False
+HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = "latin-1"
-IMAGES_STORE_S3_ACL = "private"
IMAGES_STORE_GCS_ACL = ""
-
-ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
+IMAGES_STORE_S3_ACL = "private"
ITEM_PIPELINES = {}
ITEM_PIPELINES_BASE = {}
+ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
+
JOBDIR = None
LOG_ENABLED = True
-LOG_ENCODING = "utf-8"
-LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
-LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"
-LOG_STDOUT = False
-LOG_LEVEL = "DEBUG"
+LOG_ENCODING = "utf-8"
LOG_FILE = None
LOG_FILE_APPEND = True
+LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
+LOG_LEVEL = "DEBUG"
LOG_SHORT_NAMES = False
+LOG_STDOUT = False
LOG_VERSIONS = [
"lxml",
"libxml2",
@@ -236,21 +234,19 @@ LOG_VERSIONS = [
"Platform",
]
-SCHEDULER_DEBUG = False
-
LOGSTATS_INTERVAL = 60.0
+MAIL_FROM = "scrapy@localhost"
MAIL_HOST = "localhost"
MAIL_PORT = 25
-MAIL_FROM = "scrapy@localhost"
-MAIL_PASS = None
MAIL_USER = None
+MAIL_PASS = None
MEMDEBUG_ENABLED = False # enable memory debugging
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
-MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
MEMUSAGE_ENABLED = True
+MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
MEMUSAGE_LIMIT_MB = 0
MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_WARNING_MB = 0
@@ -280,9 +276,6 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
RETRY_ENABLED = True
-RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
-RETRY_PRIORITY_ADJUST = -1
RETRY_EXCEPTIONS = [
"twisted.internet.defer.TimeoutError",
"twisted.internet.error.TimeoutError",
@@ -298,12 +291,16 @@ RETRY_EXCEPTIONS = [
OSError,
"scrapy.core.downloader.handlers.http11.TunnelError",
]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
+RETRY_PRIORITY_ADJUST = -1
+RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
ROBOTSTXT_OBEY = False
ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
ROBOTSTXT_USER_AGENT = None
SCHEDULER = "scrapy.core.scheduler.Scheduler"
+SCHEDULER_DEBUG = False
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
@@ -312,11 +309,19 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
+SPIDER_CONTRACTS = {}
+SPIDER_CONTRACTS_BASE = {
+ "scrapy.contracts.default.UrlContract": 1,
+ "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+ "scrapy.contracts.default.MetadataContract": 1,
+ "scrapy.contracts.default.ReturnsContract": 2,
+ "scrapy.contracts.default.ScrapesContract": 3,
+}
+
SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
SPIDER_LOADER_WARN_ONLY = False
SPIDER_MIDDLEWARES = {}
-
SPIDER_MIDDLEWARES_BASE = {
# Engine side
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
@@ -334,27 +339,18 @@ STATS_DUMP = True
STATSMAILER_RCPTS = []
-TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
-
-URLLENGTH_LIMIT = 2083
-
-USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
-
TELNETCONSOLE_ENABLED = 1
-TELNETCONSOLE_PORT = [6023, 6073]
TELNETCONSOLE_HOST = "127.0.0.1"
+TELNETCONSOLE_PORT = [6023, 6073]
TELNETCONSOLE_USERNAME = "scrapy"
TELNETCONSOLE_PASSWORD = None
+TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
+
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
-SPIDER_CONTRACTS = {}
-SPIDER_CONTRACTS_BASE = {
- "scrapy.contracts.default.UrlContract": 1,
- "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
- "scrapy.contracts.default.MetadataContract": 1,
- "scrapy.contracts.default.ReturnsContract": 2,
- "scrapy.contracts.default.ScrapesContract": 3,
-}
+URLLENGTH_LIMIT = 2083
+
+USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
WARN_ON_GENERATOR_RETURN_VALUE = True