Scrapy, a fast high-level web crawling & scraping framework for Python.
fixing settings order (#6849)
Mehraz Hossain Rummanc6698b9fe8d3
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 4a27017a6..7cd470f11 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -1,16 +1,16 @@-"""-This module contains the default values for all settings used by Scrapy.+"""This module contains the default values for all settings used by Scrapy.For more information about these settings you can read the settingsdocumentation in docs/topics/settings.rstScrapy developers, if you add a setting here remember to:-* add it in alphabetical order+* add it in alphabetical order, with the exception that enabling flags and+ other high-level settings for a group should come first in their group+ and pairs like host/port and user/password should be in the usual order* group similar settings without leaving blank lines* add its documentation to the available settings documentation(docs/topics/settings.rst)-"""import sys@@ -31,10 +31,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0BOT_NAME = "scrapybot"-CLOSESPIDER_TIMEOUT = 0-CLOSESPIDER_PAGECOUNT = 0-CLOSESPIDER_ITEMCOUNT = 0CLOSESPIDER_ERRORCOUNT = 0+CLOSESPIDER_ITEMCOUNT = 0+CLOSESPIDER_PAGECOUNT = 0+CLOSESPIDER_TIMEOUT = 0COMMANDS_MODULE = ""@@ -59,8 +59,8 @@ DEFAULT_REQUEST_HEADERS = {}DEPTH_LIMIT = 0-DEPTH_STATS_VERBOSE = FalseDEPTH_PRIORITY = 0+DEPTH_STATS_VERBOSE = FalseDNSCACHE_ENABLED = TrueDNSCACHE_SIZE = 10000@@ -69,6 +69,8 @@ DNS_TIMEOUT = 60DOWNLOAD_DELAY = 0+DOWNLOAD_FAIL_ON_DATALOSS = True+DOWNLOAD_HANDLERS = {}DOWNLOAD_HANDLERS_BASE = {"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",@@ -79,18 +81,13 @@ DOWNLOAD_HANDLERS_BASE = {"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",}-DOWNLOAD_TIMEOUT = 180 # 3mins-DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024mDOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m-DOWNLOAD_FAIL_ON_DATALOSS = True+DOWNLOAD_TIMEOUT = 180 # 3minsDOWNLOADER = "scrapy.core.downloader.Downloader"-DOWNLOADER_HTTPCLIENTFACTORY = (- "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"-)DOWNLOADER_CLIENTCONTEXTFACTORY = ("scrapy.core.downloader.contextfactory.ScrapyClientContextFactory")@@ -99,8 +96,11 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"DOWNLOADER_CLIENT_TLS_METHOD = "TLS"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False-DOWNLOADER_MIDDLEWARES = {}+DOWNLOADER_HTTPCLIENTFACTORY = (+ "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"+)+DOWNLOADER_MIDDLEWARES = {}DOWNLOADER_MIDDLEWARES_BASE = {# Engine side"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,@@ -130,7 +130,6 @@ if sys.platform == "win32":EDITOR = "%s -m idlelib.idle"EXTENSIONS = {}-EXTENSIONS_BASE = {"scrapy.extensions.corestats.CoreStats": 0,"scrapy.extensions.telnet.TelnetConsole": 0,@@ -143,22 +142,11 @@ EXTENSIONS_BASE = {"scrapy.extensions.throttle.AutoThrottle": 0,}-FEED_TEMPDIR = NoneFEEDS = {}-FEED_URI_PARAMS = None # a function to extend uri arguments-FEED_STORE_EMPTY = True+FEED_EXPORT_BATCH_ITEM_COUNT = 0FEED_EXPORT_ENCODING = NoneFEED_EXPORT_FIELDS = None-FEED_STORAGES = {}-FEED_STORAGES_BASE = {- "": "scrapy.extensions.feedexport.FileFeedStorage",- "file": "scrapy.extensions.feedexport.FileFeedStorage",- "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",- "gs": "scrapy.extensions.feedexport.GCSFeedStorage",- "s3": "scrapy.extensions.feedexport.S3FeedStorage",- "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",-}-FEED_EXPORT_BATCH_ITEM_COUNT = 0+FEED_EXPORT_INDENT = 0FEED_EXPORTERS = {}FEED_EXPORTERS_BASE = {"json": "scrapy.exporters.JsonItemExporter",@@ -170,59 +158,69 @@ FEED_EXPORTERS_BASE = {"marshal": "scrapy.exporters.MarshalItemExporter","pickle": "scrapy.exporters.PickleItemExporter",}-FEED_EXPORT_INDENT = 0-+FEED_STORE_EMPTY = True+FEED_STORAGES = {}+FEED_STORAGES_BASE = {+ "": "scrapy.extensions.feedexport.FileFeedStorage",+ "file": "scrapy.extensions.feedexport.FileFeedStorage",+ "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",+ "gs": "scrapy.extensions.feedexport.GCSFeedStorage",+ "s3": "scrapy.extensions.feedexport.S3FeedStorage",+ "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",+}FEED_STORAGE_FTP_ACTIVE = FalseFEED_STORAGE_GCS_ACL = ""FEED_STORAGE_S3_ACL = ""+FEED_TEMPDIR = None+FEED_URI_PARAMS = None # a function to extend uri arguments-FILES_STORE_S3_ACL = "private"FILES_STORE_GCS_ACL = ""+FILES_STORE_S3_ACL = "private"FORCE_CRAWLER_PROCESS = False+FTP_PASSIVE_MODE = TrueFTP_USER = "anonymous"FTP_PASSWORD = "guest" # noqa: S105-FTP_PASSIVE_MODE = TrueGCS_PROJECT_ID = NoneHTTPCACHE_ENABLED = False+HTTPCACHE_ALWAYS_STORE = False+HTTPCACHE_DBM_MODULE = "dbm"HTTPCACHE_DIR = "httpcache"-HTTPCACHE_IGNORE_MISSING = False-HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"HTTPCACHE_EXPIRATION_SECS = 0-HTTPCACHE_ALWAYS_STORE = False+HTTPCACHE_GZIP = FalseHTTPCACHE_IGNORE_HTTP_CODES = []-HTTPCACHE_IGNORE_SCHEMES = ["file"]+HTTPCACHE_IGNORE_MISSING = FalseHTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []-HTTPCACHE_DBM_MODULE = "dbm"+HTTPCACHE_IGNORE_SCHEMES = ["file"]HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"-HTTPCACHE_GZIP = False+HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"HTTPPROXY_ENABLED = TrueHTTPPROXY_AUTH_ENCODING = "latin-1"-IMAGES_STORE_S3_ACL = "private"IMAGES_STORE_GCS_ACL = ""--ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"+IMAGES_STORE_S3_ACL = "private"ITEM_PIPELINES = {}ITEM_PIPELINES_BASE = {}+ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"+JOBDIR = NoneLOG_ENABLED = True-LOG_ENCODING = "utf-8"-LOG_FORMATTER = "scrapy.logformatter.LogFormatter"-LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"-LOG_STDOUT = False-LOG_LEVEL = "DEBUG"+LOG_ENCODING = "utf-8"LOG_FILE = NoneLOG_FILE_APPEND = True+LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"+LOG_FORMATTER = "scrapy.logformatter.LogFormatter"+LOG_LEVEL = "DEBUG"LOG_SHORT_NAMES = False+LOG_STDOUT = FalseLOG_VERSIONS = ["lxml","libxml2",@@ -236,21 +234,19 @@ LOG_VERSIONS = ["Platform",]-SCHEDULER_DEBUG = False-LOGSTATS_INTERVAL = 60.0+MAIL_FROM = "scrapy@localhost"MAIL_HOST = "localhost"MAIL_PORT = 25-MAIL_FROM = "scrapy@localhost"-MAIL_PASS = NoneMAIL_USER = None+MAIL_PASS = NoneMEMDEBUG_ENABLED = False # enable memory debuggingMEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown-MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0MEMUSAGE_ENABLED = True+MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0MEMUSAGE_LIMIT_MB = 0MEMUSAGE_NOTIFY_MAIL = []MEMUSAGE_WARNING_MB = 0@@ -280,9 +276,6 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"RETRY_ENABLED = True-RETRY_TIMES = 2 # initial response + 2 retries = 3 requests-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]-RETRY_PRIORITY_ADJUST = -1RETRY_EXCEPTIONS = ["twisted.internet.defer.TimeoutError","twisted.internet.error.TimeoutError",@@ -298,12 +291,16 @@ RETRY_EXCEPTIONS = [OSError,"scrapy.core.downloader.handlers.http11.TunnelError",]+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]+RETRY_PRIORITY_ADJUST = -1+RETRY_TIMES = 2 # initial response + 2 retries = 3 requestsROBOTSTXT_OBEY = FalseROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"ROBOTSTXT_USER_AGENT = NoneSCHEDULER = "scrapy.core.scheduler.Scheduler"+SCHEDULER_DEBUG = FalseSCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"@@ -312,11 +309,19 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000+SPIDER_CONTRACTS = {}+SPIDER_CONTRACTS_BASE = {+ "scrapy.contracts.default.UrlContract": 1,+ "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,+ "scrapy.contracts.default.MetadataContract": 1,+ "scrapy.contracts.default.ReturnsContract": 2,+ "scrapy.contracts.default.ScrapesContract": 3,+}+SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"SPIDER_LOADER_WARN_ONLY = FalseSPIDER_MIDDLEWARES = {}-SPIDER_MIDDLEWARES_BASE = {# Engine side"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,@@ -334,27 +339,18 @@ STATS_DUMP = TrueSTATSMAILER_RCPTS = []-TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())--URLLENGTH_LIMIT = 2083--USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"-TELNETCONSOLE_ENABLED = 1-TELNETCONSOLE_PORT = [6023, 6073]TELNETCONSOLE_HOST = "127.0.0.1"+TELNETCONSOLE_PORT = [6023, 6073]TELNETCONSOLE_USERNAME = "scrapy"TELNETCONSOLE_PASSWORD = None+TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"-SPIDER_CONTRACTS = {}-SPIDER_CONTRACTS_BASE = {- "scrapy.contracts.default.UrlContract": 1,- "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,- "scrapy.contracts.default.MetadataContract": 1,- "scrapy.contracts.default.ReturnsContract": 2,- "scrapy.contracts.default.ScrapesContract": 3,-}+URLLENGTH_LIMIT = 2083++USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"WARN_ON_GENERATOR_RETURN_VALUE = True