Scrapy, a fast high-level web crawling & scraping framework for Python.
Type scrapy/settings/default_settings.py, drop mypy override (#8077)
Adrian941d7eb41900
diff --git a/pyproject.toml b/pyproject.tomlindex 513f0f27c..551b07ee7 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -134,11 +134,6 @@ follow_imports = "skip"module = "twisted"implicit_reexport = true-# TODO-[[tool.mypy.overrides]]-module = "scrapy.settings.default_settings"-ignore_errors = true-# usually no type hints[[tool.mypy.overrides]]module = [diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 85e9944f5..1bfff1c9f 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -224,7 +224,7 @@ __all__ = ["WARN_ON_GENERATOR_RETURN_VALUE",]-ADDONS = {}+ADDONS: dict[str, int] = {}ASYNCIO_EVENT_LOOP = None@@ -290,7 +290,7 @@ DOWNLOAD_DELAY = 0DOWNLOAD_FAIL_ON_DATALOSS = True-DOWNLOAD_HANDLERS = {}+DOWNLOAD_HANDLERS: dict[str, str] = {}DOWNLOAD_HANDLERS_BASE = {"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler","file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",@@ -303,7 +303,7 @@ DOWNLOAD_HANDLERS_BASE = {DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024mDOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m-DOWNLOAD_SLOTS = {}+DOWNLOAD_SLOTS: dict[str, dict[str, Any]] = {}DOWNLOAD_TIMEOUT = 180 # 3mins@@ -319,7 +319,7 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"DOWNLOADER_CLIENT_TLS_METHOD = "TLS"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False-DOWNLOADER_MIDDLEWARES = {}+DOWNLOADER_MIDDLEWARES: dict[str, int] = {}DOWNLOADER_MIDDLEWARES_BASE = {# Engine side"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,@@ -348,7 +348,7 @@ EDITOR = "vi"if sys.platform == "win32":EDITOR = "%s -m idlelib.idle"-EXTENSIONS = {}+EXTENSIONS: dict[str, int] = {}EXTENSIONS_BASE = {"scrapy.extensions.corestats.CoreStats": 0,"scrapy.extensions.logcount.LogCount": 0,@@ -363,12 +363,12 @@ EXTENSIONS_BASE = {"scrapy.extensions.remote_control.RemoteControl": 0,}-FEEDS = {}+FEEDS: dict[str | Path, dict[str, Any]] = {}FEED_EXPORT_BATCH_ITEM_COUNT = 0FEED_EXPORT_ENCODING = NoneFEED_EXPORT_FIELDS = NoneFEED_EXPORT_INDENT = 0-FEED_EXPORTERS = {}+FEED_EXPORTERS: dict[str, str] = {}FEED_EXPORTERS_BASE = {"json": "scrapy.exporters.JsonItemExporter","jsonlines": "scrapy.exporters.JsonLinesItemExporter",@@ -381,7 +381,7 @@ FEED_EXPORTERS_BASE = {}FEED_FORMAT = "jsonlines"FEED_STORE_EMPTY = True-FEED_STORAGES = {}+FEED_STORAGES: dict[str, str] = {}FEED_STORAGES_BASE = {"": "scrapy.extensions.feedexport.FileFeedStorage","file": "scrapy.extensions.feedexport.FileFeedStorage",@@ -422,15 +422,15 @@ HTTPCACHE_DBM_MODULE = "dbm"HTTPCACHE_DIR = "httpcache"HTTPCACHE_EXPIRATION_SECS = 0HTTPCACHE_GZIP = False-HTTPCACHE_IGNORE_HTTP_CODES = []+HTTPCACHE_IGNORE_HTTP_CODES: list[int] = []HTTPCACHE_IGNORE_MISSING = False-HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []+HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS: list[str] = []HTTPCACHE_IGNORE_SCHEMES = ["file"]HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"HTTPERROR_ALLOW_ALL = False-HTTPERROR_ALLOWED_CODES = []+HTTPERROR_ALLOWED_CODES: list[int] = []HTTPPROXY_ENABLED = TrueHTTPPROXY_AUTH_ENCODING = "latin-1"@@ -441,8 +441,8 @@ IMAGES_STORE = NoneIMAGES_STORE_GCS_ACL = ""IMAGES_STORE_S3_ACL = "private"-ITEM_PIPELINES = {}-ITEM_PIPELINES_BASE = {}+ITEM_PIPELINES: dict[str, int] = {}+ITEM_PIPELINES_BASE: dict[str, int] = {}ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"@@ -486,7 +486,7 @@ MEMDEBUG_ENABLED = False # enable memory debuggingMEMUSAGE_ENABLED = TrueMEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0MEMUSAGE_LIMIT_MB = 0-MEMUSAGE_NOTIFY_MAIL = []+MEMUSAGE_NOTIFY_MAIL: list[str] = []MEMUSAGE_WARNING_MB = 0METAREFRESH_ENABLED = True@@ -509,7 +509,7 @@ REDIRECT_PRIORITY_ADJUST = +2REFERER_ENABLED = TrueREFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"-REFERRER_POLICIES = {}+REFERRER_POLICIES: dict[str, str | None] = {}REMOTE_CONTROL_ENABLED = TrueREMOTE_CONTROL_JOBS_DIR = None@@ -554,7 +554,7 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000-SPIDER_CONTRACTS = {}+SPIDER_CONTRACTS: dict[str, int] = {}SPIDER_CONTRACTS_BASE = {"scrapy.contracts.default.UrlContract": 1,"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,@@ -566,7 +566,7 @@ SPIDER_CONTRACTS_BASE = {SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"SPIDER_LOADER_WARN_ONLY = False-SPIDER_MIDDLEWARES = {}+SPIDER_MIDDLEWARES: dict[str, int] = {}SPIDER_MIDDLEWARES_BASE = {# Engine side"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,@@ -578,12 +578,12 @@ SPIDER_MIDDLEWARES_BASE = {# Spider side}-SPIDER_MODULES = []+SPIDER_MODULES: list[str] = []STATS_CLASS = "scrapy.statscollectors.MemoryStatsCollector"STATS_DUMP = True-STATSMAILER_RCPTS = []+STATSMAILER_RCPTS: list[str] = []TELNETCONSOLE_ENABLED = 1TELNETCONSOLE_HOST = "127.0.0.1"