Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type scrapy/settings/default_settings.py, drop mypy override (#8077)

Adrian941d7eb41900
diff --git a/pyproject.toml b/pyproject.toml
index 513f0f27c..551b07ee7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -134,11 +134,6 @@ follow_imports = "skip"
module = "twisted"
implicit_reexport = true
-# TODO
-[[tool.mypy.overrides]]
-module = "scrapy.settings.default_settings"
-ignore_errors = true
-
# usually no type hints
[[tool.mypy.overrides]]
module = [
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 85e9944f5..1bfff1c9f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -224,7 +224,7 @@ __all__ = [
"WARN_ON_GENERATOR_RETURN_VALUE",
]
-ADDONS = {}
+ADDONS: dict[str, int] = {}
ASYNCIO_EVENT_LOOP = None
@@ -290,7 +290,7 @@ DOWNLOAD_DELAY = 0
DOWNLOAD_FAIL_ON_DATALOSS = True
-DOWNLOAD_HANDLERS = {}
+DOWNLOAD_HANDLERS: dict[str, str] = {}
DOWNLOAD_HANDLERS_BASE = {
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
"file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
@@ -303,7 +303,7 @@ DOWNLOAD_HANDLERS_BASE = {
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
-DOWNLOAD_SLOTS = {}
+DOWNLOAD_SLOTS: dict[str, dict[str, Any]] = {}
DOWNLOAD_TIMEOUT = 180 # 3mins
@@ -319,7 +319,7 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
-DOWNLOADER_MIDDLEWARES = {}
+DOWNLOADER_MIDDLEWARES: dict[str, int] = {}
DOWNLOADER_MIDDLEWARES_BASE = {
# Engine side
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
@@ -348,7 +348,7 @@ EDITOR = "vi"
if sys.platform == "win32":
EDITOR = "%s -m idlelib.idle"
-EXTENSIONS = {}
+EXTENSIONS: dict[str, int] = {}
EXTENSIONS_BASE = {
"scrapy.extensions.corestats.CoreStats": 0,
"scrapy.extensions.logcount.LogCount": 0,
@@ -363,12 +363,12 @@ EXTENSIONS_BASE = {
"scrapy.extensions.remote_control.RemoteControl": 0,
}
-FEEDS = {}
+FEEDS: dict[str | Path, dict[str, Any]] = {}
FEED_EXPORT_BATCH_ITEM_COUNT = 0
FEED_EXPORT_ENCODING = None
FEED_EXPORT_FIELDS = None
FEED_EXPORT_INDENT = 0
-FEED_EXPORTERS = {}
+FEED_EXPORTERS: dict[str, str] = {}
FEED_EXPORTERS_BASE = {
"json": "scrapy.exporters.JsonItemExporter",
"jsonlines": "scrapy.exporters.JsonLinesItemExporter",
@@ -381,7 +381,7 @@ FEED_EXPORTERS_BASE = {
}
FEED_FORMAT = "jsonlines"
FEED_STORE_EMPTY = True
-FEED_STORAGES = {}
+FEED_STORAGES: dict[str, str] = {}
FEED_STORAGES_BASE = {
"": "scrapy.extensions.feedexport.FileFeedStorage",
"file": "scrapy.extensions.feedexport.FileFeedStorage",
@@ -422,15 +422,15 @@ HTTPCACHE_DBM_MODULE = "dbm"
HTTPCACHE_DIR = "httpcache"
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_GZIP = False
-HTTPCACHE_IGNORE_HTTP_CODES = []
+HTTPCACHE_IGNORE_HTTP_CODES: list[int] = []
HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
+HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS: list[str] = []
HTTPCACHE_IGNORE_SCHEMES = ["file"]
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
HTTPERROR_ALLOW_ALL = False
-HTTPERROR_ALLOWED_CODES = []
+HTTPERROR_ALLOWED_CODES: list[int] = []
HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = "latin-1"
@@ -441,8 +441,8 @@ IMAGES_STORE = None
IMAGES_STORE_GCS_ACL = ""
IMAGES_STORE_S3_ACL = "private"
-ITEM_PIPELINES = {}
-ITEM_PIPELINES_BASE = {}
+ITEM_PIPELINES: dict[str, int] = {}
+ITEM_PIPELINES_BASE: dict[str, int] = {}
ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
@@ -486,7 +486,7 @@ MEMDEBUG_ENABLED = False # enable memory debugging
MEMUSAGE_ENABLED = True
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
MEMUSAGE_LIMIT_MB = 0
-MEMUSAGE_NOTIFY_MAIL = []
+MEMUSAGE_NOTIFY_MAIL: list[str] = []
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
@@ -509,7 +509,7 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
-REFERRER_POLICIES = {}
+REFERRER_POLICIES: dict[str, str | None] = {}
REMOTE_CONTROL_ENABLED = True
REMOTE_CONTROL_JOBS_DIR = None
@@ -554,7 +554,7 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
-SPIDER_CONTRACTS = {}
+SPIDER_CONTRACTS: dict[str, int] = {}
SPIDER_CONTRACTS_BASE = {
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
@@ -566,7 +566,7 @@ SPIDER_CONTRACTS_BASE = {
SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
SPIDER_LOADER_WARN_ONLY = False
-SPIDER_MIDDLEWARES = {}
+SPIDER_MIDDLEWARES: dict[str, int] = {}
SPIDER_MIDDLEWARES_BASE = {
# Engine side
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
@@ -578,12 +578,12 @@ SPIDER_MIDDLEWARES_BASE = {
# Spider side
}
-SPIDER_MODULES = []
+SPIDER_MODULES: list[str] = []
STATS_CLASS = "scrapy.statscollectors.MemoryStatsCollector"
STATS_DUMP = True
-STATSMAILER_RCPTS = []
+STATSMAILER_RCPTS: list[str] = []
TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_HOST = "127.0.0.1"