Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Clean up setting getter defaults (#6892)

Adrián Chaves85aeda365db0
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 9293d7b78..4b5654826 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -124,7 +124,7 @@ class Downloader:
)
self._slot_gc_loop.start(60)
self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
- "DOWNLOAD_SLOTS", {}
+ "DOWNLOAD_SLOTS"
)
@inlineCallbacks
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index e7a8962a1..a23deaa45 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -43,7 +43,7 @@ class AjaxCrawlMiddleware:
# middleware parses first 4k. 4k turns out to be insufficient
# for this middleware, and parsing 100k could be slow.
# We use something in between (32K) by default.
- self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
+ self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE")
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index aba455bdd..fbd737970 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -38,10 +38,8 @@ class RobotsTxtMiddleware:
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
- self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
- self._robotstxt_useragent: str | None = crawler.settings.get(
- "ROBOTSTXT_USER_AGENT", None
- )
+ self._default_useragent: str = crawler.settings["USER_AGENT"]
+ self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
self.crawler: Crawler = crawler
self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
self._parserimpl: RobotParser = load_object(
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c39a9c92e..d9e9ea775 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -479,7 +479,7 @@ class FeedExporter:
uri = self.settings["FEED_URI"]
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
- feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
+ feed_options = {"format": self.settings["FEED_FORMAT"]}
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 9158482fa..98210990a 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -78,7 +78,7 @@ class PeriodicLog:
)
ext_timing_enabled: bool = crawler.settings.getbool(
- "PERIODIC_LOG_TIMING_ENABLED", False
+ "PERIODIC_LOG_TIMING_ENABLED"
)
if not (ext_stats or ext_delta or ext_timing_enabled):
raise NotConfigured
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7cd470f11..b6f47f1c3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -20,6 +20,7 @@ from pathlib import Path
ADDONS = {}
AJAXCRAWL_ENABLED = False
+AJAXCRAWL_MAXSIZE = 32768
ASYNCIO_EVENT_LOOP = None
@@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0
COOKIES_ENABLED = True
COOKIES_DEBUG = False
+CRAWLSPIDER_FOLLOW_LINKS = True
+
DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"
DEFAULT_ITEM_CLASS = "scrapy.item.Item"
@@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = {
"marshal": "scrapy.exporters.MarshalItemExporter",
"pickle": "scrapy.exporters.PickleItemExporter",
}
+FEED_FORMAT = "jsonlines"
FEED_STORE_EMPTY = True
FEED_STORAGES = {}
FEED_STORAGES_BASE = {
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f44f70e40..98e7b23c0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -213,7 +213,5 @@ class CrawlSpider(Spider):
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
- spider._follow_links = crawler.settings.getbool(
- "CRAWLSPIDER_FOLLOW_LINKS", True
- )
+ spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
return spider