Scrapy, a fast high-level web crawling & scraping framework for Python.
Clean up setting getter defaults (#6892)
Adrián Chaves85aeda365db0
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 9293d7b78..4b5654826 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -124,7 +124,7 @@ class Downloader:)self._slot_gc_loop.start(60)self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(- "DOWNLOAD_SLOTS", {}+ "DOWNLOAD_SLOTS")@inlineCallbacksdiff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex e7a8962a1..a23deaa45 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -43,7 +43,7 @@ class AjaxCrawlMiddleware:# middleware parses first 4k. 4k turns out to be insufficient# for this middleware, and parsing 100k could be slow.# We use something in between (32K) by default.- self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)+ self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE")@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex aba455bdd..fbd737970 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -38,10 +38,8 @@ class RobotsTxtMiddleware:def __init__(self, crawler: Crawler):if not crawler.settings.getbool("ROBOTSTXT_OBEY"):raise NotConfigured- self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")- self._robotstxt_useragent: str | None = crawler.settings.get(- "ROBOTSTXT_USER_AGENT", None- )+ self._default_useragent: str = crawler.settings["USER_AGENT"]+ self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]self.crawler: Crawler = crawlerself._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}self._parserimpl: RobotParser = load_object(diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c39a9c92e..d9e9ea775 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -479,7 +479,7 @@ class FeedExporter:uri = self.settings["FEED_URI"]# handle pathlib.Path objectsuri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()- feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}+ feed_options = {"format": self.settings["FEED_FORMAT"]}self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex 9158482fa..98210990a 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -78,7 +78,7 @@ class PeriodicLog:)ext_timing_enabled: bool = crawler.settings.getbool(- "PERIODIC_LOG_TIMING_ENABLED", False+ "PERIODIC_LOG_TIMING_ENABLED")if not (ext_stats or ext_delta or ext_timing_enabled):raise NotConfigureddiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 7cd470f11..b6f47f1c3 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -20,6 +20,7 @@ from pathlib import PathADDONS = {}AJAXCRAWL_ENABLED = False+AJAXCRAWL_MAXSIZE = 32768ASYNCIO_EVENT_LOOP = None@@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0COOKIES_ENABLED = TrueCOOKIES_DEBUG = False+CRAWLSPIDER_FOLLOW_LINKS = True+DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"DEFAULT_ITEM_CLASS = "scrapy.item.Item"@@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = {"marshal": "scrapy.exporters.MarshalItemExporter","pickle": "scrapy.exporters.PickleItemExporter",}+FEED_FORMAT = "jsonlines"FEED_STORE_EMPTY = TrueFEED_STORAGES = {}FEED_STORAGES_BASE = {diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex f44f70e40..98e7b23c0 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -213,7 +213,5 @@ class CrawlSpider(Spider):@classmethoddef from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:spider = super().from_crawler(crawler, *args, **kwargs)- spider._follow_links = crawler.settings.getbool(- "CRAWLSPIDER_FOLLOW_LINKS", True- )+ spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")return spider