Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

periodic_log: Exception handling on init updated

Georgiy Zatserklianyi2ce4856508f7
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 0f01c441c..b1f5b8894 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -39,29 +39,16 @@ class PeriodicLog:
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
try:
ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")
- except ValueError:
+ except (TypeError, ValueError):
ext_stats = (
{"enabled": True}
if crawler.settings.getbool("PERIODIC_LOG_STATS")
else None
)
- except TypeError:
- ext_stats = (
- {"enabled": True}
- if crawler.settings.getbool("PERIODIC_LOG_STATS")
- else None
- )
-
try:
ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")
- except ValueError:
+ except (TypeError, ValueError):
ext_delta = (
- {"enabled": True}
- if crawler.settings.getdict("PERIODIC_LOG_DELTA")
- else None
- )
- except TypeError:
- ext_stats = (
{"enabled": True}
if crawler.settings.getbool("PERIODIC_LOG_DELTA")
else None
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 9f7ec7b23..4dccc687b 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -51,20 +51,78 @@ stats_dump_2 = {
}
+def extension(settings=None):
+ return PeriodicLog.from_crawler(
+ Crawler(
+ MetaSpider,
+ settings=settings,
+ )
+ )
+
+
class TestPeriodicLog(unittest.TestCase):
def test_extension_enabled(self):
- extension = PeriodicLog.from_crawler(
- Crawler(
- MetaSpider,
- settings={"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60},
- )
- )
- # Test enabled
- assert extension
+ # Expected that settings for this extension loaded succesfully
+ # And on certain conditions - extension raising NotConfigured
+
+ # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
+ # due to TypeError exception from settings.getdict
+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})
+
+ # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}
+ # due to JSONDecodeError(ValueError) exception from settings.getdict
+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})
+
+ # The ame for PERIODIC_LOG_DELTA:
+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
# Raise not configured if not set by settings
with self.assertRaises(NotConfigured):
- PeriodicLog.from_crawler(Crawler(MetaSpider))
+ extension()
+
+ # Regular use cases:
+ assert extension(
+ {
+ "PERIODIC_LOG_STATS": {
+ "include": [
+ "downloader/",
+ "scheduler/",
+ "log_count/",
+ "item_scraped_count/",
+ ],
+ "exclude": ["scheduler/"],
+ }
+ }
+ )
+
+ assert extension(
+ {
+ "PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
+ "PERIODIC_LOG_TIMING_ENABLED": True,
+ }
+ )
+
+ assert extension(
+ {
+ "PERIODIC_LOG_TIMING_ENABLED": True,
+ }
+ )
+
+ assert extension(
+ {
+ "PERIODIC_LOG_STATS": {
+ "include": [
+ "downloader/",
+ "scheduler/",
+ "log_count/",
+ "item_scraped_count/",
+ ],
+ "exclude": ["scheduler/"],
+ },
+ "PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
+ }
+ )
def test_periodic_log_stats(self):
pass