Scrapy, a fast high-level web crawling & scraping framework for Python.
periodic_log: Exception handling on init updated
Georgiy Zatserklianyi2ce4856508f7
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex 0f01c441c..b1f5b8894 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -39,29 +39,16 @@ class PeriodicLog:interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")try:ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")- except ValueError:+ except (TypeError, ValueError):ext_stats = ({"enabled": True}if crawler.settings.getbool("PERIODIC_LOG_STATS")else None)- except TypeError:- ext_stats = (- {"enabled": True}- if crawler.settings.getbool("PERIODIC_LOG_STATS")- else None- )-try:ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")- except ValueError:+ except (TypeError, ValueError):ext_delta = (- {"enabled": True}- if crawler.settings.getdict("PERIODIC_LOG_DELTA")- else None- )- except TypeError:- ext_stats = ({"enabled": True}if crawler.settings.getbool("PERIODIC_LOG_DELTA")else Nonediff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex 9f7ec7b23..4dccc687b 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -51,20 +51,78 @@ stats_dump_2 = {}+def extension(settings=None):+ return PeriodicLog.from_crawler(+ Crawler(+ MetaSpider,+ settings=settings,+ )+ )++class TestPeriodicLog(unittest.TestCase):def test_extension_enabled(self):- extension = PeriodicLog.from_crawler(- Crawler(- MetaSpider,- settings={"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60},- )- )- # Test enabled- assert extension+ # Expected that settings for this extension loaded succesfully+ # And on certain conditions - extension raising NotConfigured++ # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}+ # due to TypeError exception from settings.getdict+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})++ # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}+ # due to JSONDecodeError(ValueError) exception from settings.getdict+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})++ # The ame for PERIODIC_LOG_DELTA:+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})# Raise not configured if not set by settingswith self.assertRaises(NotConfigured):- PeriodicLog.from_crawler(Crawler(MetaSpider))+ extension()++ # Regular use cases:+ assert extension(+ {+ "PERIODIC_LOG_STATS": {+ "include": [+ "downloader/",+ "scheduler/",+ "log_count/",+ "item_scraped_count/",+ ],+ "exclude": ["scheduler/"],+ }+ }+ )++ assert extension(+ {+ "PERIODIC_LOG_DELTA": {"include": ["downloader/"]},+ "PERIODIC_LOG_TIMING_ENABLED": True,+ }+ )++ assert extension(+ {+ "PERIODIC_LOG_TIMING_ENABLED": True,+ }+ )++ assert extension(+ {+ "PERIODIC_LOG_STATS": {+ "include": [+ "downloader/",+ "scheduler/",+ "log_count/",+ "item_scraped_count/",+ ],+ "exclude": ["scheduler/"],+ },+ "PERIODIC_LOG_DELTA": {"include": ["downloader/"]},+ }+ )def test_periodic_log_stats(self):pass