Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

periodic_log: tests for logging deltas added

Georgiy Zatserklianyiebce5b4bcb8b
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 4dccc687b..1df030548 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -51,8 +51,16 @@ stats_dump_2 = {
}
+class TestExtPeriodicLog(PeriodicLog):
+ def set_a(self):
+ self.stats._stats = stats_dump_1
+
+ def set_b(self):
+ self.stats._stats = stats_dump_2
+
+
def extension(settings=None):
- return PeriodicLog.from_crawler(
+ return TestExtPeriodicLog.from_crawler(
Crawler(
MetaSpider,
settings=settings,
@@ -124,14 +132,57 @@ class TestPeriodicLog(unittest.TestCase):
}
)
- def test_periodic_log_stats(self):
- pass
-
def test_log_delta(self):
- pass
+ def emulate(settings=None):
+ ext = extension(settings)
+ ext.spider_opened(MetaSpider)
+ ext.set_a()
+ a = ext.log_delta()
+ ext.set_a()
+ b = ext.log_delta()
+ return ext, a, b
+
+ def check(settings: dict, condition: callable):
+ ext, a, b = emulate(settings)
+ assert list(a["delta"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
+ ]
+ assert list(b["delta"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
+ ]
+
+ # Including all
+ check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))
+
+ # include:
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},
+ lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,
+ )
- def test_settings_include(self):
- pass
+ # include multiple
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},
+ lambda k, v: isinstance(v, (int, float))
+ and ("downloader/" in k or "scheduler/" in k),
+ )
- def test_settings_exclude(self):
- pass
+ # exclude
+ check(
+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},
+ lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,
+ )
+
+ # exclude multiple
+ check(
+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},
+ lambda k, v: isinstance(v, (int, float))
+ and ("downloader/" not in k and "scheduler/" not in k),
+ )
+
+ # include exclude combined
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},
+ lambda k, v: isinstance(v, (int, float))
+ and ("downloader/" in k and "scheduler/" not in k),
+ )