Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add unit tests for the StatsMailer extension (#7013)

M Ikram Ullah Khan00b2be0943a4
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 221628642..9e900308d 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -40,7 +40,7 @@ class StatsMailer:
return o
def spider_closed(self, spider: Spider) -> Deferred[None] | None:
- spider_stats = self.stats.get_stats(spider)
+ spider_stats = self.stats.get_stats()
body = "Global stats\n\n"
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
body += f"\n\n{spider.name} stats\n\n"
diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py
new file mode 100644
index 000000000..b6322325c
--- /dev/null
+++ b/tests/test_extension_statsmailer.py
@@ -0,0 +1,82 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from scrapy import signals
+from scrapy.exceptions import NotConfigured
+from scrapy.extensions import statsmailer
+from scrapy.mail import MailSender
+from scrapy.signalmanager import SignalManager
+from scrapy.statscollectors import StatsCollector
+from scrapy.utils.spider import DefaultSpider
+
+
+def dummy_stats():
+ class DummyStats(StatsCollector):
+ def __init__(self):
+ # pylint: disable=super-init-not-called
+ self._stats = {"global_item_scraped_count": 42}
+
+ def get_stats(self):
+ return {"item_scraped_count": 10, **self._stats}
+
+ return DummyStats()
+
+
+def test_from_crawler_without_recipients_raises_notconfigured():
+ crawler = MagicMock()
+ crawler.settings.getlist.return_value = []
+ crawler.stats = MagicMock()
+
+ with pytest.raises(NotConfigured):
+ statsmailer.StatsMailer.from_crawler(crawler)
+
+
+def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch):
+ crawler = MagicMock()
+ crawler.settings.getlist.return_value = ["[email protected]"]
+ crawler.stats = dummy_stats
+ crawler.signals = SignalManager(crawler)
+
+ mailer = MagicMock(spec=MailSender)
+ monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
+
+ ext = statsmailer.StatsMailer.from_crawler(crawler)
+
+ assert isinstance(ext, statsmailer.StatsMailer)
+ assert ext.recipients == ["[email protected]"]
+ assert ext.mail is mailer
+
+
+def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):
+ crawler = MagicMock()
+ crawler.settings.getlist.return_value = ["[email protected]"]
+ crawler.stats = dummy_stats
+ crawler.signals = SignalManager(crawler)
+
+ mailer = MagicMock(spec=MailSender)
+ monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
+
+ statsmailer.StatsMailer.from_crawler(crawler)
+
+ connected = crawler.signals.send_catch_log(
+ signals.spider_closed, spider=DefaultSpider(name="dummy")
+ )
+ assert connected is not None
+
+
+def test_spider_closed_sends_email(dummy_stats):
+ recipients = ["[email protected]"]
+ mail = MagicMock(spec=MailSender)
+ ext = statsmailer.StatsMailer(dummy_stats, recipients, mail)
+
+ spider = DefaultSpider(name="dummy")
+ ext.spider_closed(spider)
+
+ args, kwargs = mail.send.call_args
+ to, subject, body = args
+ assert to == recipients
+ assert "Scrapy stats for: dummy" in subject
+ assert "global_item_scraped_count" in body
+ assert "item_scraped_count" in body