Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

3.0 KB
Raw Blame
import warnings
from typing import Any
from unittest.mock import MagicMock

import pytest

from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider

with warnings.catch_warnings():
    warnings.filterwarnings(
        "ignore",
        r"The scrapy\.extensions\.statsmailer module is deprecated",
        ScrapyDeprecationWarning,
    )
    warnings.filterwarnings(
        "ignore",
        r"The scrapy\.mail module is deprecated",
        ScrapyDeprecationWarning,
    )
    from scrapy.extensions import statsmailer
    from scrapy.mail import MailSender


@pytest.fixture
def dummy_stats():
    class DummyStats(StatsCollector):
        def __init__(self) -> None:
            # pylint: disable=super-init-not-called
            self._stats = {"global_item_scraped_count": 42}

        def get_stats(self, spider: Spider | None = None) -> dict[str, Any]:
            return {"item_scraped_count": 10, **self._stats}

    return DummyStats()


def test_from_crawler_without_recipients_raises_notconfigured():
    crawler = MagicMock()
    crawler.settings.getlist.return_value = []
    crawler.stats = MagicMock()

    with pytest.raises(NotConfigured):
        build_from_crawler(statsmailer.StatsMailer, crawler)


def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch):
    crawler = MagicMock()
    crawler.settings.getlist.return_value = ["[email protected]"]
    crawler.stats = dummy_stats
    crawler.signals = SignalManager(crawler)

    mailer = MagicMock(spec=MailSender)
    monkeypatch.setattr(MailSender, "from_crawler", lambda _: mailer)

    ext = build_from_crawler(statsmailer.StatsMailer, crawler)

    assert isinstance(ext, statsmailer.StatsMailer)
    assert ext.recipients == ["[email protected]"]
    assert ext.mail is mailer


def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):
    crawler = MagicMock()
    crawler.settings.getlist.return_value = ["[email protected]"]
    crawler.stats = dummy_stats
    crawler.signals = SignalManager(crawler)

    mailer = MagicMock(spec=MailSender)
    monkeypatch.setattr(MailSender, "from_crawler", lambda _: mailer)

    build_from_crawler(statsmailer.StatsMailer, crawler)

    connected = crawler.signals.send_catch_log(
        signals.spider_closed, spider=DefaultSpider(name="dummy")
    )
    assert connected is not None


def test_spider_closed_sends_email(dummy_stats):
    recipients = ["[email protected]"]
    mail = MagicMock(spec=MailSender)
    ext = statsmailer.StatsMailer(dummy_stats, recipients, mail)

    spider = DefaultSpider(name="dummy")
    ext.spider_closed(spider)

    args, _ = mail.send.call_args
    to, subject, body = args
    assert to == recipients
    assert "Scrapy stats for: dummy" in subject
    assert "global_item_scraped_count" in body
    assert "item_scraped_count" in body