Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

3.3 KB
Raw Blame
from __future__ import annotations

from datetime import UTC, datetime
from typing import TYPE_CHECKING

import pytest

from scrapy.extensions.logstats import LogStats
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from scrapy.crawler import Crawler
    from scrapy.statscollectors import StatsCollector


@pytest.fixture
def crawler() -> Crawler:
    crawler = get_crawler(SimpleSpider)
    crawler.spider = crawler._create_spider("spidey")
    return crawler


@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
    stats = crawler.stats
    stats.set_value("response_received_count", 4802)
    stats.set_value("item_scraped_count", 3201)
    return stats


@coroutine_test
async def test_stats_calculations(crawler: Crawler, stats: StatsCollector) -> None:
    logstats = build_from_crawler(LogStats, crawler)

    with pytest.raises(AttributeError):
        logstats.pagesprev
    with pytest.raises(AttributeError):
        logstats.itemsprev

    assert crawler.spider
    logstats.spider_opened(crawler.spider)
    assert logstats.pagesprev == 4802
    assert logstats.itemsprev == 3201

    logstats.calculate_stats()
    assert logstats.items == 3201
    assert logstats.pages == 4802
    assert logstats.irate == 0.0
    assert logstats.prate == 0.0
    assert logstats.pagesprev == 4802
    assert logstats.itemsprev == 3201

    # Simulate what happens after a minute
    stats.set_value("response_received_count", 5187)
    stats.set_value("item_scraped_count", 3492)
    logstats.calculate_stats()
    assert logstats.items == 3492
    assert logstats.pages == 5187
    assert logstats.irate == 291.0
    assert logstats.prate == 385.0
    assert logstats.pagesprev == 5187
    assert logstats.itemsprev == 3492

    # Simulate when spider closes after running for 30 mins
    stats.set_value("start_time", datetime.fromtimestamp(1655100172).astimezone(UTC))
    stats.set_value("finish_time", datetime.fromtimestamp(1655101972).astimezone(UTC))
    logstats.spider_closed(crawler.spider, "test reason")
    assert stats.get_value("responses_per_minute") == 172.9
    assert stats.get_value("items_per_minute") == 116.4


def test_stats_calculations_no_time(crawler: Crawler, stats: StatsCollector) -> None:
    """The stat values should be None since the start and finish time are
    not available.
    """
    logstats = build_from_crawler(LogStats, crawler)
    assert crawler.spider
    logstats.spider_closed(crawler.spider, "test reason")
    assert stats.get_value("responses_per_minute") is None
    assert stats.get_value("items_per_minute") is None


def test_stats_calculation_no_elapsed_time(
    crawler: Crawler, stats: StatsCollector
) -> None:
    """The stat values should be None since the elapsed time is 0."""
    logstats = build_from_crawler(LogStats, crawler)
    stats.set_value("start_time", datetime.fromtimestamp(1655100172).astimezone(UTC))
    stats.set_value("finish_time", datetime.fromtimestamp(1655100172).astimezone(UTC))
    assert crawler.spider
    logstats.spider_closed(crawler.spider, "test reason")
    assert stats.get_value("responses_per_minute") is None
    assert stats.get_value("items_per_minute") is None