Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

2.9 KB
Raw Blame
from __future__ import annotations

from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any

import pytest

from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from pathlib import Path


def test_deprecated_methods(tmp_path: Path) -> None:
    crawler = get_crawler(Spider, {"JOBDIR": str(tmp_path)})
    ss = build_from_crawler(SpiderState, crawler)
    spider = Spider(name="default")
    with pytest.warns(ScrapyDeprecationWarning, match="spider_opened"):
        ss.spider_opened(spider)
    with pytest.warns(ScrapyDeprecationWarning, match="spider_closed"):
        ss.spider_closed(spider)


def test_not_configured() -> None:
    crawler = get_crawler(Spider)
    with pytest.raises(NotConfigured):
        build_from_crawler(SpiderState, crawler)


@coroutine_test
async def test_store_load(tmp_path: Path) -> None:
    jobdir = str(tmp_path)
    spider = Spider(name="default")
    dt = datetime.now(tz=UTC)

    crawler = get_crawler(Spider, {"JOBDIR": jobdir})
    ss = build_from_crawler(SpiderState, crawler)
    await ss._spider_opened(spider)
    assert hasattr(spider, "state")
    spider.state["one"] = 1
    spider.state["dt"] = dt
    await ss._spider_closed(spider)

    spider2 = Spider(name="default")
    crawler2 = get_crawler(Spider, {"JOBDIR": jobdir})
    ss2 = build_from_crawler(SpiderState, crawler2)
    await ss2._spider_opened(spider2)
    assert hasattr(spider2, "state")
    assert spider2.state == {"one": 1, "dt": dt}
    await ss2._spider_closed(spider2)


@coroutine_test
async def test_spider_state_loaded_signal_fires(tmp_path: Path) -> None:
    crawler = get_crawler(Spider, {"JOBDIR": str(tmp_path)})
    ss = build_from_crawler(SpiderState, crawler)

    received: list[dict[str, Any]] = []

    def on_loaded(state: dict[str, Any]) -> None:
        received.append(dict(state))

    crawler.signals.connect(on_loaded, signal=signals.spider_state_loaded, weak=False)

    spider = Spider(name="default")
    await ss._spider_opened(spider)

    assert received == [{}]
    assert spider.state == {}


@coroutine_test
async def test_spider_state_saving_signal_fires(tmp_path: Path) -> None:
    crawler = get_crawler(Spider, {"JOBDIR": str(tmp_path)})
    ss = build_from_crawler(SpiderState, crawler)

    spider = Spider(name="default")
    await ss._spider_opened(spider)
    spider.state["key"] = "value"

    saving_calls: list[dict[str, Any]] = []

    def on_saving(state: dict[str, Any]) -> None:
        saving_calls.append(dict(state))

    crawler.signals.connect(on_saving, signal=signals.spider_state_saving, weak=False)
    await ss._spider_closed(spider)

    assert saving_calls == [{"key": "value"}]