Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

3.2 KB
Raw Blame
from __future__ import annotations

from asyncio import sleep
from typing import TYPE_CHECKING, Any

import pytest

from scrapy import Spider, signals
from scrapy.utils.asyncio import sleep as scrapy_sleep
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler

from .utils import twisted_sleep
from .utils.decorators import coroutine_test

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Callable

SLEEP_SECONDS = 0.1

ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}


async def _test_spider(
    spider: type[Spider],
    expected_items: list[Any] | None = None,
    settings: dict[str, Any] | None = None,
    *,
    ordered: bool = True,
) -> None:
    actual_items: list[Any] = []
    expected_items = [] if expected_items is None else expected_items

    def track_item(item: Any, response: Any, spider: Spider) -> None:
        actual_items.append(item)

    crawler = get_crawler(spider, settings)
    crawler.signals.connect(track_item, signals.item_scraped)
    await crawler.crawl_async()
    assert crawler.stats.get_value("finish_reason") == "finished"
    if not ordered:
        actual_items = sorted(actual_items, key=repr)
        expected_items = sorted(expected_items, key=repr)

    assert actual_items == expected_items


@coroutine_test
async def test_start_urls() -> None:
    class TestSpider(Spider):
        name = "test"
        start_urls = ["data:,"]

        async def parse(self, response):
            yield ITEM_A

    await _test_spider(TestSpider, [ITEM_A])


@coroutine_test
async def test_start() -> None:
    class TestSpider(Spider):
        name = "test"

        async def start(self):
            yield ITEM_A

    await _test_spider(TestSpider, [ITEM_A])


@coroutine_test
async def test_start_subclass() -> None:
    class BaseSpider(Spider):
        async def start(self):
            yield ITEM_A

    class TestSpider(BaseSpider):
        name = "test"

    await _test_spider(TestSpider, [ITEM_A])


async def _test_start(
    start_: Callable[[Any], AsyncIterator[Any]],
    expected_items: list[Any] | None = None,
) -> None:
    class TestSpider(Spider):
        name = "test"
        start = start_

    await _test_spider(TestSpider, expected_items)


@pytest.mark.only_asyncio
@coroutine_test
async def test_asyncio_delayed() -> None:
    async def start(spider):
        await sleep(SLEEP_SECONDS)
        yield ITEM_A

    await _test_start(start, [ITEM_A])


@pytest.mark.requires_reactor  # needs a reactor for twisted_sleep()
@coroutine_test
async def test_twisted_delayed() -> None:
    async def start(spider):
        await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
        yield ITEM_A

    await _test_start(start, [ITEM_A])


@coroutine_test
async def test_slow_pipeline() -> None:
    class SlowPipeline:
        async def process_item(self, item):
            await scrapy_sleep(SLEEP_SECONDS)
            return item

    class TestSpider(Spider):
        name = "test"

        async def start(self):
            yield ITEM_A
            yield ITEM_B

    await _test_spider(
        TestSpider,
        [ITEM_A, ITEM_B],
        {"ITEM_PIPELINES": {SlowPipeline: 0}},
        ordered=False,
    )