Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

22.9 KB
Raw Blame
from __future__ import annotations

import asyncio
import logging
import subprocess
import sys
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, Mock, patch

import pytest
from twisted.internet.defer import succeed
from twisted.python.failure import Failure

from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.engine import EngineState, ExecutionEngine, _Slot
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import (
    CloseSpider,
    DontCloseSpider,
    DownloadCancelledError,
    IgnoreRequest,
)
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
    _schedule_coro,
    deferred_from_coro,
    maybe_deferred_to_future,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.spiders import SingleRequestSpider
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.engine import (
    AttrsItemsSpider,
    CrawlerRun,
    DataClassItemsSpider,
    DictItemsSpider,
    MySpider,
)

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Generator

    from twisted.internet.defer import Deferred

    from scrapy.crawler import Crawler
    from tests.mockserver.http import MockServer


class DupeFilterSpider(MySpider):
    async def start(self):
        for url in self.start_urls:
            yield Request(url)  # no dont_filter=True


class ItemZeroDivisionErrorSpider(MySpider):
    custom_settings = {
        "ITEM_PIPELINES": {
            "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
        }
    }


class ChangeCloseReasonSpider(MySpider):
    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = cls(*args, **kwargs)
        spider._set_crawler(crawler)
        crawler.signals.connect(spider.spider_idle, signals.spider_idle)
        return spider

    def spider_idle(self):
        raise CloseSpider(reason="custom_reason")


class ErrorCloseReasonSpider(ChangeCloseReasonSpider):
    def spider_idle(self):
        raise CloseSpider(reason="custom_reason", error=True)


class TestEngine(TestEngineBase):
    @pytest.mark.parametrize(
        "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]
    )
    @coroutine_test
    async def test_crawler(
        self, spider: type[MySpider], mockserver: MockServer
    ) -> None:
        run = CrawlerRun(spider)
        await run.run(mockserver)
        self._assert_visited_urls(run)
        self._assert_scheduled_requests(run, count=9)
        self._assert_downloaded_responses(run, count=9)
        self._assert_scraped_items(run)
        self._assert_signals_caught(run)
        self._assert_headers_received(run)
        self._assert_bytes_received(run)

    @coroutine_test
    async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
        run = CrawlerRun(DupeFilterSpider)
        await run.run(mockserver)
        self._assert_scheduled_requests(run, count=8)
        self._assert_dropped_requests(run)

    @coroutine_test
    async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
        run = CrawlerRun(ItemZeroDivisionErrorSpider)
        await run.run(mockserver)
        self._assert_items_error(run)

    @coroutine_test
    async def test_crawler_change_close_reason_on_idle(
        self, mockserver: MockServer
    ) -> None:
        run = CrawlerRun(ChangeCloseReasonSpider)
        await run.run(mockserver)
        assert {
            "spider": run.crawler.spider,
            "reason": "custom_reason",
            "error": False,
        } == run.signals_caught[signals.spider_closed]

    @coroutine_test
    async def test_crawler_change_close_reason_and_error_on_idle(
        self, mockserver: MockServer
    ) -> None:
        run = CrawlerRun(ErrorCloseReasonSpider)
        await run.run(mockserver)
        assert {
            "spider": run.crawler.spider,
            "reason": "custom_reason",
            "error": True,
        } == run.signals_caught[signals.spider_closed]
        assert run.crawler.stats.get_value("finish_reason_error") is True

    @coroutine_test
    async def test_close_downloader(self):
        e = ExecutionEngine(get_crawler(MySpider))
        await e.close_async()

    def test_close_without_downloader(self):
        class CustomException(Exception):
            pass

        class BadDownloader:
            def __init__(self, crawler):
                raise CustomException

        with pytest.raises(CustomException):
            ExecutionEngine(get_crawler(MySpider, {"DOWNLOADER": BadDownloader}))

    @inline_callbacks_test
    def test_start_already_running_exception(self):
        crawler = get_crawler(DefaultSpider)
        crawler.spider = crawler._create_spider()
        e = ExecutionEngine(crawler)
        crawler.engine = e
        yield deferred_from_coro(e.open_spider_async())
        _schedule_coro(e.start_async())
        with pytest.raises(RuntimeError, match=r"in the (STARTING|RUNNING) state"):
            yield deferred_from_coro(e.start_async())
        yield deferred_from_coro(e.stop_async())

    @coroutine_test
    async def test_stop_async_force_mode_not_supported(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))

        with pytest.raises(ValueError, match="force stop mode is not supported"):
            await engine.stop_async(mode="force")

    @coroutine_test
    async def test_stop_async_not_running_raises(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))

        with pytest.raises(RuntimeError, match="Spider not opened"):
            await engine.stop_async()

    @coroutine_test
    async def test_stop_async_reentrant_fast_drops_downloads(self) -> None:
        """A fast stop requested while the spider is already closing drops
        the in-flight downloads, and returns without waiting for the close."""
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        engine._state = EngineState.SPIDER_CLOSING

        with patch.object(
            engine, "_fast_stop_downloader", new_callable=AsyncMock
        ) as fast_stop:
            await engine.stop_async(mode="fast")

        fast_stop.assert_awaited_once()
        assert engine._stop_mode == "fast"

    @coroutine_test
    async def test_stop_async_reentrant_graceful_is_noop(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        engine._state = EngineState.SPIDER_CLOSING

        with patch.object(
            engine, "_fast_stop_downloader", new_callable=AsyncMock
        ) as fast_stop:
            await engine.stop_async(mode="graceful")

        fast_stop.assert_not_called()

    @coroutine_test
    async def test_handle_downloader_output_ignores_fast_cancelled_failures(
        self,
    ) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        engine.spider = Mock()
        engine._stop_mode = "fast"

        enqueue_scrape = Mock()
        engine.scraper.enqueue_scrape = enqueue_scrape  # type: ignore[method-assign]

        result = Failure(DownloadCancelledError("dropped during fast stop"))
        await maybe_deferred_to_future(
            engine._handle_downloader_output(result, Request("https://example.com"))
        )

        enqueue_scrape.assert_not_called()

    @pytest.mark.only_asyncio
    @coroutine_test
    async def test_start_already_running_exception_asyncio(self):
        crawler = get_crawler(DefaultSpider)
        crawler.spider = crawler._create_spider()
        e = ExecutionEngine(crawler)
        crawler.engine = e
        await e.open_spider_async()
        with pytest.raises(RuntimeError, match=r"in the (STARTING|RUNNING) state"):
            await asyncio.gather(e.start_async(), e.start_async())
        await e.stop_async()

    @coroutine_test
    async def test_start_request_processing_exception(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        class BadRequestFingerprinter:
            def fingerprint(self, request):
                raise ValueError  # to make Scheduler.enqueue_request() fail

        class SimpleSpider(Spider):
            name = "simple"

            async def start(self):
                yield Request("data:,")

        crawler = get_crawler(
            SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter}
        )
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async()
        assert "Error while processing requests from start()" in caplog.text
        assert "Spider closed (shutdown)" in caplog.text

    def test_short_timeout(self):
        args = (
            sys.executable,
            "-m",
            "scrapy.cmdline",
            "fetch",
            "-s",
            "CLOSESPIDER_TIMEOUT=0.001",
            "-s",
            "LOG_LEVEL=DEBUG",
            "http://toscrape.com",
        )
        p = subprocess.Popen(
            args,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
        )

        try:
            _, stderr = p.communicate(timeout=15)
        except subprocess.TimeoutExpired:
            p.kill()
            p.communicate()
            pytest.fail("Command took too much time to complete")

        stderr_str = stderr.decode()
        assert "AttributeError" not in stderr_str, stderr_str
        assert "AssertionError" not in stderr_str, stderr_str


@coroutine_test
async def test_request_scheduled_signal():
    class TestScheduler(BaseScheduler):
        def __init__(self) -> None:
            self.enqueued: list[Request] = []

        def enqueue_request(self, request: Request) -> bool:
            self.enqueued.append(request)
            return True

    def signal_handler(request: Request, spider: Spider) -> None:
        if "drop" in request.url:
            raise IgnoreRequest

    crawler = get_crawler(MySpider)
    engine = ExecutionEngine(crawler)
    scheduler = build_from_crawler(TestScheduler, crawler)

    async def start() -> AsyncIterator[Any]:
        return
        yield

    engine._start = start()
    engine._slot = _Slot(False, Mock(), scheduler)
    crawler.signals.connect(signal_handler, signals.request_scheduled)
    keep_request = Request("https://keep.example")
    engine._schedule_request(keep_request)
    drop_request = Request("https://drop.example")
    engine._schedule_request(drop_request)
    assert scheduler.enqueued == [keep_request], (
        f"{scheduler.enqueued!r} != [{keep_request!r}]"
    )
    crawler.signals.disconnect(signal_handler, signals.request_scheduled)


class ClosingPipeline:
    def open_spider(self):
        raise CloseSpider("pipeline_reason")


class TestCloseSpiderOnStartup:
    @coroutine_test
    async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None:
        closed: list[str] = []

        def spider_closed(reason: str) -> None:
            closed.append(reason)

        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
        crawler.signals.connect(spider_closed, signals.spider_closed)
        with caplog.at_level(logging.INFO):
            await crawler.crawl_async()
        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
        assert closed == ["pipeline_reason"]
        assert "Traceback" not in caplog.text

    @coroutine_test
    async def test_spider_opened(self) -> None:
        def spider_opened(spider: Spider) -> None:
            raise CloseSpider("signal_reason")

        crawler = get_crawler(DefaultSpider)
        crawler.signals.connect(spider_opened, signals.spider_opened)
        await crawler.crawl_async()
        assert crawler.stats.get_value("finish_reason") == "signal_reason"

    @coroutine_test
    async def test_startup_wins_over_spider_opened(self) -> None:
        def spider_opened(spider: Spider) -> None:
            raise CloseSpider("signal_reason")

        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
        crawler.signals.connect(spider_opened, signals.spider_opened)
        await crawler.crawl_async()
        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"

    @inline_callbacks_test
    def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]:
        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
        yield crawler.crawl()
        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"


class TestMisuse:
    """The engine raises on operations that its state does not allow."""

    @coroutine_test
    async def test_stop_not_running(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        try:
            with pytest.raises(RuntimeError, match="Spider not opened"):
                await engine.stop_async()
        finally:
            await engine.close_async()

    def test_invalid_scheduler_class(self) -> None:
        class NotAScheduler:
            pass

        with pytest.raises(
            TypeError, match="does not fully implement the scheduler interface"
        ):
            ExecutionEngine(get_crawler(DefaultSpider, {"SCHEDULER": NotAScheduler}))

    @coroutine_test
    async def test_spider_is_idle_without_slot(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        try:
            with pytest.raises(RuntimeError, match="Engine slot not assigned"):
                engine.spider_is_idle()
        finally:
            await engine.close_async()

    @coroutine_test
    async def test_crawl_without_spider(self) -> None:
        engine = ExecutionEngine(get_crawler(DefaultSpider))
        try:
            with pytest.raises(RuntimeError, match="No open spider to crawl"):
                engine.crawl(Request("data:,"))
        finally:
            await engine.close_async()

    @coroutine_test
    async def test_open_spider_twice(self) -> None:
        crawler = get_crawler(DefaultSpider)
        crawler.spider = crawler._create_spider()
        engine = crawler.engine = ExecutionEngine(crawler)
        await engine.open_spider_async()
        try:
            with pytest.raises(RuntimeError, match="No free spider slot"):
                await engine.open_spider_async()
        finally:
            await engine.close_async()


@coroutine_test
async def test_progress_before_open() -> None:
    engine = ExecutionEngine(get_crawler(DefaultSpider))
    try:
        assert engine.requests_in_progress == frozenset()
        assert engine.items_in_progress == 0
        assert engine.response_active_size == 0
    finally:
        await engine.close_async()


class BrokenScheduler(BaseScheduler):
    """A scheduler that cannot be built."""

    @classmethod
    def from_crawler(cls, crawler: Any) -> BrokenScheduler:
        raise ValueError("broken scheduler")

    def has_pending_requests(self) -> bool:
        return False

    def enqueue_request(self, request: Request) -> bool:
        return True

    def next_request(self) -> Request | None:
        return None


@coroutine_test
async def test_scheduler_creation_error() -> None:
    """An error while building the scheduler is reported directly, instead of
    other errors such as "Engine slot not assigned"."""
    crawler = get_crawler(DefaultSpider, {"SCHEDULER": BrokenScheduler})
    with pytest.raises(ValueError, match="broken scheduler"):
        await crawler.crawl_async()
    assert crawler.engine is not None
    assert crawler.engine.spider is None


@coroutine_test
async def test_start_without_spider() -> None:
    engine = ExecutionEngine(get_crawler(DefaultSpider))
    try:
        with pytest.raises(RuntimeError, match="in the CREATED state"):
            await engine.start_async()
    finally:
        await engine.close_async()


@coroutine_test
async def test_stop_while_item_in_pipelines() -> None:
    """A stop that starts while an item is going through the item pipelines
    completes once that item is done."""

    class StoppingPipeline:
        async def process_item(self, item: Any) -> Any:
            _schedule_coro(crawler.stop_async())
            await sleep(0.1)
            return item

    class ItemSpider(Spider):
        name = "item"

        async def start(self) -> AsyncIterator[Any]:
            yield {"a": "b"}

    crawler = get_crawler(ItemSpider, {"ITEM_PIPELINES": {StoppingPipeline: 1}})
    await crawler.crawl_async()
    assert crawler.stats.get_value("item_scraped_count") == 1
    assert crawler.stats.get_value("finish_reason") == "shutdown"


@coroutine_test
async def test_pause_unpause() -> None:
    engine = ExecutionEngine(get_crawler(DefaultSpider))
    try:
        engine.pause()
        assert engine.paused
        engine.unpause()
        assert not engine.paused
    finally:
        await engine.close_async()


class TestSpiderIdle:
    @coroutine_test
    async def test_dont_close_spider(self) -> None:
        """A ``DontCloseSpider`` handler keeps the spider open for one more loop."""
        idle_calls = 0

        def spider_idle() -> None:
            nonlocal idle_calls
            idle_calls += 1
            if idle_calls == 1:
                # Schedule a request so that the engine loops again soon
                # instead of waiting for the slot heartbeat.
                crawler.engine.crawl(Request("data:,"))
                raise DontCloseSpider

        crawler = get_crawler(DefaultSpider)
        crawler.signals.connect(spider_idle, signals.spider_idle)
        await crawler.crawl_async()
        assert idle_calls == 2
        assert crawler.stats.get_value("finish_reason") == "finished"

    @coroutine_test
    async def test_not_idle_anymore(self) -> None:
        """A handler that schedules a request keeps the spider open."""
        urls: list[str] = []

        def spider_idle() -> None:
            if not urls:
                urls.append("data:,")
                crawler.engine.crawl(Request(urls[0]))

        crawler = get_crawler(DefaultSpider)
        crawler.signals.connect(spider_idle, signals.spider_idle)
        await crawler.crawl_async()
        assert crawler.stats.get_value("downloader/request_count") == 1

    @coroutine_test
    async def test_request_not_in_downloader(self) -> None:
        """The spider is not idle while a request taken from the scheduler is
        not in the downloader yet."""

        class SlowFetchDownloader(Downloader):
            def fetch(self, request: Request) -> Deferred[Response | Request]:  # type: ignore[override]
                return deferred_from_coro(self._slow_fetch(request))

            async def _slow_fetch(self, request: Request) -> Response | Request:
                await sleep(0.1)
                return await maybe_deferred_to_future(super().fetch(request))

        events: list[str] = []

        def response_received() -> None:
            events.append("response_received")

        def spider_idle() -> None:
            events.append("spider_idle")

        crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER": SlowFetchDownloader})
        crawler.signals.connect(response_received, signals.response_received)
        crawler.signals.connect(spider_idle, signals.spider_idle)
        await crawler.crawl_async(seed="data:,")
        assert events == ["response_received", "spider_idle"]

    @coroutine_test
    async def test_request_only_in_downloader(self) -> None:
        """The spider is not idle while the downloader has a request that was
        not sent through the engine."""

        class SlowDownloadHandler(BaseDownloadHandler):
            async def download_request(self, request: Request) -> Response:
                await sleep(0.1)
                return Response(request.url)

        events: list[str] = []

        class DirectFetchSpider(Spider):
            name = "direct_fetch"

            async def start(self) -> AsyncIterator[Any]:
                yield Request("data:,")

            def parse(self, response: Response) -> None:
                assert self.crawler.engine is not None
                d = self.crawler.engine.downloader.fetch(Request("slow://example.com"))
                d.addCallback(self.fetched)

            def fetched(self, result: Response | Request) -> None:
                events.append("fetched")
                # Schedule a request so that the engine loops again soon
                # instead of waiting for the slot heartbeat.
                assert self.crawler.engine is not None
                self.crawler.engine.crawl(Request("data:,b", callback=self.noop))

            def noop(self, response: Response) -> None:
                pass

        def spider_idle() -> None:
            events.append("spider_idle")

        crawler = get_crawler(
            DirectFetchSpider, {"DOWNLOAD_HANDLERS": {"slow": SlowDownloadHandler}}
        )
        crawler.signals.connect(spider_idle, signals.spider_idle)
        await crawler.crawl_async()
        assert events == ["fetched", "spider_idle"]


@coroutine_test
async def test_download_wrong_type(caplog: pytest.LogCaptureFixture) -> None:
    class WrongTypeDownloader(Downloader):
        def fetch(self, request: Request) -> str:  # type: ignore[override]
            return "not a response"

    class WrongTypeSpider(Spider):
        name = "wrong_type"

        async def start(self):
            yield Request("data:,")

    crawler = get_crawler(WrongTypeSpider, {"DOWNLOADER": WrongTypeDownloader})
    await crawler.crawl_async()
    assert "expected Response or Request, got <class 'str'>" in caplog.text


@coroutine_test
async def test_download_async_custom_downloader() -> None:
    class CustomDownloader:
        def __init__(self, crawler: Crawler) -> None:
            self.active: set[Request] = set()

        def fetch(self, request: Request) -> Deferred[Response]:
            return succeed(Response(request.url))

        def needs_backout(self) -> bool:
            return False

        def close(self) -> None:
            pass

    responses: list[Response] = []

    class DownloadAsyncSpider(Spider):
        name = "download_async"

        async def start(self):
            assert self.crawler.engine is not None
            responses.append(
                await self.crawler.engine.download_async(Request("data:,"))
            )
            return
            yield

    crawler = get_crawler(DownloadAsyncSpider, {"DOWNLOADER": CustomDownloader})
    await crawler.crawl_async()
    assert [response.url for response in responses] == ["data:,"]


@coroutine_test
async def test_enqueue_scrape_error(
    caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
    def enqueue_scrape(self, result, request):
        raise ValueError("enqueue error")

    monkeypatch.setattr("scrapy.core.engine.Scraper.enqueue_scrape", enqueue_scrape)

    class SimpleSpider(Spider):
        name = "simple"

        async def start(self):
            yield Request("data:,")

    crawler = get_crawler(SimpleSpider)
    await crawler.crawl_async()
    assert "Error while enqueuing scrape" in caplog.text