Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

8.2 KB
Raw Blame
"""
Queues that handle requests
"""

from __future__ import annotations

from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any

import pytest

from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.squeues import (
    FifoMemoryQueue,
    LifoMemoryQueue,
    MarshalFifoDiskQueue,
    MarshalFifoSQLiteQueue,
    MarshalLifoDiskQueue,
    MarshalLifoSQLiteQueue,
    PickleFifoDiskQueue,
    PickleFifoSQLiteQueue,
    PickleLifoDiskQueue,
    PickleLifoSQLiteQueue,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler

if TYPE_CHECKING:
    from collections.abc import Generator
    from pathlib import Path

    import queuelib

    from scrapy.crawler import Crawler


@pytest.fixture
def crawler() -> Crawler:
    return get_crawler(Spider)


class TestRequestQueueBase(ABC):
    @property
    @abstractmethod
    def is_fifo(self) -> bool:
        raise NotImplementedError

    def test_one_element(self, q: queuelib.queue.BaseQueue) -> None:
        assert len(q) == 0
        assert q.peek() is None
        assert q.pop() is None
        req = Request("http://www.example.com")
        q.push(req)
        assert len(q) == 1
        result = q.peek()
        assert result is not None
        assert result.url == req.url
        result = q.pop()
        assert result is not None
        assert result.url == req.url
        assert len(q) == 0
        assert q.peek() is None
        assert q.pop() is None

    def test_headers(self, q: queuelib.queue.BaseQueue) -> None:
        req = Request(
            "http://www.example.com",
            headers=[("X-Multi", "a"), ("x-multi", "b"), ("Accept", None)],
        )
        q.push(req)
        result = q.pop()
        assert result is not None
        assert result.headers == req.headers

    def test_order(self, q: queuelib.queue.BaseQueue) -> None:
        assert len(q) == 0
        assert q.peek() is None
        assert q.pop() is None
        req1 = Request("http://www.example.com/1")
        req2 = Request("http://www.example.com/2")
        req3 = Request("http://www.example.com/3")
        q.push(req1)
        q.push(req2)
        q.push(req3)
        reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1]
        for i, req in enumerate(reqs):
            assert len(q) == 3 - i
            result = q.peek()
            assert result is not None
            assert result.url == req.url
            result = q.pop()
            assert result is not None
            assert result.url == req.url
        assert len(q) == 0
        assert q.peek() is None
        assert q.pop() is None


class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
    is_fifo = True

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
    is_fifo = False

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
    is_fifo = True

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
    is_fifo = False

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestPickleFifoSQLiteQueueRequest(TestRequestQueueBase):
    is_fifo = True

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            PickleFifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestPickleLifoSQLiteQueueRequest(TestRequestQueueBase):
    is_fifo = False

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            PickleLifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestMarshalFifoSQLiteQueueRequest(TestRequestQueueBase):
    is_fifo = True

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            MarshalFifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
        )
        try:
            yield queue
        finally:
            queue.close()


class TestMarshalLifoSQLiteQueueRequest(TestRequestQueueBase):
    is_fifo = False

    @pytest.fixture
    def q(
        self, crawler: Crawler, tmp_path: Path
    ) -> Generator[queuelib.queue.BaseQueue]:
        queue = build_from_crawler(
            MarshalLifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
        )
        try:
            yield queue
        finally:
            queue.close()


@pytest.mark.parametrize(
    "queue_cls",
    [
        PickleFifoSQLiteQueue,
        PickleLifoSQLiteQueue,
        MarshalFifoSQLiteQueue,
        MarshalLifoSQLiteQueue,
    ],
)
def test_sqlite_queue_survives_unclean_shutdown(
    crawler: Crawler, tmp_path: Path, queue_cls: Any
) -> None:
    key = str(tmp_path / "queue")
    queue = queue_cls.from_crawler(crawler=crawler, key=key)
    queue.push(Request("https://toscrape.com/"))
    # No close(), as if the process had been killed. The connection is closed
    # directly instead, since on Windows the file cannot be removed later while
    # it is still open.
    queue._db.close()

    queue = queue_cls.from_crawler(crawler=crawler, key=key)
    try:
        assert len(queue) == 1
        request = queue.pop()
        assert request is not None
        assert request.url == "https://toscrape.com/"
    finally:
        queue.close()


@pytest.mark.parametrize(
    ("queue_cls", "is_fifo"),
    [
        (PickleFifoDiskQueue, True),
        (PickleLifoDiskQueue, False),
        (MarshalFifoDiskQueue, True),
        (MarshalLifoDiskQueue, False),
        (PickleFifoSQLiteQueue, True),
        (PickleLifoSQLiteQueue, False),
        (MarshalFifoSQLiteQueue, True),
        (MarshalLifoSQLiteQueue, False),
    ],
)
def test_close_reopen(
    crawler: Crawler, tmp_path: Path, queue_cls: Any, is_fifo: bool
) -> None:
    key = str(tmp_path / "queue")
    urls = [f"https://example.com/{i}" for i in range(4)]
    queue = build_from_crawler(queue_cls, crawler, key=key)
    for url in urls:
        queue.push(Request(url))
    queue.pop()
    queue.close()

    expected = urls[1:] if is_fifo else urls[-2::-1]
    queue = build_from_crawler(queue_cls, crawler, key=key)
    try:
        assert len(queue) == 3
        assert queue.peek().url == expected[0]
        assert [queue.pop().url for _ in expected] == expected
        assert queue.pop() is None
    finally:
        queue.close()


class TestFifoMemoryQueueRequest(TestRequestQueueBase):
    is_fifo = True

    @pytest.fixture
    def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:
        return build_from_crawler(FifoMemoryQueue, crawler)


class TestLifoMemoryQueueRequest(TestRequestQueueBase):
    is_fifo = False

    @pytest.fixture
    def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:
        return build_from_crawler(LifoMemoryQueue, crawler)