Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

23.8 KB
Raw Blame
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import MagicMock

import pytest
from twisted.internet.defer import succeed
from twisted.python.failure import Failure

from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.pipelines.media import (
    FileInfo,
    FileInfoOrError,
    MediaPipeline,
    _MediaRequestFiltered,
)
from scrapy.utils.defer import _process_pending_io
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.media_pipelines import mocked_download_func

if TYPE_CHECKING:
    from collections.abc import Awaitable

    from twisted.internet.defer import Deferred

    from scrapy.crawler import Crawler


class UserDefinedPipeline(MediaPipeline):
    def media_to_download(
        self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
    ) -> Deferred[FileInfo | None] | None:
        return None

    def get_media_requests(
        self, item: Any, info: MediaPipeline.SpiderInfo
    ) -> list[Request]:
        return []

    def media_downloaded(
        self,
        response: Response,
        request: Request,
        info: MediaPipeline.SpiderInfo,
        *,
        item: Any = None,
    ) -> FileInfo | Awaitable[FileInfo]:
        return cast("FileInfo", {})

    def media_failed(
        self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
    ) -> Failure:
        failure.raiseException()

    def file_path(
        self,
        request: Request,
        response: Response | None = None,
        info: MediaPipeline.SpiderInfo | None = None,
        *,
        item: Any = None,
    ) -> str:
        return ""


class TestBaseMediaPipeline:
    pipeline_class = UserDefinedPipeline
    settings = None

    def setup_method(self):
        crawler = get_crawler(DefaultSpider, self.settings)
        crawler.spider = crawler._create_spider()
        crawler.engine = MagicMock(download_async=mocked_download_func)
        self.pipe = build_from_crawler(self.pipeline_class, crawler)
        self.pipe.open_spider()
        self.info = self.pipe.spiderinfo
        self.fingerprint = crawler.request_fingerprinter.fingerprint

    @property
    def mocked_pipe(self) -> MockedMediaPipeline:
        assert isinstance(self.pipe, MockedMediaPipeline)
        return self.pipe

    def teardown_method(self):
        for name, signal in vars(signals).items():
            if not name.startswith("_"):
                disconnect_all(signal)

    def test_modify_media_request(self):
        request = Request("http://url")
        self.pipe._modify_media_request(request)
        assert request.meta == {"dont_redirect": True}

    def test_modify_media_request_keeps_meta(self):
        request = Request("http://url", meta={"dont_redirect": False})
        self.pipe._modify_media_request(request)
        assert request.meta == {"dont_redirect": False}

    def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:
        item = {"name": "name"}
        assert self.pipe.item_completed([], item, self.info) is item

        # Check that failures are logged by default
        fail = Failure(Exception())
        results: Any = [(True, 1), (False, fail)]

        caplog.clear()
        new_item = self.pipe.item_completed(results, item, self.info)
        assert new_item is item
        assert len(caplog.records) == 1
        record = caplog.records[0]
        assert record.levelname == "ERROR"
        assert record.exc_info == failure_to_exc_info(fail)

        # disable failure logging and check again
        caplog.clear()
        self.pipe.LOG_FAILED_RESULTS = False
        new_item = self.pipe.item_completed(results, item, self.info)
        assert new_item is item
        assert len(caplog.records) == 0

    def test_item_completed_filtered_request_not_logged(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """Filtered media requests (e.g. offsite ones) are not logged as errors
        by item_completed(), as they are not download errors."""
        item = {"name": "name"}
        fail = Failure(_MediaRequestFiltered("Filtered offsite request"))
        results: Any = [(True, 1), (False, fail)]

        with caplog.at_level(logging.DEBUG):
            new_item = self.pipe.item_completed(results, item, self.info)

        assert new_item is item
        assert len(caplog.records) == 0

    @coroutine_test
    async def test_default_process_item(self):
        item = {"name": "name"}
        new_item = await self.pipe.process_item(item)
        assert new_item is item


class DeferredItemCompletedPipeline(UserDefinedPipeline):
    def item_completed(
        self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
    ) -> Deferred[Any]:
        return succeed(item)


class AsyncItemCompletedPipeline(UserDefinedPipeline):
    async def item_completed(
        self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
    ) -> Any:
        return item


@pytest.mark.parametrize(
    "pipeline_class", [DeferredItemCompletedPipeline, AsyncItemCompletedPipeline]
)
@coroutine_test
async def test_awaitable_item_completed(pipeline_class: type[MediaPipeline]) -> None:
    crawler = get_crawler(DefaultSpider)
    crawler.spider = crawler._create_spider()
    pipe = build_from_crawler(pipeline_class, crawler)
    pipe.open_spider()
    item = {"name": "name"}
    assert await pipe.process_item(item) is item


class MockedMediaPipeline(UserDefinedPipeline):
    def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any):
        super().__init__(*args, crawler=crawler, **kwargs)
        self._mockcalled: list[str] = []

    def media_to_download(
        self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
    ) -> Deferred[FileInfo | None] | None:
        self._mockcalled.append("media_to_download")
        if "result" in request.meta:
            return request.meta.get("result")
        return super().media_to_download(request, info)

    def get_media_requests(
        self, item: Any, info: MediaPipeline.SpiderInfo
    ) -> list[Request]:
        self._mockcalled.append("get_media_requests")
        return item.get("requests")  # type: ignore[no-any-return]

    def media_downloaded(
        self,
        response: Response,
        request: Request,
        info: MediaPipeline.SpiderInfo,
        *,
        item: Any = None,
    ) -> FileInfo | Awaitable[FileInfo]:
        self._mockcalled.append("media_downloaded")
        return super().media_downloaded(response, request, info)

    def media_failed(
        self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
    ) -> Failure:
        self._mockcalled.append("media_failed")
        return super().media_failed(failure, request, info)

    def item_completed(
        self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
    ) -> Any:
        self._mockcalled.append("item_completed")
        item = super().item_completed(results, item, info)
        item["results"] = results
        return item


class AsyncMediaDownloadedPipeline(MockedMediaPipeline):
    async def media_downloaded(  # type: ignore[override]
        self,
        response: Response,
        request: Request,
        info: MediaPipeline.SpiderInfo,
        *,
        item: Any = None,
    ) -> FileInfo | Awaitable[FileInfo]:
        return super().media_downloaded(response, request, info)


class TestMediaPipeline(TestBaseMediaPipeline):
    pipeline_class = MockedMediaPipeline

    def _errback(self, result):
        self.mocked_pipe._mockcalled.append("request_errback")
        return result

    @coroutine_test
    async def test_result_succeed(self):
        rsp = Response("http://url1")
        req = Request(
            "http://url1",
            meta={"response": rsp},
            errback=self._errback,
        )
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, {})]
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_downloaded",
            "item_completed",
        ]

    @coroutine_test
    async def test_result_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        req = Request(
            "http://url1",
            meta={"response": exc},
            errback=self._errback,
        )
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 1
        assert new_item["results"][0][0] is False
        assert isinstance(new_item["results"][0][1], Failure)
        assert new_item["results"][0][1].value == exc
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_failed",
            "request_errback",
            "item_completed",
        ]

    @coroutine_test
    async def test_mix_of_success_and_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        exc = Exception("foo")
        req2 = Request("http://url2", meta={"response": exc})
        item = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 2
        assert new_item["results"][0] == (True, {})
        assert new_item["results"][1][0] is False
        assert isinstance(new_item["results"][1][1], Failure)
        assert new_item["results"][1][1].value == exc
        m = self.mocked_pipe._mockcalled
        # only once
        assert m[0] == "get_media_requests"  # first hook called
        assert m.count("get_media_requests") == 1
        assert m.count("item_completed") == 1
        assert m[-1] == "item_completed"  # last hook called
        # twice, one per request
        assert m.count("media_to_download") == 2
        # one to handle success and other for failure
        assert m.count("media_downloaded") == 1
        assert m.count("media_failed") == 1

    @coroutine_test
    async def test_get_media_requests(self):
        # returns single Request (without callback)
        req = Request("http://url")
        item = {"requests": req}  # pass a single item
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req) in self.info.downloaded

        # returns iterable of Requests
        req1 = Request("http://url1")
        req2 = Request("http://url2")
        item = {"requests": iter([req1, req2])}  # type: ignore[dict-item]
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req1) in self.info.downloaded
        assert self.fingerprint(req2) in self.info.downloaded

    @coroutine_test
    async def test_results_are_cached_across_multiple_items(self):
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        item: dict[str, Any] = {"requests": req1}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert new_item["results"] == [(True, {})]

        # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
        req2 = Request(
            req1.url, meta={"response": Response("http://donot.download.me")}
        )
        item = {"requests": req2}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req1) == self.fingerprint(req2)
        assert new_item["results"] == [(True, {})]

    @coroutine_test
    async def test_failures_are_cached_across_multiple_items(self):
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        req1 = Request("http://url1", meta={"response": exc})
        new_item = await self.pipe.process_item({"requests": req1})
        assert new_item["results"][0][1].value is exc

        # rsp2 is ignored, the cached failure must be reused because request
        # fingerprints are the same
        req2 = Request(
            req1.url, meta={"response": Response("http://donot.download.me")}
        )
        new_item = await self.pipe.process_item({"requests": req2})
        assert new_item["results"][0][0] is False
        assert new_item["results"][0][1].value is exc
        assert self.mocked_pipe._mockcalled.count("media_to_download") == 1

    @coroutine_test
    async def test_cached_failure_calls_errback(self):
        """The errback of a request is called for a cached failure as well."""
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        await self.pipe.process_item(
            {"requests": Request("http://url1", meta={"response": exc})}
        )

        def errback(failure):
            self.mocked_pipe._mockcalled.append("request_errback")
            return {"recovered": failure.value}

        req = Request("http://url1", errback=errback)
        new_item = await self.pipe.process_item({"requests": req})
        assert new_item["results"] == [(True, {"recovered": exc})]
        assert self.mocked_pipe._mockcalled.count("request_errback") == 1

    @coroutine_test
    async def test_results_are_cached_for_requests_of_single_item(self):
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        req2 = Request(
            req1.url, meta={"response": Response("http://donot.download.me")}
        )
        item: dict[str, Any] = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert new_item["results"] == [(True, {}), (True, {})]

    @coroutine_test
    async def test_wait_if_request_is_downloading(self):
        def _check_downloading(response: Response) -> Response:
            fp = self.fingerprint(req1)
            assert fp in self.info.downloading
            assert fp in self.info.waiting
            assert fp not in self.info.downloaded
            assert len(self.info.waiting[fp]) == 2
            return response

        rsp1 = Response("http://url")

        async def rsp1_func():
            await _process_pending_io()
            _check_downloading(rsp1)

        async def rsp2_func():
            pytest.fail("it must cache rsp1 result and must not try to redownload")

        req1 = Request("http://url", meta={"response": rsp1_func})
        req2 = Request(req1.url, meta={"response": rsp2_func})
        item = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, {}), (True, {})]

    @pytest.mark.parametrize(
        ("size", "prefixed", "cached_urls"),
        [
            (2, False, ["http://url1", "http://url3"]),
            (2, True, ["http://url1", "http://url3"]),
            (0, False, []),
            (-1, False, ["http://url2", "http://url1", "http://url3"]),
        ],
    )
    @coroutine_test
    async def test_cache_size(self, size: int, prefixed: bool, cached_urls: list[str]):
        key = "MEDIA_CACHE_SIZE"
        if prefixed:
            key = f"{self.pipeline_class.__name__.upper()}_{key}"
        crawler = get_crawler(DefaultSpider, {key: size})
        crawler.spider = crawler._create_spider()
        crawler.engine = MagicMock(download_async=mocked_download_func)
        pipe = build_from_crawler(self.pipeline_class, crawler)
        pipe.open_spider()
        for url in ["http://url1", "http://url2", "http://url1", "http://url3"]:
            req = Request(url, meta={"response": Response(url)})
            await pipe.process_item({"requests": req})
        assert list(pipe.spiderinfo.downloaded) == [
            self.fingerprint(Request(url)) for url in cached_urls
        ]

    @coroutine_test
    async def test_use_media_to_download_result(self):
        req = Request("http://url", meta={"result": "ITSME"})
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, "ITSME")]
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "item_completed",
        ]

    def test_key_for_pipe(self):
        assert (
            self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
            == "MOCKEDMEDIAPIPELINE_IMAGES"
        )


class TestAsyncMediaDownloaded(TestMediaPipeline):
    pipeline_class = AsyncMediaDownloadedPipeline

    def test_key_for_pipe(self):
        assert (
            self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
            == "ASYNCMEDIADOWNLOADEDPIPELINE_IMAGES"
        )


class TestMediaPipelineAllowRedirectSettings:
    @pytest.mark.parametrize(
        "settings",
        [
            {"MEDIA_ALLOW_REDIRECTS": True},
            {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True},
        ],
    )
    def test_allow_redirects(self, settings: dict[str, Any]) -> None:
        pipe = UserDefinedPipeline(crawler=get_crawler(None, settings))
        request = Request("http://url")
        pipe._modify_media_request(request)
        assert request.meta == {}

    def test_deprecated_attribute_get(self):
        pipe = UserDefinedPipeline(crawler=get_crawler())
        with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
            assert pipe.handle_httpstatus_list is None
        crawler = get_crawler(
            None,
            {
                "MEDIA_ALLOW_REDIRECTS": True,
                "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],
            },
        )
        pipe = UserDefinedPipeline(crawler=crawler)
        with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
            codes = pipe.handle_httpstatus_list
        assert codes is not None
        assert 404 in codes
        assert 201 not in codes
        assert 302 not in codes

    @pytest.mark.parametrize(("value", "expected"), [(None, True), ([404], [404])])
    def test_deprecated_attribute_set(self, value: Any, expected: Any) -> None:
        pipe = UserDefinedPipeline(crawler=get_crawler())
        with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
            pipe.handle_httpstatus_list = value
        with pytest.warns(ScrapyDeprecationWarning, match="handle_httpstatus_list"):
            assert pipe.handle_httpstatus_list == value
        request = Request("http://url")
        pipe._modify_media_request(request)
        assert request.meta == {"handle_http_codes": expected}


class TestBuildFromCrawler:
    def setup_method(self):
        self.crawler = get_crawler(None, {"FILES_STORE": "/foo"})

    def test_simple(self):
        class Pipeline(UserDefinedPipeline):
            pass

        pipe = build_from_crawler(Pipeline, self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter

    def test_has_from_crawler_and_init(self):
        class Pipeline(UserDefinedPipeline):
            _from_crawler_called = False

            def __init__(self, store_uri, settings, *, crawler):
                super().__init__(crawler=crawler)
                self._init_called = True

            @classmethod
            def from_crawler(cls, crawler: Crawler) -> Pipeline:
                settings = crawler.settings
                store_uri = settings["FILES_STORE"]
                o = cls(store_uri, settings=settings, crawler=crawler)
                o._from_crawler_called = True
                return o

        pipe = build_from_crawler(Pipeline, self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe._from_crawler_called
        assert pipe._init_called

    def test_has_from_crawler(self):
        class Pipeline(UserDefinedPipeline):
            _from_crawler_called = False
            store_uri: str

            @classmethod
            def from_crawler(cls, crawler: Crawler) -> Pipeline:
                settings = crawler.settings
                o = super().from_crawler(crawler)
                o._from_crawler_called = True
                o.store_uri = settings["FILES_STORE"]
                return o

        pipe = build_from_crawler(Pipeline, self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe._from_crawler_called


class MediaDownloadedErrorPipeline(MockedMediaPipeline):
    def media_downloaded(
        self,
        response: Response,
        request: Request,
        info: MediaPipeline.SpiderInfo,
        *,
        item: Any = None,
    ) -> FileInfo:
        self._mockcalled.append("media_downloaded")
        raise ValueError("foo")


class TestMediaDownloadedError(TestBaseMediaPipeline):
    pipeline_class = MediaDownloadedErrorPipeline

    @coroutine_test
    async def test_error_skips_media_failed(self):
        url = "https://example.com"
        req = Request(url, meta={"response": Response(url)})
        new_item = await self.pipe.process_item({"requests": req})
        ((ok, failure),) = new_item["results"]
        assert ok is False
        assert isinstance(failure.value, ValueError)
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_downloaded",
            "item_completed",
        ]


class MediaFailedNonePipeline(MockedMediaPipeline):
    def media_failed(  # type: ignore[override]
        self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
    ) -> None:
        self._mockcalled.append("media_failed")


class TestMediaFailedNone(TestBaseMediaPipeline):
    """Test what happens when media_failed() neither raises an exception nor
    returns a failure."""

    pipeline_class = MediaFailedNonePipeline

    @coroutine_test
    async def test_result_none(self):
        req = Request("http://url1", meta={"response": Exception("foo")})
        new_item = await self.pipe.process_item({"requests": req})
        assert new_item["results"] == [(True, None)]
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_failed",
            "item_completed",
        ]


class MediaFailedFailurePipeline(MockedMediaPipeline):
    def media_failed(
        self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
    ) -> Failure:
        self._mockcalled.append("media_failed")
        return failure  # deprecated


class TestMediaFailedFailure(TestBaseMediaPipeline):
    """Test that media_failed() can return a failure instead of raising."""

    pipeline_class = MediaFailedFailurePipeline

    def _errback(self, result):
        self.mocked_pipe._mockcalled.append("request_errback")
        return result

    @coroutine_test
    async def test_result_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        req = Request(
            "http://url1",
            meta={"response": exc},
            errback=self._errback,
        )
        item = {"requests": req}
        with pytest.warns(
            ScrapyDeprecationWarning, match="media_failed returned a Failure instance"
        ):
            new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 1
        assert new_item["results"][0][0] is False
        assert isinstance(new_item["results"][0][1], Failure)
        assert new_item["results"][0][1].value == exc
        assert self.mocked_pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_failed",
            "request_errback",
            "item_completed",
        ]