Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

12.8 KB
Raw Blame
from __future__ import annotations

import logging
import shutil
from pathlib import Path
from tempfile import mkdtemp
from typing import TYPE_CHECKING, Any

import pytest
from w3lib.url import add_or_replace_parameter

from scrapy import Spider, signals
from scrapy.pipelines.media import FileException
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from scrapy.crawler import Crawler
    from scrapy.http import Response
    from tests.mockserver.http import MockServer


class MediaDownloadSpider(SimpleSpider):
    name = "mediadownload"

    media_key: str
    media_urls_key: str

    def _process_url(self, url: str) -> str:
        return url

    def parse(self, response):
        self.logger.info(response.headers)
        self.logger.info(response.text)
        item = {
            self.media_key: [],
            self.media_urls_key: [
                self._process_url(response.urljoin(href))
                for href in response.xpath(
                    '//table[thead/tr/th="Filename"]/tbody//a/@href'
                ).getall()
            ],
        }
        yield item


class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
    name = "brokenmedia"

    def _process_url(self, url: str) -> str:
        return url + ".foo"


class RedirectedMediaDownloadSpider(MediaDownloadSpider):
    name = "redirectedmedia"

    def _process_url(self, url: str) -> str:
        assert self.mockserver
        return add_or_replace_parameter(
            self.mockserver.url("/redirect-to"), "goto", url
        )


class CreatedMediaDownloadSpider(MediaDownloadSpider):
    """Requests media files from a server that creates them on the fly, reports
    that with an empty 201 response, and points at the created file through the
    Location header."""

    name = "createdmedia"

    def _process_url(self, url: str) -> str:
        assert self.mockserver
        return add_or_replace_parameter(self.mockserver.url("/created"), "goto", url)


class TestFileDownloadCrawl:
    pipeline_class = "scrapy.pipelines.files.FilesPipeline"
    store_setting_key = "FILES_STORE"
    media_key = "files"
    media_urls_key = "file_urls"
    expected_checksums: set[str] | None = {
        "5547178b89448faf0015a13f904c936e",
        "c2281c83670e31d8aaab7cb642b824db",
        "ed3f6538dc15d4d9179dae57319edc5f",
    }

    def setup_method(self):
        # prepare a directory for storing files
        self.tmpmediastore = Path(mkdtemp())
        self.settings = {
            "ITEM_PIPELINES": {self.pipeline_class: 1},
            self.store_setting_key: str(self.tmpmediastore),
        }
        self.items = []

    def teardown_method(self):
        shutil.rmtree(self.tmpmediastore)
        self.items = []

    def _on_item_scraped(self, item):
        self.items.append(item)

    def _create_crawler(
        self, spider_class: type[Spider], settings: dict[str, Any] | None = None
    ) -> Crawler:
        if settings is None:
            settings = self.settings
        crawler = get_crawler(spider_class, settings)
        crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
        return crawler

    def _assert_files_downloaded(self, items: list[Any], logs: str) -> None:
        assert len(items) == 1
        assert self.media_key in items[0]

        # check that logs show the expected number of successful file downloads
        file_dl_success = "File (downloaded): Downloaded file from"
        assert logs.count(file_dl_success) == 3

        # check that the images/files status is `downloaded`
        for item in items:
            for i in item[self.media_key]:
                assert i["status"] == "downloaded"

        # check that the images/files checksums are what we know they should be
        if self.expected_checksums is not None:
            checksums = {i["checksum"] for item in items for i in item[self.media_key]}
            assert checksums == self.expected_checksums

        # check that the image files where actually written to the media store
        for item in items:
            for i in item[self.media_key]:
                assert (self.tmpmediastore / i["path"]).exists()

    def _assert_files_download_failure(
        self, crawler: Crawler, items: list[Any], code: int, logs: str
    ) -> None:
        # check that the item does NOT have the "images/files" field populated
        assert len(items) == 1
        assert self.media_key in items[0]
        assert not items[0][self.media_key]

        # check that there was 1 successful fetch and 3 other responses with non-200 code
        assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
        assert crawler.stats.get_value("downloader/response_count") == 4
        assert crawler.stats.get_value("downloader/response_status_count/200") == 1
        assert crawler.stats.get_value(f"downloader/response_status_count/{code}") == 3

        # check that logs do show the failure on the file downloads
        file_dl_failure = f"File (code: {code}): Error downloading file from"
        assert logs.count(file_dl_failure) == 3
        assert "File (unknown-error)" not in logs

        # check that no files were written to the media store
        assert not list(self.tmpmediastore.iterdir())

    @coroutine_test
    async def test_download_media(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        crawler = self._create_crawler(MediaDownloadSpider)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
            )
        self._assert_files_downloaded(self.items, caplog.text)

    @coroutine_test
    async def test_download_media_wrong_urls(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
            )
        self._assert_files_download_failure(crawler, self.items, 404, caplog.text)

    @coroutine_test
    async def test_download_media_failed_response(self, mockserver: MockServer) -> None:
        class ResponseFileException(FileException):
            def __init__(self, response: Response):
                super().__init__(f"download-error ({response.status})")
                self.response = response

        class ResponsePipeline(load_object(self.pipeline_class)):  # type: ignore[misc]
            async def media_downloaded(self, response, request, info, *, item=None):
                try:
                    return await super().media_downloaded(
                        response, request, info, item=item
                    )
                except FileException as e:
                    raise ResponseFileException(response) from e

            def item_completed(self, results, item, info):
                item["download_errors"] = [
                    value.value.response.status
                    for ok, value in results
                    if not ok and value.check(ResponseFileException)
                ]
                return super().item_completed(results, item, info)

        settings = {**self.settings, "ITEM_PIPELINES": {ResponsePipeline: 1}}
        crawler = self._create_crawler(BrokenLinksMediaDownloadSpider, settings)
        await crawler.crawl_async(
            mockserver.url("/static/files/images/"),
            media_key=self.media_key,
            media_urls_key=self.media_urls_key,
        )
        assert len(self.items) == 1
        assert self.items[0]["download_errors"] == [404, 404, 404]

    @coroutine_test
    async def test_download_media_redirected_default_failure(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ):
        crawler = self._create_crawler(RedirectedMediaDownloadSpider)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
                mockserver=mockserver,
            )
        self._assert_files_download_failure(crawler, self.items, 302, caplog.text)

    @coroutine_test
    async def test_download_media_redirected_allowed(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        settings = {
            **self.settings,
            "MEDIA_ALLOW_REDIRECTS": True,
        }
        crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
                mockserver=mockserver,
            )
        self._assert_files_downloaded(self.items, caplog.text)
        assert crawler.stats.get_value("downloader/response_status_count/302") == 3

    @pytest.mark.parametrize(
        "settings",
        [
            {},
            # Following the Location header of a 201 response requires both
            # settings.
            {"REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308]},
            {"MEDIA_ALLOW_REDIRECTS": True},
        ],
    )
    @coroutine_test
    async def test_download_media_created_default_failure(
        self,
        settings: dict[str, Any],
        caplog: pytest.LogCaptureFixture,
        mockserver: MockServer,
    ) -> None:
        crawler = self._create_crawler(
            CreatedMediaDownloadSpider, {**self.settings, **settings}
        )
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
                mockserver=mockserver,
            )
        assert len(self.items) == 1
        assert not self.items[0][self.media_key]
        assert crawler.stats
        assert crawler.stats.get_value("downloader/response_status_count/201") == 3
        assert caplog.text.count("File (empty-content): Empty file from") == 3
        assert not list(self.tmpmediastore.iterdir())

    @coroutine_test
    async def test_download_media_created_allowed(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        settings = {
            **self.settings,
            "REDIRECT_HTTP_CODES": [201, 301, 302, 303, 307, 308],
            "MEDIA_ALLOW_REDIRECTS": True,
        }
        crawler = self._create_crawler(CreatedMediaDownloadSpider, settings)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
                mockserver=mockserver,
            )
        self._assert_files_downloaded(self.items, caplog.text)
        assert crawler.stats
        assert crawler.stats.get_value("downloader/response_status_count/201") == 3
        assert caplog.text.count("Redirecting (201)") == 3

    @coroutine_test
    async def test_download_media_file_path_error(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        cls = load_object(self.pipeline_class)

        class ExceptionRaisingMediaPipeline(cls):  # type: ignore[misc,valid-type]
            def file_path(self, request, response=None, info=None, *, item=None):
                return 1 / 0

        settings = {
            **self.settings,
            "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
        }
        crawler = self._create_crawler(MediaDownloadSpider, settings)
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(
                mockserver.url("/static/files/images/"),
                media_key=self.media_key,
                media_urls_key=self.media_urls_key,
                mockserver=mockserver,
            )
        assert "ZeroDivisionError" in caplog.text


pillow_available: bool
try:
    from PIL import Image  # noqa: F401
except ImportError:
    pillow_available = False
else:
    pillow_available = True


@pytest.mark.skipif(
    not pillow_available,
    reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow",
)
class TestImageDownloadCrawl(TestFileDownloadCrawl):
    pipeline_class = "scrapy.pipelines.images.ImagesPipeline"
    store_setting_key = "IMAGES_STORE"
    media_key = "images"
    media_urls_key = "image_urls"

    # somehow checksums for images are different for Python 3.3
    expected_checksums = None