Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

20.3 KB
Raw Blame
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, Any

from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.spiders import MockServerSpider
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Iterator

    import pytest

    from scrapy.http import Response
    from tests.mockserver.http import MockServer


class _BaseSpiderMiddleware:
    def __init__(self, crawler):
        self.crawler = crawler

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)


class LogExceptionMiddleware(_BaseSpiderMiddleware):
    def process_spider_exception(self, response, exception):
        self.crawler.spider.logger.info(
            "Middleware: %s exception caught", exception.__class__.__name__
        )


# ================================================================================
# (0) recover from an exception on a spider callback
class RecoveryMiddleware(_BaseSpiderMiddleware):
    def process_spider_exception(self, response, exception):
        self.crawler.spider.logger.info(
            "Middleware: %s exception caught", exception.__class__.__name__
        )
        return [
            {"from": "process_spider_exception"},
            Request(response.url, meta={"dont_fail": True}, dont_filter=True),
        ]


class RecoverySpider(MockServerSpider):
    name = "RecoverySpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES_BASE": {},
        "SPIDER_MIDDLEWARES": {
            RecoveryMiddleware: 10,
        },
    }

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response: Response) -> Iterator[Any]:
        yield {"test": 1}
        self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
        if not response.meta.get("dont_fail"):
            raise TabError


class RecoveryAsyncGenSpider(RecoverySpider):
    name = "RecoveryAsyncGenSpider"

    async def parse(self, response: Response) -> AsyncIterator[Any]:  # type: ignore[override]
        for r in super().parse(response):
            yield r


# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
    def process_spider_input(self, response):
        self.crawler.spider.logger.info("Middleware: will raise IndexError")
        raise IndexError


class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider):
    name = "ProcessSpiderInputSpiderWithoutErrback"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            # spider
            FailProcessSpiderInputMiddleware: 8,
            LogExceptionMiddleware: 6,
            # engine
        }
    }

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)

    def parse(self, response):
        return {"from": "callback"}


class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
    name = "ProcessSpiderInputSpiderWithErrback"

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(
            self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
        )

    def errback(self, failure):
        self.logger.info("Got a Failure on the Request errback")
        return {"from": "errback"}


# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(MockServerSpider):
    name = "GeneratorCallbackSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 10,
        },
    }

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"test": 1}
        yield {"test": 2}
        raise ImportError


class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
    async def parse(self, response):
        yield {"test": 1}
        yield {"test": 2}
        raise ImportError


# ================================================================================
# (2.1) exceptions from a spider callback (generator, middleware right after callback)
class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
    name = "GeneratorCallbackSpiderMiddlewareRightAfterSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 100000,
        },
    }


# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotGeneratorCallbackSpider(MockServerSpider):
    name = "NotGeneratorCallbackSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 10,
        },
    }

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        return [{"test": 1}, {"test": 1 / 0}]


# ================================================================================
# (3.1) exceptions from a spider callback (not a generator, middleware right after callback)
class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider):
    name = "NotGeneratorCallbackSpiderMiddlewareRightAfterSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 100000,
        },
    }


# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )


class GeneratorFailMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r
            raise LookupError

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )
        yield {"processed": [method]}


class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
    pass


class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )
        yield {"processed": [method]}


class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
    pass


class GeneratorOutputChainSpider(MockServerSpider):
    name = "GeneratorOutputChainSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            GeneratorFailMiddleware: 10,
            GeneratorDoNothingAfterFailureMiddleware: 8,
            GeneratorRecoverMiddleware: 5,
            GeneratorDoNothingAfterRecoveryMiddleware: 3,
        },
    }

    async def start(self) -> AsyncIterator[Any]:
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"processed": ["parse-first-item"]}
        yield {"processed": ["parse-second-item"]}


# ================================================================================
# (5) an exception from a spider callback (generator) that no process_spider_exception
# method handles, with middlewares that define a process_spider_output method
class FirstUnhandledMiddleware(_GeneratorDoNothingMiddleware):
    pass


class SecondUnhandledMiddleware(_GeneratorDoNothingMiddleware):
    pass


class ThirdUnhandledMiddleware(_GeneratorDoNothingMiddleware):
    pass


class UnhandledExceptionSpider(MockServerSpider):
    name = "UnhandledExceptionSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            FirstUnhandledMiddleware: 30,
            SecondUnhandledMiddleware: 20,
            ThirdUnhandledMiddleware: 10,
        },
    }

    async def start(self):
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"processed": ["parse-first-item"]}
        raise ImportError


# ================================================================================
# (6) errback output after a download error
class LogOutputMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for o in result:
            self.crawler.spider.logger.info(
                f"Middleware: output {o} with response {response}"
            )
            yield o

    def process_spider_exception(self, response, exception):
        self.crawler.spider.logger.info(
            f"Middleware: {exception.__class__.__name__} exception caught"
            f" with response {response}"
        )
        return []


class DownloadErrorSpider(MockServerSpider):
    name = "DownloadErrorSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogOutputMiddleware: 10,
        },
    }

    async def start(self):
        assert self.mockserver
        yield Request(self.mockserver.url("/drop?abort=1"), errback=self.errback)

    def errback(self, failure):
        yield {"from": "errback"}
        assert self.mockserver
        yield Request(self.mockserver.url("/status?n=200"), callback=self.parse)

    def parse(self, response):
        self.logger.info(f"is_start_request: {response.meta.get('is_start_request')}")


class DownloadErrorFailSpider(DownloadErrorSpider):
    name = "DownloadErrorFailSpider"

    def errback(self, failure):
        yield {"from": "errback"}
        raise LookupError


# ================================================================================
class TestSpiderMiddleware:
    @staticmethod
    async def crawl_log(
        spider: type[Spider], caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> str:
        crawler = get_crawler(spider)
        caplog.clear()
        with caplog.at_level(logging.DEBUG):
            await crawler.crawl_async(mockserver=mockserver)
        return caplog.text

    @coroutine_test
    async def test_recovery(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (0) Recover from an exception in a spider callback. The final item count should be 3
        (one yielded from the callback method before the exception is raised, one directly
        from the recovery middleware and one from the spider when processing the request that
        was enqueued from the recovery middleware)
        """
        log = await self.crawl_log(RecoverySpider, caplog, mockserver)
        assert "Middleware: TabError exception caught" in log
        assert log.count("Middleware: TabError exception caught") == 1
        assert "'item_scraped_count': 3" in log

    @coroutine_test
    async def test_recovery_asyncgen(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        Same as test_recovery but with an async callback.
        """
        log = await self.crawl_log(RecoveryAsyncGenSpider, caplog, mockserver)
        assert "Middleware: TabError exception caught" in log
        assert log.count("Middleware: TabError exception caught") == 1
        assert "'item_scraped_count': 3" in log

    @coroutine_test
    async def test_process_spider_input_without_errback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (1.1) An exception from the process_spider_input chain should be caught by the
        process_spider_exception chain from the start if the Request has no errback
        """
        log1 = await self.crawl_log(
            ProcessSpiderInputSpiderWithoutErrback, caplog, mockserver
        )
        assert "Middleware: will raise IndexError" in log1
        assert "Middleware: IndexError exception caught" in log1

    @coroutine_test
    async def test_process_spider_input_with_errback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (1.2) An exception from the process_spider_input chain should not be caught by the
        process_spider_exception chain if the Request has an errback
        """
        log1 = await self.crawl_log(
            ProcessSpiderInputSpiderWithErrback, caplog, mockserver
        )
        assert "Middleware: IndexError exception caught" not in log1
        assert "Middleware: will raise IndexError" in log1
        assert "Got a Failure on the Request errback" in log1
        assert "{'from': 'errback'}" in log1
        assert "{'from': 'callback'}" not in log1
        assert "'item_scraped_count': 1" in log1

    @coroutine_test
    async def test_generator_callback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (2) An exception from a spider callback (returning a generator) should
        be caught by the process_spider_exception chain. Items yielded before the
        exception is raised should be processed normally.
        """
        log2 = await self.crawl_log(GeneratorCallbackSpider, caplog, mockserver)
        assert "Middleware: ImportError exception caught" in log2
        assert "'item_scraped_count': 2" in log2

    @coroutine_test
    async def test_async_generator_callback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        Same as test_generator_callback but with an async callback.
        """
        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider, caplog, mockserver)
        assert "Middleware: ImportError exception caught" in log2
        assert "'item_scraped_count': 2" in log2

    @coroutine_test
    async def test_generator_callback_right_after_callback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (2.1) Special case of (2): Exceptions should be caught
        even if the middleware is placed right after the spider
        """
        log21 = await self.crawl_log(
            GeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog, mockserver
        )
        assert "Middleware: ImportError exception caught" in log21
        assert "'item_scraped_count': 2" in log21

    @coroutine_test
    async def test_not_a_generator_callback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (3) An exception from a spider callback (returning a list) should
        be caught by the process_spider_exception chain. No items should be processed.
        """
        log3 = await self.crawl_log(NotGeneratorCallbackSpider, caplog, mockserver)
        assert "Middleware: ZeroDivisionError exception caught" in log3
        assert "item_scraped_count" not in log3

    @coroutine_test
    async def test_not_a_generator_callback_right_after_callback(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (3.1) Special case of (3): Exceptions should be caught
        even if the middleware is placed right after the spider
        """
        log31 = await self.crawl_log(
            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog, mockserver
        )
        assert "Middleware: ZeroDivisionError exception caught" in log31
        assert "item_scraped_count" not in log31

    @coroutine_test
    async def test_generator_output_chain(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (4) An exception from a middleware's process_spider_output method should be sent
        to the process_spider_exception method from the next middleware in the chain.
        The result of the recovery by the process_spider_exception method should be handled
        by the process_spider_output method from the next middleware.
        The final item count should be 2 (one from the spider callback and one from the
        process_spider_exception chain)
        """
        log4 = await self.crawl_log(GeneratorOutputChainSpider, caplog, mockserver)
        assert "'item_scraped_count': 2" in log4
        assert (
            "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
            in log4
        )
        assert (
            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
            in log4
        )
        assert (
            "GeneratorFailMiddleware.process_spider_exception: LookupError caught"
            not in log4
        )
        assert (
            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
            not in log4
        )
        item_from_callback = {
            "processed": [
                "parse-first-item",
                "GeneratorFailMiddleware.process_spider_output",
                "GeneratorDoNothingAfterFailureMiddleware.process_spider_output",
                "GeneratorRecoverMiddleware.process_spider_output",
                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
            ]
        }
        item_recovered = {
            "processed": [
                "GeneratorRecoverMiddleware.process_spider_exception",
                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
            ]
        }
        assert str(item_from_callback) in log4
        assert str(item_recovered) in log4
        assert "parse-second-item" not in log4

    @coroutine_test
    async def test_unhandled_exception(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (5) An exception that no process_spider_exception method handles should be
        offered to each of them only once, and then reach the spider error log.
        """
        log5 = await self.crawl_log(UnhandledExceptionSpider, caplog, mockserver)
        for middleware in (
            FirstUnhandledMiddleware,
            SecondUnhandledMiddleware,
            ThirdUnhandledMiddleware,
        ):
            method = f"{middleware.__name__}.process_spider_exception"
            assert log5.count(f"{method}: ImportError caught") == 1
        assert "Spider error processing" in log5
        assert "'item_scraped_count': 1" in log5

    @coroutine_test
    async def test_download_error_errback_output(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (6.1) The output of an errback called because a download failed goes
        through the process_spider_output chain, with None as the response.
        """
        log6 = await self.crawl_log(DownloadErrorSpider, caplog, mockserver)
        assert "Middleware: output {'from': 'errback'} with response None" in log6
        assert "'item_scraped_count': 1" in log6
        assert "Crawled (200)" in log6
        assert "is_start_request: None" in log6

    @coroutine_test
    async def test_download_error_errback_exception(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        """
        (6.2) An exception from such an errback goes through the
        process_spider_exception chain.
        """
        log6 = await self.crawl_log(DownloadErrorFailSpider, caplog, mockserver)
        assert "Middleware: output {'from': 'errback'} with response None" in log6
        assert "Middleware: LookupError exception caught with response None" in log6
        assert "'item_scraped_count': 1" in log6