Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

21.0 KB
Raw Blame
from __future__ import annotations

import asyncio
from contextlib import asynccontextmanager
from gzip import compress
from typing import TYPE_CHECKING, Any
from unittest import mock

import pytest
from twisted.internet.defer import Deferred, succeed

from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import (
    CloseSpider,
    DecompressionError,
    IgnoreRequest,
    ScrapyDeprecationWarning,
    _InvalidOutput,
)
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from collections.abc import AsyncGenerator


def _invalid_gzipped_response(request: Request) -> Response:
    body = b"<p>You are being redirected</p>"
    return Response(
        request.url,
        status=200,
        body=body,
        headers={
            "Content-Length": str(len(body)),
            "Content-Type": "text/html",
            "Content-Encoding": "gzip",
            "Location": "http://example.com/login",
        },
    )


def _truncated_gzipped_response(request: Request) -> Response:
    body = compress(b"<p>You are being redirected</p>")[:10]
    return Response(
        request.url,
        status=200,
        body=body,
        headers={
            "Content-Length": str(len(body)),
            "Content-Type": "text/html",
            "Content-Encoding": "gzip",
        },
    )


class TestManagerBase:
    settings_dict: dict[str, Any] | None = None

    # should be a fixture but async fixtures that use Futures are problematic with pytest-twisted
    @asynccontextmanager
    async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]:
        crawler = get_crawler(Spider, self.settings_dict)
        crawler.spider = crawler._create_spider("foo")
        mwman = build_from_crawler(DownloaderMiddlewareManager, crawler)
        crawler.engine = ExecutionEngine(crawler)
        await crawler.engine.open_spider_async()
        try:
            yield mwman
        finally:
            await crawler.engine.close_spider_async()

    @staticmethod
    async def _download(
        mwman: DownloaderMiddlewareManager,
        request: Request,
        response: Response | None = None,
    ) -> Response | Request:
        """Executes downloader mw manager's download method and returns
        the result (Request or Response) or raises exception in case of
        failure.
        """
        if not response:
            response = Response(request.url)

        async def download_func(request: Request) -> Response:
            return response

        return await mwman.download_async(download_func, request)


class TestDefaults(TestManagerBase):
    """Tests default behavior with default settings"""

    @coroutine_test
    async def test_request_response(self):
        req = Request("http://example.com/index.html")
        resp = Response(req.url, status=200)
        async with self.get_mwman() as mwman:
            ret = await self._download(mwman, req, resp)
        assert isinstance(ret, Response), "Non-response returned"

    @coroutine_test
    async def test_none_from_download_func(self):
        async def download_func(request: Request) -> None:
            return None

        async with self.get_mwman() as mwman:
            with pytest.raises(TypeError, match="Received None in process_response"):
                await mwman.download_async(
                    download_func,  # type: ignore[arg-type]
                    Request("http://example.com/index.html"),
                )

    @coroutine_test
    async def test_3xx_and_invalid_gzipped_body_must_redirect(self):
        """Regression test for a failure when redirecting a compressed
        request.

        This happens when httpcompression middleware is executed before redirect
        middleware and attempts to decompress a non-compressed body.
        In particular when some website returns a 30x response with header
        'Content-Encoding: gzip' giving as result the error below:

            BadGzipFile: Not a gzipped file (...)

        """
        req = Request("http://example.com")
        body = b"<p>You are being redirected</p>"
        resp = Response(
            req.url,
            status=302,
            body=body,
            headers={
                "Content-Length": str(len(body)),
                "Content-Type": "text/html",
                "Content-Encoding": "gzip",
                "Location": "http://example.com/login",
            },
        )
        async with self.get_mwman() as mwman:
            ret = await self._download(mwman, req, resp)
        assert isinstance(ret, Request), f"Not redirected: {ret!r}"
        assert ret.url.encode() == resp.headers["Location"], (
            "Not redirected to location header"
        )

    @coroutine_test
    async def test_200_and_invalid_gzipped_body_must_fail(self):
        req = Request("http://example.com")
        resp = _invalid_gzipped_response(req)
        with pytest.raises(DecompressionError):
            async with self.get_mwman() as mwman:
                await self._download(mwman, req, resp)


class TestResponseFromProcessRequest(TestManagerBase):
    """Tests middleware returning a response from process_request."""

    @coroutine_test
    async def test_download_func_not_called(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        download_func = mock.MagicMock()

        class ResponseMiddleware:
            def process_request(self, request):
                return resp

        async with self.get_mwman() as mwman:
            mwman._add_middleware(ResponseMiddleware())
            result = await mwman.download_async(download_func, req)
        assert result is resp
        assert not download_func.called


class TestResponseFromProcessException(TestManagerBase):
    """Tests middleware returning a response from process_exception."""

    @coroutine_test
    async def test_process_response_called(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        calls = []

        def download_func(request):
            raise ValueError("test")

        class ResponseMiddleware:
            def process_response(self, request, response):
                calls.append("process_response")
                return resp

            def process_exception(self, request, exception):
                calls.append("process_exception")
                return resp

        async with self.get_mwman() as mwman:
            mwman._add_middleware(ResponseMiddleware())
            result = await mwman.download_async(download_func, req)
        assert result is resp
        assert calls == [
            "process_exception",
            "process_response",
        ]


class TestResponseExceptions(TestManagerBase):
    """Tests exceptions from process_response reaching process_exception."""

    settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": True}

    @coroutine_test
    async def test_request_from_process_exception(self):
        req = Request("http://example.com/index.html")
        retry_req = Request("http://example.com/index.html")
        calls = []

        class OuterMiddleware:
            def process_response(self, request, response):
                calls.append("outer.process_response")
                return response

            def process_exception(self, request, exception):
                calls.append("outer.process_exception")
                return retry_req

        class InnerMiddleware:
            def process_response(self, request, response):
                calls.append("inner.process_response")
                raise ValueError("test")

            def process_exception(self, request, exception):
                calls.append("inner.process_exception")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(OuterMiddleware())
            mwman._add_middleware(InnerMiddleware())
            result = await self._download(mwman, req)
        assert result is retry_req
        assert calls == ["inner.process_response", "outer.process_exception"]

    @coroutine_test
    async def test_response_from_process_exception(self):
        req = Request("http://example.com/index.html")
        recovered = Response("http://example.com/index.html")
        calls = []

        class OuterMiddleware:
            def process_response(self, request, response):
                calls.append("outer.process_response")
                return response

        class MiddleMiddleware:
            def process_response(self, request, response):
                calls.append("middle.process_response")
                return response

            def process_exception(self, request, exception):
                calls.append("middle.process_exception")
                return recovered

        class InnerMiddleware:
            def process_response(self, request, response):
                calls.append("inner.process_response")
                raise ValueError("test")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(OuterMiddleware())
            mwman._add_middleware(MiddleMiddleware())
            mwman._add_middleware(InnerMiddleware())
            result = await self._download(mwman, req)
        assert result is recovered
        assert calls == [
            "inner.process_response",
            "middle.process_exception",
            "outer.process_response",
        ]

    @coroutine_test
    async def test_processed_middleware_not_called(self):
        req = Request("http://example.com/index.html")
        calls = []

        class RaisingMiddleware:
            def process_response(self, request, response):
                calls.append("raising.process_response")
                raise ValueError("test")

        class InnerMiddleware:
            def process_response(self, request, response):
                calls.append("inner.process_response")
                return response

            def process_exception(self, request, exception):
                calls.append("inner.process_exception")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(RaisingMiddleware())
            mwman._add_middleware(InnerMiddleware())
            with pytest.raises(ValueError, match="test"):
                await self._download(mwman, req)
        assert calls == ["inner.process_response", "raising.process_response"]

    @coroutine_test
    async def test_unhandled(self):
        req = Request("http://example.com/index.html")

        class RaisingMiddleware:
            def process_response(self, request, response):
                raise ValueError("test")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(RaisingMiddleware())
            with pytest.raises(ValueError, match="test"):
                await self._download(mwman, req)

    @pytest.mark.parametrize(
        "response_func", [_invalid_gzipped_response, _truncated_gzipped_response]
    )
    @coroutine_test
    async def test_retry(self, response_func):
        """Decompression failures are retried, as RETRY_EXCEPTIONS promises."""
        req = Request("http://example.com")
        async with self.get_mwman() as mwman:
            result = await self._download(mwman, req, response_func(req))
        assert isinstance(result, Request)
        assert result.url == req.url

    @coroutine_test
    async def test_close_spider(self):
        req = Request("http://example.com/index.html")
        calls = []

        class OuterMiddleware:
            def process_exception(self, request, exception):
                calls.append("outer.process_exception")

        class InnerMiddleware:
            def process_response(self, request, response):
                raise CloseSpider("test")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(OuterMiddleware())
            mwman._add_middleware(InnerMiddleware())
            with pytest.raises(CloseSpider):
                await self._download(mwman, req)
        assert not calls

    @coroutine_test
    async def test_close_spider_from_process_request(self):
        req = Request("http://example.com/index.html")
        calls = []

        class Middleware:
            def process_request(self, request):
                raise CloseSpider("test")

            def process_exception(self, request, exception):
                calls.append("process_exception")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(Middleware())
            with pytest.raises(CloseSpider):
                await self._download(mwman, req)
        assert not calls


class TestResponseExceptionsDisabled(TestManagerBase):
    """Tests the deprecated behavior, which the default value still gives."""

    @pytest.mark.parametrize("exception", [ValueError, IgnoreRequest])
    @coroutine_test
    async def test_not_passed_to_process_exception(self, exception):
        req = Request("http://example.com/index.html")
        calls = []

        class OuterMiddleware:
            def process_exception(self, request, exception):
                calls.append("outer.process_exception")

        class InnerMiddleware:
            def process_response(self, request, response):
                raise exception("test")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(OuterMiddleware())
            mwman._add_middleware(InnerMiddleware())
            with (
                pytest.raises(exception, match="test"),
                pytest.warns(
                    ScrapyDeprecationWarning,
                    match="OuterMiddleware.process_exception",
                ),
            ):
                await self._download(mwman, req)
        assert not calls

    @coroutine_test
    async def test_only_built_in_process_exception(self):
        """Nothing is warned about when only Scrapy's own process_exception()
        methods would get the exception."""
        req = Request("http://example.com/index.html")

        class RaisingMiddleware:
            def process_response(self, request, response):
                raise ValueError("test")

        async with self.get_mwman() as mwman:
            mwman._add_middleware(RaisingMiddleware())
            with pytest.raises(ValueError, match="test"):
                await self._download(mwman, req)


class TestResponseExceptionsDisabledExplicitly(TestResponseExceptionsDisabled):
    """Asking for the deprecated behavior does not silence the warning."""

    settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": False}


class TestInvalidOutput(TestManagerBase):
    @coroutine_test
    async def test_invalid_process_request(self):
        """Invalid return value for process_request method should raise an exception"""
        req = Request("http://example.com/index.html")

        class InvalidProcessRequestMiddleware:
            def process_request(self, request):
                return 1

        async with self.get_mwman() as mwman:
            mwman._add_middleware(InvalidProcessRequestMiddleware())
            with pytest.raises(_InvalidOutput):
                await self._download(mwman, req)

    @coroutine_test
    async def test_invalid_process_response(self):
        """Invalid return value for process_response method should raise an exception"""
        req = Request("http://example.com/index.html")

        class InvalidProcessResponseMiddleware:
            def process_response(self, request, response):
                return 1

        async with self.get_mwman() as mwman:
            mwman._add_middleware(InvalidProcessResponseMiddleware())
            with pytest.raises(_InvalidOutput):
                await self._download(mwman, req)

    @coroutine_test
    async def test_invalid_process_exception(self):
        """Invalid return value for process_exception method should raise an exception"""
        req = Request("http://example.com/index.html")

        class InvalidProcessExceptionMiddleware:
            def process_request(self, request):
                raise RuntimeError

            def process_exception(self, request, exception):
                return 1

        async with self.get_mwman() as mwman:
            mwman._add_middleware(InvalidProcessExceptionMiddleware())
            with pytest.raises(_InvalidOutput):
                await self._download(mwman, req)


class TestMiddlewareUsingDeferreds(TestManagerBase):
    """Middlewares using Deferreds (deprecated) should work"""

    @coroutine_test
    async def test_deferred(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        download_func = mock.MagicMock()

        class DeferredMiddleware:
            def cb(self, result):
                return result

            def process_request(self, request):
                d: Deferred[Response] = Deferred()
                d.addCallback(self.cb)
                d.callback(resp)
                return d

        async with self.get_mwman() as mwman:
            mwman._add_middleware(DeferredMiddleware())
            with pytest.warns(
                ScrapyDeprecationWarning,
                match="returned a Deferred, this is deprecated",
            ):
                result = await mwman.download_async(download_func, req)
        assert result is resp
        assert not download_func.called


class TestMiddlewareUsingCoro(TestManagerBase):
    """Middlewares using asyncio coroutines should work"""

    @coroutine_test
    async def test_asyncdef(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        download_func = mock.MagicMock()

        class CoroMiddleware:
            async def process_request(self, request):
                await succeed(42)
                return resp

        async with self.get_mwman() as mwman:
            mwman._add_middleware(CoroMiddleware())
            result = await mwman.download_async(download_func, req)
        assert result is resp
        assert not download_func.called

    @pytest.mark.only_asyncio
    @coroutine_test
    async def test_asyncdef_asyncio(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        download_func = mock.MagicMock()

        class CoroMiddleware:
            async def process_request(self, request):
                await asyncio.sleep(0.1)
                return await get_from_asyncio_queue(resp)

        async with self.get_mwman() as mwman:
            mwman._add_middleware(CoroMiddleware())
            result = await mwman.download_async(download_func, req)
        assert result is resp
        assert not download_func.called


class TestDownloadDeprecated(TestManagerBase):
    @coroutine_test
    async def test_mwman_download(self):
        req = Request("http://example.com/index.html")
        resp = Response(req.url, status=200)

        def download_func(request: Request, spider: Spider) -> Deferred[Response]:
            return succeed(resp)

        async with self.get_mwman() as mwman:
            assert mwman.crawler
            assert mwman.crawler.spider
            with pytest.warns(
                ScrapyDeprecationWarning,
                match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",
            ):
                ret = await maybe_deferred_to_future(
                    mwman.download(download_func, req, mwman.crawler.spider)
                )
        assert isinstance(ret, Response)


class TestDeprecatedSpiderArg(TestManagerBase):
    @coroutine_test
    async def test_deprecated_spider_arg(self):
        req = Request("http://example.com/index.html")
        resp = Response("http://example.com/index.html")
        download_func = mock.MagicMock()

        class DeprecatedSpiderArgMiddleware:
            def process_request(self, request, spider):
                1 / 0

            def process_response(self, request, response, spider):
                return response

            def process_exception(self, request, exception, spider):
                return resp

        async with self.get_mwman() as mwman:
            with (
                pytest.warns(
                    ScrapyDeprecationWarning,
                    match=r"process_request\(\) requires a spider argument",
                ),
                pytest.warns(
                    ScrapyDeprecationWarning,
                    match=r"process_response\(\) requires a spider argument",
                ),
                pytest.warns(
                    ScrapyDeprecationWarning,
                    match=r"process_exception\(\) requires a spider argument",
                ),
            ):
                mwman._add_middleware(DeprecatedSpiderArgMiddleware())
            result = await mwman.download_async(download_func, req)
        assert result is resp
        assert not download_func.called