Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

36.1 KB
Raw Blame
from __future__ import annotations

import warnings
from time import get_clock_info, monotonic, time
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import patch

import OpenSSL.SSL
import pytest
from pytest_twisted import async_yield_fixture
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.endpoints import HostnameEndpoint
from twisted.internet.protocol import Factory
from twisted.internet.protocol import Protocol as TxProtocol
from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web import server, static
from twisted.web.client import (
    URI,
    Agent,
    BrowserLikePolicyForHTTPS,
    _StandardEndpointFactory,
    readBody,
)
from twisted.web.client import Response as TxResponse

from scrapy import Request, Spider, signals
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader._idna_patch import (
    _install_twisted_idna_fallbacks,
    _safe_hostname_bytes,
)
from scrapy.core.downloader.contextfactory import (
    _get_creator_context,
    _load_context_factory_from_settings,
    _ScrapyClientContextFactory,
)
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer
from scrapy.exceptions import DownloadCancelledError, ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
    _schedule_coro,
    deferred_from_coro,
    maybe_deferred_to_future,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import IDNA_REJECTED_HOSTNAMES
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.spiders import MockServerSpider, SimpleSpider
from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from pathlib import Path

    from twisted.internet.interfaces import IConsumer, IListeningPort
    from twisted.python.failure import Failure
    from twisted.web.iweb import IBodyProducer

    from scrapy.crawler import Crawler
    from tests.mockserver.http import MockServer


@pytest.fixture(autouse=True, scope="module")
def _twisted_idna_fallbacks() -> None:
    """The download handlers install these when built; the tests below exercise
    the patched Twisted helpers directly, so they need them installed too."""
    _install_twisted_idna_fallbacks()


class TestSlot:
    def test_repr(self):
        slot = Slot(concurrency=8, delay=0.1, jitter=0.5)
        assert repr(slot) == "Slot(concurrency=8, delay=0.1, jitter=0.5)"

    def test_download_delay_without_jitter(self):
        slot = Slot(concurrency=8, delay=2.0, jitter=0)
        assert slot.download_delay() == 2.0

    def test_download_delay_with_jitter(self):
        slot = Slot(concurrency=8, delay=2.0, jitter=0.25)
        assert all(1.5 <= slot.download_delay() <= 2.5 for _ in range(100))

    def test_download_delay_with_jitter_above_one(self):
        slot = Slot(concurrency=8, delay=2.0, jitter=3.0)
        assert all(slot.download_delay() >= 0 for _ in range(100))

    @pytest.mark.parametrize(("value", "expected"), [(True, 0.5), (False, 0.0)])
    @pytest.mark.parametrize("positional", [False, True])
    def test_deprecated_randomize_delay_param(
        self, positional: bool, value: bool, expected: float
    ):
        with pytest.warns(
            ScrapyDeprecationWarning,
            match="The randomize_delay parameter of Slot is deprecated",
        ):
            slot = (
                Slot(8, 2.0, value)
                if positional
                else Slot(concurrency=8, delay=2.0, randomize_delay=value)
            )
        assert slot.jitter == expected

    @pytest.mark.parametrize(("jitter", "expected"), [(0, False), (0.2, True)])
    def test_deprecated_randomize_delay(self, jitter: float, expected: bool):
        slot = Slot(concurrency=8, delay=2.0, jitter=jitter)
        with pytest.warns(
            ScrapyDeprecationWarning, match="Slot.randomize_delay is deprecated"
        ):
            assert slot.randomize_delay is expected

    @pytest.mark.parametrize(("value", "expected"), [(True, 0.5), (False, 0.0)])
    def test_deprecated_randomize_delay_setter(self, value: bool, expected: float):
        slot = Slot(concurrency=8, delay=2.0, jitter=0.2)
        with pytest.warns(
            ScrapyDeprecationWarning, match="Slot.randomize_delay is deprecated"
        ):
            slot.randomize_delay = value
        assert slot.jitter == expected


class TestJitterSetting:
    @staticmethod
    def _jitter(**settings: Any) -> float:
        crawler = get_crawler(settings_dict=settings)
        downloader = Downloader(crawler)
        downloader.close()
        return downloader._jitter

    def test_default(self):
        assert self._jitter() == 0.5

    @pytest.mark.parametrize(("value", "expected"), [(0, 0.0), ("0.2", 0.2), (1, 1.0)])
    def test_value(self, value: Any, expected: float):
        assert self._jitter(DOWNLOAD_DELAY_JITTER=value) == pytest.approx(expected)

    @pytest.mark.parametrize(("value", "expected"), [(True, 0.5), (False, 0.0)])
    def test_deprecated_setting(self, value: bool, expected: float):
        with pytest.warns(
            ScrapyDeprecationWarning,
            match="The RANDOMIZE_DOWNLOAD_DELAY setting is deprecated",
        ):
            assert self._jitter(RANDOMIZE_DOWNLOAD_DELAY=value) == expected

    @pytest.mark.parametrize(("jitter", "expected"), [(0, False), (0.2, True)])
    def test_deprecated_downloader_attribute(self, jitter: float, expected: bool):
        crawler = get_crawler(settings_dict={"DOWNLOAD_DELAY_JITTER": jitter})
        downloader = Downloader(crawler)
        downloader.close()
        with pytest.warns(
            ScrapyDeprecationWarning, match="Downloader.randomize_delay is deprecated"
        ):
            assert downloader.randomize_delay is expected

    def test_deprecated_setting_loses_on_tie(self):
        with pytest.warns(ScrapyDeprecationWarning):
            jitter = self._jitter(
                RANDOMIZE_DOWNLOAD_DELAY=False, DOWNLOAD_DELAY_JITTER=0.2
            )
        assert jitter == pytest.approx(0.2)


class TestDownloaderStop:
    @coroutine_test
    async def test_waits_for_cancelled_downloads_to_settle(self) -> None:
        """stop() cancels in-flight downloads before returning.

        The engine relies on their cleanup (removal from ``_download_tasks``)
        having already happened by the time it returns, or it may consider the
        crawl finished while a download is still being torn down.
        """
        crawler = get_crawler(DefaultSpider)
        downloader = Downloader(crawler)

        requests = [Request(f"data:,{i}") for i in range(5)]
        for request in requests:
            download_dfd: Deferred[None] = Deferred()
            download_dfd.addBoth(downloader._download_task_done, request)
            downloader._download_tasks[request] = download_dfd

        await downloader.stop()

        assert not downloader._download_tasks


def _downloader() -> Downloader:
    crawler = get_crawler(DefaultSpider, {"DOWNLOADER_MIDDLEWARES_BASE": {}})
    crawler.spider = crawler._create_spider()
    return Downloader(crawler)


# asyncio runs timers that are due within one tick of the monotonic clock, so a
# delay can end up to one tick early, which is 15.6 ms on Windows before Python
# 3.13.
_DELAY_TOLERANCE = get_clock_info("monotonic").resolution


@coroutine_test
async def test_delay_meta() -> None:
    delay = 0.2
    downloader = _downloader()
    request = Request("data:,", meta={"delay": delay})
    try:
        start = monotonic()
        await maybe_deferred_to_future(downloader.fetch(request))
        delayed = monotonic() - start
        await maybe_deferred_to_future(downloader.fetch(request))
        held_again = monotonic() - start - delayed
        request.meta["delay"] = delay
        await maybe_deferred_to_future(downloader.fetch(request))
        held_for_the_new_delay = monotonic() - start - delayed - held_again
    finally:
        downloader.close()
    assert delayed >= delay - _DELAY_TOLERANCE
    assert "delay" not in request.meta
    # Downloading the same request again does not hold it again, but setting the
    # key again does.
    assert held_again < delay
    assert held_for_the_new_delay >= delay - _DELAY_TOLERANCE
    assert request.meta["delayed"] == [delay, delay]


@coroutine_test
async def test_delay_meta_stale_deadline() -> None:
    delay = 0.2
    downloader = _downloader()
    # A deadline from a different run of the crawl, restored from a disk queue,
    # is capped at the delay.
    request = Request("data:,", meta={"_delay_deadline": (delay, time() + 10 * delay)})
    try:
        start = monotonic()
        await maybe_deferred_to_future(downloader.fetch(request))
        elapsed = monotonic() - start
    finally:
        downloader.close()
    assert delay - _DELAY_TOLERANCE <= elapsed < 2 * delay


class DelaySpider(MockServerSpider):
    name = "delay_meta"
    custom_settings = {"CONCURRENT_REQUESTS": 1}

    def __init__(self, *args: Any, **kwargs: Any):
        super().__init__(*args, **kwargs)
        self.times: list[float] = []
        self.start_time: float = monotonic()

    async def start(self):
        assert self.mockserver
        # The slow request keeps the delayed one waiting for its turn for longer
        # than its delay.
        yield Request(self.mockserver.url("/delay?n=1"))
        yield Request(self.mockserver.url("/status?n=200"), meta={"delay": 0.9})

    def parse(self, response):
        self.times.append(monotonic() - self.start_time)


@coroutine_test
async def test_delay_meta_countdown_starts_when_scheduled(
    mockserver: MockServer,
) -> None:
    crawler = get_crawler(DelaySpider)
    await crawler.crawl_async(mockserver=mockserver)
    spider = crawler.spider
    assert isinstance(spider, DelaySpider)
    assert len(spider.times) == 2
    # The delay ran while the request waited for its turn, so it was sent as
    # soon as it got one, rather than 0.9s later.
    assert spider.times[1] < spider.times[0] + 0.5


@pytest.mark.requires_reactor  # this test is related to the Twisted HTTP code
class TestContextFactoryBase:
    @async_yield_fixture
    async def server_url(self, tmp_path):
        (tmp_path / "file").write_bytes(b"0123456789")
        r = static.File(str(tmp_path))
        put_child(r, b"payload", PayloadResource())
        site = server.Site(r, timeout=None)
        port = self._listen(site)
        portno = port.getHost().port

        yield f"https://127.0.0.1:{portno}/"

        await port.stopListening()

    def _listen(self, site: server.Site) -> IListeningPort:
        from twisted.internet import reactor

        return reactor.listenSSL(
            0,
            site,
            contextFactory=ssl_context_factory(),
            interface="127.0.0.1",
        )

    @staticmethod
    async def get_page(
        url: str,
        client_context_factory: BrowserLikePolicyForHTTPS,
        body: str | None = None,
    ) -> bytes:
        from twisted.internet import reactor

        agent = Agent(reactor, contextFactory=client_context_factory)
        body_producer = _RequestBodyProducer(body.encode()) if body else None
        response: TxResponse = cast(
            "TxResponse",
            await maybe_deferred_to_future(
                agent.request(
                    b"GET",
                    url.encode(),
                    bodyProducer=cast("IBodyProducer", body_producer),
                )
            ),
        )
        with warnings.catch_warnings():
            # https://github.com/twisted/twisted/issues/8227
            warnings.filterwarnings(
                "ignore",
                category=DeprecationWarning,
                message=r".*does not have an abortConnection method",
            )
            d: Deferred[bytes] = readBody(response)  # type: ignore[arg-type]
        return await maybe_deferred_to_future(d)


class TestContextFactory(TestContextFactoryBase):
    @pytest.fixture
    def factory(self) -> _ScrapyClientContextFactory:
        crawler = get_crawler()
        factory = _load_context_factory_from_settings(crawler)
        assert isinstance(factory, _ScrapyClientContextFactory)
        return factory

    @staticmethod
    def _get_dummy_protocol() -> TLSMemoryBIOProtocol:
        # from Twisted src/twisted/web/test/test_agent.py::dummyTLSProtocol()
        factory = TLSMemoryBIOFactory(
            optionsForClientTLS("example.com"), True, Factory.forProtocol(TxProtocol)
        )
        return factory.buildProtocol(None)

    @coroutine_test
    async def test_payload(
        self, factory: _ScrapyClientContextFactory, server_url: str
    ) -> None:
        s = "0123456789" * 10
        body = await self.get_page(server_url + "payload", factory, body=s)
        assert body == s.encode()

    def test_no_context_sharing(self, factory: _ScrapyClientContextFactory) -> None:
        """Every call to creatorForNetloc() should give a fresh context."""
        creator1 = factory.creatorForNetloc(b"website1.tld", 443)
        assert creator1._hostnameBytes == b"website1.tld"
        creator2 = factory.creatorForNetloc(b"website2.tld", 443)
        assert creator2._hostnameBytes == b"website2.tld"
        assert _get_creator_context(creator1) is not _get_creator_context(creator2)

    def test_no_context_sharing_with_conn(
        self, factory: _ScrapyClientContextFactory
    ) -> None:
        """Like test_no_context_sharing() but get the context from a connection."""
        creator1 = factory.creatorForNetloc(b"website1.tld", 443)
        assert creator1._hostnameBytes == b"website1.tld"
        conn1 = creator1.clientConnectionForTLS(self._get_dummy_protocol())

        creator2 = factory.creatorForNetloc(b"website2.tld", 443)
        assert creator2._hostnameBytes == b"website2.tld"
        conn2 = creator2.clientConnectionForTLS(self._get_dummy_protocol())

        assert conn1.get_context() is not conn2.get_context()

    @pytest.mark.skipif(
        PYOPENSSL_SET_CIPHER_LIST_TMP_CONN,
        reason="Fails or doesn't make sense on this pyOpenSSL version",
    )
    def test_no_immutable_ctx_warning(
        self, factory: _ScrapyClientContextFactory
    ) -> None:
        """There should be no pyOpenSSL context modification warning.

        pyOpenSSL < 25.1.0 doesn't produce this warning, and on 25.1.0 it's
        always produced due to
        https://github.com/scrapy/scrapy/issues/6859#issuecomment-4294917851.
        """
        with warnings.catch_warnings():
            warnings.filterwarnings(
                "error",
                category=DeprecationWarning,
                message="Attempting to mutate a Context after a Connection was created",
            )
            factory.creatorForNetloc(b"website.tld", 443)

    @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])
    def test_idna_rejected_hostname(
        self, factory: _ScrapyClientContextFactory, hostname: bytes
    ) -> None:
        """Hostnames rejected by the idna package, such as punycode emoji
        domains or domains with underscores, should work."""
        creator = factory.creatorForNetloc(hostname, 443)
        assert creator._hostnameBytes == hostname
        assert creator._hostnameASCII == hostname.decode("ascii")
        assert creator._hostnameIsDnsName is True
        conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
        assert conn is not None

    @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])
    def test_idna_rejected_hostname_verify_certificates(self, hostname: bytes) -> None:
        """Same as test_idna_rejected_hostname() but with certificate
        verification enabled, which uses plain optionsForClientTLS()."""
        crawler = get_crawler(settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": True})
        factory = cast(
            "_ScrapyClientContextFactory", _load_context_factory_from_settings(crawler)
        )
        creator = factory.creatorForNetloc(hostname, 443)
        assert creator._hostnameBytes == hostname
        assert creator._hostnameASCII == hostname.decode("ascii")

    @pytest.mark.parametrize("verify_certs", [False, True])
    def test_keylog(
        self,
        monkeypatch: pytest.MonkeyPatch,
        tmp_path: Path,
        verify_certs: bool,
    ) -> None:
        """The key log callback should be set with and without verification."""
        monkeypatch.setenv("SSLKEYLOGFILE", str(tmp_path / "keylog"))
        crawler = get_crawler(
            settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": verify_certs}
        )
        factory = _load_context_factory_from_settings(crawler)
        creator = factory.creatorForNetloc(b"website.tld", 443)
        conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
        assert conn.get_context()._keylog_callback is not None

    def test_keylog_ignore_environment(
        self,
        monkeypatch: pytest.MonkeyPatch,
        mockserver: MockServer,
        tmp_path: Path,
    ) -> None:
        keylog_file = tmp_path / "keylog"
        monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file))
        _, out, err = proc(
            "fetch", mockserver.url("/text", is_secure=True), python_args=("-E",)
        )
        assert out.strip() == "Works", err
        assert not keylog_file.exists()

    def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None:
        """The context should have the expected flags set."""
        creator = factory.creatorForNetloc(b"website.tld", 443)
        conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
        ctx = conn.get_context()
        # fragile but pyOpenSSL doesn't have Context.get_options()
        options = OpenSSL.SSL._lib.SSL_CTX_get_options(ctx._context)  # type: ignore[attr-defined]
        assert options & 0x4  # OP_LEGACY_SERVER_CONNECT


class TestContextFactoryCiphers(TestContextFactoryBase):
    async def _assert_factory_works(
        self, server_url: str, client_context_factory: _ScrapyClientContextFactory
    ) -> None:
        s = "0123456789" * 10
        body = await self.get_page(
            server_url + "payload", client_context_factory, body=s
        )
        assert body == s.encode()

    def test_default(self) -> None:
        """The default 'DEFAULT' value is passed to Twisted as is."""
        crawler = get_crawler()
        factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
        assert factory.tls_ciphers is not None
        # OpenSSLAcceptableCiphers has no __eq__, so compare the parsed ciphers.
        assert (
            factory.tls_ciphers._ciphers
            == AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
        )
        assert factory._get_cert_options_kwargs()["acceptableCiphers"] is not None

    def test_custom(self) -> None:
        crawler = get_crawler(
            settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
        )
        factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
        assert factory.tls_ciphers is not None
        assert (
            factory.tls_ciphers._ciphers
            == AcceptableCiphers.fromOpenSSLCipherString("CAMELLIA256-SHA")._ciphers
        )

    @coroutine_test
    async def test_none(self, server_url: str) -> None:
        """A None value enables the Twisted default ciphers."""
        crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": None})
        factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
        assert factory.tls_ciphers is None
        assert factory._get_cert_options_kwargs()["acceptableCiphers"] is None
        await self._assert_factory_works(server_url, factory)


class TestContextFactoryTLSMethod(TestContextFactoryBase):
    async def _assert_factory_works(
        self, server_url: str, client_context_factory: _ScrapyClientContextFactory
    ) -> None:
        s = "0123456789" * 10
        body = await self.get_page(
            server_url + "payload", client_context_factory, body=s
        )
        assert body == s.encode()

    @coroutine_test
    async def test_setting_default(self, server_url: str) -> None:
        crawler = get_crawler()
        client_context_factory = _load_context_factory_from_settings(crawler)
        assert isinstance(client_context_factory, _ScrapyClientContextFactory)
        assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
        await self._assert_factory_works(server_url, client_context_factory)

    def test_setting_none(self):
        crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": None})
        with (
            pytest.warns(
                ScrapyDeprecationWarning,
                match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
            ),
            pytest.raises(KeyError),
        ):
            _load_context_factory_from_settings(crawler)

    def test_setting_bad(self):
        crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
        with (
            pytest.warns(
                ScrapyDeprecationWarning,
                match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
            ),
            pytest.raises(KeyError),
        ):
            _load_context_factory_from_settings(crawler)

    @pytest.mark.filterwarnings(
        r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning"
    )
    @coroutine_test
    async def test_setting_explicit(self, server_url: str) -> None:
        crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})
        with pytest.warns(
            ScrapyDeprecationWarning,
            match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
        ):
            client_context_factory = _load_context_factory_from_settings(crawler)
        assert isinstance(client_context_factory, _ScrapyClientContextFactory)
        assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
        await self._assert_factory_works(server_url, client_context_factory)

    @coroutine_test
    async def test_direct_from_crawler(self, server_url: str) -> None:
        # the setting is ignored
        crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
        client_context_factory = build_from_crawler(
            _ScrapyClientContextFactory, crawler
        )
        assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
        await self._assert_factory_works(server_url, client_context_factory)

    @pytest.mark.filterwarnings(
        r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning"
    )
    @coroutine_test
    async def test_direct_init(self, server_url: str) -> None:
        client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)
        assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
        await self._assert_factory_works(server_url, client_context_factory)


@pytest.mark.parametrize(
    ("concurrency", "active", "expected"),
    [
        (2, 1, False),
        (2, 2, True),
        (0, 0, False),
        (0, 2, False),
    ],
)
def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
    crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
    downloader = Downloader(crawler)
    downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
    assert downloader.needs_backout() is expected
    downloader.close()


class TestSafeHostnameBytes:
    """Tests for the workarounds for hostnames rejected by the idna
    package."""

    @pytest.mark.parametrize(
        ("hostname", "expected"),
        [
            ("example.com", b"example.com"),
            ("xn--i-7iq.ws", b"xn--i-7iq.ws"),  # i❤.ws
            ("i❤.ws", b"xn--i-7iq.ws"),
            ("ü.example", b"xn--tda.example"),
            ("foo_bar.example", b"foo_bar.example"),
            # overlong ASCII label, rejected by the stdlib codec
            ("a" * 64 + ".com", b"a" * 64 + b".com"),
        ],
    )
    def test_valid(self, hostname: str, expected: bytes) -> None:
        assert _safe_hostname_bytes(hostname) == expected

    @pytest.mark.parametrize(
        "hostname",
        [
            "❤" * 200 + ".ws",
            # not hostnames at all: Twisted parses ":" out of a bracketless
            # IPv6 netloc on old versions
            ":",
            "[::1]",
        ],
    )
    def test_invalid(self, hostname: str) -> None:
        with pytest.raises(UnicodeError):
            _safe_hostname_bytes(hostname)

    @pytest.mark.parametrize(
        ("host", "expected"),
        [
            ("xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "xn--i-7iq.ws")),
            ("i❤.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),
            ("foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),
            ("example.com", (False, b"example.com", "example.com")),
            # bytes hostnames are decoded instead of encoded
            (b"xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),
            (b"foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),
            (b"example.com", (False, b"example.com", "example.com")),
            # invalid hostnames must stay invalid, or Twisted resolves them
            # instead of failing early
            (":", (True, b":", ":")),
            ("[::1]", (True, b"[::1]", "[::1]")),
            (b":", (True, b":", ":")),
        ],
    )
    def test_host_as_bytes_and_text(
        self, host: bytes | str, expected: tuple[bool, bytes, str]
    ) -> None:
        """HostnameEndpoint should consider only invalid hostnames invalid."""
        assert HostnameEndpoint._hostAsBytesAndText(host) == expected


@pytest.mark.requires_reactor
class TestIdnaRejectedEndpoints:
    """Endpoints for hostnames rejected by the idna package, such as emoji
    domains or domains with underscores, should be connectable."""

    @pytest.mark.parametrize("scheme", ["http", "https"])
    @pytest.mark.parametrize("hostname", IDNA_REJECTED_HOSTNAMES)
    def test_endpoint_for_uri(self, scheme: str, hostname: str) -> None:
        from twisted.internet import reactor

        crawler = get_crawler()
        endpoint_factory = _StandardEndpointFactory(
            reactor, _load_context_factory_from_settings(crawler), 10, None
        )
        endpoint = endpoint_factory.endpointForURI(
            URI.fromBytes(f"{scheme}://{hostname}/".encode())
        )
        # https endpoints are wrapped by wrapClientTLS()
        hostname_endpoint = getattr(endpoint, "_wrappedEndpoint", endpoint)
        assert hostname_endpoint._badHostname is False
        assert hostname_endpoint._hostBytes == hostname.encode("ascii")
        assert hostname_endpoint._hostText == hostname


@coroutine_test
async def test_fetch_deprecated_spider_arg():
    class CustomDownloader(Downloader):
        # requiring the spider argument is what triggers the deprecation
        def fetch(  # type: ignore[override]  # pylint: disable=signature-differs
            self, request: Request, spider: Spider
        ) -> Deferred[Response | Request]:
            return super().fetch(request, spider)

    crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})
    with pytest.warns(
        ScrapyDeprecationWarning,
        match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument",
    ):
        await crawler.crawl_async()


@coroutine_test
async def test_stop_drops_queued_requests() -> None:
    crawler = get_crawler(DefaultSpider)
    crawler.spider = crawler._create_spider()
    downloader = Downloader(crawler)
    slot = Slot(concurrency=1, delay=0, jitter=0)
    downloader.slots["example.com"] = slot

    request = Request("https://example.com")
    queue_dfd: Deferred[Any] = Deferred()
    failures: list[Failure] = []
    queue_dfd.addErrback(failures.append)
    slot.queue.append((request, queue_dfd))

    dropped = await downloader.stop()
    assert dropped == 1
    assert len(failures) == 1
    assert failures[0].check(DownloadCancelledError)


@coroutine_test
async def test_stop_rejects_new_requests() -> None:
    crawler = get_crawler(DefaultSpider)
    crawler.spider = crawler._create_spider()
    downloader = Downloader(crawler)

    await downloader.stop()

    with pytest.raises(
        DownloadCancelledError,
        match="not accepting new requests",
    ):
        await downloader._enqueue_request(Request("https://example.com"))


@coroutine_test
async def test_wait_for_download_errbacks_queue_deferred_on_error() -> None:
    crawler = get_crawler(DefaultSpider)
    downloader = Downloader(crawler)
    slot = Slot(concurrency=1, delay=0, jitter=0)

    queue_dfd: Deferred[Any] = Deferred()
    failures: list[Failure] = []
    queue_dfd.addErrback(failures.append)

    with patch.object(downloader, "_download", side_effect=RuntimeError("boom")):
        await downloader._wait_for_download(
            slot,
            Request("https://example.com"),
            queue_dfd,
        )

    assert len(failures) == 1
    assert failures[0].check(RuntimeError)


@coroutine_test
async def test_wait_for_download_keeps_called_queue_deferred_on_error() -> None:
    crawler = get_crawler(DefaultSpider)
    downloader = Downloader(crawler)
    slot = Slot(concurrency=1, delay=0, jitter=0)

    queue_dfd: Deferred[Any] = Deferred()
    queue_dfd.callback(None)

    with patch.object(downloader, "_download", side_effect=RuntimeError("boom")):
        await downloader._wait_for_download(
            slot,
            Request("https://example.com"),
            queue_dfd,
        )

    assert queue_dfd.called


@coroutine_test
async def test_stop_skips_called_queued_deferred() -> None:
    crawler = get_crawler(DefaultSpider)
    crawler.spider = crawler._create_spider()
    downloader = Downloader(crawler)
    slot = Slot(concurrency=1, delay=0, jitter=0)
    downloader.slots["example.com"] = slot

    queue_dfd: Deferred[Any] = Deferred()
    queue_dfd.callback(None)
    slot.queue.append((Request("https://example.com"), queue_dfd))

    dropped = await downloader.stop()
    assert dropped == 1


@coroutine_test
async def test_stop_cancels_pending_download_tasks() -> None:
    crawler = get_crawler(DefaultSpider)
    downloader = Downloader(crawler)

    done_dfd: Deferred[None] = Deferred()
    done_dfd.callback(None)

    pending_dfd: Deferred[None] = Deferred()
    failures: list[Failure] = []
    pending_dfd.addErrback(failures.append)

    downloader._download_tasks[Request("https://done.example")] = done_dfd
    downloader._download_tasks[Request("https://pending.example")] = pending_dfd

    dropped = await downloader.stop()

    assert dropped == 1
    assert len(failures) == 1
    assert failures[0].check(CancelledError)


@coroutine_test
async def test_fast_stop_with_download_in_flight(mockserver: MockServer) -> None:
    crawler = get_crawler(SimpleSpider)
    crawl_dfd = deferred_from_coro(
        crawler.crawl_async(url=mockserver.url("/delay?n=10"))
    )
    await crawler.signals.wait_for(signals.headers_received)
    # Let the download handler start waiting for the response body.
    await sleep(0)
    await crawler.stop_async(mode="fast")
    await maybe_deferred_to_future(crawl_dfd)
    assert crawler.stats
    assert crawler.stats.get_value("downloader/request_dropped_count") == 1
    assert crawler.stats.get_value("response_received_count") is None


@coroutine_test
async def test_fast_stop_before_download_starts() -> None:
    """A fast stop that cancels a download before the download starts ends
    the crawl and frees the slot."""
    crawler = get_crawler(SimpleSpider)

    def stop(request: Request, spider: Spider) -> None:
        # With asyncio, the stop runs before the download task that the
        # downloader creates right after this signal.
        _schedule_coro(crawler.stop_async(mode="fast"))

    crawler.signals.connect(stop, signals.request_reached_downloader)
    await crawler.crawl_async(url="data:,")
    (slot,) = crawler.engine.downloader.slots.values()
    assert not slot.active
    assert not slot.transferring


class ConcurrencyRecordingHandler(BaseDownloadHandler):
    """Download handler that keeps each download in progress for a moment and
    records the most downloads it had in progress at once."""

    def __init__(self, crawler: Crawler):
        super().__init__(crawler)
        self._in_progress = 0

    async def download_request(self, request: Request) -> Response:
        self._in_progress += 1
        self.crawler.stats.max_value("test/max_in_progress", self._in_progress)
        await sleep(0.05)
        self._in_progress -= 1
        return Response(request.url)


class ManyRequestsSpider(Spider):
    name = "many_requests"

    async def start(self):
        for i in range(10):
            yield Request(f"https://example.com/{i}")

    def parse(self, response):
        pass


@coroutine_test
async def test_slot_concurrency() -> None:
    crawler = get_crawler(
        ManyRequestsSpider,
        {
            "CONCURRENT_REQUESTS_PER_DOMAIN": 2,
            "DOWNLOAD_HANDLERS": {"https": ConcurrencyRecordingHandler},
        },
    )
    await crawler.crawl_async()
    assert crawler.stats.get_value("test/max_in_progress") == 2
    assert crawler.stats.get_value("response_received_count") == 10


def test_deprecated_tls_module_names() -> None:
    with pytest.warns(
        ScrapyDeprecationWarning,
        match="scrapy.core.downloader.tls.METHOD_TLS is deprecated",
    ):
        assert tls.METHOD_TLS == "TLS"
    with pytest.warns(
        ScrapyDeprecationWarning,
        match="scrapy.core.downloader.tls.openssl_methods is deprecated",
    ):
        assert isinstance(tls.openssl_methods, dict)
    with pytest.warns(
        ScrapyDeprecationWarning,
        match="scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated",
    ):
        assert tls.DEFAULT_CIPHERS._ciphers == (
            AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
        )


def test_slot_str() -> None:
    slot = Slot(concurrency=2, delay=0.5, jitter=0)
    assert str(slot).startswith(
        "<downloader.Slot concurrency=2 delay=0.50 jitter=0"
        " len(active)=0 len(queue)=0 len(transferring)=0 lastseen="
    )


def test_get_slot_key_by_ip() -> None:
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
            category=ScrapyDeprecationWarning,
            message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
        )
        crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS_PER_IP": 1})
        downloader = Downloader(crawler)
    dnscache["example.com"] = "127.0.0.1"
    try:
        assert downloader.get_slot_key(Request("http://example.com")) == "127.0.0.1"
    finally:
        del dnscache["example.com"]
        downloader.close()


@coroutine_test
async def test_slot_gc() -> None:
    """Slots that have been idle for long enough are discarded."""
    crawler = get_crawler(DefaultSpider)
    crawler.spider = crawler._create_spider()
    downloader = Downloader(crawler)
    try:
        await maybe_deferred_to_future(downloader.fetch(Request("data:,")))
        assert set(downloader.slots) == {""}
        downloader._slot_gc()
        assert set(downloader.slots) == {""}
        downloader._slot_gc(age=0)
        assert not downloader.slots
    finally:
        downloader.close()
        await downloader.handlers._close()


@coroutine_test
async def test_request_body_producer() -> None:
    class Consumer:
        def __init__(self) -> None:
            self.written: list[bytes] = []

        def write(self, data: bytes) -> None:
            self.written.append(data)

    producer = _RequestBodyProducer(b"body")
    assert producer.length == 4
    consumer = Consumer()
    await maybe_deferred_to_future(producer.startProducing(cast("IConsumer", consumer)))
    assert consumer.written == [b"body"]
    # The body is written in a single call, so there is nothing to pause or stop.
    producer.pauseProducing()
    producer.stopProducing()