Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

10.1 KB
Raw Blame
"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler."""

from __future__ import annotations

import sys
from importlib.util import find_spec
from typing import TYPE_CHECKING, Any, ClassVar

import pytest

from scrapy import Request
from scrapy.core.downloader.handlers._httpx import (  # type: ignore[attr-defined]
    HAS_HTTP2,
    HAS_SOCKS,
    HttpxDownloadHandler,
    httpx,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import IDNA_REJECTED_HOSTNAMES
from tests.utils.bases.download_handlers_http import (
    TestHttpBase,
    TestHttpProxyBase,
    TestHttpsBase,
    TestHttpsCustomCiphersBase,
    TestHttpsDefaultCiphersBase,
    TestHttpsInvalidDNSIdBase,
    TestHttpsInvalidDNSPatternBase,
    TestHttpsTLSVersionBase,
    TestHttpsWrongHostnameBase,
    TestHttpWithCrawlerBase,
    TestMitmProxyBase,
    TestRealWebsiteBase,
    TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from pathlib import Path

    from scrapy.core.downloader.handlers import DownloadHandlerProtocol
    from tests.mockserver.http import MockServer
    from tests.mockserver.proxy_echo import ProxyEchoMockServer


pytestmark = pytest.mark.only_asyncio

if find_spec("httpx2") is None:
    pytest.skip("httpx2 is not installed", allow_module_level=True)

# httpx2 < 2.4.0 reads URL.host through idna.decode() without display=True when
# building the Host header, which raises for punycode labels that IDNA 2008
# rejects. https://github.com/pydantic/httpx2/pull/1018
# This check can be dropped once the httpx2 requirement is bumped to >= 2.4.0.
try:
    for _hostname in IDNA_REJECTED_HOSTNAMES:
        _ = httpx.URL(f"http://{_hostname}").host
except Exception:
    HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = False
else:
    HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = True


class HttpxDownloadHandlerMixin:
    # httpx does not expose the response head size limit of its HTTP client, so
    # DOWNLOAD_HEADERS_MAXSIZE and DOWNLOAD_HEADERS_WARNSIZE do not apply. Over
    # HTTP/1.1 the limit is httpcore's MAX_INCOMPLETE_EVENT_SIZE (100 KiB),
    # which h11 only enforces once the head exceeds it and is still incomplete.
    headers_maxsize: int | None = 100 * 1024
    handler_supports_partial_response_on_timeout = True

    @property
    def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
        return HttpxDownloadHandler

    @property
    def settings_dict(self) -> dict[str, Any] | None:
        return {
            "DOWNLOAD_HANDLERS": {
                "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
                "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
            }
        }


class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
    handler_supports_bindaddress_meta = False
    handler_bad_header_handling = "fail"
    handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES

    @pytest.mark.skipif(
        sys.platform == "darwin",
        reason="127.0.0.2 is not available on macOS by default",
    )
    @coroutine_test
    async def test_bind_address_port_warning(
        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
    ) -> None:
        request = Request(mockserver.url("/client-ip"))
        async with self.get_dh(
            {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 12345)}
        ) as download_handler:
            response = await download_handler.download_request(request)
        assert response.body == b"127.0.0.2"
        assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text
        assert "Ignoring the port" in caplog.text


class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
    handler_supports_bindaddress_meta = False
    handler_bad_header_handling = "fail"
    handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES
    tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"

    @pytest.mark.skip(reason="The check is Twisted-specific")
    def test_verify_certs_deprecated(self) -> None:  # type: ignore[override]
        pass


@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")
class TestHttp2(TestHttps):
    http2 = True
    # Over HTTP/2 the limit is instead h2's DEFAULT_MAX_HEADER_LIST_SIZE.
    headers_maxsize = 64 * 1024

    default_handler_settings: ClassVar[dict[str, Any]] = {
        "HTTPX_HTTP2_ENABLED": True,
    }

    @coroutine_test
    async def test_protocol(self, mockserver: MockServer) -> None:
        request = Request(mockserver.url("/host", is_secure=self.is_secure))
        async with self.get_dh() as download_handler:
            response = await download_handler.download_request(request)
        assert response.protocol == "HTTP/2"


class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
    pass


class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
    pass


class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
    pass


class TestHttpsInvalidDNSPattern(
    HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
):
    pass


class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
    pass


class TestHttpsDefaultCiphers(HttpxDownloadHandlerMixin, TestHttpsDefaultCiphersBase):
    pass


class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase):
    pass


class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
    pass


class TestHttpsWithCrawler(TestHttpWithCrawler):
    is_secure = True


class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
    pass


class TestHttpsProxy(TestHttpProxy):
    is_secure = True

    @coroutine_test
    async def test_keylog(
        self,
        monkeypatch: pytest.MonkeyPatch,
        proxy_mockserver: ProxyEchoMockServer,
        tmp_path: Path,
    ) -> None:
        keylog_file = tmp_path / "keylog"
        monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file))
        http_proxy = proxy_mockserver.url("", is_secure=True)
        request = Request("http://example.com", meta={"proxy": http_proxy})
        async with self.get_dh() as download_handler:
            response = await download_handler.download_request(request)
        assert response.body == b"http://example.com/"
        assert keylog_file.read_text()


@pytest.mark.requires_mitmproxy
class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
    handler_supports_socks = HAS_SOCKS
    # httpx sends an unbracketed IPv6 authority to the proxy, so the proxy
    # answers 400; reproducible with httpx alone, without Scrapy involved.
    # https://github.com/pydantic/httpx2/pull/1091
    handler_supports_ipv6_destination = False


@pytest.mark.requires_internet
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
    pass


@coroutine_test
async def test_proxy_clients() -> None:
    proxy = "http://proxy.example:8080"
    handler = build_from_crawler(HttpxDownloadHandler, get_crawler())
    try:
        client = handler._get_client(proxy, (("X-A", "1"),))
        assert handler._get_client(proxy, (("X-A", "1"),)) is client
        assert handler._get_client(proxy, (("X-A", "2"),)) is not client
        assert handler._get_client(proxy) is not client
        assert len(handler._proxy_clients) == 3
    finally:
        await handler.close()


@coroutine_test
async def test_proxy_headers_ignored_for_socks(
    caplog: pytest.LogCaptureFixture,
) -> None:
    if not HAS_SOCKS:  # pragma: no cover
        pytest.skip("SOCKS support is not installed")
    handler = build_from_crawler(HttpxDownloadHandler, get_crawler())
    try:
        with caplog.at_level("WARNING"):
            handler._get_client("socks5://proxy.example:1080", (("X-A", "1"),))
    finally:
        await handler.close()
    assert "the SOCKS protocol has no headers" in caplog.text


@pytest.mark.parametrize(("limit", "expected"), [(16, 16), (0, None)])
@coroutine_test
async def test_pool_limits(limit: int, expected: int | None) -> None:
    crawler = get_crawler(settings_dict={"CONCURRENT_CONNECTIONS_PER_HANDLER": limit})
    handler = build_from_crawler(HttpxDownloadHandler, crawler)
    try:
        assert handler._limits.max_connections == expected
        assert handler._limits.max_keepalive_connections == expected
    finally:
        await handler.close()


@coroutine_test
async def test_keepalive_timeout() -> None:
    crawler = get_crawler(settings_dict={"CONNECTION_KEEPALIVE_TIMEOUT": 5})
    handler = build_from_crawler(HttpxDownloadHandler, crawler)
    try:
        assert handler._limits.keepalive_expiry == 5
    finally:
        await handler.close()


class StableHandler(HttpxDownloadHandler):
    experimental = False


@pytest.mark.parametrize(
    ("handler_cls", "expected"), [(HttpxDownloadHandler, True), (StableHandler, False)]
)
@coroutine_test
async def test_experimental_warning(
    handler_cls: type[HttpxDownloadHandler],
    expected: bool,
    caplog: pytest.LogCaptureFixture,
) -> None:
    handler = build_from_crawler(handler_cls, get_crawler())
    await handler.close()
    assert ("is experimental" in caplog.text) is expected


@coroutine_test
async def test_proxies_not_supported() -> None:
    class NoProxyHandler(HttpxDownloadHandler):
        supports_proxies = False

    crawler = get_crawler()
    handler = build_from_crawler(NoProxyHandler, crawler)
    request = Request("http://example.com", meta={"proxy": "http://proxy.example:8080"})
    try:
        with pytest.raises(NotImplementedError, match="doesn't support proxies"):
            await handler.download_request(request)
    finally:
        await handler.close()


@coroutine_test
async def test_non_basic_proxy_auth() -> None:
    crawler = get_crawler()
    handler = build_from_crawler(HttpxDownloadHandler, crawler)
    request = Request(
        "http://example.com",
        meta={"proxy": "http://proxy.example:8080"},
        headers={"Proxy-Authorization": "Digest deadbeef"},
    )
    try:
        with pytest.raises(ValueError, match="got Digest"):
            await handler.download_request(request)
    finally:
        await handler.close()