Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import sys
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import Mock
import pytest
from twisted.internet.address import IPv6Address
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import ConnectingCancelledError, ConnectionClosed
from twisted.internet.interfaces import (
IAddress,
IHostnameResolver,
IHostResolution,
IReactorPluggableNameResolver,
IResolutionReceiver,
)
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.testing import MemoryReactorClock, StringTransport
from twisted.python.failure import Failure
from twisted.web._newclient import Request as TxClientRequest
from twisted.web._newclient import RequestNotSent
from twisted.web.client import URI, BrowserLikePolicyForHTTPS
from twisted.web.http_headers import Headers as TxHeaders
from zope.interface import implementer
try:
import resource
except ImportError:
resource = None # type: ignore[assignment]
from scrapy import Request, Spider
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers._base_http import _auto_connection_limit
from scrapy.core.downloader.handlers.http11 import (
HTTP11DownloadHandler,
TunnelError,
_ScrapyAgent,
_ScrapyHTTP11ClientProtocol,
_tunnel_request_data,
_TunnelEndpoint,
_TunnelingAgent,
_TunnelProtocol,
)
from scrapy.crawler import Crawler
from scrapy.exceptions import (
CannotResolveHostError,
DownloadConnectionRefusedError,
DownloadFailedError,
NotConfigured,
ResponseHeadersTooLargeError,
)
from scrapy.resolver import HostResolution
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.utils import _free_port
from tests.utils import ipv6_loopback_available
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
TestHttpsBase,
TestHttpsCustomCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsTLSVersionBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator, Sequence
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IProtocol, IStreamClientEndpoint
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
from tests.utils.bases.download_handlers_http import HeaderNameCase
pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a reactor
class HTTP11DownloadHandlerMixin:
handler_supports_partial_response_on_timeout = True
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return HTTP11DownloadHandler
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
"https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
}
}
handler_supports_custom_content_length = False
handler_request_header_name_case: HeaderNameCase = "canonical"
handler_response_header_name_case: HeaderNameCase = "canonical"
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
build_from_crawler(HTTP11DownloadHandler, crawler)
def _tunnel_endpoint(reactor: MemoryReactorClock) -> _TunnelEndpoint:
return _TunnelEndpoint(
TCP4ClientEndpoint(reactor, "127.0.0.1", 8080),
host=b"example.com",
port=443,
proxyHost="127.0.0.1",
proxyPort=8080,
)
@asynccontextmanager
async def _get_dh(
settings_dict: dict[str, Any] | None = None,
) -> AsyncGenerator[HTTP11DownloadHandler]:
crawler = get_crawler(DefaultSpider, settings_dict)
crawler.spider = crawler._create_spider()
dh = build_from_crawler(HTTP11DownloadHandler, crawler)
try:
yield dh
finally:
await dh.close()
@coroutine_test
async def test_connection_limit_auto() -> None:
async with _get_dh({}) as dh:
assert dh._pool._limit == _auto_connection_limit()
@coroutine_test
@pytest.mark.parametrize("limit", [0, 20])
async def test_connection_limit_explicit(limit: int) -> None:
async with _get_dh({"CONCURRENT_CONNECTIONS_PER_HANDLER": limit}) as dh:
assert dh._pool._limit == limit
@coroutine_test
async def test_connection_limit_below_concurrent_requests(
caplog: pytest.LogCaptureFixture,
) -> None:
settings_dict = {
"CONCURRENT_CONNECTIONS_PER_HANDLER": 4,
"CONCURRENT_REQUESTS": 8,
}
with caplog.at_level("WARNING"):
async with _get_dh(settings_dict):
pass
assert "CONCURRENT_CONNECTIONS_PER_HANDLER (4)" in caplog.text
@pytest.mark.skipif(resource is None, reason="No file descriptor limit")
@coroutine_test
async def test_connection_limit_auto_without_file_descriptor_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
infinity = resource.RLIM_INFINITY
monkeypatch.setattr(resource, "getrlimit", lambda _: (infinity, infinity))
async with _get_dh({}) as dh:
assert dh._pool._limit == 0
@coroutine_test
async def test_keepalive_timeout() -> None:
async with _get_dh({"CONNECTION_KEEPALIVE_TIMEOUT": 5}) as dh:
assert dh._pool.cachedConnectionTimeout == 5
@pytest.mark.parametrize(
("proxy_headers", "proxy_auth", "expected"),
[
(None, None, ()),
({"X-B": "2", "X-A": "1"}, None, (("X-A", "1"), ("X-B", "2"))),
(None, "Basic Zm9v", (("Proxy-Authorization", "Basic Zm9v"),)),
(
{"X-A": "1"},
"Basic Zm9v",
(("Proxy-Authorization", "Basic Zm9v"), ("X-A", "1")),
),
(
{"Proxy-Authorization": "Custom foo"},
"Basic Zm9v",
(("Proxy-Authorization", "Custom foo"),),
),
],
ids=[
"no headers",
"sorted headers",
"auth header",
"auth header and headers",
"auth header overridden",
],
)
def test_tunnel_proxy_headers(
proxy_headers: dict[str, str] | None,
proxy_auth: str | None,
expected: tuple[tuple[str, str], ...],
) -> None:
meta: dict[str, Any] = {"proxy": "http://proxy.example:8080"}
if proxy_headers is not None:
meta["proxy_headers"] = proxy_headers
request = Request(
"https://example.com",
meta=meta,
headers={"Proxy-Authorization": proxy_auth} if proxy_auth else None,
)
crawler = get_crawler(Spider)
agent = _ScrapyAgent(
contextFactory=_load_context_factory_from_settings(crawler), crawler=crawler
)
tunneling_agent = agent._get_agent(request, 10)
assert isinstance(tunneling_agent, _TunnelingAgent)
assert tunneling_agent._proxyConf == ("proxy.example", 8080, expected, False)
def test_tunneling_cancelled_before_connected() -> None:
"""Cancelling a tunneled download before the connection to the proxy is
established stops the connection attempt.
A download cannot reach this scenario, because the connection timeout of
the endpoint, which uses the same value as the download timeout, always
fires earlier than the cancellation. See also
``test_download_with_proxy_stalled_connect``, which covers cancelling once
the connection to the proxy is established.
"""
reactor = MemoryReactorClock()
results: list[IProtocol | Failure] = []
d = _tunnel_endpoint(reactor).connect(Factory.forProtocol(Protocol))
d.addBoth(results.append)
d.cancel()
assert reactor.connectors[0].stoppedConnecting
assert isinstance(results[0], Failure)
assert isinstance(results[0].value, ConnectingCancelledError)
def test_tunneling_cancelled_before_connect_response() -> None:
"""A CONNECT response received after cancelling the tunnel, but before the
connection to the proxy is closed, is ignored."""
reactor = MemoryReactorClock()
results: list[IProtocol | Failure] = []
d = _tunnel_endpoint(reactor).connect(Factory.forProtocol(Protocol))
d.addBoth(results.append)
factory = reactor.tcpClients[0][2]
protocol = factory.buildProtocol(None)
assert protocol is not None
transport = StringTransport()
protocol.makeConnection(transport)
d.cancel()
protocol.dataReceived(b"HTTP/1.1 200 OK\r\n\r\n")
assert transport.disconnecting
assert len(results) == 1
assert isinstance(results[0], Failure)
assert isinstance(results[0].value, CancelledError)
class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):
pass
class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase):
pass
class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase):
pass
class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
pass
class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
pass
class TestHttpsInvalidDNSPattern(
HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
):
pass
class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
pass
class TestHttpsTLSVersion(HTTP11DownloadHandlerMixin, TestHttpsTLSVersionBase):
pass
class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase):
pass
class TestHttpsWithCrawler(TestHttpWithCrawler):
is_secure = True
class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
pass
class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
@pytest.mark.requires_mitmproxy
class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
pass
@pytest.mark.skipif(
not ipv6_loopback_available(), reason="IPv6 loopback is not available"
)
class TestMitmProxyIPv6(TestMitmProxy):
proxy_host = "::1"
@pytest.mark.parametrize(
("url", "expected_authority"),
[
(b"https://[::1]:8443/x", b"[::1]:8443"),
(b"https://[::ffff:192.168.0.1]/x", b"[::ffff:192.168.0.1]:443"),
(b"https://example.com:8443/x", b"example.com:8443"),
(b"https://127.0.0.1:8443/x", b"127.0.0.1:8443"),
],
)
def test_connect_authority(url: bytes, expected_authority: bytes) -> None:
uri = URI.fromBytes(url)
tunnel_req = _tunnel_request_data(_TunnelProtocol._format_host(uri.host), uri.port)
assert tunnel_req == (
b"CONNECT " + expected_authority + b" HTTP/1.1\r\n"
b"Host: " + expected_authority + b"\r\n\r\n"
)
@implementer(IHostnameResolver)
class _SentinelIPv6Resolver:
"""Resolves one sentinel hostname to ::1, delegating everything else.
There is no portable hostname that resolves only to an IPv6 address, so the
test that needs one provides it here.
"""
hostname = "ipv6-only-proxy.invalid"
def __init__(self, original: IHostnameResolver):
self._original = original
def resolveHostName(
self,
resolutionReceiver: IResolutionReceiver,
hostName: str,
portNumber: int = 0,
addressTypes: Sequence[type[IAddress]] | None = None,
transportSemantics: str = "TCP",
) -> IHostResolution:
if hostName != self.hostname:
return self._original.resolveHostName(
resolutionReceiver,
hostName,
portNumber,
addressTypes,
transportSemantics,
)
resolutionReceiver.resolutionBegan(HostResolution(hostName))
# Honor addressTypes, as a real resolver does: a caller that asks only
# for IPv4 addresses gets no results for this name.
if addressTypes is None or IPv6Address in addressTypes:
resolutionReceiver.addressResolved(IPv6Address("TCP", "::1", portNumber))
resolutionReceiver.resolutionComplete()
return resolutionReceiver
@implementer(IHostnameResolver)
class _UnresolvingResolver:
"""Resolves nothing, and records what it was asked to resolve.
Stands in for Scrapy's default resolver, which cannot resolve an IPv6
literal: it only ever reports IPv4 addresses.
"""
def __init__(self) -> None:
self.names: list[str] = []
def resolveHostName(
self,
resolutionReceiver: IResolutionReceiver,
hostName: str,
portNumber: int = 0,
addressTypes: Sequence[type[IAddress]] | None = None,
transportSemantics: str = "TCP",
) -> IHostResolution:
self.names.append(hostName)
resolution = HostResolution(hostName)
resolutionReceiver.resolutionBegan(resolution)
resolutionReceiver.resolutionComplete()
return resolution
@implementer(IReactorPluggableNameResolver)
class _ReactorWithNameResolver:
"""The given reactor, but resolving names with the given resolver.
``HostnameEndpoint`` reads ``nameResolver`` off the reactor when it
connects, so injecting one here keeps the test from having to install a
resolver on the global reactor and restore it afterwards.
"""
def __init__(self, reactor: object, nameResolver: IHostnameResolver):
self._reactor = reactor
self.nameResolver = nameResolver
def installNameResolver(
self, resolver: IHostnameResolver
) -> IHostnameResolver: # pragma: no cover
raise NotImplementedError
def __getattr__(self, name: str) -> Any:
return getattr(self._reactor, name)
class _FakeProxyProtocol(Protocol):
"""Answers any CONNECT request with a refusal, so no TLS is attempted."""
def dataReceived(self, data: bytes) -> None:
assert self.transport
self.transport.write(b"HTTP/1.1 403 Forbidden\r\n\r\n")
@pytest.fixture
def fake_proxy_port() -> Generator[int]:
"""The port of a CONNECT-answering server listening on ::1."""
from twisted.internet import reactor
listening_port = reactor.listenTCP(
0, Factory.forProtocol(_FakeProxyProtocol), interface="::1"
)
try:
yield listening_port.getHost().port
finally:
listening_port.stopListening()
@pytest.fixture
def ipv6_only_proxy(fake_proxy_port: int) -> tuple[ReactorBase, str, int]:
"""That server, reachable only by a name that resolves to ::1."""
from twisted.internet import reactor
resolving_reactor = _ReactorWithNameResolver(
reactor, _SentinelIPv6Resolver(reactor.nameResolver)
)
return (
cast("ReactorBase", resolving_reactor),
_SentinelIPv6Resolver.hostname,
fake_proxy_port,
)
def _proxy_tunnel_endpoint(
reactor: ReactorBase, proxy_host: str, proxy_port: int
) -> IStreamClientEndpoint:
agent = _TunnelingAgent(
reactor=reactor,
proxyConf=(proxy_host, proxy_port, (), False),
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=10,
)
return agent._getEndpoint(URI.fromBytes(b"https://example.com"))
@pytest.mark.skipif(
not ipv6_loopback_available(), reason="IPv6 loopback is not available"
)
@coroutine_test
async def test_tunnel_to_proxy_reachable_only_over_ipv6(
ipv6_only_proxy: tuple[ReactorBase, str, int],
) -> None:
"""The CONNECT tunnel must reach a proxy whose name resolves only to IPv6.
Getting as far as a TunnelError means the proxy was reached and answered; a
connection error instead would mean the endpoint picked the wrong address
family for the proxy.
"""
reactor, proxy_host, proxy_port = ipv6_only_proxy
endpoint = _proxy_tunnel_endpoint(reactor, proxy_host, proxy_port)
with pytest.raises(TunnelError, match="Could not open CONNECT tunnel"):
await maybe_deferred_to_future(endpoint.connect(Factory.forProtocol(Protocol)))
@pytest.mark.skipif(
not ipv6_loopback_available(), reason="IPv6 loopback is not available"
)
@coroutine_test
async def test_tunnel_to_ipv6_literal_proxy(fake_proxy_port: int) -> None:
"""The CONNECT tunnel must reach a proxy given as an IPv6 address literal
without resolving it, as the default resolver cannot resolve one.
"""
from twisted.internet import reactor
resolver = _UnresolvingResolver()
endpoint = _proxy_tunnel_endpoint(
cast("ReactorBase", _ReactorWithNameResolver(reactor, resolver)),
"::1",
fake_proxy_port,
)
with pytest.raises(TunnelError, match="Could not open CONNECT tunnel"):
await maybe_deferred_to_future(endpoint.connect(Factory.forProtocol(Protocol)))
assert not resolver.names
@pytest.mark.requires_internet
class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase):
@property
def platform_cert_store_works(self) -> bool:
return sys.platform != "win32"
class TestTunnelingHeadersMaxsize:
"""Tests for the DOWNLOAD_HEADERS_MAXSIZE limit that ``_TunnelProtocol``
applies to the response head of the proxy, which no HTTP client parses for
us."""
@staticmethod
def _get_protocol(
headers_maxsize: int,
) -> tuple[_TunnelProtocol, StringTransport, list[Failure]]:
deferred: Deferred[IProtocol] = Deferred()
failures: list[Failure] = []
deferred.addErrback(failures.append)
protocol = _TunnelProtocol(
deferred,
Mock(),
b"example.com",
443,
"proxy.example.com",
8080,
headersMaxsize=headers_maxsize,
)
transport = StringTransport()
protocol.makeConnection(transport) # type: ignore[no-untyped-call]
return protocol, transport, failures
def test_over_maxsize(self) -> None:
protocol, transport, failures = self._get_protocol(1024)
# A proxy response head that never ends.
for _ in range(3):
protocol.dataReceived(b"a" * 512)
assert len(failures) == 1
assert failures[0].check(ResponseHeadersTooLargeError)
assert "1024" in str(failures[0].value)
assert "CONNECT example.com:443" in str(failures[0].value)
assert transport.disconnecting
def test_under_maxsize(self) -> None:
protocol, transport, failures = self._get_protocol(1024)
protocol.dataReceived(b"a" * 512)
assert not failures
assert not transport.disconnecting
def test_maxsize_disabled(self) -> None:
protocol, transport, failures = self._get_protocol(0)
for _ in range(3):
protocol.dataReceived(b"a" * 512)
assert not failures
assert not transport.disconnecting
class TestScrapyHTTP11ClientProtocol:
"""Tests for the response header size limiting that
``_ScrapyHTTP11ClientProtocol`` installs on the response parser that
Twisted builds for each request."""
@staticmethod
def _get_protocol() -> _ScrapyHTTP11ClientProtocol:
protocol = _ScrapyHTTP11ClientProtocol(lambda _: None, 64 * 1024, 32 * 1024)
protocol.makeConnection(StringTransport()) # type: ignore[no-untyped-call]
return protocol
@staticmethod
def _get_request() -> TxClientRequest:
return TxClientRequest(
b"GET", b"/", TxHeaders({b"host": [b"example.com"]}), None
)
def test_parser_is_limited(self) -> None:
protocol = self._get_protocol()
protocol.request(self._get_request())
assert protocol._parser is not None
assert protocol._parser.MAX_LENGTH == 64 * 1024
# _limit_response_headers() overrides these on the instance.
assert "lineReceived" in vars(protocol._parser)
assert "lineLengthExceeded" in vars(protocol._parser)
def test_refused_request_keeps_previous_parser_limits(self) -> None:
"""A request that Twisted refuses leaves the parser of the previous
request untouched, so that its size counter is not reset while its
response is still being read."""
protocol = self._get_protocol()
protocol.request(self._get_request())
parser = protocol._parser
assert parser is not None
line_received = vars(parser)["lineReceived"]
# A second request over the same connection is refused, as the first one
# is still in progress.
deferred = protocol.request(self._get_request())
failures: list[Failure] = []
deferred.addErrback(failures.append)
assert len(failures) == 1
assert failures[0].check(RequestNotSent)
assert protocol._parser is parser
assert vars(parser)["lineReceived"] is line_received
class _CannedConnectProxy(Protocol):
"""Answers any request with a canned response, or drops the connection when
the response is ``None``."""
factory: _CannedConnectProxyFactory
def dataReceived(self, data: bytes) -> None:
from twisted.internet import reactor
assert self.transport
response = self.factory.response
if response is None:
self.transport.loseConnection()
return
if not self.factory.split_at:
self.transport.write(response)
return
self.transport.write(response[: self.factory.split_at])
reactor.callLater(0.1, self.transport.write, response[self.factory.split_at :])
class _CannedConnectProxyFactory(Factory):
protocol = _CannedConnectProxy # type: ignore[assignment]
def __init__(self, response: bytes | None, split_at: int = 0):
self.response = response
self.split_at = split_at
@asynccontextmanager
async def _canned_proxy(
response: bytes | None, split_at: int = 0
) -> AsyncGenerator[str]:
from twisted.internet import reactor
port = reactor.listenTCP(
0, _CannedConnectProxyFactory(response, split_at), interface="127.0.0.1"
)
try:
yield f"http://127.0.0.1:{port.getHost().port}"
finally:
await maybe_deferred_to_future(port.stopListening())
class TestTunnelingErrors:
@coroutine_test
async def test_response_in_two_packets(self) -> None:
"""A CONNECT response split across packets is buffered until complete."""
response = b"HTTP/1.1 407 Proxy Authentication Required\r\n\r\n"
async with _canned_proxy(response, split_at=len(response) - 2) as proxy:
request = Request("https://example.com", meta={"proxy": proxy})
async with _get_dh() as dh:
with pytest.raises(TunnelError, match="407"):
await dh.download_request(request)
@coroutine_test
async def test_unparsable_response(self) -> None:
async with _canned_proxy(b"not an HTTP response\r\n\r\n") as proxy:
request = Request("https://example.com", meta={"proxy": proxy})
async with _get_dh() as dh:
with pytest.raises(TunnelError, match="not an HTTP response"):
await dh.download_request(request)
@coroutine_test
async def test_proxy_closes_connection(self) -> None:
"""The proxy drops the connection instead of answering CONNECT."""
async with _canned_proxy(None) as proxy:
request = Request("https://example.com", meta={"proxy": proxy})
async with _get_dh() as dh:
with pytest.raises(ConnectionClosed):
await dh.download_request(request)
@coroutine_test
async def test_trailing_bytes(self) -> None:
"""Bytes right after a successful CONNECT response belong to the
tunneled protocol, whose TLS handshake they corrupt."""
response = b"HTTP/1.1 200 Connection established\r\n\r\ntrailing-bytes"
async with _canned_proxy(response) as proxy:
request = Request("https://example.com", meta={"proxy": proxy})
async with _get_dh() as dh:
with pytest.raises(DownloadFailedError):
await dh.download_request(request)
@coroutine_test
async def test_proxy_connection_refused(self) -> None:
request = Request(
"https://example.com", meta={"proxy": f"http://127.0.0.1:{_free_port()}"}
)
async with _get_dh() as dh:
with pytest.raises(DownloadConnectionRefusedError):
await dh.download_request(request)
@coroutine_test
async def test_proxy_without_port(self) -> None:
if NON_EXISTING_RESOLVABLE:
pytest.skip("Non-existing hosts are resolvable")
request = Request(
"https://example.com", meta={"proxy": "http://no-such-domain.nosuch"}
)
async with _get_dh() as dh:
with pytest.raises(CannotResolveHostError):
await dh.download_request(request)