Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bracket IPv6 destinations in HTTP proxy CONNECT requests (#8343)

Adrian377fc4ef613b
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 3f5309666..af6317030 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -205,11 +205,22 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._contextFactory: IPolicyForHTTPS = contextFactory
self._connectBuffer: bytearray = bytearray()
+ @staticmethod
+ def _format_host(host: bytes) -> bytes:
+ """Return *host* as it goes into the authority of the CONNECT request,
+ i.e. with square brackets around IPv6 literals.
+ """
+ # URI.host strips the brackets of IPv6 literals, and only those contain
+ # colons.
+ return b"[" + host + b"]" if b":" in host else host
+
def requestTunnel(self, protocol: Protocol) -> Protocol:
"""Asks the proxy to open a tunnel."""
assert protocol.transport
tunnelReq = _tunnel_request_data(
- self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
+ self._format_host(self._tunneledHost),
+ self._tunneledPort,
+ self._proxyAuthHeader,
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 9e58e3105..e956d1c09 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -9,12 +9,14 @@ from twisted.internet.error import ConnectingCancelledError
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.testing import MemoryReactorClock
from twisted.python.failure import Failure
+from twisted.web.client import URI
from scrapy import Request, Spider
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers.http11 import (
HTTP11DownloadHandler,
TunnelError,
+ _tunnel_request_data,
_TunnelingTCP4ClientEndpoint,
)
from scrapy.crawler import Crawler
@@ -167,6 +169,26 @@ class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
handler_supports_tls_in_tls = False
+ ("url", "expected_authority"),
+ [
+ (b"https://[::1]:8443/x", b"[::1]:8443"),
+ (b"https://[::ffff:192.168.0.1]/x", b"[::ffff:192.168.0.1]:443"),
+ (b"https://example.com:8443/x", b"example.com:8443"),
+ (b"https://127.0.0.1:8443/x", b"127.0.0.1:8443"),
+ ],
+)
+def test_connect_authority(url: bytes, expected_authority: bytes) -> None:
+ uri = URI.fromBytes(url)
+ tunnel_req = _tunnel_request_data(
+ _TunnelingTCP4ClientEndpoint._format_host(uri.host), uri.port
+ )
+ assert tunnel_req == (
+ b"CONNECT " + expected_authority + b" HTTP/1.1\r\n"
+ b"Host: " + expected_authority + b"\r\n\r\n"
+ )
+
+
@pytest.mark.requires_internet
class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase):
@property