Scrapy, a fast high-level web crawling & scraping framework for Python.
Bracket IPv6 destinations in HTTP proxy CONNECT requests (#8343)
Adrian377fc4ef613b
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 3f5309666..af6317030 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -205,11 +205,22 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):self._contextFactory: IPolicyForHTTPS = contextFactoryself._connectBuffer: bytearray = bytearray()+ @staticmethod+ def _format_host(host: bytes) -> bytes:+ """Return *host* as it goes into the authority of the CONNECT request,+ i.e. with square brackets around IPv6 literals.+ """+ # URI.host strips the brackets of IPv6 literals, and only those contain+ # colons.+ return b"[" + host + b"]" if b":" in host else host+def requestTunnel(self, protocol: Protocol) -> Protocol:"""Asks the proxy to open a tunnel."""assert protocol.transporttunnelReq = _tunnel_request_data(- self._tunneledHost, self._tunneledPort, self._proxyAuthHeader+ self._format_host(self._tunneledHost),+ self._tunneledPort,+ self._proxyAuthHeader,)protocol.transport.write(tunnelReq)self._protocolDataReceived = protocol.dataReceiveddiff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex 9e58e3105..e956d1c09 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -9,12 +9,14 @@ from twisted.internet.error import ConnectingCancelledErrorfrom twisted.internet.protocol import Factory, Protocolfrom twisted.internet.testing import MemoryReactorClockfrom twisted.python.failure import Failure+from twisted.web.client import URIfrom scrapy import Request, Spiderfrom scrapy.core.downloader.contextfactory import _load_context_factory_from_settingsfrom scrapy.core.downloader.handlers.http11 import (HTTP11DownloadHandler,TunnelError,+ _tunnel_request_data,_TunnelingTCP4ClientEndpoint,)from scrapy.crawler import Crawler@@ -167,6 +169,26 @@ class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):handler_supports_tls_in_tls = False+ ("url", "expected_authority"),+ [+ (b"https://[::1]:8443/x", b"[::1]:8443"),+ (b"https://[::ffff:192.168.0.1]/x", b"[::ffff:192.168.0.1]:443"),+ (b"https://example.com:8443/x", b"example.com:8443"),+ (b"https://127.0.0.1:8443/x", b"127.0.0.1:8443"),+ ],+)+def test_connect_authority(url: bytes, expected_authority: bytes) -> None:+ uri = URI.fromBytes(url)+ tunnel_req = _tunnel_request_data(+ _TunnelingTCP4ClientEndpoint._format_host(uri.host), uri.port+ )+ assert tunnel_req == (+ b"CONNECT " + expected_authority + b" HTTP/1.1\r\n"+ b"Host: " + expected_authority + b"\r\n\r\n"+ )++@pytest.mark.requires_internetclass TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase):@property