Scrapy, a fast high-level web crawling & scraping framework for Python.
Add "Host" header in CONNECT requests to HTTPS proxies
Paul Tremberthd9343463cb69
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 88c6b9480..d02027dd1 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):for it."""- _responseMatcher = re.compile(b'HTTP/1\.. 200')+ _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')def __init__(self, reactor, host, port, proxyConf, contextFactory,timeout=30, bindAddress=None):@@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):self._protocol = protocolreturn protocol- def processProxyResponse(self, bytes):+ def processProxyResponse(self, rcvd_bytes):"""Processes the response from the proxy. If the tunnel is successfullycreated, notifies the client that we are ready to send requests. If notraises a TunnelError."""self._protocol.dataReceived = self._protocolDataReceived- if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):+ respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)+ if respm and int(respm.group('status')) == 200:try:# this sets proper Server Name Indication extension# but is only available for Twisted>=14.0@@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):self._protocolFactory)self._tunnelReadyDeferred.callback(self._protocol)else:+ if respm:+ extra = {'status': int(respm.group('status')),+ 'reason': respm.group('reason').strip()}+ else:+ extra = rcvd_bytes[:32]self._tunnelReadyDeferred.errback(- TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (- self._host, self._port)))+ TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (+ self._host, self._port, extra)))def connectFailed(self, reason):"""Propagates the errback to the appropriate deferred."""@@ -151,23 +157,22 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):return self._tunnelReadyDeferred-def tunnel_request_data(host, port, proxy_auth_header=None):+def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):r"""Return binary content of a CONNECT request.>>> from scrapy.utils.python import to_native_str as s>>> s(tunnel_request_data("example.com", 8080))- 'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'>>> s(tunnel_request_data("example.com", 8080, b"123"))- 'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'>>> s(tunnel_request_data(b"example.com", "8090"))- 'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'+ 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'"""- tunnel_req = (- b'CONNECT ' +- to_bytes(host, encoding='ascii') + b':' +- to_bytes(str(port)) +- b' HTTP/1.1\r\n')+ host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))+ tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'+ if host_header:+ tunnel_req += b'Host: ' + host_value + b'\r\n'if proxy_auth_header:tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'tunnel_req += b'\r\n'