Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add "Host" header in CONNECT requests to HTTPS proxies

Paul Tremberthd9343463cb69
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 88c6b9480..d02027dd1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
- _responseMatcher = re.compile(b'HTTP/1\.. 200')
+ _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
def __init__(self, reactor, host, port, proxyConf, contextFactory,
timeout=30, bindAddress=None):
@@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocol = protocol
return protocol
- def processProxyResponse(self, bytes):
+ def processProxyResponse(self, rcvd_bytes):
"""Processes the response from the proxy. If the tunnel is successfully
created, notifies the client that we are ready to send requests. If not
raises a TunnelError.
"""
self._protocol.dataReceived = self._protocolDataReceived
- if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
+ respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
+ if respm and int(respm.group('status')) == 200:
try:
# this sets proper Server Name Indication extension
# but is only available for Twisted>=14.0
@@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
+ if respm:
+ extra = {'status': int(respm.group('status')),
+ 'reason': respm.group('reason').strip()}
+ else:
+ extra = rcvd_bytes[:32]
self._tunnelReadyDeferred.errback(
- TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
- self._host, self._port)))
+ TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
+ self._host, self._port, extra)))
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@@ -151,23 +157,22 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
return self._tunnelReadyDeferred
-def tunnel_request_data(host, port, proxy_auth_header=None):
+def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):
r"""
Return binary content of a CONNECT request.
>>> from scrapy.utils.python import to_native_str as s
>>> s(tunnel_request_data("example.com", 8080))
- 'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'
+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
- 'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'
+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
>>> s(tunnel_request_data(b"example.com", "8090"))
- 'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'
+ 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
"""
- tunnel_req = (
- b'CONNECT ' +
- to_bytes(host, encoding='ascii') + b':' +
- to_bytes(str(port)) +
- b' HTTP/1.1\r\n')
+ host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
+ tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
+ if host_header:
+ tunnel_req += b'Host: ' + host_value + b'\r\n'
if proxy_auth_header:
tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
tunnel_req += b'\r\n'