Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add HTTPS tests with non-hostname-maching server certificate

Paul Tremberthcd979ace40f2
diff --git a/tests/keys/example-com.cert.pem b/tests/keys/example-com.cert.pem
new file mode 100644
index 000000000..af87198aa
--- /dev/null
+++ b/tests/keys/example-com.cert.pem
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEVTCCAz2gAwIBAgIJANuZ/6fbAJNcMA0GCSqGSIb3DQEBCwUAMH0xCzAJBgNV
+BAYTAlhXMQswCQYDVQQIDAJYVzEVMBMGA1UEBwwMVGhlIEludGVybmV0MQ8wDQYD
+VQQKDAZTY3JhcHkxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTEfMB0GCSqGSIb3
+DQEJARYQdGVzdEBleGFtcGxlLmNvbTAgFw0xNjA0MjAxMjExNTZaGA8yMTE2MDMy
+NzEyMTE1NlowfTELMAkGA1UEBhMCWFcxCzAJBgNVBAgMAlhXMRUwEwYDVQQHDAxU
+aGUgSW50ZXJuZXQxDzANBgNVBAoMBlNjcmFweTEYMBYGA1UEAwwPd3d3LmV4YW1w
+bGUuY29tMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUuY29tMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5r2BzxXivtQXvIwrTGug8l6vjuDhf0WD
+HBU5yIHWvX1rT2MQFuZpu120iFGOK7MBYBBdmAIGsR5cHZ03cqQkAXOGQ5ug5O/u
+d/GZUbcgyv8WCwW71MWLb5CNiDWj/vZq6CUqRg2QkUpkjr/DcQrKsm67yadKLgyd
+G85OyZO6NBuAukQcKrNhspk/Ms55X7RbgYPUbZ0bBee4b3GRnE7PLltIsHo/tloV
+ynC0Sd3T1taYyyG7IJd2LWJELzK0Ww+QUV2qoOdZjl8db1x5c99OR6xY0+Mjf14r
+6kkXOBpPkrJ990qU40+z406u2HPf2abR4D/DUoe9qw+fElCeiuXFFQIDAQABo4HV
+MIHSMB0GA1UdDgQWBBTY3DPInWZxrmQfPHA5w2R+AsbnOjAfBgNVHSMEGDAWgBTY
+3DPInWZxrmQfPHA5w2R+AsbnOjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDBKBgNV
+HREEQzBBggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tghBtYWlsLmV4YW1w
+bGUuY29tgg9mdHAuZXhhbXBsZS5jb20wLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wg
+R2VuZXJhdGVkIENlcnRpZmljYXRlMA0GCSqGSIb3DQEBCwUAA4IBAQDmnzdIu9XV
+/Tnn5/mt9E98YEsF/eaXBSKG+f2oZc3n2errKwY5qYqULlu8mhajGWTd5EfFCjbD
+lH7nmBdRUAhjzKWntc1G84eaWwHyv+N/5WJrtUfa2A1Ps3Mu9Vz4k2M9HGi/s6KX
+IshezlHe3/TyhIT/WC+MZhpFTL73dpuIgHmp2NjjlJqtG25eC6zmonsc2RApJPa8
+6J0WY/ISH9OwhDSbI9+TIE8QwdC7draiCKK/oid3Jg9fzaEQW+Pr3/4AmYWH1j8s
+iaOVYIXYtt3urQ2Q+qfh34kfRfX5IqAdd1r/fnUjaOLhPNJxGP2KuFaYbdSC9p+n
+bfExzQHUvH2n
+-----END CERTIFICATE-----
diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf
new file mode 100644
index 000000000..8aa338cd5
--- /dev/null
+++ b/tests/keys/example-com.conf
@@ -0,0 +1,84 @@
+# this is copied from http://stackoverflow.com/a/27931596
+[ req ]
+default_bits = 2048
+default_keyfile = server-key.pem
+distinguished_name = subject
+req_extensions = req_ext
+x509_extensions = x509_ext
+string_mask = utf8only
+
+# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description).
+# Its sort of a mashup. For example, RFC 4514 does not provide emailAddress.
+[ subject ]
+countryName = Country Name (2 letter code)
+countryName_default = US
+
+stateOrProvinceName = State or Province Name (full name)
+stateOrProvinceName_default = NY
+
+localityName = Locality Name (eg, city)
+localityName_default = New York
+
+organizationName = Organization Name (eg, company)
+organizationName_default = Example, LLC
+
+# Use a friendly name here because its presented to the user. The server's DNS
+# names are placed in Subject Alternate Names. Plus, DNS names here is deprecated
+# by both IETF and CA/Browser Forums. If you place a DNS name here, then you
+# must include the DNS name in the SAN too (otherwise, Chrome and others that
+# strictly follow the CA/Browser Baseline Requirements will fail).
+commonName = Common Name (e.g. server FQDN or YOUR name)
+commonName_default = Example Company
+
+emailAddress = Email Address
+emailAddress_default = [email protected]
+
+# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ...
+[ x509_ext ]
+
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+
+# You only need digitalSignature below. *If* you don't allow
+# RSA Key transport (i.e., you use ephemeral cipher suites), then
+# omit keyEncipherment because that's key transport.
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+subjectAltName = @alternate_names
+nsComment = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+# In either case, you probably only need serverAuth.
+# extendedKeyUsage = serverAuth, clientAuth
+
+# Section req_ext is used when generating a certificate signing request. I.e., openssl req ...
+[ req_ext ]
+
+subjectKeyIdentifier = hash
+
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+subjectAltName = @alternate_names
+nsComment = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+# In either case, you probably only need serverAuth.
+# extendedKeyUsage = serverAuth, clientAuth
+
+[ alternate_names ]
+
+DNS.1 = example.com
+DNS.2 = www.example.com
+DNS.3 = mail.example.com
+DNS.4 = ftp.example.com
+
+# Add these if you need them. But usually you don't want them or
+# need them in production. You may need them for development.
+# DNS.5 = localhost
+# DNS.6 = localhost.localdomain
+# DNS.7 = 127.0.0.1
+
+# IPv6 localhost
+# DNS.8 = ::1
diff --git a/tests/keys/example-com.gen.README b/tests/keys/example-com.gen.README
new file mode 100644
index 000000000..955e7b057
--- /dev/null
+++ b/tests/keys/example-com.gen.README
@@ -0,0 +1,24 @@
+$ openssl req -config example-com.conf \
+ -new -x509 -sha256 -newkey rsa:2048 -nodes \
+ -keyout example-com.key.pem \
+ -days 36500 \
+ -out example-com.cert.pem
+Generating a 2048 bit RSA private key
+....+++
+.....................+++
+writing new private key to 'example-com.key.pem'
+-----
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [US]:XW
+State or Province Name (full name) [NY]:XW
+Locality Name (eg, city) [New York]:The Internet
+Organization Name (eg, company) [Example, LLC]:Scrapy
+Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com
+Email Address [[email protected]]:
+
diff --git a/tests/keys/example-com.key.pem b/tests/keys/example-com.key.pem
new file mode 100644
index 000000000..56e805df8
--- /dev/null
+++ b/tests/keys/example-com.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDmvYHPFeK+1Be8
+jCtMa6DyXq+O4OF/RYMcFTnIgda9fWtPYxAW5mm7XbSIUY4rswFgEF2YAgaxHlwd
+nTdypCQBc4ZDm6Dk7+538ZlRtyDK/xYLBbvUxYtvkI2INaP+9mroJSpGDZCRSmSO
+v8NxCsqybrvJp0ouDJ0bzk7Jk7o0G4C6RBwqs2GymT8yznlftFuBg9RtnRsF57hv
+cZGcTs8uW0iwej+2WhXKcLRJ3dPW1pjLIbsgl3YtYkQvMrRbD5BRXaqg51mOXx1v
+XHlz305HrFjT4yN/XivqSRc4Gk+Ssn33SpTjT7PjTq7Yc9/ZptHgP8NSh72rD58S
+UJ6K5cUVAgMBAAECggEAEVxi3vTzmY4Vtx8Ixvg2JOZQ6TwsW0ocoklKjraONLWy
+FEgM1txBSlKzmaohO1J9oP+6Owyz+jGhlqzUljCZcO6DRKT6Bx+yXp3z/jz8H6nv
+u6aTyl+OrTdAHuaCT1W1F4BsXNb6cKQbSs5M4z1/oMtKH9MRdsOGMqhkLzCJSxA+
+E+rRomrP6E9XooLxqpSJooxmX772XPHE7+ZILzRF0viXJ6z0Jd1cOuAQqyIYvmHM
+4313kYJdAKYgJNxe8M8mYLeZcGwImAz/pNQ5R+uVyZlW3kXVzvS7B2m+KcW7Olu8
+r4Ocpdyh20GStpw1f+tk7PLl+SkwFslK+uI7Wl/ygQKBgQD+QsREu1sv507CnvYC
+FZnFryhHUzxMWIX5bvH3YoPbVptqwG5Nj05zIQPCZAEanW4HENCujO9oWbZqLANO
+Th5sNOkBJiC4X6+1NIzQIszaZs4nKIyWNLIOcP4p20k3cR0sS8wLSLwcuW2is27N
+ACKa8u93X1Gb27V0qUhmEqP3NQKBgQDoUY3HRtOVQnpoi5zbTiSKwlBKEkMAawaW
+Q8VSZmrNQZXpcwa2JYN0IeiHnVjctdLul1u9qj5goghTV4XMQ8LSZs0emhvgJxMa
+QpsDLTRr0mBtmduOwZW9a8EcbI2NCth/Irsdl892+y8UVoAO2G6Fgr0DhgXWOJEo
+RcUUkGHyYQKBgHCkT6NLhYhhZykdl0sxGqDTinqey3XfOetZVWUNhfDkG5JdkgBW
+XqDunWW/PCX4XMhQkMLjuSR3qjK7MPO99AhoIFnb9F76NTOIBmInKK4RIX+DnVTm
+H4P6Vv9gIL2pJQ18vva40G3BUGrmJ042ox4WRjSSS+tKmMcIQATIeU+JAoGBAIpr
+TRZW1Oox2c1Aogzo2kzyyfPYPaOaISqW5pLaAviZA0E9D9qnL1OagHmM/s1CaJNQ
+C5FORiw9XsiJdWbnWMUUC1MYb1N29KLI3KNf48P6bQngijjcjuN1uHG/G/fVZnkZ
+sHNJaItzzfFIOLSfr/pMk2HuELw6qAJez7YY8MMBAoGAf5sC2S7duDH2KliMSYhd
+yByHEVTbf7JdJVd/7kq5MPZauJtYztGTSwyyM+gBv+lxdY1jGu/iNu9xAD2DJlE3
+RTPgTIID/PaSmUVotHLq86hbazKPBorx6UWkbGsthEaSF/cTY2eFFFsK/Awoj0yU
+ZhraPWu8S27Pcr9HvZvh5tg=
+-----END PRIVATE KEY-----
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 365ec81fd..a40e2e501 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -204,10 +204,10 @@ class MockServer():
time.sleep(0.2)
-def ssl_context_factory():
+def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'):
return ssl.DefaultOpenSSLContextFactory(
- os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
- os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+ os.path.join(os.path.dirname(__file__), keyfile),
+ os.path.join(os.path.dirname(__file__), certfile),
)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e13becf1b..45a806f2e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -119,6 +119,10 @@ class HttpTestCase(unittest.TestCase):
scheme = 'http'
download_handler_cls = HTTPDownloadHandler
+ # only used for HTTPS tests
+ keyfile = 'keys/cert.pem'
+ certfile = 'keys/cert.pem'
+
def setUp(self):
name = self.mktemp()
os.mkdir(name)
@@ -137,7 +141,8 @@ class HttpTestCase(unittest.TestCase):
self.host = 'localhost'
if self.scheme == 'https':
self.port = reactor.listenSSL(
- 0, self.wrapper, ssl_context_factory(), interface=self.host)
+ 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),
+ interface=self.host)
else:
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
self.portno = self.port.getHost().port
@@ -318,6 +323,18 @@ class Https11TestCase(Http11TestCase):
scheme = 'https'
+class Https11WrongHostnameTestCase(Http11TestCase):
+ scheme = 'https'
+
+ # above tests use a server certificate for "localhost",
+ # client connection to "localhost" too.
+ # here we test that even if the server certificate is for another domain,
+ # "www.example.com" in this case,
+ # the tests still pass
+ keyfile = 'keys/example-com.key.pem'
+ certfile = 'keys/example-com.cert.pem'
+
+
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
if twisted_version < (11, 1, 0):