Scrapy, a fast high-level web crawling & scraping framework for Python.
Add HTTPS tests with non-hostname-maching server certificate
Paul Tremberthcd979ace40f2
diff --git a/tests/keys/example-com.cert.pem b/tests/keys/example-com.cert.pemnew file mode 100644index 000000000..af87198aa--- /dev/null+++ b/tests/keys/example-com.cert.pem@@ -0,0 +1,26 @@+-----BEGIN CERTIFICATE-----+MIIEVTCCAz2gAwIBAgIJANuZ/6fbAJNcMA0GCSqGSIb3DQEBCwUAMH0xCzAJBgNV+BAYTAlhXMQswCQYDVQQIDAJYVzEVMBMGA1UEBwwMVGhlIEludGVybmV0MQ8wDQYD+VQQKDAZTY3JhcHkxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTEfMB0GCSqGSIb3+DQEJARYQdGVzdEBleGFtcGxlLmNvbTAgFw0xNjA0MjAxMjExNTZaGA8yMTE2MDMy+NzEyMTE1NlowfTELMAkGA1UEBhMCWFcxCzAJBgNVBAgMAlhXMRUwEwYDVQQHDAxU+aGUgSW50ZXJuZXQxDzANBgNVBAoMBlNjcmFweTEYMBYGA1UEAwwPd3d3LmV4YW1w+bGUuY29tMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUuY29tMIIBIjANBgkq+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5r2BzxXivtQXvIwrTGug8l6vjuDhf0WD+HBU5yIHWvX1rT2MQFuZpu120iFGOK7MBYBBdmAIGsR5cHZ03cqQkAXOGQ5ug5O/u+d/GZUbcgyv8WCwW71MWLb5CNiDWj/vZq6CUqRg2QkUpkjr/DcQrKsm67yadKLgyd+G85OyZO6NBuAukQcKrNhspk/Ms55X7RbgYPUbZ0bBee4b3GRnE7PLltIsHo/tloV+ynC0Sd3T1taYyyG7IJd2LWJELzK0Ww+QUV2qoOdZjl8db1x5c99OR6xY0+Mjf14r+6kkXOBpPkrJ990qU40+z406u2HPf2abR4D/DUoe9qw+fElCeiuXFFQIDAQABo4HV+MIHSMB0GA1UdDgQWBBTY3DPInWZxrmQfPHA5w2R+AsbnOjAfBgNVHSMEGDAWgBTY+3DPInWZxrmQfPHA5w2R+AsbnOjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDBKBgNV+HREEQzBBggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tghBtYWlsLmV4YW1w+bGUuY29tgg9mdHAuZXhhbXBsZS5jb20wLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wg+R2VuZXJhdGVkIENlcnRpZmljYXRlMA0GCSqGSIb3DQEBCwUAA4IBAQDmnzdIu9XV+/Tnn5/mt9E98YEsF/eaXBSKG+f2oZc3n2errKwY5qYqULlu8mhajGWTd5EfFCjbD+lH7nmBdRUAhjzKWntc1G84eaWwHyv+N/5WJrtUfa2A1Ps3Mu9Vz4k2M9HGi/s6KX+IshezlHe3/TyhIT/WC+MZhpFTL73dpuIgHmp2NjjlJqtG25eC6zmonsc2RApJPa8+6J0WY/ISH9OwhDSbI9+TIE8QwdC7draiCKK/oid3Jg9fzaEQW+Pr3/4AmYWH1j8s+iaOVYIXYtt3urQ2Q+qfh34kfRfX5IqAdd1r/fnUjaOLhPNJxGP2KuFaYbdSC9p+n+bfExzQHUvH2n+-----END CERTIFICATE-----diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.confnew file mode 100644index 000000000..8aa338cd5--- /dev/null+++ b/tests/keys/example-com.conf@@ -0,0 +1,84 @@+# this is copied from http://stackoverflow.com/a/27931596+[ req ]+default_bits = 2048+default_keyfile = server-key.pem+distinguished_name = subject+req_extensions = req_ext+x509_extensions = x509_ext+string_mask = utf8only++# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description).+# Its sort of a mashup. For example, RFC 4514 does not provide emailAddress.+[ subject ]+countryName = Country Name (2 letter code)+countryName_default = US++stateOrProvinceName = State or Province Name (full name)+stateOrProvinceName_default = NY++localityName = Locality Name (eg, city)+localityName_default = New York++organizationName = Organization Name (eg, company)+organizationName_default = Example, LLC++# Use a friendly name here because its presented to the user. The server's DNS+# names are placed in Subject Alternate Names. Plus, DNS names here is deprecated+# by both IETF and CA/Browser Forums. If you place a DNS name here, then you+# must include the DNS name in the SAN too (otherwise, Chrome and others that+# strictly follow the CA/Browser Baseline Requirements will fail).+commonName = Common Name (e.g. server FQDN or YOUR name)+commonName_default = Example Company++emailAddress = Email Address+emailAddress_default = [email protected]++# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ...+[ x509_ext ]++subjectKeyIdentifier = hash+authorityKeyIdentifier = keyid,issuer++# You only need digitalSignature below. *If* you don't allow+# RSA Key transport (i.e., you use ephemeral cipher suites), then+# omit keyEncipherment because that's key transport.+basicConstraints = CA:FALSE+keyUsage = digitalSignature, keyEncipherment+subjectAltName = @alternate_names+nsComment = "OpenSSL Generated Certificate"++# RFC 5280, Section 4.2.1.12 makes EKU optional+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused+# In either case, you probably only need serverAuth.+# extendedKeyUsage = serverAuth, clientAuth++# Section req_ext is used when generating a certificate signing request. I.e., openssl req ...+[ req_ext ]++subjectKeyIdentifier = hash++basicConstraints = CA:FALSE+keyUsage = digitalSignature, keyEncipherment+subjectAltName = @alternate_names+nsComment = "OpenSSL Generated Certificate"++# RFC 5280, Section 4.2.1.12 makes EKU optional+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused+# In either case, you probably only need serverAuth.+# extendedKeyUsage = serverAuth, clientAuth++[ alternate_names ]++DNS.1 = example.com+DNS.2 = www.example.com+DNS.3 = mail.example.com+DNS.4 = ftp.example.com++# Add these if you need them. But usually you don't want them or+# need them in production. You may need them for development.+# DNS.5 = localhost+# DNS.6 = localhost.localdomain+# DNS.7 = 127.0.0.1++# IPv6 localhost+# DNS.8 = ::1diff --git a/tests/keys/example-com.gen.README b/tests/keys/example-com.gen.READMEnew file mode 100644index 000000000..955e7b057--- /dev/null+++ b/tests/keys/example-com.gen.README@@ -0,0 +1,24 @@+$ openssl req -config example-com.conf \+ -new -x509 -sha256 -newkey rsa:2048 -nodes \+ -keyout example-com.key.pem \+ -days 36500 \+ -out example-com.cert.pem+Generating a 2048 bit RSA private key+....++++.....................++++writing new private key to 'example-com.key.pem'+-----+You are about to be asked to enter information that will be incorporated+into your certificate request.+What you are about to enter is what is called a Distinguished Name or a DN.+There are quite a few fields but you can leave some blank+For some fields there will be a default value,+If you enter '.', the field will be left blank.+-----+Country Name (2 letter code) [US]:XW+State or Province Name (full name) [NY]:XW+Locality Name (eg, city) [New York]:The Internet+Organization Name (eg, company) [Example, LLC]:Scrapy+Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com+Email Address [[email protected]]:+diff --git a/tests/keys/example-com.key.pem b/tests/keys/example-com.key.pemnew file mode 100644index 000000000..56e805df8--- /dev/null+++ b/tests/keys/example-com.key.pem@@ -0,0 +1,28 @@+-----BEGIN PRIVATE KEY-----+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDmvYHPFeK+1Be8+jCtMa6DyXq+O4OF/RYMcFTnIgda9fWtPYxAW5mm7XbSIUY4rswFgEF2YAgaxHlwd+nTdypCQBc4ZDm6Dk7+538ZlRtyDK/xYLBbvUxYtvkI2INaP+9mroJSpGDZCRSmSO+v8NxCsqybrvJp0ouDJ0bzk7Jk7o0G4C6RBwqs2GymT8yznlftFuBg9RtnRsF57hv+cZGcTs8uW0iwej+2WhXKcLRJ3dPW1pjLIbsgl3YtYkQvMrRbD5BRXaqg51mOXx1v+XHlz305HrFjT4yN/XivqSRc4Gk+Ssn33SpTjT7PjTq7Yc9/ZptHgP8NSh72rD58S+UJ6K5cUVAgMBAAECggEAEVxi3vTzmY4Vtx8Ixvg2JOZQ6TwsW0ocoklKjraONLWy+FEgM1txBSlKzmaohO1J9oP+6Owyz+jGhlqzUljCZcO6DRKT6Bx+yXp3z/jz8H6nv+u6aTyl+OrTdAHuaCT1W1F4BsXNb6cKQbSs5M4z1/oMtKH9MRdsOGMqhkLzCJSxA++E+rRomrP6E9XooLxqpSJooxmX772XPHE7+ZILzRF0viXJ6z0Jd1cOuAQqyIYvmHM+4313kYJdAKYgJNxe8M8mYLeZcGwImAz/pNQ5R+uVyZlW3kXVzvS7B2m+KcW7Olu8+r4Ocpdyh20GStpw1f+tk7PLl+SkwFslK+uI7Wl/ygQKBgQD+QsREu1sv507CnvYC+FZnFryhHUzxMWIX5bvH3YoPbVptqwG5Nj05zIQPCZAEanW4HENCujO9oWbZqLANO+Th5sNOkBJiC4X6+1NIzQIszaZs4nKIyWNLIOcP4p20k3cR0sS8wLSLwcuW2is27N+ACKa8u93X1Gb27V0qUhmEqP3NQKBgQDoUY3HRtOVQnpoi5zbTiSKwlBKEkMAawaW+Q8VSZmrNQZXpcwa2JYN0IeiHnVjctdLul1u9qj5goghTV4XMQ8LSZs0emhvgJxMa+QpsDLTRr0mBtmduOwZW9a8EcbI2NCth/Irsdl892+y8UVoAO2G6Fgr0DhgXWOJEo+RcUUkGHyYQKBgHCkT6NLhYhhZykdl0sxGqDTinqey3XfOetZVWUNhfDkG5JdkgBW+XqDunWW/PCX4XMhQkMLjuSR3qjK7MPO99AhoIFnb9F76NTOIBmInKK4RIX+DnVTm+H4P6Vv9gIL2pJQ18vva40G3BUGrmJ042ox4WRjSSS+tKmMcIQATIeU+JAoGBAIpr+TRZW1Oox2c1Aogzo2kzyyfPYPaOaISqW5pLaAviZA0E9D9qnL1OagHmM/s1CaJNQ+C5FORiw9XsiJdWbnWMUUC1MYb1N29KLI3KNf48P6bQngijjcjuN1uHG/G/fVZnkZ+sHNJaItzzfFIOLSfr/pMk2HuELw6qAJez7YY8MMBAoGAf5sC2S7duDH2KliMSYhd+yByHEVTbf7JdJVd/7kq5MPZauJtYztGTSwyyM+gBv+lxdY1jGu/iNu9xAD2DJlE3+RTPgTIID/PaSmUVotHLq86hbazKPBorx6UWkbGsthEaSF/cTY2eFFFsK/Awoj0yU+ZhraPWu8S27Pcr9HvZvh5tg=+-----END PRIVATE KEY-----diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 365ec81fd..a40e2e501 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -204,10 +204,10 @@ class MockServer():time.sleep(0.2)-def ssl_context_factory():+def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'):return ssl.DefaultOpenSSLContextFactory(- os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),- os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),+ os.path.join(os.path.dirname(__file__), keyfile),+ os.path.join(os.path.dirname(__file__), certfile),)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex e13becf1b..45a806f2e 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -119,6 +119,10 @@ class HttpTestCase(unittest.TestCase):scheme = 'http'download_handler_cls = HTTPDownloadHandler+ # only used for HTTPS tests+ keyfile = 'keys/cert.pem'+ certfile = 'keys/cert.pem'+def setUp(self):name = self.mktemp()os.mkdir(name)@@ -137,7 +141,8 @@ class HttpTestCase(unittest.TestCase):self.host = 'localhost'if self.scheme == 'https':self.port = reactor.listenSSL(- 0, self.wrapper, ssl_context_factory(), interface=self.host)+ 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),+ interface=self.host)else:self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)self.portno = self.port.getHost().port@@ -318,6 +323,18 @@ class Https11TestCase(Http11TestCase):scheme = 'https'+class Https11WrongHostnameTestCase(Http11TestCase):+ scheme = 'https'++ # above tests use a server certificate for "localhost",+ # client connection to "localhost" too.+ # here we test that even if the server certificate is for another domain,+ # "www.example.com" in this case,+ # the tests still pass+ keyfile = 'keys/example-com.key.pem'+ certfile = 'keys/example-com.cert.pem'++class Http11MockServerTestCase(unittest.TestCase):"""HTTP 1.1 test case with MockServer"""if twisted_version < (11, 1, 0):