Scrapy, a fast high-level web crawling & scraping framework for Python.
Tests for setting SSL ciphers.
Andrey Rakhmatullin9a8edf2bf117
diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 3fa4bc0f0..8be8a36bb 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -3,6 +3,8 @@ import sys, time, random, os, jsonfrom six.moves.urllib.parse import urlencodefrom subprocess import Popen, PIPE+from OpenSSL import SSL+from twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.resource import Resourcefrom twisted.web.static import File@@ -222,6 +224,14 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c)+def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'):+ factory = ssl_context_factory(keyfile, certfile)+ ctx = factory.getContext()+ ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2)+ ctx.set_cipher_list(cipher_string)+ return factory++if __name__ == "__main__":root = Root()factory = Site(root)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex efef4192c..5df2ffe45 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_botofrom scrapy.utils.python import to_bytesfrom scrapy.exceptions import NotConfigured-from tests.mockserver import MockServer, ssl_context_factory, Echo+from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factoryfrom tests.spiders import SingleRequestSpider@@ -553,6 +553,47 @@ class Https11InvalidDNSPattern(Https11TestCase):super(Https11InvalidDNSPattern, self).setUp()+class Https11BadCiphers(unittest.TestCase):+ scheme = 'https'+ download_handler_cls = HTTP11DownloadHandler++ keyfile = 'keys/localhost.key'+ certfile = 'keys/localhost.crt'++ def setUp(self):+ self.tmpname = self.mktemp()+ os.mkdir(self.tmpname)+ FilePath(self.tmpname).child("file").setContent(b"0123456789")+ r = static.File(self.tmpname)+ self.site = server.Site(r, timeout=None)+ self.wrapper = WrappingFactory(self.site)+ self.host = 'localhost'+ self.port = reactor.listenSSL(+ 0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),+ interface=self.host)+ self.portno = self.port.getHost().port+ self.download_handler = self.download_handler_cls(+ Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))+ self.download_request = self.download_handler.download_request++ @defer.inlineCallbacks+ def tearDown(self):+ yield self.port.stopListening()+ if hasattr(self.download_handler, 'close'):+ yield self.download_handler.close()+ shutil.rmtree(self.tmpname)++ def getURL(self, path):+ return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)++ def test_download(self):+ request = Request(self.getURL('file'))+ d = self.download_request(request, Spider('foo'))+ d.addCallback(lambda r: r.body)+ d.addCallback(self.assertEqual, b"0123456789")+ return d++class Http11MockServerTestCase(unittest.TestCase):"""HTTP 1.1 test case with MockServer"""diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 766329b57..2ebe075ab 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -8,15 +8,18 @@ import shutilfrom twisted.trial import unittestfrom twisted.web import server, static, util, resource-from twisted.internet import reactor, defer+from twisted.internet import reactor, defer, sslfrom twisted.test.proto_helpers import StringTransportfrom twisted.python.filepath import FilePathfrom twisted.protocols.policies import WrappingFactoryfrom twisted.internet.defer import inlineCallbacksfrom scrapy.core.downloader import webclient as client+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactoryfrom scrapy.http import Request, Headers+from scrapy.settings import Settingsfrom scrapy.utils.python import to_bytes, to_unicode+from tests.mockserver import ssl_context_factory, broken_ssl_context_factorydef getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):@@ -363,3 +366,48 @@ class WebClientTestCase(unittest.TestCase):self.assertEqual(content_encoding, EncodingResource.out_encoding)self.assertEqual(response.body.decode(content_encoding), to_unicode(original_body))+++class WebClientSSLTestCase(unittest.TestCase):+ context_factory = None++ def _listen(self, site):+ return reactor.listenSSL(+ 0, site,+ contextFactory=self.context_factory or ssl_context_factory(),+ interface="127.0.0.1")++ def getURL(self, path):+ return "https://127.0.0.1:%d/%s" % (self.portno, path)++ def setUp(self):+ self.tmpname = self.mktemp()+ os.mkdir(self.tmpname)+ FilePath(self.tmpname).child("file").setContent(b"0123456789")+ r = static.File(self.tmpname)+ r.putChild(b"payload", PayloadResource())+ self.site = server.Site(r, timeout=None)+ self.wrapper = WrappingFactory(self.site)+ self.port = self._listen(self.wrapper)+ self.portno = self.port.getHost().port++ @inlineCallbacks+ def tearDown(self):+ yield self.port.stopListening()+ shutil.rmtree(self.tmpname)++ def testPayload(self):+ s = "0123456789" * 10+ return getPage(self.getURL("payload"), body=s).addCallback(+ self.assertEqual, to_bytes(s))+++class WebClientBrokenSSLTestCase(WebClientSSLTestCase):+ context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA')++ def testPayload(self):+ s = "0123456789" * 10+ settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})+ return getPage(self.getURL("payload"), body=s,+ contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual,+ to_bytes(s))