Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Tests for setting SSL ciphers.

Andrey Rakhmatullin9a8edf2bf117
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 3fa4bc0f0..8be8a36bb 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,6 +3,8 @@ import sys, time, random, os, json
from six.moves.urllib.parse import urlencode
from subprocess import Popen, PIPE
+from OpenSSL import SSL
+
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File
@@ -222,6 +224,14 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
)
+def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'):
+ factory = ssl_context_factory(keyfile, certfile)
+ ctx = factory.getContext()
+ ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2)
+ ctx.set_cipher_list(cipher_string)
+ return factory
+
+
if __name__ == "__main__":
root = Root()
factory = Site(root)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index efef4192c..5df2ffe45 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
from scrapy.exceptions import NotConfigured
-from tests.mockserver import MockServer, ssl_context_factory, Echo
+from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factory
from tests.spiders import SingleRequestSpider
@@ -553,6 +553,47 @@ class Https11InvalidDNSPattern(Https11TestCase):
super(Https11InvalidDNSPattern, self).setUp()
+class Https11BadCiphers(unittest.TestCase):
+ scheme = 'https'
+ download_handler_cls = HTTP11DownloadHandler
+
+ keyfile = 'keys/localhost.key'
+ certfile = 'keys/localhost.crt'
+
+ def setUp(self):
+ self.tmpname = self.mktemp()
+ os.mkdir(self.tmpname)
+ FilePath(self.tmpname).child("file").setContent(b"0123456789")
+ r = static.File(self.tmpname)
+ self.site = server.Site(r, timeout=None)
+ self.wrapper = WrappingFactory(self.site)
+ self.host = 'localhost'
+ self.port = reactor.listenSSL(
+ 0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
+ interface=self.host)
+ self.portno = self.port.getHost().port
+ self.download_handler = self.download_handler_cls(
+ Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))
+ self.download_request = self.download_handler.download_request
+
+ @defer.inlineCallbacks
+ def tearDown(self):
+ yield self.port.stopListening()
+ if hasattr(self.download_handler, 'close'):
+ yield self.download_handler.close()
+ shutil.rmtree(self.tmpname)
+
+ def getURL(self, path):
+ return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
+
+ def test_download(self):
+ request = Request(self.getURL('file'))
+ d = self.download_request(request, Spider('foo'))
+ d.addCallback(lambda r: r.body)
+ d.addCallback(self.assertEqual, b"0123456789")
+ return d
+
+
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 766329b57..2ebe075ab 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -8,15 +8,18 @@ import shutil
from twisted.trial import unittest
from twisted.web import server, static, util, resource
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, ssl
from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
from twisted.protocols.policies import WrappingFactory
from twisted.internet.defer import inlineCallbacks
from scrapy.core.downloader import webclient as client
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.http import Request, Headers
+from scrapy.settings import Settings
from scrapy.utils.python import to_bytes, to_unicode
+from tests.mockserver import ssl_context_factory, broken_ssl_context_factory
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
@@ -363,3 +366,48 @@ class WebClientTestCase(unittest.TestCase):
self.assertEqual(content_encoding, EncodingResource.out_encoding)
self.assertEqual(
response.body.decode(content_encoding), to_unicode(original_body))
+
+
+class WebClientSSLTestCase(unittest.TestCase):
+ context_factory = None
+
+ def _listen(self, site):
+ return reactor.listenSSL(
+ 0, site,
+ contextFactory=self.context_factory or ssl_context_factory(),
+ interface="127.0.0.1")
+
+ def getURL(self, path):
+ return "https://127.0.0.1:%d/%s" % (self.portno, path)
+
+ def setUp(self):
+ self.tmpname = self.mktemp()
+ os.mkdir(self.tmpname)
+ FilePath(self.tmpname).child("file").setContent(b"0123456789")
+ r = static.File(self.tmpname)
+ r.putChild(b"payload", PayloadResource())
+ self.site = server.Site(r, timeout=None)
+ self.wrapper = WrappingFactory(self.site)
+ self.port = self._listen(self.wrapper)
+ self.portno = self.port.getHost().port
+
+ @inlineCallbacks
+ def tearDown(self):
+ yield self.port.stopListening()
+ shutil.rmtree(self.tmpname)
+
+ def testPayload(self):
+ s = "0123456789" * 10
+ return getPage(self.getURL("payload"), body=s).addCallback(
+ self.assertEqual, to_bytes(s))
+
+
+class WebClientBrokenSSLTestCase(WebClientSSLTestCase):
+ context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA')
+
+ def testPayload(self):
+ s = "0123456789" * 10
+ settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+ return getPage(self.getURL("payload"), body=s,
+ contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual,
+ to_bytes(s))