Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use best practices for TLS connections when using Twisted>=14.0

Paul Tremberth30a27effa9b2
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 0b39b89d8..347b74648 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,28 +1,72 @@
from OpenSSL import SSL
from twisted.internet.ssl import ClientContextFactory
-try:
- # available since twisted 14.0
- from twisted.internet._sslverify import ClientTLSOptions
-except ImportError:
- ClientTLSOptions = None
-
-
-class ScrapyClientContextFactory(ClientContextFactory):
- "A SSL context factory which is more permissive against SSL bugs."
- # see https://github.com/scrapy/scrapy/issues/82
- # and https://github.com/scrapy/scrapy/issues/26
- # and https://github.com/scrapy/scrapy/issues/981
-
- def __init__(self):
- # see this issue on why we use TLSv1_METHOD by default
- # https://github.com/scrapy/scrapy/issues/194
- self.method = SSL.TLSv1_METHOD
-
- def getContext(self, hostname=None, port=None):
- ctx = ClientContextFactory.getContext(self)
- # Enable all workarounds to SSL bugs as documented by
- # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
- ctx.set_options(SSL.OP_ALL)
- if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
- ClientTLSOptions(hostname, ctx)
- return ctx
+
+from scrapy import twisted_version
+
+if twisted_version >= (14, 0, 0):
+
+ from twisted.web.client import BrowserLikePolicyForHTTPS
+ from twisted.internet.ssl import optionsForClientTLS
+
+ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+ """
+ Using Twisted recommended context factory for twisted.web.client.Agent
+
+ Quoting:
+ "The default is to use a BrowserLikePolicyForHTTPS,
+ so unless you have special requirements you can leave this as-is."
+
+ See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html
+ """
+
+
+ class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
+
+ openssl_method = SSL.SSLv23_METHOD
+
+ def creatorForNetloc(self, hostname, port):
+ return optionsForClientTLS(hostname.decode("ascii"),
+ trustRoot=self._trustRoot,
+ extraCertificateOptions={
+ 'method': self.openssl_method
+ })
+
+
+else:
+
+ class OpenSSLMethodContextFactory(ClientContextFactory):
+ "A SSL context factory which is more permissive against SSL bugs."
+ # see https://github.com/scrapy/scrapy/issues/82
+ # and https://github.com/scrapy/scrapy/issues/26
+ # and https://github.com/scrapy/scrapy/issues/981
+ openssl_method = SSL.SSLv23_METHOD
+
+ def __init__(self):
+ self.method = self.openssl_method
+
+ def getContext(self, hostname=None, port=None):
+ ctx = ClientContextFactory.getContext(self)
+ # Enable all workarounds to SSL bugs as documented by
+ # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
+ ctx.set_options(SSL.OP_ALL)
+ if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
+ ClientTLSOptions(hostname, ctx)
+ return ctx
+
+ ScrapyClientContextFactory = OpenSSLMethodContextFactory
+
+
+class SSLv3ContextFactory(OpenSSLMethodContextFactory):
+ openssl_method = SSL.SSLv3_METHOD
+
+
+class TLSv1ContextFactory(OpenSSLMethodContextFactory):
+ openssl_method = SSL.TLSv1_METHOD
+
+
+class TLSv11ContextFactory(OpenSSLMethodContextFactory):
+ openssl_method = SSL.TLSv1_1_METHOD
+
+
+class TLSv12ContextFactory(OpenSSLMethodContextFactory):
+ openssl_method = SSL.TLSv1_2_METHOD