Scrapy, a fast high-level web crawling & scraping framework for Python.
Use best practices for TLS connections when using Twisted>=14.0
Paul Tremberth30a27effa9b2
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 0b39b89d8..347b74648 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -1,28 +1,72 @@from OpenSSL import SSLfrom twisted.internet.ssl import ClientContextFactory-try:- # available since twisted 14.0- from twisted.internet._sslverify import ClientTLSOptions-except ImportError:- ClientTLSOptions = None---class ScrapyClientContextFactory(ClientContextFactory):- "A SSL context factory which is more permissive against SSL bugs."- # see https://github.com/scrapy/scrapy/issues/82- # and https://github.com/scrapy/scrapy/issues/26- # and https://github.com/scrapy/scrapy/issues/981-- def __init__(self):- # see this issue on why we use TLSv1_METHOD by default- # https://github.com/scrapy/scrapy/issues/194- self.method = SSL.TLSv1_METHOD-- def getContext(self, hostname=None, port=None):- ctx = ClientContextFactory.getContext(self)- # Enable all workarounds to SSL bugs as documented by- # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html- ctx.set_options(SSL.OP_ALL)- if hostname and ClientTLSOptions is not None: # workaround for TLS SNI- ClientTLSOptions(hostname, ctx)- return ctx++from scrapy import twisted_version++if twisted_version >= (14, 0, 0):++ from twisted.web.client import BrowserLikePolicyForHTTPS+ from twisted.internet.ssl import optionsForClientTLS++ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):+ """+ Using Twisted recommended context factory for twisted.web.client.Agent++ Quoting:+ "The default is to use a BrowserLikePolicyForHTTPS,+ so unless you have special requirements you can leave this as-is."++ See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html+ """+++ class OpenSSLMethodContextFactory(ScrapyClientContextFactory):++ openssl_method = SSL.SSLv23_METHOD++ def creatorForNetloc(self, hostname, port):+ return optionsForClientTLS(hostname.decode("ascii"),+ trustRoot=self._trustRoot,+ extraCertificateOptions={+ 'method': self.openssl_method+ })+++else:++ class OpenSSLMethodContextFactory(ClientContextFactory):+ "A SSL context factory which is more permissive against SSL bugs."+ # see https://github.com/scrapy/scrapy/issues/82+ # and https://github.com/scrapy/scrapy/issues/26+ # and https://github.com/scrapy/scrapy/issues/981+ openssl_method = SSL.SSLv23_METHOD++ def __init__(self):+ self.method = self.openssl_method++ def getContext(self, hostname=None, port=None):+ ctx = ClientContextFactory.getContext(self)+ # Enable all workarounds to SSL bugs as documented by+ # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html+ ctx.set_options(SSL.OP_ALL)+ if hostname and ClientTLSOptions is not None: # workaround for TLS SNI+ ClientTLSOptions(hostname, ctx)+ return ctx++ ScrapyClientContextFactory = OpenSSLMethodContextFactory+++class SSLv3ContextFactory(OpenSSLMethodContextFactory):+ openssl_method = SSL.SSLv3_METHOD+++class TLSv1ContextFactory(OpenSSLMethodContextFactory):+ openssl_method = SSL.TLSv1_METHOD+++class TLSv11ContextFactory(OpenSSLMethodContextFactory):+ openssl_method = SSL.TLSv1_1_METHOD+++class TLSv12ContextFactory(OpenSSLMethodContextFactory):+ openssl_method = SSL.TLSv1_2_METHOD