Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove fallback ScrapyClientContextFactory class (used in Twisted < 14.0.0)
Eugenio Lacuesta3164543ed165
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 5ac20c0bb..127a246f5 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -1,112 +1,85 @@from OpenSSL import SSL-from twisted.internet.ssl import ClientContextFactory--from scrapy import twisted_version--if twisted_version >= (14, 0, 0):-- from zope.interface.declarations import implementer-- from twisted.internet.ssl import (optionsForClientTLS,- CertificateOptions,- platformTrust)- from twisted.web.client import BrowserLikePolicyForHTTPS- from twisted.web.iweb import IPolicyForHTTPS-- from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS--- @implementer(IPolicyForHTTPS)- class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):- """- Non-peer-certificate verifying HTTPS context factory-- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)- which allows TLS protocol negotiation-- 'A TLS/SSL connection established with [this method] may- understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'- """-- def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)- self._ssl_method = method- self.tls_verbose_logging = tls_verbose_logging-- @classmethod- def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):- tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')- return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)-- def getCertificateOptions(self):- # setting verify=True will require you to provide CAs- # to verify against; in other words: it's not that simple-- # backward-compatible SSL/TLS method:- #- # * this will respect `method` attribute in often recommended- # `ScrapyClientContextFactory` subclass- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)- #- # * getattr() for `_ssl_method` attribute for context factories- # not calling super(..., self).__init__- return CertificateOptions(verify=False,- method=getattr(self, 'method',- getattr(self, '_ssl_method', None)),- fixBrokenPeers=True,- acceptableCiphers=DEFAULT_CIPHERS)-- # kept for old-style HTTP/1.0 downloader context twisted calls,- # e.g. connectSSL()- def getContext(self, hostname=None, port=None):- return self.getCertificateOptions().getContext()-- def creatorForNetloc(self, hostname, port):- return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),- verbose_logging=self.tls_verbose_logging)--- @implementer(IPolicyForHTTPS)- class BrowserLikeContextFactory(ScrapyClientContextFactory):- """- Twisted-recommended context factory for web clients.-- Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:- "The default is to use a BrowserLikePolicyForHTTPS,- so unless you have special requirements you can leave this as-is."-- creatorForNetloc() is the same as BrowserLikePolicyForHTTPS- except this context factory allows setting the TLS/SSL method to use.-- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)- which allows TLS protocol negotiation.- """- def creatorForNetloc(self, hostname, port):-- # trustRoot set to platformTrust() will use the platform's root CAs.- #- # This means that a website like https://www.cacert.org will be rejected- # by default, since CAcert.org CA certificate is seldom shipped.- return optionsForClientTLS(hostname.decode("ascii"),- trustRoot=platformTrust(),- extraCertificateOptions={- 'method': self._ssl_method,- })--else:-- class ScrapyClientContextFactory(ClientContextFactory):- "A SSL context factory which is more permissive against SSL bugs."- # see https://github.com/scrapy/scrapy/issues/82- # and https://github.com/scrapy/scrapy/issues/26- # and https://github.com/scrapy/scrapy/issues/981-- def __init__(self, method=SSL.SSLv23_METHOD):- self.method = method-- def getContext(self, hostname=None, port=None):- ctx = ClientContextFactory.getContext(self)- # Enable all workarounds to SSL bugs as documented by- # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html- ctx.set_options(SSL.OP_ALL)- return ctx+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust+from twisted.web.client import BrowserLikePolicyForHTTPS+from twisted.web.iweb import IPolicyForHTTPS+from zope.interface.declarations import implementer++from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS+++@implementer(IPolicyForHTTPS)+class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):+ """+ Non-peer-certificate verifying HTTPS context factory++ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)+ which allows TLS protocol negotiation++ 'A TLS/SSL connection established with [this method] may+ understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'+ """++ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):+ super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)+ self._ssl_method = method+ self.tls_verbose_logging = tls_verbose_logging++ @classmethod+ def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):+ tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')+ return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)++ def getCertificateOptions(self):+ # setting verify=True will require you to provide CAs+ # to verify against; in other words: it's not that simple++ # backward-compatible SSL/TLS method:+ #+ # * this will respect `method` attribute in often recommended+ # `ScrapyClientContextFactory` subclass+ # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)+ #+ # * getattr() for `_ssl_method` attribute for context factories+ # not calling super(..., self).__init__+ return CertificateOptions(verify=False,+ method=getattr(self, 'method',+ getattr(self, '_ssl_method', None)),+ fixBrokenPeers=True,+ acceptableCiphers=DEFAULT_CIPHERS)++ # kept for old-style HTTP/1.0 downloader context twisted calls,+ # e.g. connectSSL()+ def getContext(self, hostname=None, port=None):+ return self.getCertificateOptions().getContext()++ def creatorForNetloc(self, hostname, port):+ return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),+ verbose_logging=self.tls_verbose_logging)+++@implementer(IPolicyForHTTPS)+class BrowserLikeContextFactory(ScrapyClientContextFactory):+ """+ Twisted-recommended context factory for web clients.++ Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:+ "The default is to use a BrowserLikePolicyForHTTPS,+ so unless you have special requirements you can leave this as-is."++ creatorForNetloc() is the same as BrowserLikePolicyForHTTPS+ except this context factory allows setting the TLS/SSL method to use.++ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)+ which allows TLS protocol negotiation.+ """+ def creatorForNetloc(self, hostname, port):++ # trustRoot set to platformTrust() will use the platform's root CAs.+ #+ # This means that a website like https://www.cacert.org will be rejected+ # by default, since CAcert.org CA certificate is seldom shipped.+ return optionsForClientTLS(hostname.decode("ascii"),+ trustRoot=platformTrust(),+ extraCertificateOptions={+ 'method': self._ssl_method,+ })diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex d6b7967da..995f8cbba 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -10,12 +10,14 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_infologger = logging.getLogger(__name__)+METHOD_SSLv3 = 'SSLv3'METHOD_TLS = 'TLS'METHOD_TLSv10 = 'TLSv1.0'METHOD_TLSv11 = 'TLSv1.1'METHOD_TLSv12 = 'TLSv1.2'+openssl_methods = {METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)@@ -25,11 +27,6 @@ openssl_methods = {}-# ClientTLSOptions requires a recent-enough version of Twisted (14.0.0+)-# Not having ScrapyClientTLSOptions should not matter for older-# Twisted versions because it is not used in the fallback-# ScrapyClientContextFactory.-try:# XXX: this import would fail on Debian jessie with system installed# service_identity library, due to lack of cryptography.x509 dependency