Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove fallback ScrapyClientContextFactory class (used in Twisted < 14.0.0)

Eugenio Lacuesta3164543ed165
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 5ac20c0bb..127a246f5 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,112 +1,85 @@
from OpenSSL import SSL
-from twisted.internet.ssl import ClientContextFactory
-
-from scrapy import twisted_version
-
-if twisted_version >= (14, 0, 0):
-
- from zope.interface.declarations import implementer
-
- from twisted.internet.ssl import (optionsForClientTLS,
- CertificateOptions,
- platformTrust)
- from twisted.web.client import BrowserLikePolicyForHTTPS
- from twisted.web.iweb import IPolicyForHTTPS
-
- from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
-
-
- @implementer(IPolicyForHTTPS)
- class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
- """
- Non-peer-certificate verifying HTTPS context factory
-
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation
-
- 'A TLS/SSL connection established with [this method] may
- understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
- """
-
- def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
- self._ssl_method = method
- self.tls_verbose_logging = tls_verbose_logging
-
- @classmethod
- def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
- tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
- return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
-
- def getCertificateOptions(self):
- # setting verify=True will require you to provide CAs
- # to verify against; in other words: it's not that simple
-
- # backward-compatible SSL/TLS method:
- #
- # * this will respect `method` attribute in often recommended
- # `ScrapyClientContextFactory` subclass
- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
- #
- # * getattr() for `_ssl_method` attribute for context factories
- # not calling super(..., self).__init__
- return CertificateOptions(verify=False,
- method=getattr(self, 'method',
- getattr(self, '_ssl_method', None)),
- fixBrokenPeers=True,
- acceptableCiphers=DEFAULT_CIPHERS)
-
- # kept for old-style HTTP/1.0 downloader context twisted calls,
- # e.g. connectSSL()
- def getContext(self, hostname=None, port=None):
- return self.getCertificateOptions().getContext()
-
- def creatorForNetloc(self, hostname, port):
- return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
- verbose_logging=self.tls_verbose_logging)
-
-
- @implementer(IPolicyForHTTPS)
- class BrowserLikeContextFactory(ScrapyClientContextFactory):
- """
- Twisted-recommended context factory for web clients.
-
- Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
- "The default is to use a BrowserLikePolicyForHTTPS,
- so unless you have special requirements you can leave this as-is."
-
- creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
- except this context factory allows setting the TLS/SSL method to use.
-
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation.
- """
- def creatorForNetloc(self, hostname, port):
-
- # trustRoot set to platformTrust() will use the platform's root CAs.
- #
- # This means that a website like https://www.cacert.org will be rejected
- # by default, since CAcert.org CA certificate is seldom shipped.
- return optionsForClientTLS(hostname.decode("ascii"),
- trustRoot=platformTrust(),
- extraCertificateOptions={
- 'method': self._ssl_method,
- })
-
-else:
-
- class ScrapyClientContextFactory(ClientContextFactory):
- "A SSL context factory which is more permissive against SSL bugs."
- # see https://github.com/scrapy/scrapy/issues/82
- # and https://github.com/scrapy/scrapy/issues/26
- # and https://github.com/scrapy/scrapy/issues/981
-
- def __init__(self, method=SSL.SSLv23_METHOD):
- self.method = method
-
- def getContext(self, hostname=None, port=None):
- ctx = ClientContextFactory.getContext(self)
- # Enable all workarounds to SSL bugs as documented by
- # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
- ctx.set_options(SSL.OP_ALL)
- return ctx
+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS
+from zope.interface.declarations import implementer
+
+from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+
+
+@implementer(IPolicyForHTTPS)
+class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+ """
+ Non-peer-certificate verifying HTTPS context factory
+
+ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+ which allows TLS protocol negotiation
+
+ 'A TLS/SSL connection established with [this method] may
+ understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+ """
+
+ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
+ super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+ self._ssl_method = method
+ self.tls_verbose_logging = tls_verbose_logging
+
+ @classmethod
+ def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+ tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+ return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
+
+ def getCertificateOptions(self):
+ # setting verify=True will require you to provide CAs
+ # to verify against; in other words: it's not that simple
+
+ # backward-compatible SSL/TLS method:
+ #
+ # * this will respect `method` attribute in often recommended
+ # `ScrapyClientContextFactory` subclass
+ # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
+ #
+ # * getattr() for `_ssl_method` attribute for context factories
+ # not calling super(..., self).__init__
+ return CertificateOptions(verify=False,
+ method=getattr(self, 'method',
+ getattr(self, '_ssl_method', None)),
+ fixBrokenPeers=True,
+ acceptableCiphers=DEFAULT_CIPHERS)
+
+ # kept for old-style HTTP/1.0 downloader context twisted calls,
+ # e.g. connectSSL()
+ def getContext(self, hostname=None, port=None):
+ return self.getCertificateOptions().getContext()
+
+ def creatorForNetloc(self, hostname, port):
+ return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+ verbose_logging=self.tls_verbose_logging)
+
+
+@implementer(IPolicyForHTTPS)
+class BrowserLikeContextFactory(ScrapyClientContextFactory):
+ """
+ Twisted-recommended context factory for web clients.
+
+ Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
+ "The default is to use a BrowserLikePolicyForHTTPS,
+ so unless you have special requirements you can leave this as-is."
+
+ creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
+ except this context factory allows setting the TLS/SSL method to use.
+
+ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+ which allows TLS protocol negotiation.
+ """
+ def creatorForNetloc(self, hostname, port):
+
+ # trustRoot set to platformTrust() will use the platform's root CAs.
+ #
+ # This means that a website like https://www.cacert.org will be rejected
+ # by default, since CAcert.org CA certificate is seldom shipped.
+ return optionsForClientTLS(hostname.decode("ascii"),
+ trustRoot=platformTrust(),
+ extraCertificateOptions={
+ 'method': self._ssl_method,
+ })
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index d6b7967da..995f8cbba 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -10,12 +10,14 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
+
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
METHOD_TLSv12 = 'TLSv1.2'
+
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
@@ -25,11 +27,6 @@ openssl_methods = {
}
-# ClientTLSOptions requires a recent-enough version of Twisted (14.0.0+)
-# Not having ScrapyClientTLSOptions should not matter for older
-# Twisted versions because it is not used in the fallback
-# ScrapyClientContextFactory.
-
try:
# XXX: this import would fail on Debian jessie with system installed
# service_identity library, due to lack of cryptography.x509 dependency