Scrapy, a fast high-level web crawling & scraping framework for Python.
Catch VerificationError but keep the rest of ClientTLSOptions
Paul Tremberth25ee023561a5
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 3e147ae36..b643d935b 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -1,21 +1,19 @@from OpenSSL import SSLfrom twisted.internet.ssl import ClientContextFactory-try:from zope.interface.declarations import implementer# the following should be available from Twisted 14.0.0- from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust- from twisted.internet._sslverify import ClientTLSOptions+ from twisted.internet.ssl import (optionsForClientTLS,+ CertificateOptions,+ platformTrust)+from twisted.web.client import BrowserLikePolicyForHTTPSfrom twisted.web.iweb import IPolicyForHTTPS-- class ScrapyClientTLSOptions(ClientTLSOptions):- def _identityVerifyingInfoCallback(self, connection, where, ret):- pass+ from scrapy.core.downloader.tls import ScrapyClientTLSOptions@implementer(IPolicyForHTTPS)diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex 64ebb0714..2cf8ba869 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -1,6 +1,9 @@+import loggingfrom OpenSSL import SSL+logger = logging.getLogger(__name__)+METHOD_SSLv3 = 'SSLv3'METHOD_TLS = 'TLS'METHOD_TLSv10 = 'TLSv1.0'@@ -14,3 +17,36 @@ openssl_methods = {METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 onlyMETHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only}++# ClientTLSOptions requires a recent-enough version of Twisted+try:++ # taken from twisted/twisted/internet/_sslverify.py+ try:+ from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START+ except ImportError:+ SSL_CB_HANDSHAKE_START = 0x10+ SSL_CB_HANDSHAKE_DONE = 0x20++ from twisted.internet._sslverify import (ClientTLSOptions,+ _maybeSetHostNameIndication,+ verifyHostname,+ VerificationError)++ class ScrapyClientTLSOptions(ClientTLSOptions):+ # same as Twisted's ClientTLSOptions,+ # except that VerificationError is caught+ # and doesn't close the connection+ def _identityVerifyingInfoCallback(self, connection, where, ret):+ if where & SSL_CB_HANDSHAKE_START:+ _maybeSetHostNameIndication(connection, self._hostnameBytes)+ elif where & SSL_CB_HANDSHAKE_DONE:+ try:+ verifyHostname(connection, self._hostnameASCII)+ except VerificationError as e:+ logger.warning(e)++except ImportError:+ # ImportError should not matter for older Twisted versions+ # as the above is not used in the fallback ScrapyClientContextFactory+ pass