Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Catch VerificationError but keep the rest of ClientTLSOptions

Paul Tremberth25ee023561a5
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 3e147ae36..b643d935b 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,21 +1,19 @@
from OpenSSL import SSL
from twisted.internet.ssl import ClientContextFactory
-
try:
from zope.interface.declarations import implementer
# the following should be available from Twisted 14.0.0
- from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
- from twisted.internet._sslverify import ClientTLSOptions
+ from twisted.internet.ssl import (optionsForClientTLS,
+ CertificateOptions,
+ platformTrust)
+
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
-
- class ScrapyClientTLSOptions(ClientTLSOptions):
- def _identityVerifyingInfoCallback(self, connection, where, ret):
- pass
+ from scrapy.core.downloader.tls import ScrapyClientTLSOptions
@implementer(IPolicyForHTTPS)
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 64ebb0714..2cf8ba869 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,6 +1,9 @@
+import logging
from OpenSSL import SSL
+logger = logging.getLogger(__name__)
+
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
@@ -14,3 +17,36 @@ openssl_methods = {
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
}
+
+# ClientTLSOptions requires a recent-enough version of Twisted
+try:
+
+ # taken from twisted/twisted/internet/_sslverify.py
+ try:
+ from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
+ except ImportError:
+ SSL_CB_HANDSHAKE_START = 0x10
+ SSL_CB_HANDSHAKE_DONE = 0x20
+
+ from twisted.internet._sslverify import (ClientTLSOptions,
+ _maybeSetHostNameIndication,
+ verifyHostname,
+ VerificationError)
+
+ class ScrapyClientTLSOptions(ClientTLSOptions):
+ # same as Twisted's ClientTLSOptions,
+ # except that VerificationError is caught
+ # and doesn't close the connection
+ def _identityVerifyingInfoCallback(self, connection, where, ret):
+ if where & SSL_CB_HANDSHAKE_START:
+ _maybeSetHostNameIndication(connection, self._hostnameBytes)
+ elif where & SSL_CB_HANDSHAKE_DONE:
+ try:
+ verifyHostname(connection, self._hostnameASCII)
+ except VerificationError as e:
+ logger.warning(e)
+
+except ImportError:
+ # ImportError should not matter for older Twisted versions
+ # as the above is not used in the fallback ScrapyClientContextFactory
+ pass