Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Catch and ignore TLS verification exception for IP-address hosts

Paul Tremberth37efdde3e3dd
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 5fa8e2723..9bf6a575b 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -48,6 +48,11 @@ try:
'Remote certificate is not valid for hostname "{}"; {}'.format(
self._hostnameASCII, e))
+ except ValueError as e:
+ logger.warning(
+ 'SSL/TLS verification failed for hostname "{}"; {}'.format(
+ self._hostnameASCII, e))
+
except ImportError:
# ImportError should not matter for older Twisted versions
# as the above is not used in the fallback ScrapyClientContextFactory
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45a806f2e..fb73c43ee 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -335,6 +335,14 @@ class Https11WrongHostnameTestCase(Http11TestCase):
certfile = 'keys/example-com.cert.pem'
+class Https11InvalidDNSId(Https11TestCase):
+ """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
+
+ def setUp(self):
+ super(Https11InvalidDNSId, self).setUp()
+ self.host = '127.0.0.1'
+
+
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
if twisted_version < (11, 1, 0):