Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix deprecation warnings with pyOpenSSL 26.3.0. (#7619)
Andrey Rakhmatullin9893a7fac62a
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.pyindex 08e8cad24..17957aba4 100644--- a/scrapy/utils/_deps_compat.py+++ b/scrapy/utils/_deps_compat.py@@ -18,8 +18,8 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)-# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object-PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")+# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred+PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutablePYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.pyindex 22e9414b8..828a99e23 100644--- a/scrapy/utils/ssl.py+++ b/scrapy/utils/ssl.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport loggingimport ssl+import warningsfrom typing import TYPE_CHECKING, Any, TypedDict, TypeVarimport OpenSSL._util as pyOpenSSLutil@@ -9,7 +10,11 @@ import OpenSSL.SSLimport OpenSSL.versionfrom twisted.internet.ssl import CertificateOptions, TLSVersion-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._deps_compat import (+ PYOPENSSL_X509_DEPRECATED,+ TWISTED_TLS_LIMITS_OFFBY1,+)from scrapy.utils.python import to_unicodeif TYPE_CHECKING:@@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:# pyOpenSSL utils-def ffi_buf_to_string(buf: Any) -> str:+def _ffi_buf_to_string(buf: Any) -> str:return to_unicode(pyOpenSSLutil.ffi.string(buf))-def x509name_to_string(x509name: X509Name) -> str:+def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover+ warnings.warn(+ "ffi_buf_to_string() is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return ffi_buf_to_string(buf)+++def _x509name_to_string(x509name: X509Name) -> str:# from OpenSSL.crypto.X509Name.__repr__+ # only used on pyOpenSSL < 24.3.0result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))- return ffi_buf_to_string(result_buffer)+ return _ffi_buf_to_string(result_buffer)+++def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover+ warnings.warn(+ "x509name_to_string() is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _x509name_to_string(x509name)-def get_temp_key_info(ssl_object: Any) -> str | None:+def _get_temp_key_info(ssl_object: Any) -> str | None:# adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):- # removed in cryptography 40.0.0+ # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)return Nonetemp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):@@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None:cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)if cname == pyOpenSSLutil.ffi.NULL:cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)- key_info.append(ffi_buf_to_string(cname))+ key_info.append(_ffi_buf_to_string(cname))else:- key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))+ key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")return ", ".join(key_info)+def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover+ warnings.warn(+ "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _get_temp_key_info(ssl_object)++def get_openssl_version() -> str:system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)system_openssl = system_openssl_bytes.decode("ascii", errors="replace")@@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)connection.get_protocol_version_name(),connection.get_cipher_name(),)- server_cert = connection.get_peer_certificate()- if server_cert:- logger.debug(- 'SSL connection certificate: issuer "%s", subject "%s"',- x509name_to_string(server_cert.get_issuer()),- x509name_to_string(server_cert.get_subject()),- )- key_info = get_temp_key_info(connection._ssl)+ if PYOPENSSL_X509_DEPRECATED:+ if server_cert := connection.get_peer_certificate(as_cryptography=True):+ logger.debug(+ 'SSL connection certificate: issuer "%s", subject "%s"',+ server_cert.issuer.rfc4514_string(),+ server_cert.subject.rfc4514_string(),+ )+ else: # noqa: PLR5501+ if server_cert_pyopenssl := connection.get_peer_certificate():+ logger.debug(+ 'SSL connection certificate: issuer "%s", subject "%s"',+ _x509name_to_string(server_cert_pyopenssl.get_issuer()),+ _x509name_to_string(server_cert_pyopenssl.get_subject()),+ )+ key_info = _get_temp_key_info(connection._ssl)if key_info:logger.debug("SSL temp key: %s", key_info)diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.pyindex 7aa656780..5c4ca7457 100644--- a/tests/mockserver/utils.py+++ b/tests/mockserver/utils.py@@ -10,7 +10,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekeyfrom twisted.internet.ssl import CertificateOptions, ContextFactoryfrom scrapy.core.downloader.tls import _TWISTED_VERSION_MAP-from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY+from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATEDfrom scrapy.utils.python import to_bytesfrom scrapy.utils.ssl import _get_cert_options_version_kwargs@@ -29,7 +29,7 @@ def ssl_context_factory(keyfile_path = Path(__file__).parent.parent / keyfilecertfile_path = Path(__file__).parent.parent / certfile- if not PYOPENSSL_WANTS_X509_PKEY:+ if PYOPENSSL_X509_DEPRECATED:cert = load_pem_x509_certificate(certfile_path.read_bytes())key = load_pem_private_key(keyfile_path.read_bytes(), password=None)else:diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 223f288bf..25a200817 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -33,7 +33,10 @@ from scrapy.exceptions import (UnsupportedURLSchemeError,)from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1+from scrapy.utils._deps_compat import (+ PYOPENSSL_X509_DEPRECATED,+ TWISTED_TLS_LIMITS_OFFBY1,+)from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpider@@ -831,8 +834,15 @@ class TestHttpsBase(TestHttpBase):is_secure = Truetls_log_message = (- 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '- 'subject "/C=IE/O=Scrapy/CN=localhost"'+ (+ 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '+ 'subject "CN=localhost,O=Scrapy,C=IE"'+ )+ if PYOPENSSL_X509_DEPRECATED+ else (+ 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '+ 'subject "/C=IE/O=Scrapy/CN=localhost"'+ ))def test_download_conn_lost(self) -> None: # type: ignore[override]diff --git a/tox.ini b/tox.iniindex 50ac7aa86..44e240e03 100644--- a/tox.ini+++ b/tox.ini@@ -82,7 +82,7 @@ deps =ptpython==3.0.32# newer ones require newer Pythonipython==8.39.0- pyOpenSSL==26.2.0+ pyOpenSSL==26.3.0pytest==9.0.3socksio==1.0.0types-Pygments==2.20.0.20260508