Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make DOWNLOADER_CLIENT_TLS_CIPHERS=None enable Twisted defaults (#7665)

Adrian74e6b61071c9
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c452f88d1..b92733ede 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -763,6 +763,9 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
+Set this setting to ``None`` to use the default ciphers of the underlying TLS
+implementation.
+
.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
.. note::
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 40f09ffb2..c1796c723 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -17,7 +17,6 @@ from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
_TWISTED_VERSION_MAP,
- DEFAULT_CIPHERS,
_openssl_methods,
_ScrapyClientTLSOptions,
_ScrapyClientTLSOptions26,
@@ -68,11 +67,11 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
self.tls_min_version: TLSVersion | None = tls_min_version
self.tls_max_version: TLSVersion | None = tls_max_version
self.tls_verbose_logging: bool = tls_verbose_logging # unused
- self.tls_ciphers: AcceptableCiphers
- if tls_ciphers:
- self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
- else:
- self.tls_ciphers = DEFAULT_CIPHERS
+ self.tls_ciphers: AcceptableCiphers | None = (
+ AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
+ if tls_ciphers
+ else None
+ )
self._verify_certificates = verify_certificates
@classmethod
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 3b903b39d..c5cf1156d 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -43,6 +43,13 @@ _openssl_methods: dict[str, int] = {
def __getattr__(name: str) -> Any:
+ if name == "DEFAULT_CIPHERS":
+ warnings.warn(
+ "scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")
deprecated = {
"METHOD_TLS": "TLS",
"METHOD_TLSv10": "TLSv1.0",
@@ -177,8 +184,3 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
return True
return verifyCallback
-
-
-DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString(
- "DEFAULT"
-)
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index e348bfb7f..fdd5edc27 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -8,7 +8,7 @@ import pytest
from pytest_twisted import async_yield_fixture
from twisted.internet.protocol import Factory
from twisted.internet.protocol import Protocol as TxProtocol
-from twisted.internet.ssl import optionsForClientTLS
+from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
@@ -180,6 +180,49 @@ class TestContextFactory(TestContextFactoryBase):
assert options & 0x4 # OP_LEGACY_SERVER_CONNECT
+class TestContextFactoryCiphers(TestContextFactoryBase):
+ async def _assert_factory_works(
+ self, server_url: str, client_context_factory: _ScrapyClientContextFactory
+ ) -> None:
+ s = "0123456789" * 10
+ body = await self.get_page(
+ server_url + "payload", client_context_factory, body=s
+ )
+ assert body == to_bytes(s)
+
+ def test_default(self) -> None:
+ """The default 'DEFAULT' value is passed to Twisted as is."""
+ crawler = get_crawler()
+ factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
+ assert factory.tls_ciphers is not None
+ # OpenSSLAcceptableCiphers has no __eq__, so compare the parsed ciphers.
+ assert (
+ factory.tls_ciphers._ciphers
+ == AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
+ )
+ assert factory._get_cert_options_kwargs()["acceptableCiphers"] is not None
+
+ def test_custom(self) -> None:
+ crawler = get_crawler(
+ settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
+ )
+ factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
+ assert factory.tls_ciphers is not None
+ assert (
+ factory.tls_ciphers._ciphers
+ == AcceptableCiphers.fromOpenSSLCipherString("CAMELLIA256-SHA")._ciphers
+ )
+
+ @coroutine_test
+ async def test_none(self, server_url: str) -> None:
+ """A None value enables the Twisted default ciphers."""
+ crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": None})
+ factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
+ assert factory.tls_ciphers is None
+ assert factory._get_cert_options_kwargs()["acceptableCiphers"] is None
+ await self._assert_factory_works(server_url, factory)
+
+
class TestContextFactoryTLSMethod(TestContextFactoryBase):
async def _assert_factory_works(
self, server_url: str, client_context_factory: _ScrapyClientContextFactory
@@ -278,3 +321,10 @@ def test_deprecated_tls_module_names() -> None:
match="scrapy.core.downloader.tls.openssl_methods is deprecated",
):
assert isinstance(tls.openssl_methods, dict)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated",
+ ):
+ assert tls.DEFAULT_CIPHERS._ciphers == (
+ AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
+ )