Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests for DOWNLOADER_CLIENT_TLS_METHOD, remove dead code.
Andrey Rakhmatullin4dcc04be48b5
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 8e17eab9a..d44c663bb 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -111,18 +111,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):def getCertificateOptions(self) -> CertificateOptions:# setting verify=True will require you to provide CAs# to verify against; in other words: it's not that simple-- # backward-compatible SSL/TLS method:- #- # * this will respect `method` attribute in often recommended- # `ScrapyClientContextFactory` subclass- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)- #- # * getattr() for `_ssl_method` attribute for context factories- # not calling super().__init__return CertificateOptions(verify=False,- method=getattr(self, "method", getattr(self, "_ssl_method", None)),+ method=self._ssl_method,fixBrokenPeers=True,acceptableCiphers=self.tls_ciphers,)diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 1797d5e1f..1cad68b9c 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -3,21 +3,29 @@ from twisted.internet import deferTests borrowed from the twisted.web.client tests."""+from __future__ import annotations+import shutilfrom pathlib import Pathfrom tempfile import mkdtemp+from typing import Anyimport OpenSSL.SSL+from pytest import raisesfrom twisted.internet import defer, reactor-from twisted.internet.defer import inlineCallbacks+from twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.internet.testing import StringTransportfrom twisted.protocols.policies import WrappingFactoryfrom twisted.trial import unittestfrom twisted.web import resource, server, static, utilfrom scrapy.core.downloader import webclient as client-from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory+from scrapy.core.downloader.contextfactory import (+ ScrapyClientContextFactory,+ load_context_factory_from_settings,+)from scrapy.http import Headers, Request+from scrapy.settings import Settingsfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes, to_unicodefrom scrapy.utils.test import get_crawler@@ -482,3 +490,51 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):self.getURL("payload"), body=s, contextFactory=client_context_factory)return self.assertFailure(d, OpenSSL.SSL.Error)+++class WebClientTLSMethodTestCase(WebClientSSLTestCase):+ def _assert_factory_works(+ self, client_context_factory: ScrapyClientContextFactory+ ) -> Deferred[Any]:+ s = "0123456789" * 10+ return getPage(+ self.getURL("payload"), body=s, contextFactory=client_context_factory+ ).addCallback(self.assertEqual, to_bytes(s))++ def test_setting_default(self):+ crawler = get_crawler()+ settings = Settings()+ client_context_factory = load_context_factory_from_settings(settings, crawler)+ assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD+ return self._assert_factory_works(client_context_factory)++ def test_setting_none(self):+ crawler = get_crawler()+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None})+ with raises(KeyError):+ load_context_factory_from_settings(settings, crawler)++ def test_setting_bad(self):+ crawler = get_crawler()+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})+ with raises(KeyError):+ load_context_factory_from_settings(settings, crawler)++ def test_setting_explicit(self):+ crawler = get_crawler()+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})+ client_context_factory = load_context_factory_from_settings(settings, crawler)+ assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD+ return self._assert_factory_works(client_context_factory)++ def test_direct_from_crawler(self):+ # the setting is ignored+ crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})+ client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)+ assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD+ return self._assert_factory_works(client_context_factory)++ def test_direct_init(self):+ client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)+ assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD+ return self._assert_factory_works(client_context_factory)