Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add tests for DOWNLOADER_CLIENT_TLS_METHOD, remove dead code.

Andrey Rakhmatullin4dcc04be48b5
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 8e17eab9a..d44c663bb 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -111,18 +111,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
def getCertificateOptions(self) -> CertificateOptions:
# setting verify=True will require you to provide CAs
# to verify against; in other words: it's not that simple
-
- # backward-compatible SSL/TLS method:
- #
- # * this will respect `method` attribute in often recommended
- # `ScrapyClientContextFactory` subclass
- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
- #
- # * getattr() for `_ssl_method` attribute for context factories
- # not calling super().__init__
return CertificateOptions(
verify=False,
- method=getattr(self, "method", getattr(self, "_ssl_method", None)),
+ method=self._ssl_method,
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 1797d5e1f..1cad68b9c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,21 +3,29 @@ from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
+from __future__ import annotations
+
import shutil
from pathlib import Path
from tempfile import mkdtemp
+from typing import Any
import OpenSSL.SSL
+from pytest import raises
from twisted.internet import defer, reactor
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.testing import StringTransport
from twisted.protocols.policies import WrappingFactory
from twisted.trial import unittest
from twisted.web import resource, server, static, util
from scrapy.core.downloader import webclient as client
-from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
+from scrapy.core.downloader.contextfactory import (
+ ScrapyClientContextFactory,
+ load_context_factory_from_settings,
+)
from scrapy.http import Headers, Request
+from scrapy.settings import Settings
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.test import get_crawler
@@ -482,3 +490,51 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
self.getURL("payload"), body=s, contextFactory=client_context_factory
)
return self.assertFailure(d, OpenSSL.SSL.Error)
+
+
+class WebClientTLSMethodTestCase(WebClientSSLTestCase):
+ def _assert_factory_works(
+ self, client_context_factory: ScrapyClientContextFactory
+ ) -> Deferred[Any]:
+ s = "0123456789" * 10
+ return getPage(
+ self.getURL("payload"), body=s, contextFactory=client_context_factory
+ ).addCallback(self.assertEqual, to_bytes(s))
+
+ def test_setting_default(self):
+ crawler = get_crawler()
+ settings = Settings()
+ client_context_factory = load_context_factory_from_settings(settings, crawler)
+ assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
+ return self._assert_factory_works(client_context_factory)
+
+ def test_setting_none(self):
+ crawler = get_crawler()
+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None})
+ with raises(KeyError):
+ load_context_factory_from_settings(settings, crawler)
+
+ def test_setting_bad(self):
+ crawler = get_crawler()
+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
+ with raises(KeyError):
+ load_context_factory_from_settings(settings, crawler)
+
+ def test_setting_explicit(self):
+ crawler = get_crawler()
+ settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})
+ client_context_factory = load_context_factory_from_settings(settings, crawler)
+ assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
+ return self._assert_factory_works(client_context_factory)
+
+ def test_direct_from_crawler(self):
+ # the setting is ignored
+ crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
+ client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)
+ assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
+ return self._assert_factory_works(client_context_factory)
+
+ def test_direct_init(self):
+ client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)
+ assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
+ return self._assert_factory_works(client_context_factory)