Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Silence the CertificateOptions method warning. (#7410)

Andrey Rakhmatullin2b174e348d88
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index a0e0dd933..ef0a46a8e 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import warnings
+from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, cast
from OpenSSL import SSL
@@ -25,6 +26,8 @@ from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
+ from collections.abc import Generator
+
from twisted.internet._sslverify import ClientTLSOptions
# typing.Self requires Python 3.11
@@ -34,6 +37,18 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
+@contextmanager
+def _filter_method_warning() -> Generator[None]:
+ with warnings.catch_warnings():
+ # Twisted deprecation, https://github.com/scrapy/scrapy/issues/3288
+ warnings.filterwarnings(
+ "ignore",
+ message=r"Passing method to twisted\.internet\.ssl\.CertificateOptions",
+ category=DeprecationWarning,
+ )
+ yield
+
+
@implementer(IPolicyForHTTPS)
class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""Non-peer-certificate verifying HTTPS context factory.
@@ -63,11 +78,12 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
else:
self.tls_ciphers = DEFAULT_CIPHERS
- self._certificate_options = CertificateOptions(
- method=self._ssl_method,
- fixBrokenPeers=True,
- acceptableCiphers=self.tls_ciphers,
- )
+ with _filter_method_warning():
+ self._certificate_options = CertificateOptions(
+ method=self._ssl_method,
+ fixBrokenPeers=True,
+ acceptableCiphers=self.tls_ciphers,
+ )
self._ctx = self._get_context()
self._verify_certificates = verify_certificates
@@ -110,13 +126,14 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
if not self._verify_certificates:
return _ScrapyClientTLSOptions(hostname.decode("ascii"), self._ctx) # type: ignore[no-untyped-call]
# Note that this doesn't use self._ctx
- return optionsForClientTLS(
- hostname=hostname.decode("ascii"),
- extraCertificateOptions={
- "method": self._ssl_method,
- "acceptableCiphers": self.tls_ciphers,
- },
- )
+ with _filter_method_warning():
+ return optionsForClientTLS(
+ hostname=hostname.decode("ascii"),
+ extraCertificateOptions={
+ "method": self._ssl_method,
+ "acceptableCiphers": self.tls_ciphers,
+ },
+ )
ScrapyClientContextFactory = create_deprecated_class(
@@ -160,10 +177,11 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
super().__init__(*args, **kwargs)
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
- return optionsForClientTLS(
- hostname=hostname.decode("ascii"),
- extraCertificateOptions={"method": self._ssl_method},
- )
+ with _filter_method_warning():
+ return optionsForClientTLS(
+ hostname=hostname.decode("ascii"),
+ extraCertificateOptions={"method": self._ssl_method},
+ )
@implementer(IPolicyForHTTPS)