Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve the test coverage of scrapy.utils (#7841)
Adrianfd6fb6316d6f
diff --git a/pyproject.toml b/pyproject.tomlindex 0bdcf6b51..61c23fbec 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -8,7 +8,7 @@ dynamic = ["version"]description = "A high-level Web Crawling and Web Scraping framework"dependencies = ["Twisted>=21.7.0",- "cryptography>=37.0.0",+ "cryptography>=41.0.5","cssselect>=0.9.1","defusedxml>=0.7.1","itemadapter>=0.1.0",@@ -17,9 +17,9 @@ dependencies = ["packaging","parsel>=1.5.0","protego>=0.1.15",- "pyOpenSSL>=22.0.0",+ "pyOpenSSL>=24.3.0","queuelib>=1.6.1",- "service_identity>=23.1.0",+ "service_identity>=24.2.0","tldextract","w3lib>=1.17.0","zope.interface>=5.1.0",diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex ac98a61c8..66770254d 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -44,17 +44,15 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:_check_max_size(decompressed_size, max_size)output_stream = BytesIO()output_stream.write(first_chunk)- while decompressor.unconsumed_tail:+ # Anything left in unconsumed_tail once the stream has ended is not part of+ # it, and feeding it back would neither consume it nor produce output.+ while decompressor.unconsumed_tail and not decompressor.eof:output_chunk = decompressor.decompress(decompressor.unconsumed_tail, max_length=_CHUNK_SIZE)decompressed_size += len(output_chunk)_check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- if tail := decompressor.flush():- decompressed_size += len(tail)- _check_max_size(decompressed_size, max_size)- output_stream.write(tail)return output_stream.getvalue()diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.pyindex 17957aba4..c6391dd8e 100644--- a/scrapy/utils/_deps_compat.py+++ b/scrapy/utils/_deps_compat.py@@ -18,8 +18,6 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)-# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred-PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutablePYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex 403cd54a8..4055b6820 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -33,7 +33,9 @@ def _getarg(return type_(request.args[name][0]) if name in request.args else default-if __name__ == "__main__":+# The bench command kills this process, so coverage data is never written for+# the lines below.+if __name__ == "__main__": # pragma: no coverfrom twisted.internet import reactorroot = Root() # type: ignore[no-untyped-call]diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.pyindex 4eda29e25..4ad8e5741 100644--- a/scrapy/utils/ossignal.py+++ b/scrapy/utils/ossignal.py@@ -10,12 +10,7 @@ SignalHandlerT: TypeAlias = (Callable[[int, FrameType | None], Any] | int | signal.Handlers | None)-signal_names: dict[int, str] = {}-for signame in dir(signal):- if signame.startswith("SIG") and not signame.startswith("SIG_"):- signum = getattr(signal, signame)- if isinstance(signum, int):- signal_names[signum] = signame+signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}def install_shutdown_handlers(diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 40fc05257..8485b06f3 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -101,7 +101,9 @@ def to_bytes(return text.encode(encoding, errors)-def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:+def _chunk_iter(+ text: str, chunk_size: int+) -> Iterable[tuple[str, int]]: # pragma: no coveroffset = len(text)while True:offset -= chunk_size * 1024@@ -213,17 +215,15 @@ def get_func_args_dict(except ValueError:return {}- if isinstance(func, partial):- partial_args = func.args- partial_kw = func.keywords-- args = {}- for name, param in sig.parameters.items():- if name in partial_args:- continue- if partial_kw and name in partial_kw:- continue- args[name] = param+ if isinstance(func, partial) and func.keywords:+ # The signature of a partial already omits the parameters bound to+ # positional arguments, but it keeps those bound to keyword arguments,+ # turned into keyword-only parameters with a default.+ args = {+ name: param+ for name, param in sig.parameters.items()+ if name not in func.keywords+ }else:args = sig.parametersdiff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 5499e3f48..217249ee5 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -27,7 +27,7 @@ _T = TypeVar("_T")_P = ParamSpec("_P")-def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # noqa: RET503+def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:"""Like reactor.listenTCP but tries different ports in a range."""from twisted.internet import reactor@@ -37,14 +37,14 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:raise ValueError(f"invalid portrange: {portrange}")if not portrange:return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return]- if len(portrange) == 1:- return reactor.listenTCP(portrange[0], factory, interface=host) # type: ignore[no-any-return]- for x in range(portrange[0], portrange[1] + 1):+ for x in range(portrange[0], portrange[-1]):try:return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return]except error.CannotListenError:- if x == portrange[1]:- raise+ pass+ # The last port of the range is tried outside the loop so that its+ # CannotListenError propagates to the caller.+ return reactor.listenTCP(portrange[-1], factory, interface=host) # type: ignore[no-any-return]class CallLaterOnce(Generic[_T]):diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.pyindex 828a99e23..fdb8b8f64 100644--- a/scrapy/utils/ssl.py+++ b/scrapy/utils/ssl.py@@ -11,10 +11,7 @@ import OpenSSL.versionfrom twisted.internet.ssl import CertificateOptions, TLSVersionfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils._deps_compat import (- PYOPENSSL_X509_DEPRECATED,- TWISTED_TLS_LIMITS_OFFBY1,-)+from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1from scrapy.utils.python import to_unicodeif TYPE_CHECKING:@@ -125,18 +122,17 @@ def _ffi_buf_to_string(buf: Any) -> str:return to_unicode(pyOpenSSLutil.ffi.string(buf))-def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover+def ffi_buf_to_string(buf: Any) -> str:warnings.warn("ffi_buf_to_string() is deprecated.",ScrapyDeprecationWarning,stacklevel=2,)- return ffi_buf_to_string(buf)+ return _ffi_buf_to_string(buf)def _x509name_to_string(x509name: X509Name) -> str:# from OpenSSL.crypto.X509Name.__repr__- # only used on pyOpenSSL < 24.3.0result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer)@@ -144,7 +140,7 @@ def _x509name_to_string(x509name: X509Name) -> str:return _ffi_buf_to_string(result_buffer)-def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover+def x509name_to_string(x509name: X509Name) -> str:warnings.warn("x509name_to_string() is deprecated.",ScrapyDeprecationWarning,@@ -153,48 +149,15 @@ def x509name_to_string(x509name: X509Name) -> str: # pragma: no coverreturn _x509name_to_string(x509name)-def _get_temp_key_info(ssl_object: Any) -> str | None:- # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()- if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):- # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)- return None- temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")- if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):- return None- temp_key = temp_key_p[0]- if temp_key == pyOpenSSLutil.ffi.NULL:- return None- temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)- key_info = []- key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)- if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:- key_info.append("RSA")- elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:- key_info.append("DH")- elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:- key_info.append("ECDH")- ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)- ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)- nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(- pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)- )- cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)- if cname == pyOpenSSLutil.ffi.NULL:- cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)- key_info.append(_ffi_buf_to_string(cname))- else:- key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))- key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")- return ", ".join(key_info)---def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover+def get_temp_key_info(ssl_object: Any) -> str | None:+ # A no-op: it read the negotiated ephemeral key through a binding that+ # cryptography 40.0.0 removed.warnings.warn("get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",ScrapyDeprecationWarning,stacklevel=2,)- return _get_temp_key_info(ssl_object)+ return Nonedef get_openssl_version() -> str:@@ -210,23 +173,12 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)connection.get_protocol_version_name(),connection.get_cipher_name(),)- if PYOPENSSL_X509_DEPRECATED:- if server_cert := connection.get_peer_certificate(as_cryptography=True):- logger.debug(- 'SSL connection certificate: issuer "%s", subject "%s"',- server_cert.issuer.rfc4514_string(),- server_cert.subject.rfc4514_string(),- )- else: # noqa: PLR5501- if server_cert_pyopenssl := connection.get_peer_certificate():- logger.debug(- 'SSL connection certificate: issuer "%s", subject "%s"',- _x509name_to_string(server_cert_pyopenssl.get_issuer()),- _x509name_to_string(server_cert_pyopenssl.get_subject()),- )- key_info = _get_temp_key_info(connection._ssl)- if key_info:- logger.debug("SSL temp key: %s", key_info)+ if server_cert := connection.get_peer_certificate(as_cryptography=True):+ logger.debug(+ 'SSL connection certificate: issuer "%s", subject "%s"',+ server_cert.issuer.rfc4514_string(),+ server_cert.subject.rfc4514_string(),+ )# Twisted-specificdiff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.pyindex 33c3340b9..e993bedb2 100644--- a/tests/mockserver/utils.py+++ b/tests/mockserver/utils.py@@ -7,11 +7,9 @@ from typing import TYPE_CHECKING, castfrom cryptography.hazmat.primitives.serialization import load_pem_private_keyfrom cryptography.x509 import load_pem_x509_certificatefrom OpenSSL import SSL-from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekeyfrom twisted.internet.ssl import CertificateOptions, ContextFactoryfrom scrapy.core.downloader.tls import _TWISTED_VERSION_MAP-from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATEDfrom scrapy.utils.python import to_bytesfrom scrapy.utils.ssl import _get_cert_options_version_kwargs@@ -37,12 +35,8 @@ def ssl_context_factory(keyfile_path = Path(__file__).parent.parent / keyfilecertfile_path = Path(__file__).parent.parent / certfile- if PYOPENSSL_X509_DEPRECATED:- cert = load_pem_x509_certificate(certfile_path.read_bytes())- key = load_pem_private_key(keyfile_path.read_bytes(), password=None)- else:- cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment]- key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment]+ cert = load_pem_x509_certificate(certfile_path.read_bytes())+ key = load_pem_private_key(keyfile_path.read_bytes(), password=None)tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else Nonetls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else Nonediff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 44178727e..9e93eae18 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -191,6 +191,11 @@ class MySpider(scrapy.Spider):class TestInteractiveShell:+ # Starting an interactive shell involves an interpreter start-up, Scrapy+ # imports and shell imports, which on PyPy with coverage enabled can take+ # well over 10 seconds.+ TIMEOUT = 60+def test_fetch(self, mockserver: MockServer) -> None:args = (sys.executable,@@ -201,7 +206,7 @@ class TestInteractiveShell:env = os.environ.copy()env["SCRAPY_PYTHON_SHELL"] = "python"logfile = BytesIO()- p = PopenSpawn(args, env=env, timeout=60)+ p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)p.logfile_read = logfilep.expect_exact("Available Scrapy objects")p.sendline(f"fetch('{mockserver.url('/')}')")@@ -235,7 +240,7 @@ class TestInteractiveShell:def _run_interactive_shell(self, env: dict[str, str]) -> str:args = (sys.executable, "-m", "scrapy.cmdline", "shell")logfile = BytesIO()- p = PopenSpawn(args, env=env, timeout=60)+ p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)p.logfile_read = logfilep.expect_exact("Available Scrapy objects")p.sendeof()@@ -256,7 +261,7 @@ class TestInteractiveShell:self._isolate_config(env, config_home)args = (sys.executable, "-m", "scrapy.cmdline", "shell")logfile = BytesIO()- p = PopenSpawn(args, env=env, timeout=60)+ p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)p.logfile_read = logfilep.expect_exact("Available Scrapy objects")# The standard Python shell never imports IPython, whereas the IPython@@ -276,6 +281,29 @@ class TestInteractiveShell:self._isolate_config(env, config_home)assert "Traceback" not in self._run_interactive_shell(env)+ @pytest.mark.skipif(+ importlib.util.find_spec("IPython") is None, reason="IPython is not installed"+ )+ def test_shell_ipython(self, tmp_path: Path) -> None:+ # Reaching the embedded IPython shell requires selecting it explicitly,+ # since ptpython takes precedence when both are installed.+ config_home = tmp_path / "config"+ config_home.mkdir()+ env = os.environ.copy()+ self._isolate_config(env, config_home)+ env["SCRAPY_PYTHON_SHELL"] = "ipython"+ args = (sys.executable, "-m", "scrapy.cmdline", "shell")+ logfile = BytesIO()+ p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)+ p.logfile_read = logfile+ p.expect_exact("Available Scrapy objects")+ p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")+ p.expect_exact("IPYMODULE True")+ p.sendeof()+ p.wait() # type: ignore[no-untyped-call]+ logfile.seek(0)+ assert "Traceback" not in logfile.read().decode()+@pytest.fixturedef restore_sigint():diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex d284805be..862714167 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport jsonimport logging+import refrom ipaddress import IPv4Addressfrom socket import gethostbynamefrom typing import TYPE_CHECKING, Any@@ -18,7 +19,11 @@ from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloafrom scrapy.http import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.defer import maybe_deferred_to_future-from scrapy.utils.engine import format_engine_status, get_engine_status+from scrapy.utils.engine import (+ format_engine_status,+ get_engine_status,+ print_engine_status,+)from scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLE@@ -372,6 +377,23 @@ with multiples linesassert s["engine.spider.name"] == crawler.spider.nameassert s["len(engine.scraper.slot.active)"] == 1+ @coroutine_test+ async def test_print_engine_status(+ self, mockserver: MockServer, capsys: pytest.CaptureFixture[str]+ ) -> None:+ def cb(response):+ assert crawler.engine+ print_engine_status(crawler.engine)++ crawler = get_crawler(SingleRequestSpider)+ await crawler.crawl_async(+ seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver+ )+ assert isinstance(crawler.spider, SingleRequestSpider)+ out = capsys.readouterr().out+ assert out.startswith("Execution engine status")+ assert re.search(rf"engine\.spider\.name +: {crawler.spider.name}\n", out)+@coroutine_testasync def test_format_engine_status(self, mockserver: MockServer) -> None:est: list[str] = []diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex defee1041..800ab9397 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -19,6 +19,10 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from tests.mockserver.http import MockServer+# Guards against a hung subprocess. Generous, because starting a script is+# slow on PyPy, slower still with coverage measurement on.+SCRIPT_TIMEOUT = 60+class ScriptRunnerMixin(ABC):@property@@ -216,7 +220,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):) -> None:sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]args = self.get_script_args(script, "3", *extra_args)- p = PopenSpawn(args, timeout=5, env=get_script_run_env())+ p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())p.expect_exact("Spider opened")p.expect_exact("Crawled (200)")p.kill(sig)@@ -234,7 +238,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):async def _test_shutdown_forced(self, script: str = "sleeping.py") -> None:sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]args = self.get_script_args(script, "10")- p = PopenSpawn(args, timeout=5, env=get_script_run_env())+ p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())p.expect_exact("Spider opened")p.expect_exact("Crawled (200)")p.kill(sig)diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 27a44227d..c82a9f871 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -22,6 +22,7 @@ from tests.utils.bases.download_handlers_http import (TestHttpProxyBase,TestHttpsBase,TestHttpsCustomCiphersBase,+ TestHttpsDefaultCiphersBase,TestHttpsInvalidDNSIdBase,TestHttpsInvalidDNSPatternBase,TestHttpsTLSVersionBase,@@ -137,6 +138,10 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBapass+class TestHttpsDefaultCiphers(HttpxDownloadHandlerMixin, TestHttpsDefaultCiphersBase):+ pass++class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase):passdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 8cb9308ff..8f5c13d62 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -9,15 +9,31 @@ from tempfile import mkdtemp, mkstempfrom unittest import mockimport pytest+from twisted.internet.defer import CancelledError+from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError+from twisted.internet.error import DNSLookupError+from twisted.internet.error import TimeoutError as TxTimeoutError+from twisted.web.client import ResponseFailed+from twisted.web.error import SchemeNotSupportedfrom w3lib.url import path_to_file_urifrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.handlers.datauri import DataURIDownloadHandlerfrom scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandler-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import (+ CannotResolveHostError,+ DownloadCancelledError,+ DownloadConnectionRefusedError,+ DownloadFailedError,+ DownloadTimeoutError,+ NotConfigured,+ ScrapyDeprecationWarning,+ UnsupportedURLSchemeError,+)from scrapy.http import Request, TextResponsefrom scrapy.responsetypes import responsetypes+from scrapy.utils._download_handlers import wrap_twisted_exceptionsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler@@ -407,3 +423,38 @@ class TestDataURI:request = Request("data:,")response = await self.download_request(request)assert response.protocol is None+++class TestWrapTwistedExceptions:+ """Every Twisted exception the HTTP handlers can raise has a Scrapy equivalent.++ Most mappings are also covered end to end by the HTTP handler tests, but a+ Twisted ``TimeoutError`` needs a connection attempt that is silently dropped+ rather than refused, which no mock server can offer reliably.+ """++ @pytest.mark.parametrize(+ ("twisted_exception", "scrapy_exception"),+ [+ (SchemeNotSupported, UnsupportedURLSchemeError),+ (CancelledError, DownloadCancelledError),+ (TxConnectionRefusedError, DownloadConnectionRefusedError),+ (DNSLookupError, CannotResolveHostError),+ (ResponseFailed, DownloadFailedError),+ (TxTimeoutError, DownloadTimeoutError),+ ],+ )+ def test_mapping(+ self,+ twisted_exception: type[Exception],+ scrapy_exception: type[Exception],+ ) -> None:+ original = twisted_exception("some message")+ with pytest.raises(scrapy_exception) as exc_info, wrap_twisted_exceptions():+ raise original+ assert exc_info.value.__cause__ is original+ assert str(exc_info.value) == str(original)++ def test_other_exceptions_pass_through(self) -> None:+ with pytest.raises(ZeroDivisionError), wrap_twisted_exceptions():+ 1 / 0diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 4e0fd7c8b..9290f55ae 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,8 +1,10 @@+import zlibfrom gzip import GzipFilefrom importlib.util import find_specfrom io import BytesIOfrom logging import WARNINGfrom pathlib import Path+from threading import Threadfrom typing import Anyimport pytest@@ -16,7 +18,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider-from scrapy.utils._compression import _DecompressionMaxSizeExceeded+from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceededfrom scrapy.utils.gz import gunzipfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler@@ -495,6 +497,57 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_count", None)self.assertStatsEqual("httpcompression/response_bytes", None)+ @pytest.mark.parametrize(+ "extra",+ [+ pytest.param(b"garbage", id="trailing-garbage"),+ pytest.param(zlib.compress(b"b" * 100_000), id="second-zlib-stream"),+ ],+ )+ def test_deflate_extra_data_after_stream(self, extra: bytes) -> None:+ """Bytes after the end of a deflate stream must not stall decompression.++ Such bytes stay in ``unconsumed_tail`` and produce no output, so feeding+ them back to the decompressor never makes progress. Decompressing in a+ thread lets the test fail instead of hanging if that ever regresses.+ """+ # The body must decompress to more than _CHUNK_SIZE, or the whole stream+ # is consumed by the first decompress() call and no loop is entered.+ plain = b"a" * 100_000+ response = Response(+ "http://example.com",+ body=zlib.compress(plain) + extra,+ headers={"Content-Encoding": "deflate"},+ )+ result: list[Request | Response] = []+ thread = Thread(+ target=lambda: result.append(+ self.mw.process_response(Request("http://example.com"), response)+ ),+ daemon=True,+ )+ thread.start()+ thread.join(60)+ assert not thread.is_alive(), "decompression did not terminate"+ assert result[0].body == plain++ def test_deflate_chunked_output_leaves_nothing_to_flush(self) -> None:+ """Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``.++ ``decompress(max_length=...)`` leaves the input it could not turn into+ output in ``unconsumed_tail``, so once that is empty every complete+ stream has been fully emitted. If a future zlib buffers output instead,+ this fails and ``_inflate()`` needs to flush the remainder again.+ """+ for size in (_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE):+ decompressor = zlib.decompressobj()+ decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)+ while decompressor.unconsumed_tail and not decompressor.eof:+ decompressor.decompress(+ decompressor.unconsumed_tail, max_length=_CHUNK_SIZE+ )+ assert decompressor.flush() == b"", f"pending output for size {size}"+def _test_compression_bomb_setting(self, compression_id: str) -> None:settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)@@ -509,6 +562,10 @@ class TestHttpCompression:cause = exc_info.value.__cause__assert isinstance(cause, _DecompressionMaxSizeExceeded)assert cause.decompressed_size < 1_100_000+ assert str(cause) == (+ f"The number of bytes decompressed so far ({cause.decompressed_size} B) "+ "exceeded the specified maximum (1000000 B)."+ )def test_compression_bomb_setting_br(self):self._test_compression_bomb_setting("br")diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 9d58a6e09..6f323a15c 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -70,15 +70,14 @@ class TestCrawl:yield crawler.crawl(mockserver=self.mockserver)slots = crawler.engine.downloader.slotstimes = crawler.spider.times- tolerance = 0.3-- delays_real = {k: v[1] - v[0] for k, v in times.items()}- error_delta = {- k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)- for k, v in slots.items()- }-- assert max(list(error_delta.values())) < tolerance+ # Downloading and processing a response add a roughly constant amount+ # of time on top of the configured delay, so the margin is absolute.+ # It stays well below the 1 second that separates the delays being+ # compared, so a slot using the delay of another one still fails.+ tolerance = 0.75++ for slot, (first, second) in times.items():+ assert abs((second - first) - slots[slot].delay) < tolerance@coroutine_testdiff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex a6abbd15a..4214fe680 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -172,6 +172,13 @@ class TestFTPFeedStorage:await self._store(url, b"bar", settings=settings)self._assert_stored(ftp_server.path / filename, b"bar")+ @coroutine_test+ async def test_missing_parent_directories(self):+ with MockFTPServer() as ftp_server:+ path = "missing/parent/dirs/file"+ await self._store(ftp_server.url(path), b"foo")+ self._assert_stored(ftp_server.path / path, b"foo")+@coroutine_testasync def test_tls(self, monkeypatch):monkeypatch.setenv(diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 7b73a133f..bc8a4d636 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -908,6 +908,19 @@ class TestLxmlParserLinkExtractor:Link(url="http://example.com/page.html", text="Link", nofollow=False),]+ def test_deduplicates_by_canonical_url(self):+ # With canonicalized=False, the default, URLs are canonicalized to+ # decide whether two extracted links are the same one.+ html = (+ b'<a href="http://example.com/page.html?b=2&a=1">1</a>'+ b'<a href="http://example.com/page.html?a=1&b=2">2</a>'+ )+ response = HtmlResponse("http://example.com/", body=html)+ lx = LxmlParserLinkExtractor(unique=True)+ assert lx.extract_links(response) == [+ Link(url="http://example.com/page.html?b=2&a=1", text="1", nofollow=False),+ ]+def test_strip_false(self):# With strip=False, trailing whitespace on a relative href survives urljoin# and is visible to process_value (safe_url_string cleans it up afterward).diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 8544602af..6bbc44829 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -1,5 +1,6 @@import pickleimport sys+from typing import Any, castimport pytestfrom queuelib.tests import test_queue as t@@ -13,6 +14,8 @@ from scrapy.squeues import (_MarshalLifoSerializationDiskQueue,_PickleFifoSerializationDiskQueue,_PickleLifoSerializationDiskQueue,+ _scrapy_non_serialization_queue,+ _serializable_queue,)@@ -20,6 +23,28 @@ class MyItem(Item):name = Field()+class NoPeekQueue:+ """Queue class without the optional peek method."""+++_no_peek_queue = cast("Any", NoPeekQueue)+++ "queue_class",+ [+ _serializable_queue(_no_peek_queue, pickle.dumps, pickle.loads),+ _scrapy_non_serialization_queue(_no_peek_queue),+ ],+)+def test_peek_unsupported(queue_class):+ with pytest.raises(+ NotImplementedError,+ match="The underlying queue class does not implement 'peek'",+ ):+ queue_class().peek()++def _test_procesor(x):return x + xdiff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex e94c0cc40..105f3acef 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -8,7 +8,6 @@ from abc import ABC, abstractmethodfrom typing import TYPE_CHECKINGimport pytest-import queuelibfrom scrapy.http import Requestfrom scrapy.spiders import Spider@@ -24,10 +23,9 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:- from scrapy.crawler import Crawler-+ import queuelib-HAVE_PEEK = hasattr(queuelib.queue.FifoMemoryQueue, "peek")+ from scrapy.crawler import Crawler@pytest.fixture@@ -41,46 +39,26 @@ class TestRequestQueueBase(ABC):def is_fifo(self) -> bool:raise NotImplementedError- @pytest.mark.parametrize("test_peek", [True, False])- def test_one_element(self, q: queuelib.queue.BaseQueue, test_peek: bool):- if test_peek and not HAVE_PEEK:- pytest.skip("The queuelib queues do not define peek")- if not test_peek and HAVE_PEEK:- pytest.skip("The queuelib queues define peek")+ def test_one_element(self, q: queuelib.queue.BaseQueue):assert len(q) == 0- if test_peek:- assert q.peek() is None+ assert q.peek() is Noneassert q.pop() is Nonereq = Request("http://www.example.com")q.push(req)assert len(q) == 1- if test_peek:- result = q.peek()- assert result is not None- assert result.url == req.url- else:- with pytest.raises(- NotImplementedError,- match="The underlying queue class does not implement 'peek'",- ):- q.peek()+ result = q.peek()+ assert result is not None+ assert result.url == req.urlresult = q.pop()assert result is not Noneassert result.url == req.urlassert len(q) == 0- if test_peek:- assert q.peek() is None+ assert q.peek() is Noneassert q.pop() is None- @pytest.mark.parametrize("test_peek", [True, False])- def test_order(self, q: queuelib.queue.BaseQueue, test_peek: bool):- if test_peek and not HAVE_PEEK:- pytest.skip("The queuelib queues do not define peek")- if not test_peek and HAVE_PEEK:- pytest.skip("The queuelib queues define peek")+ def test_order(self, q: queuelib.queue.BaseQueue):assert len(q) == 0- if test_peek:- assert q.peek() is None+ assert q.peek() is Noneassert q.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")@@ -88,25 +66,17 @@ class TestRequestQueueBase(ABC):q.push(req1)q.push(req2)q.push(req3)- if not test_peek:- with pytest.raises(- NotImplementedError,- match="The underlying queue class does not implement 'peek'",- ):- q.peek()reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1]for i, req in enumerate(reqs):assert len(q) == 3 - i- if test_peek:- result = q.peek()- assert result is not None- assert result.url == req.url+ result = q.peek()+ assert result is not None+ assert result.url == req.urlresult = q.pop()assert result is not Noneassert result.url == req.urlassert len(q) == 0- if test_peek:- assert q.peek() is None+ assert q.peek() is Noneassert q.pop() is Nonediff --git a/tests/test_utils_benchserver.py b/tests/test_utils_benchserver.pynew file mode 100644index 000000000..68065a9dc--- /dev/null+++ b/tests/test_utils_benchserver.py@@ -0,0 +1,40 @@+from __future__ import annotations++import re+from typing import TYPE_CHECKING, cast++from scrapy.utils.benchserver import Root, _getarg++if TYPE_CHECKING:+ from twisted.web.server import Request+++class _Request:+ def __init__(self, **args: bytes) -> None:+ self.args = {name.encode(): [value] for name, value in args.items()}+ self.written: list[bytes] = []++ def write(self, data: bytes) -> None:+ self.written.append(data)+++def test_getarg() -> None:+ request = cast("Request", _Request(total=b"5"))+ assert _getarg(request, b"total", 100, int) == 5+ assert _getarg(request, b"show", 100, int) == 100+ assert _getarg(request, b"missing") is None+++def test_render() -> None:+ root = Root() # type: ignore[no-untyped-call]+ request = _Request(total=b"5", show=b"2")+ assert root.getChild("follow", cast("Request", request)) is root+ assert root.render(cast("Request", request)) == b""+ body = b"".join(request.written).decode()+ assert body.startswith("<html><head></head><body>")+ assert body.endswith("</body></html>")+ numbers = re.findall(+ r"<a href='/follow\?total=5&show=2&n=(\d+)'>follow \1</a>", body+ )+ assert len(numbers) == 2+ assert all(1 <= int(number) <= 5 for number in numbers)diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 518cc3518..89f5cd141 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -9,8 +9,10 @@ from scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.conf import (arglist_to_dict,build_component_list,+ closest_scrapy_cfg,feed_complete_default_values_from_settings,feed_process_params_from_cli,+ get_sources,)@@ -39,6 +41,20 @@ class TestBuildComponentList:):build_component_list(duplicate_bs, convert=lambda x: x.lower())+ def test_duplicate_components_in_dict(self):+ d = {"one": 1, "ONE": 2}+ with pytest.raises(+ ValueError, match=r"Some paths in .* convert to the same object"+ ):+ build_component_list(d, convert=lambda x: x.lower())++ def test_invalid_value(self):+ d = {"one": "1"}+ with pytest.raises(+ ValueError, match=r"Invalid value 1 for component one, please provide"+ ):+ build_component_list(d, convert=lambda x: x)+def test_valid_numbers(self):# work well with None and numeric valuesd = {"a": 10, "b": None, "c": 15, "d": 5.0}@@ -51,6 +67,10 @@ class TestBuildComponentList:assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]+def test_get_sources():+ assert get_sources() == [*get_sources(use_closest=False), closest_scrapy_cfg()]++def test_arglist_to_dict():assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {"arg1": "val1",diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex fe6a7a97a..5ff1b0173 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -4,6 +4,7 @@ import subprocessimport sysfrom importlib.util import find_specfrom io import BytesIO+from types import ModuleTypefrom typing import TYPE_CHECKINGimport pytest@@ -44,6 +45,15 @@ def test_get_shell_embed_func():assert shell.__name__ == "_embed_standard_shell"+def test_get_shell_embed_func_known_shells():+ def embed(namespace: dict[str, object] | None = None, banner: str = "") -> None:+ pass++ known_shells = {"custom": lambda: embed}+ assert get_shell_embed_func(["custom"], known_shells) is embed+ assert get_shell_embed_func(["python"], known_shells) is None++def test_get_shell_embed_func_python():# the standard shell is always availableshell = get_shell_embed_func(["python"])@@ -58,6 +68,25 @@ def test_get_shell_embed_func_bpython():assert shell.__name__ == "_embed_bpython_shell"+def test_embed_bpython_shell(monkeypatch: pytest.MonkeyPatch) -> None:+ # bpython is never the default shell, so a stand-in module takes the place+ # of the interactive session the IPython tests below use.+ calls: list[tuple[dict[str, object], str]] = []+ bpython = ModuleType("bpython")+ monkeypatch.setattr(+ bpython,+ "embed",+ lambda locals_, banner: calls.append((locals_, banner)),+ raising=False,+ )+ monkeypatch.setitem(sys.modules, "bpython", bpython)++ shell = get_shell_embed_func(["bpython"])+ assert shell is not None+ shell({"a": 1}, "SHELL-READY")+ assert calls == [({"a": 1}, "SHELL-READY")]++def test_get_shell_embed_func_ipython():pytest.importorskip("IPython")shell = get_shell_embed_func(["ipython"])diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 6b30744bb..8ab6c5c2f 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -41,6 +41,26 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ def test_get_cookie_option(self):+ curl_command = 'curl "http://example.org/" -b "a=1; b=2"'+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "cookies": {"a": "1", "b": "2"},+ }+ self._test_command(curl_command, expected_result)++ def test_get_cookie_file_option(self):+ # curl reads cookies from a file when the value is not a key-value+ # pair; Scrapy ignores it.+ curl_command = 'curl "http://example.org/" -b cookies.txt -b "a=1"'+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "cookies": {"a": "1"},+ }+ self._test_command(curl_command, expected_result)+def test_get_complex(self):curl_command = ("curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 175a4fe03..e61741fec 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport random+import warningsfrom asyncio import Futurefrom typing import TYPE_CHECKING, Any@@ -16,6 +17,7 @@ from scrapy.utils.defer import (deferred_to_future,iter_errback,maybe_deferred_to_future,+ maybeDeferred_coro,mustbe_deferred,parallel_async,)@@ -357,6 +359,15 @@ class TestDeferredToFuture:assert future_result == 42[email protected]_not_asyncio+class TestDeferredToFutureNotAsyncio:+ def test_deferred(self):+ with pytest.raises(+ RuntimeError, match=r"deferred_to_future\(\) requires an installed asyncio"+ ):+ deferred_to_future(Deferred())++@pytest.mark.only_asyncioclass TestMaybeDeferredToFutureAsyncio:@coroutine_test@@ -400,3 +411,16 @@ class TestMaybeDeferredToFutureNotAsyncio:result = maybe_deferred_to_future(d)assert isinstance(result, Deferred)assert result is d+++def test_maybe_deferred_coro_deferred() -> None:+ d: Deferred[int] = Deferred()+ with warnings.catch_warnings(record=True) as records:+ warnings.simplefilter("always")+ assert maybeDeferred_coro(lambda: d) is d+ # Only the deprecation of maybeDeferred_coro() itself is reported; callables+ # that return a Deferred are the reason it exists.+ assert [str(record.message) for record in records] == [+ "maybeDeferred_coro() is deprecated and will be removed in a future"+ " Scrapy version."+ ]diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 4c8585916..5aa221414 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -7,7 +7,7 @@ from unittest import mockimport pytestfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.deprecate import create_deprecated_class, update_classpath+from scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpathclass MyWarning(UserWarning):@@ -252,6 +252,11 @@ class TestWarnWhenSubclassed:):create_deprecated_class("DeprecatedName", NewName)+ def test_unknown_parent_module(self):+ with mock.patch("inspect.getmodule", return_value=None):+ cls = create_deprecated_class("DeprecatedName", NewName)+ assert cls.__module__ == "scrapy.utils.deprecate"+@mock.patch("scrapy.utils.deprecate.DEPRECATION_RULES",@@ -284,3 +289,20 @@ class TestUpdateClassPath:def test_returns_nonstring(self):for notastring in [None, True, [1, 2, 3], object()]:assert update_classpath(notastring) == notastring+++class TestAttribute:+ class MyClass:+ pass++ def test_default_version(self):+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"MyClass\.old attribute is deprecated and will be no longer "+ r"supported in Scrapy 0\.12, use MyClass\.new attribute instead",+ ):+ attribute(self.MyClass(), "old", "new")++ def test_custom_version(self):+ with pytest.warns(ScrapyDeprecationWarning, match="in Scrapy 3.0"):+ attribute(self.MyClass(), "old", "new", "3.0")diff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex a87816c20..b8d15bcba 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -1,11 +1,14 @@from __future__ import annotationsimport builtins+import sysfrom io import StringIOfrom typing import TYPE_CHECKINGfrom unittest import mock-from scrapy.utils.display import pformat, pprint+import pytest++from scrapy.utils.display import _enable_windows_terminal_processing, pformat, pprintif TYPE_CHECKING:from collections.abc import Mapping, Sequence@@ -81,6 +84,14 @@ def test_pformat_windows(assert pformat(value) in colorized_strings[email protected](sys.platform != "win32", reason="Windows-only console API")+def test_enable_windows_terminal_processing() -> None:+ # The tests above mock this out, so this is the only place the real+ # kernel32 calls run. Whether they succeed depends on whether stdout is a+ # console, which pytest does not guarantee.+ assert isinstance(_enable_windows_terminal_processing(), bool)++@mock.patch("sys.platform", "linux")@mock.patch("sys.stdout.isatty")def test_pformat_no_pygments(isatty: mock.Mock) -> None:diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex d431b8017..672ae0be6 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -210,6 +210,31 @@ class TestXmliter:with pytest.raises(StopIteration):next(my_iter)+ def test_xmliter_namespaced_nodename_other_prefixes(self):+ body = b"""<?xml version="1.0" encoding="UTF-8"?>+ <rss version="2.0"+ xmlns:a="http://example.com/ns/a"+ xmlns:g="http://base.google.com/ns/1.0">+ <item>+ <a:id>A_1</a:id>+ <g:id>ITEM_1</g:id>+ </item>+ </rss>+ """+ response = XmlResponse(url="http://example.com", body=body)+ node = next(xmliter_lxml(response, "g:id"))+ assert node.xpath("text()").getall() == ["ITEM_1"]++ def test_xmliter_non_text_response(self):+ body = (+ b'<?xml version="1.0" encoding="UTF-8"?>'+ b"<products><product>one</product></products>"+ )+ response = Response(url="http://example.com", body=body)+ assert [+ node.xpath("text()").get() for node in xmliter_lxml(response, "product")+ ] == ["one"]+def test_xmliter_exception(self):body = ('<?xml version="1.0" encoding="UTF-8"?>'@@ -421,6 +446,9 @@ class TestUtilsCsv:{"id": "4", "name": "empty", "value": ""},]+ def test_csviter_empty(self):+ assert not list(csviter(""))+def test_csviter_exception(self):body = get_testdata("feeds", "feed-sample3.csv")@@ -481,6 +509,12 @@ class TestBodyOrStr:assert type(r1) is type(r2)assert type(r1) is not type(r3) # type: ignore[comparison-overlap]+ def test_body_or_str_unsupported_type(self) -> None:+ with pytest.raises(+ TypeError, match="must be Response or str or bytes, not int"+ ):+ _body_or_str(42) # type: ignore[call-overload]+@staticmethoddef _assert_type_and_value(a: str | bytes, b: str | bytes, obj: Response | str | bytesdiff --git a/tests/test_utils_job.py b/tests/test_utils_job.pynew file mode 100644index 000000000..31adc1514--- /dev/null+++ b/tests/test_utils_job.py@@ -0,0 +1,24 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from scrapy.settings import Settings+from scrapy.utils.job import job_dir++if TYPE_CHECKING:+ from pathlib import Path+++def test_no_jobdir() -> None:+ assert job_dir(Settings()) is None+ assert job_dir(Settings({"JOBDIR": ""})) is None+++def test_existing_jobdir(tmp_path: Path) -> None:+ assert job_dir(Settings({"JOBDIR": str(tmp_path)})) == str(tmp_path)+++def test_missing_jobdir(tmp_path: Path) -> None:+ jobdir = tmp_path / "missing" / "jobdir"+ assert job_dir(Settings({"JOBDIR": str(jobdir)})) == str(jobdir)+ assert jobdir.is_dir()diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 3ea82ba8c..d1b98d009 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -9,15 +9,21 @@ from io import StringIOfrom typing import TYPE_CHECKING, Any, castimport pytest+from twisted.python import log as twisted_logfrom twisted.python.failure import Failurefrom scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.settings import Settingsfrom scrapy.utils.log import (LogCounterHandler,SpiderLoggerAdapter,StreamLogger,TopLevelFormatter,+ _uninstall_scrapy_root_handler,+ configure_logging,failure_to_exc_info,+ get_scrapy_root_handler,+ install_scrapy_root_handler,logformatter_adapter,)from scrapy.utils.test import get_crawler@@ -122,6 +128,70 @@ class TestStreamLogger:sys.stdout = old_stdout+ def test_flush(self) -> None:+ class FlushCountingHandler(logging.Handler):+ flushes = 0++ def flush(self) -> None:+ self.flushes += 1++ handler = FlushCountingHandler()+ logger = logging.getLogger("test_flush")+ logger.addHandler(handler)+ try:+ StreamLogger(logger).flush()+ finally:+ logger.removeHandler(handler)+ assert handler.flushes == 1+++class TestConfigureLogging:+ @pytest.fixture(autouse=True)+ def restore_logging(self) -> Generator[None]:+ root_handlers = logging.root.handlers[:]+ # configure_logging() adds a Twisted log observer without giving any way+ # to remove it afterwards.+ observers = twisted_log.theLogPublisher.observers[:]+ old_stdout = sys.stdout+ old_showwarning = warnings.showwarning+ try:+ yield+ finally:+ warnings.showwarning = old_showwarning+ sys.stdout = old_stdout+ twisted_log.theLogPublisher.observers[:] = observers+ logging.root.handlers[:] = root_handlers+ _uninstall_scrapy_root_handler()++ @staticmethod+ def _warnings_are_captured() -> bool:+ return warnings.showwarning.__module__ == "logging"++ def test_log_stdout(self) -> None:+ configure_logging(settings={"LOG_STDOUT": True}, install_root_handler=False)+ assert isinstance(sys.stdout, StreamLogger)++ def test_captures_warnings(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(sys, "warnoptions", [])+ logging.captureWarnings(False)+ configure_logging(install_root_handler=False)+ assert self._warnings_are_captured()++ def test_keeps_warnoptions(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(sys, "warnoptions", ["default"])+ logging.captureWarnings(False)+ configure_logging(install_root_handler=False)+ assert not self._warnings_are_captured()++ def test_reinstall_root_handler_removed_from_root(self) -> None:+ install_scrapy_root_handler(Settings())+ handler = get_scrapy_root_handler()+ assert handler is not None+ # Something else removed the handler from the root logger.+ logging.root.removeHandler(handler)+ install_scrapy_root_handler(Settings())+ assert get_scrapy_root_handler() is not handler+@pytest.mark.parametrize(("base_extra", "log_extra", "expected_extra"),diff --git a/tests/test_utils_ossignal.py b/tests/test_utils_ossignal.pynew file mode 100644index 000000000..4c8f7d0b4--- /dev/null+++ b/tests/test_utils_ossignal.py@@ -0,0 +1,54 @@+from __future__ import annotations++import signal+from typing import TYPE_CHECKING, Any++import pytest++from scrapy.utils.ossignal import install_shutdown_handlers++if TYPE_CHECKING:+ from collections.abc import Generator+ from types import FrameType++SHUTDOWN_SIGNALS = [+ getattr(signal, name)+ for name in ("SIGTERM", "SIGINT", "SIGBREAK")+ if hasattr(signal, name)+]++[email protected](autouse=True)+def restore_handlers() -> Generator[None]:+ handlers = {sig: signal.getsignal(sig) for sig in SHUTDOWN_SIGNALS}+ try:+ yield+ finally:+ for sig, handler in handlers.items():+ signal.signal(sig, handler)+++def shutdown(signum: int, frame: FrameType | None) -> Any:+ pass+++def test_install() -> None:+ install_shutdown_handlers(shutdown)+ for sig in SHUTDOWN_SIGNALS:+ assert signal.getsignal(sig) is shutdown+++def test_keeps_custom_sigint_handler() -> None:+ def custom(signum: int, frame: FrameType | None) -> Any:+ pass++ signal.signal(signal.SIGINT, custom)+ install_shutdown_handlers(shutdown, override_sigint=False)+ assert signal.getsignal(signal.SIGINT) is custom+ assert signal.getsignal(signal.SIGTERM) is shutdown+++def test_overrides_default_sigint_handler() -> None:+ signal.signal(signal.SIGINT, signal.default_int_handler)+ install_shutdown_handlers(shutdown, override_sigint=False)+ assert signal.getsignal(signal.SIGINT) is shutdowndiff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 7eade6463..8a7f2ee03 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -6,8 +6,14 @@ from typing import TYPE_CHECKINGimport pytest+from scrapy.exceptions import NotConfiguredfrom scrapy.utils.misc import set_environ-from scrapy.utils.project import data_path, get_project_settings+from scrapy.utils.project import (+ data_path,+ get_project_settings,+ inside_project,+ project_data_dir,+)if TYPE_CHECKING:from collections.abc import Generator@@ -27,6 +33,19 @@ def proj_path(tmp_path: Path) -> Generator[Path]:os.chdir(prev_dir)+def no_proj_path(tmp_path: Path) -> Generator[Path]:+ """A working directory without a scrapy.cfg file, also isolated from the+ user-wide and system-wide Scrapy configuration files."""+ prev_dir = Path.cwd()+ try:+ os.chdir(tmp_path)+ with set_environ(HOME=str(tmp_path), XDG_CONFIG_HOME=str(tmp_path)):+ yield tmp_path+ finally:+ os.chdir(prev_dir)++def test_data_path_outside_project() -> None:assert str(Path(".scrapy", "somepath")) == data_path("somepath")abspath = str(Path(os.path.sep, "absolute", "path"))@@ -40,6 +59,51 @@ def test_data_path_inside_project(proj_path: Path) -> None:assert abspath == data_path(abspath)+def test_data_path_createdir(no_proj_path: Path) -> None:+ path = Path(data_path("somepath", createdir=True))+ assert path.is_dir()+ # An existing directory is left alone.+ assert Path(data_path("somepath", createdir=True)) == path+++def test_inside_project_unimportable_settings_module(no_proj_path: Path) -> None:+ with (+ set_environ(SCRAPY_SETTINGS_MODULE="tests.no_such_settings_module"),+ pytest.warns(+ UserWarning, match="Cannot import scrapy settings module tests.no_such"+ ),+ ):+ assert inside_project() is False+++def test_project_data_dir_outside_project(no_proj_path: Path) -> None:+ with pytest.raises(NotConfigured, match="Not inside a project"):+ project_data_dir()+++def test_project_data_dir_without_scrapy_cfg(no_proj_path: Path) -> None:+ with (+ set_environ(SCRAPY_SETTINGS_MODULE="tests.test_cmdline.settings"),+ pytest.raises(NotConfigured, match=r"Unable to find scrapy\.cfg file"),+ ):+ project_data_dir()+++def test_project_data_dir_default(proj_path: Path) -> None:+ expected = (proj_path / ".scrapy").resolve()+ assert Path(project_data_dir()) == expected+ assert expected.is_dir()+ # A second call finds the directory already created.+ assert Path(project_data_dir()) == expected+++def test_project_data_dir_from_scrapy_cfg(proj_path: Path) -> None:+ datadir = proj_path / "custom-datadir"+ Path("scrapy.cfg").write_text(f"[datadir]\ndefault = {datadir}\n")+ assert Path(project_data_dir()) == datadir+ assert datadir.is_dir()++class TestGetProjectSettings:def test_valid_envvar(self):value = "tests.test_cmdline.settings"diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 099b5ccc2..25e50960f 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -12,8 +12,10 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.defer import aiter_errbackfrom scrapy.utils.python import (MutableAsyncChain,+ _looks_like_import_path,binary_is_text,get_func_args,+ get_spec,memoizemethod_noargs,to_bytes,to_unicode,@@ -138,6 +140,11 @@ def test_binaryistext(value: bytes, expected: bool) -> None:assert binary_is_text(value) is expected+def test_binaryistext_not_bytes() -> None:+ with pytest.raises(TypeError, match="data must be bytes, got 'str'"):+ binary_is_text("hello") # type: ignore[arg-type]++def test_get_func_args():def f1(a, b, c):pass@@ -164,6 +171,8 @@ def test_get_func_args():partial_f1 = functools.partial(f1, None)partial_f2 = functools.partial(f1, b=None)partial_f3 = functools.partial(partial_f2, None)+ # a positional value that happens to match the name of a free parameter+ partial_f4 = functools.partial(f1, "b")assert get_func_args(f1) == ["a", "b", "c"]assert get_func_args(f2) == ["a", "b", "c"]@@ -173,6 +182,7 @@ def test_get_func_args():assert get_func_args(partial_f1) == ["b", "c"]assert get_func_args(partial_f2) == ["a", "c"]assert get_func_args(partial_f3) == ["c"]+ assert get_func_args(partial_f4) == ["b", "c"]assert get_func_args(cal) == ["a", "b", "c"]assert get_func_args(object) == []assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]@@ -209,6 +219,35 @@ def test_get_func_args_unresolvable_annotations():assert get_func_args(namespace["f"]) == ["a", "b"]+def test_get_func_args_not_callable() -> None:+ with pytest.raises(TypeError, match="func must be callable, got 'int'"):+ get_func_args(1) # type: ignore[arg-type]+++def test_get_spec_not_callable() -> None:+ with pytest.raises(TypeError, match="is not callable"):+ get_spec(1) # type: ignore[arg-type]+++ ("value", "expected"),+ [+ ("scrapy.Spider", True),+ ("scrapy", True),+ ("", False),+ ("scrapy Spider", False),+ ("scrapy.Spider\n", False),+ ("scrapy-Spider", False),+ (".scrapy", False),+ ("scrapy.", False),+ ("scrapy..Spider", False),+ ("scrapy.1Spider", False),+ ],+)+def test_looks_like_import_path(value: str, expected: bool) -> None:+ assert _looks_like_import_path(value) is expected++@pytest.mark.parametrize(("value", "expected"),[diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 44cb5c306..3be0a03b8 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -1,17 +1,29 @@from __future__ import annotationsimport asyncio+import sys+from typing import TYPE_CHECKINGimport pytest+from twisted.internet.error import CannotListenError+from twisted.internet.protocol import ServerFactoryfrom scrapy.utils.reactor import (_asyncio_reactor_path,install_reactor,is_asyncio_reactor_installed,+ listen_tcp,set_asyncio_event_loop,+ verify_installed_asyncio_event_loop,+ verify_installed_reactor,)from tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import Generator++ from twisted.internet.tcp import Port+class TestAsyncio:@pytest.mark.requires_reactor # needs a reactor@@ -25,3 +37,64 @@ class TestAsyncio:async def test_set_asyncio_event_loop(self):install_reactor(_asyncio_reactor_path)assert set_asyncio_event_loop(None) is asyncio.get_running_loop()+++class TestVerifyWithoutReactor:+ @pytest.fixture(autouse=True)+ def no_reactor(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delitem(sys.modules, "twisted.internet.reactor", raising=False)++ def test_verify_installed_reactor(self) -> None:+ with pytest.raises(+ RuntimeError, match=r"verify_installed_reactor\(\) called without"+ ):+ verify_installed_reactor(_asyncio_reactor_path)++ def test_verify_installed_asyncio_event_loop(self) -> None:+ with pytest.raises(+ RuntimeError,+ match=r"verify_installed_asyncio_event_loop\(\) called without",+ ):+ verify_installed_asyncio_event_loop("asyncio.SelectorEventLoop")++[email protected]_reactor # needs a reactor+class TestListenTcp:+ @pytest.fixture+ def ports(self) -> Generator[list[Port]]:+ opened: list[Port] = []+ yield opened+ for port in opened:+ port.stopListening()++ @pytest.mark.parametrize("portrange", [[1, 2, 3], [8000, 7000]])+ def test_invalid_portrange(self, portrange: list[int]) -> None:+ with pytest.raises(ValueError, match="invalid portrange"):+ listen_tcp(portrange, "127.0.0.1", ServerFactory())++ def test_empty_portrange(self, ports: list[Port]) -> None:+ port = listen_tcp([], "127.0.0.1", ServerFactory())+ ports.append(port)+ assert port.getHost().port > 0++ def test_single_port(self, ports: list[Port]) -> None:+ port = listen_tcp([0], "127.0.0.1", ServerFactory())+ ports.append(port)+ assert port.getHost().port > 0++ def test_skips_used_ports(self, ports: list[Port]) -> None:+ used = listen_tcp([], "127.0.0.1", ServerFactory())+ ports.append(used)+ used_number = used.getHost().port++ port = listen_tcp([used_number, used_number + 50], "127.0.0.1", ServerFactory())+ ports.append(port)+ assert used_number < port.getHost().port <= used_number + 50++ def test_no_free_port(self, ports: list[Port]) -> None:+ used = listen_tcp([], "127.0.0.1", ServerFactory())+ ports.append(used)+ used_number = used.getHost().port++ with pytest.raises(CannotListenError):+ listen_tcp([used_number, used_number], "127.0.0.1", ServerFactory())diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 3109bc149..cbcc070d5 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -53,6 +53,32 @@ class TestSendCatchLog:dispatcher.disconnect(self.error_handler, signal=test_signal)dispatcher.disconnect(self.ok_handler, signal=test_signal)+ @coroutine_test+ async def test_send_catch_log_dont_log(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ test_signal = object()+ handlers_called: set[Callable[..., None]] = set()++ dispatcher.connect(self.error_handler, signal=test_signal)+ caplog.clear()+ result = await ensure_awaitable(+ self._get_result(+ test_signal,+ arg="test",+ handlers_called=handlers_called,+ dont_log=ZeroDivisionError,+ )+ )++ assert self.error_handler in handlers_called+ assert not caplog.records+ assert isinstance(+ result[0][1], Exception if self.returns_exceptions else Failure+ )++ dispatcher.disconnect(self.error_handler, signal=test_signal)+def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:return send_catch_log(signal, *a, **kw)diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex 9f2dce6d6..dda13006d 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -41,6 +41,21 @@ def test_sitemap():]+def test_sitemap_str():+ xmltext = """<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">+<url><loc>http://www.example.com/</loc></url>+</urlset>"""++ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Passing `str` type as `xmltext` is deprecated",+ ):+ s = Sitemap(xmltext)+ assert s.type == "urlset"+ assert list(s) == [{"loc": "http://www.example.com/"}]++def test_sitemap_index():s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 71f0f76ad..6caa1f6a3 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -1,17 +1,38 @@from __future__ import annotations+import logging+from typing import TYPE_CHECKING++import pytest+from scrapy import Spiderfrom scrapy.http import Requestfrom scrapy.item import Item-from scrapy.utils.spider import iter_spider_classes, iterate_spider_output+from scrapy.settings import Settings+from scrapy.spiderloader import get_spider_loader+from scrapy.utils.spider import (+ iter_spider_classes,+ iterate_spider_output,+ spidercls_for_request,+)++if TYPE_CHECKING:+ from scrapy.spiderloader import SpiderLoaderProtocolclass MySpider1(Spider):name = "myspider1"+ allowed_domains = ["example.com", "myspider1.example"]class MySpider2(Spider):name = "myspider2"+ allowed_domains = ["example.com"]+++def spider_loader() -> SpiderLoaderProtocol:+ return get_spider_loader(Settings({"SPIDER_MODULES": ["tests.test_utils_spider"]}))def test_iterate_spider_output():@@ -30,3 +51,38 @@ def test_iter_spider_classes():it = iter_spider_classes(tests.test_utils_spider)assert set(it) == {MySpider1, MySpider2}+++class TestSpiderclsForRequest:+ def test_single_match(self, spider_loader: SpiderLoaderProtocol) -> None:+ request = Request("http://myspider1.example/")+ assert spidercls_for_request(spider_loader, request) is MySpider1++ def test_no_match(self, spider_loader: SpiderLoaderProtocol) -> None:+ request = Request("http://toscrape.com/")+ assert spidercls_for_request(spider_loader, request) is None+ assert spidercls_for_request(spider_loader, request, MySpider1) is MySpider1++ def test_multiple_matches(self, spider_loader: SpiderLoaderProtocol) -> None:+ request = Request("http://example.com/")+ assert spidercls_for_request(spider_loader, request) is None+ assert spidercls_for_request(spider_loader, request, MySpider2) is MySpider2++ def test_log_none(+ self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture+ ) -> None:+ request = Request("http://toscrape.com/")+ with caplog.at_level(logging.ERROR):+ assert spidercls_for_request(spider_loader, request, log_none=True) is None+ assert "Unable to find spider that handles" in caplog.text++ def test_log_multiple(+ self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture+ ) -> None:+ request = Request("http://example.com/")+ with caplog.at_level(logging.ERROR):+ assert (+ spidercls_for_request(spider_loader, request, log_multiple=True) is None+ )+ assert "More than one spider can handle" in caplog.text+ assert "myspider1, myspider2" in caplog.textdiff --git a/tests/test_utils_ssl.py b/tests/test_utils_ssl.pynew file mode 100644index 000000000..8e90f7b3e--- /dev/null+++ b/tests/test_utils_ssl.py@@ -0,0 +1,69 @@+from __future__ import annotations++import logging+import warnings+from typing import cast+from unittest import mock++import OpenSSL._util as pyOpenSSLutil+import OpenSSL.SSL+import pytest+from OpenSSL import crypto++from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.ssl import (+ _log_ssl_conn_debug_info,+ ffi_buf_to_string,+ get_openssl_version,+ get_temp_key_info,+ x509name_to_string,+)+++def test_ffi_buf_to_string() -> None:+ buf = pyOpenSSLutil.ffi.new("char[]", b"some text")+ with pytest.warns(+ ScrapyDeprecationWarning, match=r"ffi_buf_to_string\(\) is deprecated"+ ):+ assert ffi_buf_to_string(buf) == "some text"+++def test_x509name_to_string() -> None:+ with warnings.catch_warnings():+ # X509 itself is deprecated in pyOpenSSL, but it is the only way to build+ # the X509Name that the deprecated function under test takes.+ warnings.simplefilter("ignore", DeprecationWarning)+ subject = crypto.X509().get_subject()+ subject.C = "IE"+ subject.O = "Scrapy"+ subject.CN = "localhost"+ with pytest.warns(+ ScrapyDeprecationWarning, match=r"x509name_to_string\(\) is deprecated"+ ):+ assert x509name_to_string(subject) == "/C=IE/O=Scrapy/CN=localhost"+++def test_get_temp_key_info() -> None:+ with pytest.warns(+ ScrapyDeprecationWarning, match=r"get_temp_key_info\(\) is deprecated"+ ):+ assert get_temp_key_info(object()) is None+++def test_get_openssl_version() -> None:+ assert "OpenSSL" in get_openssl_version()+++def test_log_ssl_conn_debug_info_no_certificate(+ caplog: pytest.LogCaptureFixture,+) -> None:+ connection = mock.MagicMock()+ connection.get_protocol_version_name.return_value = "TLSv1.3"+ connection.get_cipher_name.return_value = "TLS_AES_256_GCM_SHA384"+ connection.get_peer_certificate.return_value = None+ with caplog.at_level(logging.DEBUG, logger="scrapy.utils.ssl"):+ _log_ssl_conn_debug_info(+ "example.com", cast("OpenSSL.SSL.Connection", connection)+ )+ assert "SSL connection to example.com using protocol TLSv1.3" in caplog.text+ assert "certificate" not in caplog.textdiff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex 9585c6d83..07daa149c 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -119,11 +119,21 @@ def test_get_oldest():assert trackref.get_oldest("Foo") is o3+def test_get_oldest_all_dead():+ o1 = Foo()+ del o1+ if _IS_PYPY:+ garbage_collect()+ # Foo is still a key of live_refs, but it no longer tracks any instance.+ assert trackref.get_oldest("Foo") is None++def test_iter_all():o1 = Foo()o2 = Bar() # noqa: F841o3 = Foo()assert set(trackref.iter_all("Foo")) == {o1, o3}+ assert list(trackref.iter_all("XXX")) == []def test_run_time_classes() -> None:diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex abf153817..f40b0126f 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -31,10 +31,7 @@ from scrapy.exceptions import (UnsupportedURLSchemeError,)from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse-from scrapy.utils._deps_compat import (- PYOPENSSL_X509_DEPRECATED,- TWISTED_TLS_LIMITS_OFFBY1,-)+from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpider@@ -870,15 +867,8 @@ class TestHttpsBase(TestHttpBase):is_secure = Truetls_log_message = (- (- 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '- 'subject "CN=localhost,O=Scrapy,C=IE"'- )- if PYOPENSSL_X509_DEPRECATED- else (- 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '- 'subject "/C=IE/O=Scrapy/CN=localhost"'- )+ 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '+ 'subject "CN=localhost,O=Scrapy,C=IE"')def test_download_conn_lost(self) -> None: # type: ignore[override]@@ -940,6 +930,9 @@ class TestSimpleHttpsBase(ABC):certfile = "keys/localhost.crt"host = "localhost"cipher_string: str | None = None+ # Crawler settings for the download handler. When unset, the client is asked+ # for the same ciphers the server is restricted to.+ client_settings: dict[str, Any] | None = None@pytest.fixture(scope="class")@classmethod@@ -961,10 +954,9 @@ class TestSimpleHttpsBase(ABC):@asynccontextmanagerasync def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]:- if self.cipher_string is not None:+ settings_dict = self.client_settings+ if settings_dict is None and self.cipher_string is not None:settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string}- else:- settings_dict = Nonecrawler = get_crawler(DefaultSpider, settings_dict=settings_dict)crawler.spider = crawler._create_spider()dh = build_from_crawler(self.download_handler_cls, crawler)@@ -1008,6 +1000,12 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase):cipher_string = "CAMELLIA256-SHA"+class TestHttpsDefaultCiphersBase(TestSimpleHttpsBase):+ """A ``None`` cipher list leaves the TLS library defaults in place."""++ client_settings: dict[str, Any] | None = {"DOWNLOADER_CLIENT_TLS_CIPHERS": None}++class TestHttpsTLSVersionBase(ABC):keyfile = "keys/localhost.key"certfile = "keys/localhost.crt"diff --git a/tox.ini b/tox.iniindex 7ea1c57d7..e807317b3 100644--- a/tox.ini+++ b/tox.ini@@ -140,15 +140,15 @@ deps =Twisted==21.7.0brotli==1.2.0; implementation_name != "pypy"brotlicffi==1.2.0.0; implementation_name == "pypy"- cryptography==37.0.0+ cryptography==41.0.5cssselect==0.9.1httpx2==2.0.0itemadapter==0.1.0lxml==4.6.4parsel==1.5.0- pyOpenSSL==22.0.0+ pyOpenSSL==24.3.0queuelib==1.6.1- service_identity==23.1.0+ service_identity==24.2.0w3lib==1.17.0zope.interface==5.1.0{[test-requirements]deps}@@ -283,8 +283,7 @@ setenv =[testenv:pypy3]basepython = pypy3commands =- ; not enabling coverage as it significantly increases the run time- pytest {posargs:--durations=10 scrapy tests}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=pypy3.junit.xml -o junit_family=legacy --durations=10 scrapy tests}[testenv:pypy3-extra-deps]basepython = pypy3@@ -307,7 +306,7 @@ deps =parsel==1.5.0pyOpenSSL==24.3.0queuelib==1.6.1- service_identity==23.1.0+ service_identity==24.2.0# w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses# an inline flag placement that Python 3.11 treats as an error: global# flags not at the start of the expression. w3lib 1.18 stopped encoding []