Scrapy, a fast high-level web crawling & scraping framework for Python.
CI: warnings → errors (#7702)
Adriane0fac4116e70
diff --git a/pyproject.toml b/pyproject.tomlindex b449dcbe1..b89a64a3b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -308,12 +308,32 @@ markers = ["requires_internet: marks tests that need real Internet access",]filterwarnings = [+ # Envs with pinned old dependencies opt out of this, see tox.ini.+ "error","ignore::DeprecationWarning:twisted.web.static",- # Jobs that do not report coverage disable it with --no-cov, which pytest-cov- # warns about because the coverage options below stay in place.- "ignore::pytest_cov.CovDisabledWarning",- # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108- "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",+ # Twisted leaves the listening socket open after a failed bind+ # (CannotListenError): https://github.com/twisted/twisted/issues/6108+ # The asyncio reactor likewise leaves client sockets and transports open on+ # teardown. These surface as unraisable exceptions during finalization; the+ # message wording varies across Python implementations and versions+ # (PyPy leaves the prefix empty), so match on the object repr.+ "ignore:.*<socket\\.socket fd=:pytest.PytestUnraisableExceptionWarning",+ "ignore:.*<function (_SelectorTransport|_ProactorBasePipeTransport|_ProactorSocketTransport|BaseEventLoop)\\.__del__:pytest.PytestUnraisableExceptionWarning",+ # PyDispatcher iterates its connection registry while a weakref callback+ # may mutate it, which raises RuntimeError on PyPy, where several weakrefs+ # can die within a single collection.+ "ignore:.*<function connect\\.<locals>\\.remove:pytest.PytestUnraisableExceptionWarning",+ # queuelib's test helpers (reused by tests/test_squeues.py) leave queue+ # files open; the resulting warning surfaces at an arbitrary GC point.+ "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning",+ # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher.+ "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",+ # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.+ "ignore:The async(hat|ore) module is deprecated:DeprecationWarning",+ # CI runs without coverage pass --no-cov, which pytest-cov reports. Matched+ # by message because some tox envs do not install pytest-cov, and pytest+ # warns when a filter names a module it cannot import.+ "ignore:Coverage disabled via --no-cov switch!",][tool.ruff.lint]diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 9e93eae18..979704a87 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -19,7 +19,7 @@ from scrapy.utils.reactor import _asyncio_reactor_pathfrom scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLE, tests_datadirfrom tests.utils.bases.commands import TestProjectBase-from tests.utils.cmdline import proc+from tests.utils.cmdline import proc, stop_spawnfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -212,12 +212,7 @@ class TestInteractiveShell:p.sendline(f"fetch('{mockserver.url('/')}')")p.sendline("type(response)")p.expect_exact("HtmlResponse")- p.sendeof()- p.wait() # type: ignore[no-untyped-call]- if p.proc.stdin:- p.proc.stdin.close()- if p.proc.stdout:- p.proc.stdout.close()+ stop_spawn(p)logfile.seek(0)assert "Traceback" not in logfile.read().decode()@@ -243,8 +238,7 @@ class TestInteractiveShell:p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)p.logfile_read = logfilep.expect_exact("Available Scrapy objects")- p.sendeof()- p.wait() # type: ignore[no-untyped-call]+ stop_spawn(p)logfile.seek(0)return logfile.read().decode()@@ -269,8 +263,7 @@ class TestInteractiveShell:# shell=python was honored, regardless of platform-specific prompts.p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")p.expect_exact("IPYMODULE False")- p.sendeof()- p.wait() # type: ignore[no-untyped-call]+ stop_spawn(p)logfile.seek(0)assert "Traceback" not in logfile.read().decode()@@ -299,8 +292,7 @@ class TestInteractiveShell:p.expect_exact("Available Scrapy objects")p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")p.expect_exact("IPYMODULE True")- p.sendeof()- p.wait() # type: ignore[no-untyped-call]+ stop_spawn(p)logfile.seek(0)assert "Traceback" not in logfile.read().decode()diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 800ab9397..cbb6f2c5b 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -14,6 +14,7 @@ from pexpect.popen_spawn import PopenSpawnfrom scrapy.utils.asyncio import sleepfrom tests.utils import get_script_run_env+from tests.utils.cmdline import stop_spawnfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -226,11 +227,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):p.kill(sig)p.expect_exact("shutting down gracefully")p.expect_exact("Spider closed (shutdown)")- p.wait() # type: ignore[no-untyped-call]- if p.proc.stdin:- p.proc.stdin.close()- if p.proc.stdout:- p.proc.stdout.close()+ stop_spawn(p)def test_shutdown_graceful(self) -> None:self._test_shutdown_graceful()@@ -247,11 +244,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):await sleep(0.01)p.kill(sig)p.expect_exact("forcing unclean shutdown")- p.wait() # type: ignore[no-untyped-call]- if p.proc.stdin:- p.proc.stdin.close()- if p.proc.stdout:- p.proc.stdout.close()+ stop_spawn(p)@coroutine_testasync def test_shutdown_forced(self) -> None:diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex 4214fe680..46e603af7 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -219,16 +219,16 @@ class TestBlockingFeedStorage:def test_default_temp_dir(self):b = MyBlockingFeedStorage()- storage_file = b.open(get_test_spider())- storage_dir = Path(storage_file.name).parent+ with b.open(get_test_spider()) as storage_file:+ storage_dir = Path(storage_file.name).parentassert str(storage_dir) == tempfile.gettempdir()def test_temp_file(self, tmp_path):b = MyBlockingFeedStorage()spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)})- storage_file = b.open(spider)- storage_dir = Path(storage_file.name).parent+ with b.open(spider) as storage_file:+ storage_dir = Path(storage_file.name).parentassert storage_dir == tmp_pathdef test_invalid_folder(self, tmp_path):diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.pyindex 095cb17a7..4020daaa1 100644--- a/tests/utils/cmdline.py+++ b/tests/utils/cmdline.py@@ -11,6 +11,8 @@ from scrapy.utils.test import get_testenvif TYPE_CHECKING:from pathlib import Path+ from pexpect.popen_spawn import PopenSpawn+def call(*args: str, **popen_kwargs: Any) -> int:args = (sys.executable, "-m", "scrapy.cmdline", *args)@@ -41,6 +43,16 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:return p.returncode, p.stdout, p.stderr+def stop_spawn(p: PopenSpawn[str]) -> None:+ p.sendeof()+ p.wait() # type: ignore[no-untyped-call]+ # PopenSpawn leaves the subprocess pipes open, which triggers+ # ResourceWarning at an arbitrary garbage collection point.+ for pipe in (p.proc.stdin, p.proc.stdout):+ if pipe:+ pipe.close()++def write_recording_editor(editor: Path) -> None:"""Create an executable editor script that writes the path it is asked toopen (its last argument) into the file given as its first argument."""diff --git a/tox.ini b/tox.iniindex c5e4ff7bf..bdad9b621 100644--- a/tox.ini+++ b/tox.ini@@ -154,6 +154,9 @@ deps ={[test-requirements]deps}setenv =_SCRAPY_MIN=true+ # Pinned old Python and library versions trigger warnings that we cannot+ # fix, so these envs do not turn warnings into errors.+ PYTEST_ADDOPTS=-W default {env:PYTEST_ADDOPTS:}commands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}