Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CI: warnings → errors (#7702)

Adriane0fac4116e70
diff --git a/pyproject.toml b/pyproject.toml
index b449dcbe1..b89a64a3b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -308,12 +308,32 @@ markers = [
"requires_internet: marks tests that need real Internet access",
]
filterwarnings = [
+ # Envs with pinned old dependencies opt out of this, see tox.ini.
+ "error",
"ignore::DeprecationWarning:twisted.web.static",
- # Jobs that do not report coverage disable it with --no-cov, which pytest-cov
- # warns about because the coverage options below stay in place.
- "ignore::pytest_cov.CovDisabledWarning",
- # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
- "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
+ # Twisted leaves the listening socket open after a failed bind
+ # (CannotListenError): https://github.com/twisted/twisted/issues/6108
+ # The asyncio reactor likewise leaves client sockets and transports open on
+ # teardown. These surface as unraisable exceptions during finalization; the
+ # message wording varies across Python implementations and versions
+ # (PyPy leaves the prefix empty), so match on the object repr.
+ "ignore:.*<socket\\.socket fd=:pytest.PytestUnraisableExceptionWarning",
+ "ignore:.*<function (_SelectorTransport|_ProactorBasePipeTransport|_ProactorSocketTransport|BaseEventLoop)\\.__del__:pytest.PytestUnraisableExceptionWarning",
+ # PyDispatcher iterates its connection registry while a weakref callback
+ # may mutate it, which raises RuntimeError on PyPy, where several weakrefs
+ # can die within a single collection.
+ "ignore:.*<function connect\\.<locals>\\.remove:pytest.PytestUnraisableExceptionWarning",
+ # queuelib's test helpers (reused by tests/test_squeues.py) leave queue
+ # files open; the resulting warning surfaces at an arbitrary GC point.
+ "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning",
+ # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher.
+ "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",
+ # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.
+ "ignore:The async(hat|ore) module is deprecated:DeprecationWarning",
+ # CI runs without coverage pass --no-cov, which pytest-cov reports. Matched
+ # by message because some tox envs do not install pytest-cov, and pytest
+ # warns when a filter names a module it cannot import.
+ "ignore:Coverage disabled via --no-cov switch!",
]
[tool.ruff.lint]
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 9e93eae18..979704a87 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -19,7 +19,7 @@ from scrapy.utils.reactor import _asyncio_reactor_path
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
from tests.utils.bases.commands import TestProjectBase
-from tests.utils.cmdline import proc
+from tests.utils.cmdline import proc, stop_spawn
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -212,12 +212,7 @@ class TestInteractiveShell:
p.sendline(f"fetch('{mockserver.url('/')}')")
p.sendline("type(response)")
p.expect_exact("HtmlResponse")
- p.sendeof()
- p.wait() # type: ignore[no-untyped-call]
- if p.proc.stdin:
- p.proc.stdin.close()
- if p.proc.stdout:
- p.proc.stdout.close()
+ stop_spawn(p)
logfile.seek(0)
assert "Traceback" not in logfile.read().decode()
@@ -243,8 +238,7 @@ class TestInteractiveShell:
p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
- p.sendeof()
- p.wait() # type: ignore[no-untyped-call]
+ stop_spawn(p)
logfile.seek(0)
return logfile.read().decode()
@@ -269,8 +263,7 @@ class TestInteractiveShell:
# shell=python was honored, regardless of platform-specific prompts.
p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")
p.expect_exact("IPYMODULE False")
- p.sendeof()
- p.wait() # type: ignore[no-untyped-call]
+ stop_spawn(p)
logfile.seek(0)
assert "Traceback" not in logfile.read().decode()
@@ -299,8 +292,7 @@ class TestInteractiveShell:
p.expect_exact("Available Scrapy objects")
p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")
p.expect_exact("IPYMODULE True")
- p.sendeof()
- p.wait() # type: ignore[no-untyped-call]
+ stop_spawn(p)
logfile.seek(0)
assert "Traceback" not in logfile.read().decode()
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 800ab9397..cbb6f2c5b 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -14,6 +14,7 @@ from pexpect.popen_spawn import PopenSpawn
from scrapy.utils.asyncio import sleep
from tests.utils import get_script_run_env
+from tests.utils.cmdline import stop_spawn
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -226,11 +227,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.kill(sig)
p.expect_exact("shutting down gracefully")
p.expect_exact("Spider closed (shutdown)")
- p.wait() # type: ignore[no-untyped-call]
- if p.proc.stdin:
- p.proc.stdin.close()
- if p.proc.stdout:
- p.proc.stdout.close()
+ stop_spawn(p)
def test_shutdown_graceful(self) -> None:
self._test_shutdown_graceful()
@@ -247,11 +244,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
await sleep(0.01)
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
- p.wait() # type: ignore[no-untyped-call]
- if p.proc.stdin:
- p.proc.stdin.close()
- if p.proc.stdout:
- p.proc.stdout.close()
+ stop_spawn(p)
@coroutine_test
async def test_shutdown_forced(self) -> None:
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 4214fe680..46e603af7 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -219,16 +219,16 @@ class TestBlockingFeedStorage:
def test_default_temp_dir(self):
b = MyBlockingFeedStorage()
- storage_file = b.open(get_test_spider())
- storage_dir = Path(storage_file.name).parent
+ with b.open(get_test_spider()) as storage_file:
+ storage_dir = Path(storage_file.name).parent
assert str(storage_dir) == tempfile.gettempdir()
def test_temp_file(self, tmp_path):
b = MyBlockingFeedStorage()
spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
- storage_file = b.open(spider)
- storage_dir = Path(storage_file.name).parent
+ with b.open(spider) as storage_file:
+ storage_dir = Path(storage_file.name).parent
assert storage_dir == tmp_path
def test_invalid_folder(self, tmp_path):
diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py
index 095cb17a7..4020daaa1 100644
--- a/tests/utils/cmdline.py
+++ b/tests/utils/cmdline.py
@@ -11,6 +11,8 @@ from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
from pathlib import Path
+ from pexpect.popen_spawn import PopenSpawn
+
def call(*args: str, **popen_kwargs: Any) -> int:
args = (sys.executable, "-m", "scrapy.cmdline", *args)
@@ -41,6 +43,16 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:
return p.returncode, p.stdout, p.stderr
+def stop_spawn(p: PopenSpawn[str]) -> None:
+ p.sendeof()
+ p.wait() # type: ignore[no-untyped-call]
+ # PopenSpawn leaves the subprocess pipes open, which triggers
+ # ResourceWarning at an arbitrary garbage collection point.
+ for pipe in (p.proc.stdin, p.proc.stdout):
+ if pipe:
+ pipe.close()
+
+
def write_recording_editor(editor: Path) -> None:
"""Create an executable editor script that writes the path it is asked to
open (its last argument) into the file given as its first argument."""
diff --git a/tox.ini b/tox.ini
index c5e4ff7bf..bdad9b621 100644
--- a/tox.ini
+++ b/tox.ini
@@ -154,6 +154,9 @@ deps =
{[test-requirements]deps}
setenv =
_SCRAPY_MIN=true
+ # Pinned old Python and library versions trigger warnings that we cannot
+ # fix, so these envs do not turn warnings into errors.
+ PYTEST_ADDOPTS=-W default {env:PYTEST_ADDOPTS:}
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}