Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn about signal handler arguments the signal never sends (#8345)
Adrian46c3eda9293b
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.pyindex f9af3c58e..72c5ad091 100644--- a/scrapy/signalmanager.py+++ b/scrapy/signalmanager.py@@ -1,14 +1,17 @@from __future__ import annotations+import inspectimport warningsfrom typing import TYPE_CHECKING, Anyfrom pydispatch import dispatcherfrom twisted.internet.defer import Deferred+from scrapy import signalsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils import signal as _signalfrom scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.python import global_object_nameif TYPE_CHECKING:from collections.abc import Callable@@ -34,6 +37,7 @@ class SignalManager::param signal: the signal to connect to:type signal: object"""+ _warn_unknown_args(receiver, signal)kwargs.setdefault("sender", self.sender)dispatcher.connect(receiver, signal, **kwargs)@@ -123,3 +127,34 @@ class SignalManager:self.connect(handle, signal)await maybe_deferred_to_future(d)+++def _warn_unknown_args(receiver: Callable[..., Any], signal: object) -> None:+ sent = signals._signal_args.get(signal)+ if sent is None:+ return+ try:+ parameters = inspect.signature(receiver).parameters.values()+ except (TypeError, ValueError):+ return+ if any(p.kind is p.VAR_KEYWORD for p in parameters):+ return+ unknown = {+ p.name+ for p in parameters+ if p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)+ and p.name not in sent+ and p.name not in {"signal", "sender"}+ }+ if not unknown:+ return+ try:+ receiver_name = global_object_name(receiver)+ except AttributeError:+ receiver_name = repr(receiver)+ signal_name = next(k for k, v in vars(signals).items() if v is signal)+ warnings.warn(+ f"Signal handler {receiver_name} declares {', '.join(sorted(unknown))},"+ f" which scrapy.signals.{signal_name} does not send.",+ stacklevel=3,+ )diff --git a/scrapy/signals.py b/scrapy/signals.pyindex a440cfba4..b703419ac 100644--- a/scrapy/signals.py+++ b/scrapy/signals.py@@ -30,3 +30,35 @@ feed_exporter_closed = object()feed_slots_initialized = object()spider_state_loaded = object()spider_state_saving = object()++_signal_args: dict[object, frozenset[str]] = {+ engine_started: frozenset(),+ engine_stopped: frozenset(),+ scheduler_empty: frozenset(),+ spider_opened: frozenset({"spider"}),+ spider_idle: frozenset({"spider"}),+ spider_closed: frozenset({"spider", "reason", "error"}),+ spider_error: frozenset({"failure", "response", "spider"}),+ memusage_warning_reached: frozenset(),+ request_scheduled: frozenset({"request", "spider"}),+ request_dropped: frozenset({"request", "spider"}),+ request_reached_downloader: frozenset({"request", "spider"}),+ request_left_downloader: frozenset({"request", "spider"}),+ response_received: frozenset({"response", "request", "spider"}),+ response_downloaded: frozenset({"response", "request", "spider"}),+ headers_received: frozenset({"headers", "body_length", "request", "spider"}),+ bytes_received: frozenset({"data", "request", "spider"}),+ robots_parsed: frozenset({"robotparser", "request", "response"}),+ item_scraped: frozenset({"item", "response", "spider"}),+ item_dropped: frozenset({"item", "response", "spider", "exception"}),+ item_error: frozenset({"item", "response", "spider", "failure"}),+ feed_slot_closed: frozenset({"slot"}),+ feed_exporter_closed: frozenset(),+ feed_slots_initialized: frozenset({"slots"}),+ spider_state_loaded: frozenset({"state"}),+ spider_state_saving: frozenset({"state"}),+}+"""Arguments that each signal sends, used to catch handlers that declare an+argument their signal never sends. ``signal`` and ``sender`` are omitted+because every signal sends them.+"""diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.pyindex cf8cd489f..2f9138bc0 100644--- a/tests/test_signalmanager.py+++ b/tests/test_signalmanager.py@@ -1,5 +1,15 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++import pytest++from scrapy import signalsfrom scrapy.signalmanager import SignalManager+if TYPE_CHECKING:+ from collections.abc import Callable+def test_disconnect_all() -> None:signal = object()@@ -18,3 +28,42 @@ def test_disconnect_all() -> None:sm.disconnect_all(signal)sm.send_catch_log(signal)assert calls == [1] # handler no longer called after disconnect_all+++class TestUnknownArgumentWarning:+ def test_warns(self) -> None:+ def handler(spider: Any, cheese: Any = None, *, ham: Any = None) -> None:+ pass++ with pytest.warns(+ UserWarning,+ match=r"handler declares cheese, ham, which scrapy\.signals\.spider_opened does not send",+ ) as record:+ SignalManager().connect(handler, signals.spider_opened)+ assert record[0].filename == __file__++ def test_callable_instance(self) -> None:+ class Handler:+ def __call__(self, cheese: Any = None) -> None:+ pass++ with pytest.warns(UserWarning, match=r"Handler object at .* declares cheese"):+ SignalManager().connect(Handler(), signals.spider_opened)++ @pytest.mark.filterwarnings("error")+ @pytest.mark.parametrize(+ ("handler", "signal"),+ [+ (lambda spider, signal, sender: None, signals.spider_opened),+ (lambda cheese, **kwargs: None, signals.spider_opened),+ (lambda cheese: None, object()),+ (type, signals.spider_opened),+ ],+ )+ def test_no_warning(self, handler: Callable[..., Any], signal: object) -> None:+ SignalManager().connect(handler, signal)+++def test_every_signal_declares_its_arguments() -> None:+ defined = {v for v in vars(signals).values() if type(v) is object}+ assert defined == set(signals._signal_args)