Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Warn about signal handler arguments the signal never sends (#8345)

Adrian46c3eda9293b
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index f9af3c58e..72c5ad091 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,14 +1,17 @@
from __future__ import annotations
+import inspect
import warnings
from typing import TYPE_CHECKING, Any
from pydispatch import dispatcher
from twisted.internet.defer import Deferred
+from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils import signal as _signal
from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Callable
@@ -34,6 +37,7 @@ class SignalManager:
:param signal: the signal to connect to
:type signal: object
"""
+ _warn_unknown_args(receiver, signal)
kwargs.setdefault("sender", self.sender)
dispatcher.connect(receiver, signal, **kwargs)
@@ -123,3 +127,34 @@ class SignalManager:
self.connect(handle, signal)
await maybe_deferred_to_future(d)
+
+
+def _warn_unknown_args(receiver: Callable[..., Any], signal: object) -> None:
+ sent = signals._signal_args.get(signal)
+ if sent is None:
+ return
+ try:
+ parameters = inspect.signature(receiver).parameters.values()
+ except (TypeError, ValueError):
+ return
+ if any(p.kind is p.VAR_KEYWORD for p in parameters):
+ return
+ unknown = {
+ p.name
+ for p in parameters
+ if p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)
+ and p.name not in sent
+ and p.name not in {"signal", "sender"}
+ }
+ if not unknown:
+ return
+ try:
+ receiver_name = global_object_name(receiver)
+ except AttributeError:
+ receiver_name = repr(receiver)
+ signal_name = next(k for k, v in vars(signals).items() if v is signal)
+ warnings.warn(
+ f"Signal handler {receiver_name} declares {', '.join(sorted(unknown))},"
+ f" which scrapy.signals.{signal_name} does not send.",
+ stacklevel=3,
+ )
diff --git a/scrapy/signals.py b/scrapy/signals.py
index a440cfba4..b703419ac 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -30,3 +30,35 @@ feed_exporter_closed = object()
feed_slots_initialized = object()
spider_state_loaded = object()
spider_state_saving = object()
+
+_signal_args: dict[object, frozenset[str]] = {
+ engine_started: frozenset(),
+ engine_stopped: frozenset(),
+ scheduler_empty: frozenset(),
+ spider_opened: frozenset({"spider"}),
+ spider_idle: frozenset({"spider"}),
+ spider_closed: frozenset({"spider", "reason", "error"}),
+ spider_error: frozenset({"failure", "response", "spider"}),
+ memusage_warning_reached: frozenset(),
+ request_scheduled: frozenset({"request", "spider"}),
+ request_dropped: frozenset({"request", "spider"}),
+ request_reached_downloader: frozenset({"request", "spider"}),
+ request_left_downloader: frozenset({"request", "spider"}),
+ response_received: frozenset({"response", "request", "spider"}),
+ response_downloaded: frozenset({"response", "request", "spider"}),
+ headers_received: frozenset({"headers", "body_length", "request", "spider"}),
+ bytes_received: frozenset({"data", "request", "spider"}),
+ robots_parsed: frozenset({"robotparser", "request", "response"}),
+ item_scraped: frozenset({"item", "response", "spider"}),
+ item_dropped: frozenset({"item", "response", "spider", "exception"}),
+ item_error: frozenset({"item", "response", "spider", "failure"}),
+ feed_slot_closed: frozenset({"slot"}),
+ feed_exporter_closed: frozenset(),
+ feed_slots_initialized: frozenset({"slots"}),
+ spider_state_loaded: frozenset({"state"}),
+ spider_state_saving: frozenset({"state"}),
+}
+"""Arguments that each signal sends, used to catch handlers that declare an
+argument their signal never sends. ``signal`` and ``sender`` are omitted
+because every signal sends them.
+"""
diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.py
index cf8cd489f..2f9138bc0 100644
--- a/tests/test_signalmanager.py
+++ b/tests/test_signalmanager.py
@@ -1,5 +1,15 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy import signals
from scrapy.signalmanager import SignalManager
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
def test_disconnect_all() -> None:
signal = object()
@@ -18,3 +28,42 @@ def test_disconnect_all() -> None:
sm.disconnect_all(signal)
sm.send_catch_log(signal)
assert calls == [1] # handler no longer called after disconnect_all
+
+
+class TestUnknownArgumentWarning:
+ def test_warns(self) -> None:
+ def handler(spider: Any, cheese: Any = None, *, ham: Any = None) -> None:
+ pass
+
+ with pytest.warns(
+ UserWarning,
+ match=r"handler declares cheese, ham, which scrapy\.signals\.spider_opened does not send",
+ ) as record:
+ SignalManager().connect(handler, signals.spider_opened)
+ assert record[0].filename == __file__
+
+ def test_callable_instance(self) -> None:
+ class Handler:
+ def __call__(self, cheese: Any = None) -> None:
+ pass
+
+ with pytest.warns(UserWarning, match=r"Handler object at .* declares cheese"):
+ SignalManager().connect(Handler(), signals.spider_opened)
+
+ @pytest.mark.filterwarnings("error")
+ @pytest.mark.parametrize(
+ ("handler", "signal"),
+ [
+ (lambda spider, signal, sender: None, signals.spider_opened),
+ (lambda cheese, **kwargs: None, signals.spider_opened),
+ (lambda cheese: None, object()),
+ (type, signals.spider_opened),
+ ],
+ )
+ def test_no_warning(self, handler: Callable[..., Any], signal: object) -> None:
+ SignalManager().connect(handler, signal)
+
+
+def test_every_signal_declares_its_arguments() -> None:
+ defined = {v for v in vars(signals).values() if type(v) is object}
+ assert defined == set(signals._signal_args)