Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't evaluate annotations when inspecting signatures (#7818)

Janit Rajkarnikar3fc7148c5ec7
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 4960dc27a..2924c81f9 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.python import _signature
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine
@@ -109,7 +110,7 @@ def _warn_spider_arg(
):
"""Decorator to warn if a ``spider`` argument is passed to a function."""
- sig = inspect.signature(func)
+ sig = _signature(func)
def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:
bound = sig.bind(*args, **kwargs)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 8a7517c1d..40fc05257 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool:
return all(c not in _BINARYCHARS for c in data)
+# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature()
+# can raise NameError for names imported only under TYPE_CHECKING. We only need
+# parameter names, kinds and defaults, so leave such annotations as ForwardRefs.
+if sys.version_info >= (3, 14):
+ from annotationlib import Format
+
+ def _signature(func: Callable[..., Any]) -> inspect.Signature:
+ return inspect.signature(func, annotation_format=Format.FORWARDREF)
+
+else:
+
+ def _signature(func: Callable[..., Any]) -> inspect.Signature:
+ return inspect.signature(func)
+
+
def get_func_args_dict(
func: Callable[..., Any], stripself: bool = False
) -> Mapping[str, inspect.Parameter]:
"""Return the argument dict of a callable object.
+ Annotations are not evaluated, so on Python 3.14 and later the ``annotation``
+ attribute of the returned parameters may be a ``ForwardRef`` instead of the
+ resolved type.
+
.. versionadded:: 2.14
"""
if not callable(func):
@@ -190,7 +209,7 @@ def get_func_args_dict(
args: Mapping[str, inspect.Parameter]
try:
- sig = inspect.signature(func)
+ sig = _signature(func)
except ValueError:
return {}
diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py
index 807294a57..4c29d2917 100644
--- a/tests/test_utils_decorators.py
+++ b/tests/test_utils_decorators.py
@@ -1,7 +1,8 @@
from __future__ import annotations
+import sys
import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred
@@ -12,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator
+ from collections.abc import AsyncGenerator, Callable
class TestDeprecated:
@@ -77,6 +78,31 @@ class TestWarnSpiderArg:
):
assert parse("response", spider="spider") == "response"
+ @pytest.mark.skipif(
+ sys.version_info < (3, 14),
+ reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)",
+ )
+ def test_sync_warns_with_unresolvable_annotations(self):
+ # dont_inherit=True, or the module's future import stringizes the annotations
+ namespace: dict[str, Any] = {}
+ exec( # pylint: disable=exec-used
+ compile(
+ "def parse(response: OnlyAtTypeCheckingTime,"
+ " spider: OnlyAtTypeCheckingTime | None = None): return response",
+ "<test>",
+ "exec",
+ dont_inherit=True,
+ ),
+ namespace,
+ )
+ parse_func: Callable[..., str] = namespace["parse"]
+ parse = _warn_spider_arg(parse_func)
+
+ with pytest.warns(
+ ScrapyDeprecationWarning, match=r"Passing a 'spider' argument"
+ ):
+ assert parse("response", spider="spider") == "response"
+
def test_sync_no_warning_without_spider_arg(self):
@_warn_spider_arg
def parse(response: str, spider: str | None = None) -> str:
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index c3b5dfc99..099b5ccc2 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -4,7 +4,7 @@ import functools
import operator
import platform
import sys
-from typing import TYPE_CHECKING, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
import pytest
@@ -190,6 +190,25 @@ def test_get_func_args():
]
+ sys.version_info < (3, 14),
+ reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)",
+)
+def test_get_func_args_unresolvable_annotations():
+ # dont_inherit=True, or the module's future import stringizes the annotations
+ namespace: dict[str, Any] = {}
+ exec( # pylint: disable=exec-used
+ compile(
+ "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass",
+ "<test>",
+ "exec",
+ dont_inherit=True,
+ ),
+ namespace,
+ )
+ assert get_func_args(namespace["f"]) == ["a", "b"]
+
+
@pytest.mark.parametrize(
("value", "expected"),
[