Scrapy, a fast high-level web crawling & scraping framework for Python.
Next refactoring pass of test_utils_*. (#7797)
Andrey Rakhmatulline7d8b34e73ef
diff --git a/pyproject.toml b/pyproject.tomlindex 9b1e64121..576a42e5c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -178,13 +178,6 @@ module = ["tests.test_squeues","tests.test_squeues_request","tests.test_stats",- "tests.test_utils_datatypes",- "tests.test_utils_decorators",- "tests.test_utils_defer",- "tests.test_utils_deprecate",- "tests.test_utils_misc.test_return_with_argument_inside_generator",- "tests.test_utils_python",- "tests.test_utils_request","tests.utils.bases.http_request","tests.utils.bases.http_response","tests.utils.bases.spider",diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 07ff4a74e..29b3e3c0f 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):headers = {"local filename": protocol.filename or b"", "size": protocol.size}body = protocol.filename or protocol.body.read()respcls = responsetypes.from_args(url=request.url, body=body)- # hints for Headers-related types may need to be fixed to not use AnyStr- return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]+ return respcls(url=request.url, status=200, body=body, headers=headers)diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 34d4ec6f2..b55ef6191 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom collections.abc import Mapping-from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast+from typing import TYPE_CHECKING, Any, TypeAlias, castfrom w3lib.http import headers_dict_to_raw@@ -25,14 +25,20 @@ class Headers(CaselessDict):def __init__(self,- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ seq: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,encoding: str = "utf-8",):self.encoding: str = encodingsuper().__init__(seq)def update( # type: ignore[override]- self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]+ self,+ seq: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]],) -> None:seq = seq.items() if isinstance(seq, Mapping) else seqiseq: dict[bytes, list[bytes]] = {}@@ -40,7 +46,7 @@ class Headers(CaselessDict):iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))super().update(iseq)- def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]+ def normkey(self, key: str | bytes) -> bytes:"""Normalize key to bytes"""return self._tobytes(key.title())@@ -67,19 +73,19 @@ class Headers(CaselessDict):return str(x).encode(self.encoding)raise TypeError(f"Unsupported value type: {type(x)}")- def __getitem__(self, key: AnyStr) -> bytes | None:+ def __getitem__(self, key: str | bytes) -> bytes | None:try:return cast("list[bytes]", super().__getitem__(key))[-1]except IndexError:return None- def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:+ def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:try:return cast("list[bytes]", super().get(key, def_val))[-1]except IndexError:return None- def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:+ def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:try:return cast("list[bytes]", super().__getitem__(key))except KeyError:@@ -87,15 +93,15 @@ class Headers(CaselessDict):return self.normvalue(def_val)return []- def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:+ def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:self[key] = list_def setlistdefault(- self, key: AnyStr, default_list: Iterable[_RawValue] = ()+ self, key: str | bytes, default_list: Iterable[_RawValue] = ()) -> Any:return self.setdefault(key, default_list)- def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:+ def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:lst = self.getlist(key)lst.extend(self.normvalue(value))self[key] = lstdiff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 73c2e7dd4..57517a65b 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -11,7 +11,6 @@ import inspectfrom typing import (TYPE_CHECKING,Any,- AnyStr,Concatenate,NoReturn,TypeAlias,@@ -125,7 +124,10 @@ class Request(object_ref):url: str,callback: CallbackT | None = None,method: str = "GET",- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes | str | None = None,cookies: CookiesT | None = None,meta: dict[str, Any] | None = None,@@ -310,7 +312,11 @@ class Request(object_ref):@headers.setterdef headers(- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None+ self,+ value: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None,) -> None:if isinstance(value, Headers):self._headers = valuediff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 09b1c8b32..f1db11488 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rstfrom __future__ import annotations-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload+from typing import TYPE_CHECKING, Any, TypeVar, overloadfrom urllib.parse import urljoinfrom scrapy.exceptions import NotSupported@@ -72,7 +72,10 @@ class Response(object_ref):self,url: str,status: int = 200,- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes = b"",flags: list[str] | None = None,request: Request | None = None,@@ -145,7 +148,11 @@ class Response(object_ref):@headers.setterdef headers(- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None+ self,+ value: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None,) -> None:if isinstance(value, Headers):self._headers = value@@ -222,7 +229,10 @@ class Response(object_ref):url: str | Link,callback: CallbackT | None = None,method: str = "GET",- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes | str | None = None,cookies: CookiesT | None = None,meta: dict[str, Any] | None = None,@@ -272,7 +282,10 @@ class Response(object_ref):urls: Iterable[str | Link],callback: CallbackT | None = None,method: str = "GET",- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes | str | None = None,cookies: CookiesT | None = None,meta: dict[str, Any] | None = None,diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 6876e35e8..d01e23e47 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -9,7 +9,7 @@ from __future__ import annotationsimport jsonfrom contextlib import suppress-from typing import TYPE_CHECKING, Any, AnyStr, cast+from typing import TYPE_CHECKING, Any, castfrom urllib.parse import urljoinimport parsel@@ -170,7 +170,10 @@ class TextResponse(Response):url: str | Link | parsel.Selector,callback: CallbackT | None = None,method: str = "GET",- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes | str | None = None,cookies: CookiesT | None = None,meta: dict[str, Any] | None = None,@@ -223,7 +226,10 @@ class TextResponse(Response):urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,callback: CallbackT | None = None,method: str = "GET",- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ headers: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,body: bytes | str | None = None,cookies: CookiesT | None = None,meta: dict[str, Any] | None = None,diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex e761a2474..9a945c61c 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -11,12 +11,12 @@ import warningsimport weakreffrom collections import OrderedDictfrom collections.abc import Mapping-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast+from typing import TYPE_CHECKING, Any, TypeVar, castfrom scrapy.exceptions import ScrapyDeprecationWarningif TYPE_CHECKING:- from collections.abc import Iterable, Sequence+ from collections.abc import Container, Iterable# typing.Self requires Python 3.11from typing_extensions import Self@@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg]def __init__(self,- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,+ seq: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]]+ | None = None,):super().__init__()if seq:self.update(seq)- def __getitem__(self, key: AnyStr) -> Any:+ def __getitem__(self, key: str | bytes) -> Any:return dict.__getitem__(self, self.normkey(key))- def __setitem__(self, key: AnyStr, value: Any) -> None:+ def __setitem__(self, key: str | bytes, value: Any) -> None:dict.__setitem__(self, self.normkey(key), self.normvalue(value))- def __delitem__(self, key: AnyStr) -> None:+ def __delitem__(self, key: str | bytes) -> None:dict.__delitem__(self, self.normkey(key))- def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]return dict.__contains__(self, self.normkey(key))has_key = __contains__@@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg]copy = __copy__- def normkey(self, key: AnyStr) -> AnyStr:+ def normkey(self, key: str | bytes) -> str | bytes:"""Method to normalize dictionary key access"""return key.lower()@@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg]"""Method to normalize values prior to be set"""return value- def get(self, key: AnyStr, def_val: Any = None) -> Any:+ def get(self, key: str | bytes, def_val: Any = None) -> Any:return dict.get(self, self.normkey(key), self.normvalue(def_val))- def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:+ def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))# doesn't fully implement MutableMapping.update()- def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override]+ def update( # type: ignore[override]+ self,+ seq: Mapping[str, Any]+ | Mapping[bytes, Any]+ | Iterable[tuple[str | bytes, Any]],+ ) -> None:seq = seq.items() if isinstance(seq, Mapping) else seqiseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)super().update(iseq)@classmethod- def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override]- return cls((k, value) for k in keys) # type: ignore[misc]+ def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]+ return cls((k, value) for k in keys)- def pop(self, key: AnyStr, *args: Any) -> Any:+ def pop(self, key: str | bytes, *args: Any) -> Any:return dict.pop(self, self.normkey(key), *args)@@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):class SequenceExclude:"""Object to test if an item is NOT within some sequence."""- def __init__(self, seq: Sequence[Any]):- self.seq: Sequence[Any] = seq+ def __init__(self, seq: Container[Any]):+ self.seq: Container[Any] = seqdef __contains__(self, item: Any) -> bool:return item not in self.seqdiff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex a5bb6fa24..4960dc27a 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -19,9 +19,19 @@ _T = TypeVar("_T")_P = ParamSpec("_P")+@overload+def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...+++@overload+def deprecated(+ use_instead: str | None = None,+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...++def deprecated(- use_instead: Any = None,-) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:+ use_instead: Callable[_P, _T] | str | None = None,+) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:"""This is a decorator which can be used to mark functionsas deprecated. It will result in a warning being emittedwhen the function is used."""@@ -38,8 +48,9 @@ def deprecated(return wrappedif callable(use_instead):- deco = deco(use_instead)+ func = use_insteaduse_instead = None+ return deco(func)return decodiff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.pyindex fc4e1c487..1d36a66fc 100644--- a/tests/test_utils_asyncgen.py+++ b/tests/test_utils_asyncgen.py@@ -1,16 +1,18 @@+from __future__ import annotations+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom tests.utils.decorators import coroutine_test-class TestAsyncgenUtils:- @coroutine_test- async def test_as_async_generator(self):- ag = as_async_generator(range(42))- results = [i async for i in ag]- assert results == list(range(42))+@coroutine_test+async def test_as_async_generator():+ ag = as_async_generator(range(42))+ results = [i async for i in ag]+ assert results == list(range(42))+- @coroutine_test- async def test_collect_asyncgen(self):- ag = as_async_generator(range(42))- results = await collect_asyncgen(ag)- assert results == list(range(42))+@coroutine_test+async def test_collect_asyncgen():+ ag = as_async_generator(range(42))+ results = await collect_asyncgen(ag)+ assert results == list(range(42))diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 7528dc51a..9b7eb22fa 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -20,11 +20,10 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator-class TestAsyncio:- @coroutine_test- async def test_is_asyncio_available(self, reactor_pytest: str) -> None:- # the result should depend only on the pytest --reactor argument- assert is_asyncio_available() == (reactor_pytest != "default")+@coroutine_test+async def test_is_asyncio_available(reactor_pytest: str) -> None:+ # the result should depend only on the pytest --reactor argument+ assert is_asyncio_available() == (reactor_pytest != "default")@pytest.mark.only_asynciodiff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex fce9fc984..6b30744bb 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -1,3 +1,5 @@+from __future__ import annotations+import warningsfrom typing import Anydiff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex f43d20e69..af203ca61 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -1,7 +1,9 @@+from __future__ import annotations+import copyfrom abc import ABC, abstractmethodfrom collections.abc import Iterator, Mapping, MutableMapping-from typing import Any+from typing import Any, Generic, TypeVarimport pytest@@ -16,11 +18,13 @@ from scrapy.utils.datatypes import ()from scrapy.utils.python import garbage_collect+_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")+-class TestCaseInsensitiveDictBase(ABC):+class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):@property@abstractmethod- def dict_class(self) -> type[MutableMapping[str, Any]]:+ def dict_class(self) -> type[_DictT]:raise NotImplementedErrordef test_init_dict(self):@@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):assert d["black"] == 3def test_init_mapping(self):- class MyMapping(Mapping):- def __init__(self, **kwargs):+ class MyMapping(Mapping[str, int]):+ def __init__(self, **kwargs: int) -> None:self._d = kwargs- def __getitem__(self, key):+ def __getitem__(self, key: str) -> int:return self._d[key]- def __iter__(self):+ def __iter__(self) -> Iterator[str]:return iter(self._d)- def __len__(self):+ def __len__(self) -> int:return len(self._d)seq = MyMapping(red=1, black=3)@@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):assert d["black"] == 3def test_init_mutable_mapping(self):- class MyMutableMapping(MutableMapping):- def __init__(self, **kwargs):+ class MyMutableMapping(MutableMapping[str, int]):+ def __init__(self, **kwargs: int) -> None:self._d = kwargs- def __getitem__(self, key):+ def __getitem__(self, key: str) -> int:return self._d[key]- def __setitem__(self, key, value):+ def __setitem__(self, key: str, value: int) -> None:self._d[key] = value- def __delitem__(self, key):+ def __delitem__(self, key: str) -> None:del self._d[key]- def __iter__(self):+ def __iter__(self) -> Iterator[str]:return iter(self._d)- def __len__(self):+ def __len__(self) -> int:return len(self._d)seq = MyMutableMapping(red=1, black=3)@@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):d.pop("A")def test_normkey(self):- class MyDict(self.dict_class):+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]def _normkey(self, key):return key.title()@@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):assert list(d.keys()) == ["Key-One"]def test_normvalue(self):- class MyDict(self.dict_class):+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]def _normvalue(self, value):if value is not None:return value + 1@@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC):assert dict(h1) == {"header1": "value1", "header2": "value2"}-class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):- dict_class = CaseInsensitiveDict # type: ignore[assignment]+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):+ dict_class = CaseInsensitiveDictdef test_repr(self):d1 = self.dict_class({"foo": "bar"})@@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):assert list(iterkeys) == ["AsDf", "FoO"]def test_copy_keeps_values(self):- class MyDict(self.dict_class):+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]def _normvalue(self, value):return value + 1@@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-class TestCaselessDict(TestCaseInsensitiveDictBase):+class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):dict_class = CaselessDictdef test_deprecation_message(self):@@ -319,7 +323,7 @@ class TestSequenceExclude:class TestLocalCache:def test_cache_with_limit(self):- cache = LocalCache(limit=2)+ cache: LocalCache[str, int] = LocalCache(limit=2)cache["a"] = 1cache["b"] = 2cache["c"] = 3@@ -332,7 +336,7 @@ class TestLocalCache:def test_cache_without_limit(self):maximum = 10**4- cache = LocalCache()+ cache: LocalCache[str, int] = LocalCache()for x in range(maximum):cache[str(x)] = xassert len(cache) == maximum@@ -341,7 +345,7 @@ class TestLocalCache:assert cache[str(x)] == xdef test_cache_with_zero_limit(self):- cache = LocalCache(limit=0)+ cache: LocalCache[str, int] = LocalCache(limit=0)cache["a"] = 1cache["b"] = 2cache["c"] = 3@@ -353,7 +357,9 @@ class TestLocalCache:class TestLocalWeakReferencedCache:def test_cache_with_limit(self):- cache = LocalWeakReferencedCache(limit=2)+ cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(+ limit=2+ )r1 = Request("https://example.org")r2 = Request("https://example.com")r3 = Request("https://example.net")@@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache:assert len(cache) == 1def test_cache_non_weak_referenceable_objects(self):- cache = LocalWeakReferencedCache()+ cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()k1 = Nonek2 = 1k3 = [1, 2, 3]@@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache:def test_cache_without_limit(self):maximum = 10**4- cache = LocalWeakReferencedCache()+ cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()refs = []for x in range(maximum):refs.append(Request(f"https://example.org/{x}"))diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.pyindex 9743e1a50..807294a57 100644--- a/tests/test_utils_decorators.py+++ b/tests/test_utils_decorators.py@@ -1,6 +1,7 @@from __future__ import annotationsimport warnings+from typing import TYPE_CHECKINGimport pytestfrom twisted.internet.defer import Deferred@@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthreadfrom scrapy.utils.defer import maybe_deferred_to_futurefrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import AsyncGenerator+class TestDeprecated:def test_warns_and_still_calls(self):@deprecated()- def add(a, b):+ def add(a: int, b: int) -> int:return a + bwith pytest.warns(@@ -26,7 +30,7 @@ class TestDeprecated:def test_use_instead_in_message(self):@deprecated(use_instead="other_function")- def old():+ def old() -> None:return Nonewith pytest.warns(@@ -37,7 +41,7 @@ class TestDeprecated:def test_applied_without_parentheses(self):@deprecated- def square(x):+ def square(x: int) -> int:return x * xwith pytest.warns(@@ -65,7 +69,7 @@ class TestInthread:class TestWarnSpiderArg:def test_sync_warns_with_spider_arg(self):@_warn_spider_arg- def parse(response, spider=None):+ def parse(response: str, spider: str | None = None) -> str:return responsewith pytest.warns(@@ -75,7 +79,7 @@ class TestWarnSpiderArg:def test_sync_no_warning_without_spider_arg(self):@_warn_spider_arg- def parse(response, spider=None):+ def parse(response: str, spider: str | None = None) -> str:return responsewith warnings.catch_warnings():@@ -85,7 +89,7 @@ class TestWarnSpiderArg:@coroutine_testasync def test_async_warns_with_spider_arg(self):@_warn_spider_arg- async def parse(response, spider=None):+ async def parse(response: str, spider: str | None = None) -> str:return responsewith pytest.warns(@@ -96,7 +100,9 @@ class TestWarnSpiderArg:@coroutine_testasync def test_asyncgen_warns_with_spider_arg(self):@_warn_spider_arg- async def parse(response, spider=None):+ async def parse(+ response: str, spider: str | None = None+ ) -> AsyncGenerator[str]:yield responsewith pytest.warns(diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 978c24f5a..175a4fe03 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_testif TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Generator+ from twisted.python.failure import Failure+@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")@@ -70,7 +72,7 @@ class TestIterErrback:def itergood() -> Generator[int, None, None]:yield from range(10)- errors = []+ errors: list[Failure] = []out = list(iter_errback(itergood(), errors.append))assert out == list(range(10))assert not errors@@ -82,7 +84,7 @@ class TestIterErrback:1 / 0yield x- errors = []+ errors: list[Failure] = []out = list(iter_errback(iterbad(), errors.append))assert out == [0, 1, 2, 3, 4]assert len(errors) == 1@@ -96,7 +98,7 @@ class TestAiterErrback:for x in range(10):yield x- errors = []+ errors: list[Failure] = []out = await collect_asyncgen(aiter_errback(itergood(), errors.append))assert out == list(range(10))assert not errors@@ -109,7 +111,7 @@ class TestAiterErrback:1 / 0yield x- errors = []+ errors: list[Failure] = []out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))assert out == [0, 1, 2, 3, 4]assert len(errors) == 1@@ -202,7 +204,7 @@ class TestParallelAsync:for length in [20, 50, 100]:parallel_count = [0]max_parallel_count = [0]- results = []+ results: list[int] = []ait = self.get_async_iterable(length)dl = parallel_async(ait,@@ -222,7 +224,7 @@ class TestParallelAsync:for length in [20, 50, 100]:parallel_count = [0]max_parallel_count = [0]- results = []+ results: list[int] = []ait = self.get_async_iterable_with_delays(length)dl = parallel_async(ait,@@ -240,7 +242,7 @@ class TestParallelAsync:class TestDeferredFromCoro:def test_deferred(self):- d = Deferred()+ d: Deferred[None] = Deferred()result = deferred_from_coro(d)assert isinstance(result, Deferred)assert result is d@@ -274,7 +276,7 @@ class TestDeferredFromCoro:@pytest.mark.only_asyncio@inline_callbacks_testdef test_future(self):- future = Future()+ future: Future[int] = Future()result = deferred_from_coro(future)assert isinstance(result, Deferred)future.set_result(42)@@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:class TestDeferredToFuture:@coroutine_testasync def test_deferred(self):- d = Deferred()+ d: Deferred[int] = Deferred()result = deferred_to_future(d)assert isinstance(result, Future)d.callback(42)@@ -359,7 +361,7 @@ class TestDeferredToFuture:class TestMaybeDeferredToFutureAsyncio:@coroutine_testasync def test_deferred(self):- d = Deferred()+ d: Deferred[int] = Deferred()result = maybe_deferred_to_future(d)assert isinstance(result, Future)d.callback(42)@@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:class TestMaybeDeferredToFutureNotAsyncio:@coroutine_testasync def test_deferred(self):- d = Deferred()+ d: Deferred[int] = Deferred()result = maybe_deferred_to_future(d)assert isinstance(result, Deferred)assert result is ddiff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 0706fec99..4c8585916 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -1,3 +1,5 @@+from __future__ import annotations+import inspectimport warningsfrom unittest import mock@@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:)with pytest.warns(MyWarning, match=msg) as w:- class UserClass(Deprecated):+ class UserClass(Deprecated): # type: ignore[misc, valid-type]passassert w[0].lineno == inspect.getsourcelines(UserClass)[1]@@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",):- class UserClass(Deprecated):+ class UserClass(Deprecated): # type: ignore[misc, valid-type]passwith pytest.warns(@@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:match="UserClass inherits from deprecated class",):- class UserClass(Deprecated):+ class UserClass(Deprecated): # type: ignore[misc, valid-type]passwith warnings.catch_warnings():@@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:match="UserClass inherits from deprecated class",):- class UserClass(Deprecated):+ class UserClass(Deprecated): # type: ignore[misc, valid-type]passwith warnings.catch_warnings():warnings.simplefilter("error", MyWarning)- class FooClass(Deprecated):+ class FooClass(Deprecated): # type: ignore[misc, valid-type]pass- class BarClass(Deprecated):+ class BarClass(Deprecated): # type: ignore[misc, valid-type]passdef test_warning_on_instance(self):@@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:"Deprecated", NewName, warn_category=MyWarning)- with pytest.warns(MyWarning) as w:- _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())-- w = [x for x in w if x.category is MyWarning]+ with pytest.warns(+ MyWarning,+ match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "+ r"instantiate tests\.test_utils_deprecate\.NewName instead\.",+ ) as w:+ _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]assert len(w) == 1- assert (- str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "- "instantiate tests.test_utils_deprecate.NewName instead."- )assert w[0].lineno == lineno# ignore subclassing warningswith warnings.catch_warnings():warnings.simplefilter("ignore", MyWarning)- class UserClass(Deprecated):+ class UserClass(Deprecated): # type: ignore[misc, valid-type]passwith warnings.catch_warnings():@@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",):- class UserClass2(Deprecated):+ class UserClass2(Deprecated): # type: ignore[misc, valid-type]passdef test_issubclass(self):@@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:class UpdatedUserClass1a(NewName):pass- class OutdatedUserClass1(DeprecatedName):+ class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]pass- class OutdatedUserClass1a(DeprecatedName):+ class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]passclass UnrelatedClass:@@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)with pytest.raises(TypeError):- issubclass(object(), DeprecatedName)+ issubclass(object(), DeprecatedName) # type: ignore[arg-type]def test_isinstance(self):with warnings.catch_warnings():@@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:class UpdatedUserClass2a(NewName):pass- class OutdatedUserClass2(DeprecatedName):+ class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]pass- class OutdatedUserClass2a(DeprecatedName):+ class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]passclass UnrelatedClass:@@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:warnings.simplefilter("ignore", ScrapyDeprecationWarning)Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})- assert Deprecated.foo == "bar"+ assert Deprecated.foo == "bar" # type: ignore[attr-defined]def test_deprecate_a_class_with_custom_metaclass(self):Meta1 = type("Meta1", (type,), {})@@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",):- class UserClass(AlsoDeprecated):+ class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]passdef test_inspect_stack(self):diff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex 9f9e24957..a87816c20 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -31,13 +31,13 @@ plain_string = "{'a': 1}"@mock.patch("sys.platform", "linux")@mock.patch("sys.stdout.isatty")-def test_pformat(isatty):+def test_pformat(isatty: mock.Mock) -> None:isatty.return_value = Trueassert pformat(value) in colorized_strings@mock.patch("sys.stdout.isatty")-def test_pformat_dont_colorize(isatty):+def test_pformat_dont_colorize(isatty: mock.Mock) -> None:isatty.return_value = Trueassert pformat(value, colorize=False) == plain_string@@ -49,7 +49,7 @@ def test_pformat_not_tty():@mock.patch("sys.platform", "win32")@mock.patch("platform.version")@mock.patch("sys.stdout.isatty")-def test_pformat_old_windows(isatty, version):+def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:isatty.return_value = Trueversion.return_value = "10.0.14392"assert pformat(value) in colorized_strings@@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")@mock.patch("platform.version")@mock.patch("sys.stdout.isatty")-def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):+def test_pformat_windows_no_terminal_processing(+ isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock+) -> None:isatty.return_value = Trueversion.return_value = "10.0.14393"terminal_processing.return_value = False@@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")@mock.patch("platform.version")@mock.patch("sys.stdout.isatty")-def test_pformat_windows(isatty, version, terminal_processing):+def test_pformat_windows(+ isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock+) -> None:isatty.return_value = Trueversion.return_value = "10.0.14393"terminal_processing.return_value = True@@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):@mock.patch("sys.platform", "linux")@mock.patch("sys.stdout.isatty")-def test_pformat_no_pygments(isatty):+def test_pformat_no_pygments(isatty: mock.Mock) -> None:isatty.return_value = Truereal_import = builtins.__import__diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.pyindex 06fdf9cba..75f8be6f6 100644--- a/tests/test_utils_gz.py+++ b/tests/test_utils_gz.py@@ -1,3 +1,5 @@+from __future__ import annotations+from gzip import BadGzipFilefrom pathlib import Pathdiff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.pyindex 0eb330461..610c463ec 100644--- a/tests/test_utils_httpobj.py+++ b/tests/test_utils_httpobj.py@@ -1,3 +1,5 @@+from __future__ import annotations+from urllib.parse import urlparsefrom scrapy.http import Requestdiff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex ab6965ed5..0775ec608 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -20,150 +20,157 @@ from scrapy.utils.misc import ()-class TestUtilsMisc:- def test_load_object_class(self):- obj = load_object(Field)- assert obj is Field- obj = load_object("scrapy.item.Field")- assert obj is Field-- def test_load_object_function(self):- obj = load_object(load_object)- assert obj is load_object- obj = load_object("scrapy.utils.misc.load_object")- assert obj is load_object-- def test_load_object_exceptions(self):- with pytest.raises(ImportError):- load_object("nomodule999.mod.function")- with pytest.raises(NameError):- load_object("scrapy.utils.misc.load_object999")- with pytest.raises(TypeError):- load_object({}) # type: ignore[arg-type]-- def test_walk_modules(self):- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")+def test_load_object_class() -> None:+ obj = load_object(Field)+ assert obj is Field+ obj = load_object("scrapy.item.Field")+ assert obj is Field+++def test_load_object_function() -> None:+ obj = load_object(load_object)+ assert obj is load_object+ obj = load_object("scrapy.utils.misc.load_object")+ assert obj is load_object+++def test_load_object_exceptions() -> None:+ with pytest.raises(ImportError):+ load_object("nomodule999.mod.function")+ with pytest.raises(NameError):+ load_object("scrapy.utils.misc.load_object999")+ with pytest.raises(TypeError):+ load_object({}) # type: ignore[arg-type]+++def test_walk_modules() -> None:+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")+ expected = [+ "tests.test_utils_misc.test_walk_modules",+ "tests.test_utils_misc.test_walk_modules.mod",+ "tests.test_utils_misc.test_walk_modules.mod.mod0",+ "tests.test_utils_misc.test_walk_modules.mod1",+ ]+ assert {m.__name__ for m in mods} == set(expected)++ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")+ expected = [+ "tests.test_utils_misc.test_walk_modules.mod",+ "tests.test_utils_misc.test_walk_modules.mod.mod0",+ ]+ assert {m.__name__ for m in mods} == set(expected)++ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")+ expected = [+ "tests.test_utils_misc.test_walk_modules.mod1",+ ]+ assert {m.__name__ for m in mods} == set(expected)++ with pytest.raises(ImportError):+ for _ in walk_modules_iter("nomodule999"):+ pass+ with (+ pytest.raises(ImportError),+ pytest.warns(+ ScrapyDeprecationWarning,+ match="The scrapy.utils.misc.walk_modules function is deprecated and will be "+ "removed in a future version of Scrapy. "+ "Use scrapy.utils.misc.walk_modules_iter instead.",+ ),+ ):+ walk_modules("nomodule999")+++def test_walk_modules_egg() -> None:+ egg = str(Path(__file__).parent / "test.egg")+ sys.path.append(egg)+ try:+ mods = walk_modules_iter("testegg")expected = [- "tests.test_utils_misc.test_walk_modules",- "tests.test_utils_misc.test_walk_modules.mod",- "tests.test_utils_misc.test_walk_modules.mod.mod0",- "tests.test_utils_misc.test_walk_modules.mod1",+ "testegg.spiders",+ "testegg.spiders.a",+ "testegg.spiders.b",+ "testegg",]assert {m.__name__ for m in mods} == set(expected)-- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")- expected = [- "tests.test_utils_misc.test_walk_modules.mod",- "tests.test_utils_misc.test_walk_modules.mod.mod0",- ]- assert {m.__name__ for m in mods} == set(expected)-- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")- expected = [- "tests.test_utils_misc.test_walk_modules.mod1",- ]- assert {m.__name__ for m in mods} == set(expected)-- with pytest.raises(ImportError):- for _ in walk_modules_iter("nomodule999"):- pass- with (- pytest.raises(ImportError),- pytest.warns(- ScrapyDeprecationWarning,- match="The scrapy.utils.misc.walk_modules function is deprecated and will be "- "removed in a future version of Scrapy. "- "Use scrapy.utils.misc.walk_modules_iter instead.",- ),- ):- walk_modules("nomodule999")-- def test_walk_modules_egg(self):- egg = str(Path(__file__).parent / "test.egg")- sys.path.append(egg)- try:- mods = walk_modules_iter("testegg")- expected = [- "testegg.spiders",- "testegg.spiders.a",- "testegg.spiders.b",- "testegg",- ]- assert {m.__name__ for m in mods} == set(expected)- finally:- sys.path.remove(egg)-- def test_arg_to_iter(self):- class TestItem(Item):- name = Field()-- assert hasattr(arg_to_iter(None), "__iter__")- assert hasattr(arg_to_iter(100), "__iter__")- assert hasattr(arg_to_iter("lala"), "__iter__")- assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")- assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")-- assert not list(arg_to_iter(None))- assert list(arg_to_iter("lala")) == ["lala"]- assert list(arg_to_iter(100)) == [100]- assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]- assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]- assert list(arg_to_iter({"a": 1})) == [{"a": 1}]- assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]-- def test_build_from_crawler(self):- crawler = mock.MagicMock(spec_set=["settings"])- args = (True, 100.0)- kwargs = {"key": "val"}-- def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:- build_from_crawler(mock, crawler, *args, **kwargs)- if hasattr(mock, "from_crawler"):- mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)- assert mock.call_count == 0- else:- mock.assert_called_once_with(*args, **kwargs)-- # Check usage of correct constructor using 2 mocks:- # 1. with no alternative constructors- # 2. with from_crawler() constructor- spec_sets = (- ["__qualname__"],- ["__qualname__", "from_crawler"],- )- for specs in spec_sets:- m = mock.MagicMock(spec_set=specs)- _test_with_crawler(m, crawler)- m.reset_mock()-- # Check adoption of crawler- m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])- m.from_crawler.return_value = None- with pytest.raises(TypeError):- build_from_crawler(m, crawler, *args, **kwargs)-- def test_set_environ(self):- assert os.environ.get("some_test_environ") is None- with set_environ(some_test_environ="test_value"):- assert os.environ.get("some_test_environ") == "test_value"- assert os.environ.get("some_test_environ") is None-- os.environ["some_test_environ"] = "test"- assert os.environ.get("some_test_environ") == "test"- with set_environ(some_test_environ="test_value"):- assert os.environ.get("some_test_environ") == "test_value"- assert os.environ.get("some_test_environ") == "test"-- def test_rel_has_nofollow(self):- assert rel_has_nofollow("ugc nofollow") is True- assert rel_has_nofollow("ugc,nofollow") is True- assert rel_has_nofollow("ugc") is False- assert rel_has_nofollow("nofollow") is True- assert rel_has_nofollow("nofollowfoo") is False- assert rel_has_nofollow("foonofollow") is False- assert rel_has_nofollow("ugc, , nofollow") is True- # rel attribute values are ASCII case-insensitive per the HTML spec- assert rel_has_nofollow("NoFollow") is True- assert rel_has_nofollow("NOFOLLOW") is True- assert rel_has_nofollow("UGC NoFollow") is True- assert rel_has_nofollow("ugc,NoFollow") is True+ finally:+ sys.path.remove(egg)+++def test_arg_to_iter() -> None:+ class TestItem(Item):+ name = Field()++ assert hasattr(arg_to_iter(None), "__iter__")+ assert hasattr(arg_to_iter(100), "__iter__")+ assert hasattr(arg_to_iter("lala"), "__iter__")+ assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")+ assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")++ assert not list(arg_to_iter(None))+ assert list(arg_to_iter("lala")) == ["lala"]+ assert list(arg_to_iter(100)) == [100]+ assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]+ assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]+ assert list(arg_to_iter({"a": 1})) == [{"a": 1}]+ assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]+++def test_build_from_crawler() -> None:+ crawler = mock.MagicMock(spec_set=["settings"])+ args = (True, 100.0)+ kwargs = {"key": "val"}++ def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:+ build_from_crawler(mock, crawler, *args, **kwargs)+ if hasattr(mock, "from_crawler"):+ mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)+ assert mock.call_count == 0+ else:+ mock.assert_called_once_with(*args, **kwargs)++ # Check usage of correct constructor using 2 mocks:+ # 1. with no alternative constructors+ # 2. with from_crawler() constructor+ spec_sets = (+ ["__qualname__"],+ ["__qualname__", "from_crawler"],+ )+ for specs in spec_sets:+ m = mock.MagicMock(spec_set=specs)+ _test_with_crawler(m, crawler)+ m.reset_mock()++ # Check adoption of crawler+ m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])+ m.from_crawler.return_value = None+ with pytest.raises(TypeError):+ build_from_crawler(m, crawler, *args, **kwargs)+++def test_set_environ() -> None:+ assert os.environ.get("some_test_environ") is None+ with set_environ(some_test_environ="test_value"):+ assert os.environ.get("some_test_environ") == "test_value"+ assert os.environ.get("some_test_environ") is None++ os.environ["some_test_environ"] = "test"+ assert os.environ.get("some_test_environ") == "test"+ with set_environ(some_test_environ="test_value"):+ assert os.environ.get("some_test_environ") == "test_value"+ assert os.environ.get("some_test_environ") == "test"+++def test_rel_has_nofollow() -> None:+ assert rel_has_nofollow("ugc nofollow") is True+ assert rel_has_nofollow("ugc,nofollow") is True+ assert rel_has_nofollow("ugc") is False+ assert rel_has_nofollow("nofollow") is True+ assert rel_has_nofollow("nofollowfoo") is False+ assert rel_has_nofollow("foonofollow") is False+ assert rel_has_nofollow("ugc, , nofollow") is True+ # rel attribute values are ASCII case-insensitive per the HTML spec+ assert rel_has_nofollow("NoFollow") is True+ assert rel_has_nofollow("NOFOLLOW") is True+ assert rel_has_nofollow("UGC NoFollow") is True+ assert rel_has_nofollow("ugc,NoFollow") is Truediff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.pyindex 1acc3aac2..7343a135a 100644--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py@@ -1,5 +1,8 @@+from __future__ import annotations+import warningsfrom functools import partial+from typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytest@@ -9,6 +12,11 @@ from scrapy.utils.misc import (warn_on_generator_with_return_value,)+if TYPE_CHECKING:+ from collections.abc import Generator++ from scrapy import Spider+def _indentation_error(*args, **kwargs):raise IndentationError@@ -35,244 +43,239 @@ https://example.orgyield url-def generator_that_returns_stuff():+def generator_that_returns_stuff() -> Generator[int, None, int]:yield 1yield 2return 3-class TestUtilsMisc:- @pytest.fixture- def mock_spider(self):- class MockSettings:- def __init__(self, settings_dict=None):- self.settings_dict = settings_dict or {- "WARN_ON_GENERATOR_RETURN_VALUE": True- }+def mock_spider() -> Spider:+ class MockSettings:+ def __init__(self, settings_dict: dict[str, Any] | None = None):+ self.settings_dict = settings_dict or {+ "WARN_ON_GENERATOR_RETURN_VALUE": True+ }- def getbool(self, name, default=False):- return self.settings_dict.get(name, default)+ def getbool(self, name, default=False):+ return self.settings_dict.get(name, default)- class MockSpider:- def __init__(self):- self.settings = MockSettings()+ class MockSpider:+ def __init__(self) -> None:+ self.settings = MockSettings()- return MockSpider()+ return MockSpider() # type: ignore[return-value]- def test_generators_return_something(self, mock_spider):- def f1():- yield 1- return 2- def g1():- yield 1- return "asdf"+def test_generators_return_something(mock_spider):+ def f1():+ yield 1+ return 2- def h1():- yield 1+ def g1():+ yield 1+ return "asdf"- def helper():- return 0+ def h1():+ yield 1- yield helper()- return 2+ def helper() -> int:+ return 0- def i1():- """- docstring- """- url = """-https://example.org+ yield helper()+ return 2++ def i1():"""- yield url- return 1-- assert is_generator_with_return_value(top_level_return_something)- assert is_generator_with_return_value(f1)- assert is_generator_with_return_value(g1)- assert is_generator_with_return_value(h1)- assert is_generator_with_return_value(i1)-- with pytest.warns(- UserWarning,- match='The "MockSpider.top_level_return_something" method is a generator',- ):- warn_on_generator_with_return_value(mock_spider, top_level_return_something)- with pytest.warns(- UserWarning, match='The "MockSpider.f1" method is a generator'- ):- warn_on_generator_with_return_value(mock_spider, f1)- with pytest.warns(- UserWarning, match='The "MockSpider.g1" method is a generator'- ):- warn_on_generator_with_return_value(mock_spider, g1)- with pytest.warns(- UserWarning, match='The "MockSpider.h1" method is a generator'- ):- warn_on_generator_with_return_value(mock_spider, h1)- with pytest.warns(- UserWarning, match='The "MockSpider.i1" method is a generator'- ):- warn_on_generator_with_return_value(mock_spider, i1)-- def test_generators_return_none(self, mock_spider):- def f2():- yield 1-- def g2():- yield 1-- def h2():- yield 1-- def i2():- yield 1- yield from generator_that_returns_stuff()-- def j2():- yield 1-- def helper():- return 0-- yield helper()-- def k2():- """- docstring- """- url = """+ docstring+ """+ url = """https://example.org+ """+ yield url+ return 1++ assert is_generator_with_return_value(top_level_return_something)+ assert is_generator_with_return_value(f1)+ assert is_generator_with_return_value(g1)+ assert is_generator_with_return_value(h1)+ assert is_generator_with_return_value(i1)++ with pytest.warns(+ UserWarning,+ match='The "MockSpider.top_level_return_something" method is a generator',+ ):+ warn_on_generator_with_return_value(mock_spider, top_level_return_something)+ with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'):+ warn_on_generator_with_return_value(mock_spider, f1)+ with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'):+ warn_on_generator_with_return_value(mock_spider, g1)+ with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'):+ warn_on_generator_with_return_value(mock_spider, h1)+ with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'):+ warn_on_generator_with_return_value(mock_spider, i1)+++def test_generators_return_none(mock_spider):+ def f2():+ yield 1++ def g2():+ yield 1++ def h2():+ yield 1++ def i2():+ yield 1+ yield from generator_that_returns_stuff()++ def j2():+ yield 1++ def helper() -> int:+ return 0++ yield helper()++ def k2():"""- yield url-- def l2():- return-- assert not is_generator_with_return_value(top_level_return_none)- assert not is_generator_with_return_value(f2)- assert not is_generator_with_return_value(g2)- assert not is_generator_with_return_value(h2)- assert not is_generator_with_return_value(i2)- assert not is_generator_with_return_value(j2) # not recursive- assert not is_generator_with_return_value(k2) # not recursive- assert not is_generator_with_return_value(l2)-- with warnings.catch_warnings():- warnings.simplefilter("error", UserWarning)- warn_on_generator_with_return_value(mock_spider, top_level_return_none)- warn_on_generator_with_return_value(mock_spider, f2)- warn_on_generator_with_return_value(mock_spider, g2)- warn_on_generator_with_return_value(mock_spider, h2)- warn_on_generator_with_return_value(mock_spider, i2)- warn_on_generator_with_return_value(mock_spider, j2)- warn_on_generator_with_return_value(mock_spider, k2)- warn_on_generator_with_return_value(mock_spider, l2)-- def test_generators_return_none_with_decorator(self, mock_spider):- def decorator(func):- def inner_func():- func()-- return inner_func-- @decorator- def f3():- yield 1-- @decorator- def g3():- yield 1-- @decorator- def h3():- yield 1-- @decorator- def i3():- yield 1- yield from generator_that_returns_stuff()-- @decorator- def j3():- yield 1-- def helper():- return 0-- yield helper()-- @decorator- def k3():- """- docstring- """- url = """+ docstring+ """+ url = """https://example.org+ """+ yield url++ def l2():+ return++ assert not is_generator_with_return_value(top_level_return_none)+ assert not is_generator_with_return_value(f2)+ assert not is_generator_with_return_value(g2)+ assert not is_generator_with_return_value(h2)+ assert not is_generator_with_return_value(i2)+ assert not is_generator_with_return_value(j2) # not recursive+ assert not is_generator_with_return_value(k2) # not recursive+ assert not is_generator_with_return_value(l2)++ with warnings.catch_warnings():+ warnings.simplefilter("error", UserWarning)+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)+ warn_on_generator_with_return_value(mock_spider, f2)+ warn_on_generator_with_return_value(mock_spider, g2)+ warn_on_generator_with_return_value(mock_spider, h2)+ warn_on_generator_with_return_value(mock_spider, i2)+ warn_on_generator_with_return_value(mock_spider, j2)+ warn_on_generator_with_return_value(mock_spider, k2)+ warn_on_generator_with_return_value(mock_spider, l2)+++def test_generators_return_none_with_decorator(mock_spider):+ def decorator(func):+ def inner_func():+ func()++ return inner_func++ @decorator+ def f3():+ yield 1++ @decorator+ def g3():+ yield 1++ @decorator+ def h3():+ yield 1++ @decorator+ def i3():+ yield 1+ yield from generator_that_returns_stuff()++ @decorator+ def j3():+ yield 1++ def helper() -> int:+ return 0++ yield helper()++ @decorator+ def k3():"""- yield url-- @decorator- def l3():- return-- assert not is_generator_with_return_value(top_level_return_none)- assert not is_generator_with_return_value(f3)- assert not is_generator_with_return_value(g3)- assert not is_generator_with_return_value(h3)- assert not is_generator_with_return_value(i3)- assert not is_generator_with_return_value(j3) # not recursive- assert not is_generator_with_return_value(k3) # not recursive- assert not is_generator_with_return_value(l3)-- with warnings.catch_warnings():- warnings.simplefilter("error", UserWarning)- warn_on_generator_with_return_value(mock_spider, top_level_return_none)- warn_on_generator_with_return_value(mock_spider, f3)- warn_on_generator_with_return_value(mock_spider, g3)- warn_on_generator_with_return_value(mock_spider, h3)- warn_on_generator_with_return_value(mock_spider, i3)- warn_on_generator_with_return_value(mock_spider, j3)- warn_on_generator_with_return_value(mock_spider, k3)- warn_on_generator_with_return_value(mock_spider, l3)-- @mock.patch(- "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error- )- def test_indentation_error(self, mock_spider):- with pytest.warns(UserWarning, match="Unable to determine"):- warn_on_generator_with_return_value(mock_spider, top_level_return_none)-- def test_partial(self):- def cb(arg1, arg2):- yield {}-- partial_cb = partial(cb, arg1=42)- assert not is_generator_with_return_value(partial_cb)-- def test_warn_on_generator_with_return_value_settings_disabled(self):- class MockSettings:- def __init__(self, settings_dict=None):- self.settings_dict = settings_dict or {}-- def getbool(self, name, default=False):- return self.settings_dict.get(name, default)-- class MockSpider:- def __init__(self):- self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})-- spider = MockSpider()-- def gen_with_return():- yield 1- return "value"-- with warnings.catch_warnings():- warnings.simplefilter("error", UserWarning)- warn_on_generator_with_return_value(spider, gen_with_return)-- spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True-- with pytest.warns(UserWarning, match="is a generator"):- warn_on_generator_with_return_value(spider, gen_with_return)+ docstring+ """+ url = """+https://example.org+ """+ yield url++ @decorator+ def l3():+ return++ assert not is_generator_with_return_value(top_level_return_none)+ assert not is_generator_with_return_value(f3)+ assert not is_generator_with_return_value(g3)+ assert not is_generator_with_return_value(h3)+ assert not is_generator_with_return_value(i3)+ assert not is_generator_with_return_value(j3) # not recursive+ assert not is_generator_with_return_value(k3) # not recursive+ assert not is_generator_with_return_value(l3)++ with warnings.catch_warnings():+ warnings.simplefilter("error", UserWarning)+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)+ warn_on_generator_with_return_value(mock_spider, f3)+ warn_on_generator_with_return_value(mock_spider, g3)+ warn_on_generator_with_return_value(mock_spider, h3)+ warn_on_generator_with_return_value(mock_spider, i3)+ warn_on_generator_with_return_value(mock_spider, j3)+ warn_on_generator_with_return_value(mock_spider, k3)+ warn_on_generator_with_return_value(mock_spider, l3)++[email protected]("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)+def test_indentation_error(mock_spider):+ with pytest.warns(UserWarning, match="Unable to determine"):+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)+++def test_partial() -> None:+ def cb(arg1, arg2):+ yield {}++ partial_cb = partial(cb, arg1=42)+ assert not is_generator_with_return_value(partial_cb)+++def test_warn_on_generator_with_return_value_settings_disabled() -> None:+ class MockSettings:+ def __init__(self, settings_dict: dict[str, Any] | None = None):+ self.settings_dict = settings_dict or {}++ def getbool(self, name, default=False):+ return self.settings_dict.get(name, default)++ class MockSpider:+ def __init__(self) -> None:+ self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})++ spider = MockSpider()++ def gen_with_return():+ yield 1+ return "value"++ with warnings.catch_warnings():+ warnings.simplefilter("error", UserWarning)+ warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]++ spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True++ with pytest.warns(UserWarning, match="is a generator"):+ warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 5333a55cb..7eade6463 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -1,14 +1,20 @@+from __future__ import annotations+import osfrom pathlib import Path+from typing import TYPE_CHECKINGimport pytestfrom scrapy.utils.misc import set_environfrom scrapy.utils.project import data_path, get_project_settings+if TYPE_CHECKING:+ from collections.abc import Generator+@pytest.fixture-def proj_path(tmp_path):+def proj_path(tmp_path: Path) -> Generator[Path]:prev_dir = Path.cwd()project_dir = tmp_path@@ -21,7 +27,7 @@ def proj_path(tmp_path):os.chdir(prev_dir)-def test_data_path_outside_project():+def test_data_path_outside_project() -> None:assert str(Path(".scrapy", "somepath")) == data_path("somepath")abspath = str(Path(os.path.sep, "absolute", "path"))assert abspath == data_path(abspath)diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 2e8047e2d..c3b5dfc99 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -22,8 +22,7 @@ from scrapy.utils.python import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from collections.abc import Iterable, Mapping-+ from collections.abc import AsyncIterator, Iterable, Mapping_KT = TypeVar("_KT")_VT = TypeVar("_VT")@@ -31,22 +30,22 @@ _VT = TypeVar("_VT")class TestMutableAsyncChain:@staticmethod- async def g1():+ async def g1() -> AsyncIterator[int]:for i in range(3):yield i@staticmethod- async def g2():+ async def g2() -> AsyncIterator[int]:returnyield@staticmethod- async def g3():+ async def g3() -> AsyncIterator[int]:for i in range(7, 10):yield i@staticmethod- async def g4():+ async def g4() -> AsyncIterator[int]:for i in range(3, 5):yield i1 / 0@@ -85,7 +84,7 @@ class TestToUnicode:def test_converting_a_strange_object_should_raise_type_error(self):with pytest.raises(TypeError):- to_unicode(423)+ to_unicode(423) # type: ignore[arg-type]def test_errors_argument(self):assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb"@@ -103,7 +102,7 @@ class TestToBytes:def test_converting_a_strange_object_should_raise_type_error(self):with pytest.raises(TypeError):- to_bytes(pytest)+ to_bytes(pytest) # type: ignore[arg-type]def test_errors_argument(self):assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"@@ -112,10 +111,10 @@ class TestToBytes:def test_memoizemethod_noargs():class A:@memoizemethod_noargs- def cached(self):+ def cached(self) -> object:return object()- def noncached(self):+ def noncached(self) -> object:return object()a = A()@@ -150,7 +149,7 @@ def test_get_func_args():passclass A:- def __init__(self, a, b, c):+ def __init__(self, a: int, b: int, c: int):passdef method(self, a, b, c):diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 7d39a478e..44cb5c306 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -1,3 +1,5 @@+from __future__ import annotations+import asyncioimport pytestdiff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 1642a932b..eb95be06c 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport jsonfrom hashlib import sha1-from typing import Any+from typing import TYPE_CHECKING, Any, Protocolfrom weakref import WeakKeyDictionaryimport pytest@@ -17,6 +17,9 @@ from scrapy.utils.request import ()from scrapy.utils.test import get_crawler+if TYPE_CHECKING:+ from collections.abc import Iterable+@pytest.mark.parametrize(("r", "expected"),@@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:request_httprepr(r)+class _FingerprintFunction(Protocol):+ def __call__(+ self,+ request: Request,+ *,+ include_headers: Iterable[bytes | str] | None = None,+ keep_fragments: bool = False,+ ) -> bytes: ...++class TestFingerprint:- function: staticmethod[[Request], bytes] = staticmethod(fingerprint)+ function: _FingerprintFunction = staticmethod(fingerprint)cache: (WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]@@ -261,6 +274,7 @@ class TestRequestFingerprinter:def test_fingerprint(self):crawler = get_crawler()request = Request("https://example.com")+ assert crawler.request_fingerprinterassert crawler.request_fingerprinter.fingerprint(request) == fingerprint(request)@@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)+ assert crawler.request_fingerprinterr1 = Request("http://www.example.com", headers={"X-ID": "1"})fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com", headers={"X-ID": "2"})@@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter:def test_dont_canonicalize(self):class RequestFingerprinter:- cache = WeakKeyDictionary()+ cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary()- def fingerprint(self, request):+ def fingerprint(self, request: Request) -> bytes:if request not in self.cache:fp = sha1()fp.update(to_bytes(request.url))@@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)+ assert crawler.request_fingerprinterr1 = Request("http://www.example.com?a=1&a=2")fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com?a=2&a=1")@@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)+ assert crawler.request_fingerprinterr1 = Request("http://www.example.com")fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com", meta={"fingerprint": "a"})@@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)+ assert crawler.request_fingerprinterrequest = Request("http://www.example.com")fingerprint = crawler.request_fingerprinter.fingerprint(request)assert fingerprint == settings["FINGERPRINT"]diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 146cdb802..608b2bbd9 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -1,3 +1,5 @@+from __future__ import annotations+from pathlib import Pathfrom time import process_timefrom urllib.parse import urlparse@@ -15,7 +17,7 @@ from scrapy.utils.response import ()-def _read_browser_output(burl: str):+def _read_browser_output(burl: str) -> bytes:path = urlparse(burl).pathif not path or not Path(path).exists():path = burl.replace("file://", "")@@ -224,7 +226,7 @@ def test_open_in_browser_redos_head():(b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),],)-def test_remove_html_comments(input_body, output_body):+def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:assert _remove_html_comments(input_body) == output_bodydiff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.pyindex 2e6a790f8..2702c2cce 100644--- a/tests/test_utils_serialize.py+++ b/tests/test_utils_serialize.py@@ -1,3 +1,5 @@+from __future__ import annotations+import dataclassesimport datetimeimport jsondiff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex dbb9caf2a..3109bc149 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from collections.abc import Callable-if TYPE_CHECKING:- from collections.abc import Callable-class TestSendCatchLog:# whether the function being tested returns exceptions or failuresdiff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex ac57e1739..9f2dce6d6 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -1,3 +1,5 @@+from __future__ import annotations+import pytestfrom scrapy.exceptions import ScrapyDeprecationWarningdiff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 4515ce36e..076edf3db 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -1,7 +1,14 @@+from __future__ import annotations++from typing import TYPE_CHECKING+from scrapy.utils.template import render_templatefile+if TYPE_CHECKING:+ from pathlib import Path+-def test_simple_render(tmp_path):+def test_simple_render(tmp_path: Path) -> None:context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}template = "from ${project_name}.spiders.${name} import ${classname}"rendered = "from proj.spiders.spi import TheSpider"diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex 2334c76e9..5458aa603 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -1,3 +1,5 @@+from __future__ import annotations+import sysfrom io import StringIOfrom unittest import mock@@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago@mock.patch("sys.stdout", new_callable=StringIO)-def test_print_live_refs_empty(stdout):+def test_print_live_refs_empty(stdout: StringIO) -> None:trackref.print_live_refs()assert stdout.getvalue() == "Live References\n\n\n"@mock.patch("sys.stdout", new_callable=StringIO)-def test_print_live_refs_with_objects(stdout):+def test_print_live_refs_with_objects(stdout: StringIO) -> None:o1 = Foo() # noqa: F841trackref.print_live_refs()assert (diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 5b98131a1..d9d162c23 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,3 +1,5 @@+from __future__ import annotations+import pytestfrom scrapy.linkextractors import IGNORED_EXTENSIONS@@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str):),],)-def test_guess_scheme_skipped(url: str, expected: str, reason: str):+def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:pytest.skip(reason)diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testingindex a431091d5..4ff562dff 100644--- a/tests_typing/test_http_request.mypy-testing+++ b/tests_typing/test_http_request.mypy-testing@@ -16,7 +16,7 @@ class MyRequest2(Request):@pytest.mark.mypy_testingdef mypy_test_headers() -> None:- Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"+ Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"Request("data:,", headers=None)Request("data:,", headers={})Request("data:,", headers=[])diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testingindex d497c2470..1c157328c 100644--- a/tests_typing/test_http_response.mypy-testing+++ b/tests_typing/test_http_response.mypy-testing@@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse@pytest.mark.mypy_testingdef mypy_test_headers() -> None:- Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"+ Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"Response("data:,", headers=None)Response("data:,", headers={})Response("data:,", headers=[])