Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Next refactoring pass of test_utils_*. (#7797)

Andrey Rakhmatulline7d8b34e73ef
diff --git a/pyproject.toml b/pyproject.toml
index 9b1e64121..576a42e5c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -178,13 +178,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
- "tests.test_utils_datatypes",
- "tests.test_utils_decorators",
- "tests.test_utils_defer",
- "tests.test_utils_deprecate",
- "tests.test_utils_misc.test_return_with_argument_inside_generator",
- "tests.test_utils_python",
- "tests.test_utils_request",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 07ff4a74e..29b3e3c0f 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
- # hints for Headers-related types may need to be fixed to not use AnyStr
- return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
+ return respcls(url=request.url, status=200, body=body, headers=headers)
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 34d4ec6f2..b55ef6191 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
@@ -25,14 +25,20 @@ class Headers(CaselessDict):
def __init__(
self,
- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
encoding: str = "utf-8",
):
self.encoding: str = encoding
super().__init__(seq)
def update( # type: ignore[override]
- self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
+ self,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq: dict[bytes, list[bytes]] = {}
@@ -40,7 +46,7 @@ class Headers(CaselessDict):
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
- def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
+ def normkey(self, key: str | bytes) -> bytes:
"""Normalize key to bytes"""
return self._tobytes(key.title())
@@ -67,19 +73,19 @@ class Headers(CaselessDict):
return str(x).encode(self.encoding)
raise TypeError(f"Unsupported value type: {type(x)}")
- def __getitem__(self, key: AnyStr) -> bytes | None:
+ def __getitem__(self, key: str | bytes) -> bytes | None:
try:
return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
- def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
+ def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
try:
return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
- def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
+ def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
try:
return cast("list[bytes]", super().__getitem__(key))
except KeyError:
@@ -87,15 +93,15 @@ class Headers(CaselessDict):
return self.normvalue(def_val)
return []
- def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
+ def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
self[key] = list_
def setlistdefault(
- self, key: AnyStr, default_list: Iterable[_RawValue] = ()
+ self, key: str | bytes, default_list: Iterable[_RawValue] = ()
) -> Any:
return self.setdefault(key, default_list)
- def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
+ def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 73c2e7dd4..57517a65b 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -11,7 +11,6 @@ import inspect
from typing import (
TYPE_CHECKING,
Any,
- AnyStr,
Concatenate,
NoReturn,
TypeAlias,
@@ -125,7 +124,10 @@ class Request(object_ref):
url: str,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -310,7 +312,11 @@ class Request(object_ref):
@headers.setter
def headers(
- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+ self,
+ value: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None,
) -> None:
if isinstance(value, Headers):
self._headers = value
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 09b1c8b32..f1db11488 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
@@ -72,7 +72,10 @@ class Response(object_ref):
self,
url: str,
status: int = 200,
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes = b"",
flags: list[str] | None = None,
request: Request | None = None,
@@ -145,7 +148,11 @@ class Response(object_ref):
@headers.setter
def headers(
- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+ self,
+ value: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@@ -222,7 +229,10 @@ class Response(object_ref):
url: str | Link,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -272,7 +282,10 @@ class Response(object_ref):
urls: Iterable[str | Link],
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6876e35e8..d01e23e47 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -9,7 +9,7 @@ from __future__ import annotations
import json
from contextlib import suppress
-from typing import TYPE_CHECKING, Any, AnyStr, cast
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
import parsel
@@ -170,7 +170,10 @@ class TextResponse(Response):
url: str | Link | parsel.Selector,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -223,7 +226,10 @@ class TextResponse(Response):
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e761a2474..9a945c61c 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -11,12 +11,12 @@ import warnings
import weakref
from collections import OrderedDict
from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
- from collections.abc import Iterable, Sequence
+ from collections.abc import Container, Iterable
# typing.Self requires Python 3.11
from typing_extensions import Self
@@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg]
def __init__(
self,
- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
):
super().__init__()
if seq:
self.update(seq)
- def __getitem__(self, key: AnyStr) -> Any:
+ def __getitem__(self, key: str | bytes) -> Any:
return dict.__getitem__(self, self.normkey(key))
- def __setitem__(self, key: AnyStr, value: Any) -> None:
+ def __setitem__(self, key: str | bytes, value: Any) -> None:
dict.__setitem__(self, self.normkey(key), self.normvalue(value))
- def __delitem__(self, key: AnyStr) -> None:
+ def __delitem__(self, key: str | bytes) -> None:
dict.__delitem__(self, self.normkey(key))
- def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
@@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg]
copy = __copy__
- def normkey(self, key: AnyStr) -> AnyStr:
+ def normkey(self, key: str | bytes) -> str | bytes:
"""Method to normalize dictionary key access"""
return key.lower()
@@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg]
"""Method to normalize values prior to be set"""
return value
- def get(self, key: AnyStr, def_val: Any = None) -> Any:
+ def get(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.get(self, self.normkey(key), self.normvalue(def_val))
- def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
+ def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
# doesn't fully implement MutableMapping.update()
- def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override]
+ def update( # type: ignore[override]
+ self,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]],
+ ) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super().update(iseq)
@classmethod
- def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override]
- return cls((k, value) for k in keys) # type: ignore[misc]
+ def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]
+ return cls((k, value) for k in keys)
- def pop(self, key: AnyStr, *args: Any) -> Any:
+ def pop(self, key: str | bytes, *args: Any) -> Any:
return dict.pop(self, self.normkey(key), *args)
@@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
- def __init__(self, seq: Sequence[Any]):
- self.seq: Sequence[Any] = seq
+ def __init__(self, seq: Container[Any]):
+ self.seq: Container[Any] = seq
def __contains__(self, item: Any) -> bool:
return item not in self.seq
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index a5bb6fa24..4960dc27a 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -19,9 +19,19 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
+@overload
+def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
+
+
+@overload
+def deprecated(
+ use_instead: str | None = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
+
+
def deprecated(
- use_instead: Any = None,
-) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+ use_instead: Callable[_P, _T] | str | None = None,
+) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emitted
when the function is used."""
@@ -38,8 +48,9 @@ def deprecated(
return wrapped
if callable(use_instead):
- deco = deco(use_instead)
+ func = use_instead
use_instead = None
+ return deco(func)
return deco
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index fc4e1c487..1d36a66fc 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,16 +1,18 @@
+from __future__ import annotations
+
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from tests.utils.decorators import coroutine_test
-class TestAsyncgenUtils:
- @coroutine_test
- async def test_as_async_generator(self):
- ag = as_async_generator(range(42))
- results = [i async for i in ag]
- assert results == list(range(42))
+@coroutine_test
+async def test_as_async_generator():
+ ag = as_async_generator(range(42))
+ results = [i async for i in ag]
+ assert results == list(range(42))
+
- @coroutine_test
- async def test_collect_asyncgen(self):
- ag = as_async_generator(range(42))
- results = await collect_asyncgen(ag)
- assert results == list(range(42))
+@coroutine_test
+async def test_collect_asyncgen():
+ ag = as_async_generator(range(42))
+ results = await collect_asyncgen(ag)
+ assert results == list(range(42))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 7528dc51a..9b7eb22fa 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -20,11 +20,10 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
-class TestAsyncio:
- @coroutine_test
- async def test_is_asyncio_available(self, reactor_pytest: str) -> None:
- # the result should depend only on the pytest --reactor argument
- assert is_asyncio_available() == (reactor_pytest != "default")
+@coroutine_test
+async def test_is_asyncio_available(reactor_pytest: str) -> None:
+ # the result should depend only on the pytest --reactor argument
+ assert is_asyncio_available() == (reactor_pytest != "default")
@pytest.mark.only_asyncio
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index fce9fc984..6b30744bb 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import warnings
from typing import Any
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index f43d20e69..af203ca61 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
import copy
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
-from typing import Any
+from typing import Any, Generic, TypeVar
import pytest
@@ -16,11 +18,13 @@ from scrapy.utils.datatypes import (
)
from scrapy.utils.python import garbage_collect
+_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
+
-class TestCaseInsensitiveDictBase(ABC):
+class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
@property
@abstractmethod
- def dict_class(self) -> type[MutableMapping[str, Any]]:
+ def dict_class(self) -> type[_DictT]:
raise NotImplementedError
def test_init_dict(self):
@@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mapping(self):
- class MyMapping(Mapping):
- def __init__(self, **kwargs):
+ class MyMapping(Mapping[str, int]):
+ def __init__(self, **kwargs: int) -> None:
self._d = kwargs
- def __getitem__(self, key):
+ def __getitem__(self, key: str) -> int:
return self._d[key]
- def __iter__(self):
+ def __iter__(self) -> Iterator[str]:
return iter(self._d)
- def __len__(self):
+ def __len__(self) -> int:
return len(self._d)
seq = MyMapping(red=1, black=3)
@@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mutable_mapping(self):
- class MyMutableMapping(MutableMapping):
- def __init__(self, **kwargs):
+ class MyMutableMapping(MutableMapping[str, int]):
+ def __init__(self, **kwargs: int) -> None:
self._d = kwargs
- def __getitem__(self, key):
+ def __getitem__(self, key: str) -> int:
return self._d[key]
- def __setitem__(self, key, value):
+ def __setitem__(self, key: str, value: int) -> None:
self._d[key] = value
- def __delitem__(self, key):
+ def __delitem__(self, key: str) -> None:
del self._d[key]
- def __iter__(self):
+ def __iter__(self) -> Iterator[str]:
return iter(self._d)
- def __len__(self):
+ def __len__(self) -> int:
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
@@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):
d.pop("A")
def test_normkey(self):
- class MyDict(self.dict_class):
+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normkey(self, key):
return key.title()
@@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):
assert list(d.keys()) == ["Key-One"]
def test_normvalue(self):
- class MyDict(self.dict_class):
+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
if value is not None:
return value + 1
@@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC):
assert dict(h1) == {"header1": "value1", "header2": "value2"}
-class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
- dict_class = CaseInsensitiveDict # type: ignore[assignment]
+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
+ dict_class = CaseInsensitiveDict
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
@@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
assert list(iterkeys) == ["AsDf", "FoO"]
def test_copy_keeps_values(self):
- class MyDict(self.dict_class):
+ class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
return value + 1
@@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(TestCaseInsensitiveDictBase):
+class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
dict_class = CaselessDict
def test_deprecation_message(self):
@@ -319,7 +323,7 @@ class TestSequenceExclude:
class TestLocalCache:
def test_cache_with_limit(self):
- cache = LocalCache(limit=2)
+ cache: LocalCache[str, int] = LocalCache(limit=2)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@@ -332,7 +336,7 @@ class TestLocalCache:
def test_cache_without_limit(self):
maximum = 10**4
- cache = LocalCache()
+ cache: LocalCache[str, int] = LocalCache()
for x in range(maximum):
cache[str(x)] = x
assert len(cache) == maximum
@@ -341,7 +345,7 @@ class TestLocalCache:
assert cache[str(x)] == x
def test_cache_with_zero_limit(self):
- cache = LocalCache(limit=0)
+ cache: LocalCache[str, int] = LocalCache(limit=0)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@@ -353,7 +357,9 @@ class TestLocalCache:
class TestLocalWeakReferencedCache:
def test_cache_with_limit(self):
- cache = LocalWeakReferencedCache(limit=2)
+ cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
+ limit=2
+ )
r1 = Request("https://example.org")
r2 = Request("https://example.com")
r3 = Request("https://example.net")
@@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache:
assert len(cache) == 1
def test_cache_non_weak_referenceable_objects(self):
- cache = LocalWeakReferencedCache()
+ cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
k1 = None
k2 = 1
k3 = [1, 2, 3]
@@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache:
def test_cache_without_limit(self):
maximum = 10**4
- cache = LocalWeakReferencedCache()
+ cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
refs = []
for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))
diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py
index 9743e1a50..807294a57 100644
--- a/tests/test_utils_decorators.py
+++ b/tests/test_utils_decorators.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import warnings
+from typing import TYPE_CHECKING
import pytest
from twisted.internet.defer import Deferred
@@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread
from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
class TestDeprecated:
def test_warns_and_still_calls(self):
@deprecated()
- def add(a, b):
+ def add(a: int, b: int) -> int:
return a + b
with pytest.warns(
@@ -26,7 +30,7 @@ class TestDeprecated:
def test_use_instead_in_message(self):
@deprecated(use_instead="other_function")
- def old():
+ def old() -> None:
return None
with pytest.warns(
@@ -37,7 +41,7 @@ class TestDeprecated:
def test_applied_without_parentheses(self):
@deprecated
- def square(x):
+ def square(x: int) -> int:
return x * x
with pytest.warns(
@@ -65,7 +69,7 @@ class TestInthread:
class TestWarnSpiderArg:
def test_sync_warns_with_spider_arg(self):
@_warn_spider_arg
- def parse(response, spider=None):
+ def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@@ -75,7 +79,7 @@ class TestWarnSpiderArg:
def test_sync_no_warning_without_spider_arg(self):
@_warn_spider_arg
- def parse(response, spider=None):
+ def parse(response: str, spider: str | None = None) -> str:
return response
with warnings.catch_warnings():
@@ -85,7 +89,7 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_async_warns_with_spider_arg(self):
@_warn_spider_arg
- async def parse(response, spider=None):
+ async def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@@ -96,7 +100,9 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_asyncgen_warns_with_spider_arg(self):
@_warn_spider_arg
- async def parse(response, spider=None):
+ async def parse(
+ response: str, spider: str | None = None
+ ) -> AsyncGenerator[str]:
yield response
with pytest.warns(
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 978c24f5a..175a4fe03 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
+ from twisted.python.failure import Failure
+
@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@@ -70,7 +72,7 @@ class TestIterErrback:
def itergood() -> Generator[int, None, None]:
yield from range(10)
- errors = []
+ errors: list[Failure] = []
out = list(iter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@@ -82,7 +84,7 @@ class TestIterErrback:
1 / 0
yield x
- errors = []
+ errors: list[Failure] = []
out = list(iter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@@ -96,7 +98,7 @@ class TestAiterErrback:
for x in range(10):
yield x
- errors = []
+ errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@@ -109,7 +111,7 @@ class TestAiterErrback:
1 / 0
yield x
- errors = []
+ errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@@ -202,7 +204,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
- results = []
+ results: list[int] = []
ait = self.get_async_iterable(length)
dl = parallel_async(
ait,
@@ -222,7 +224,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
- results = []
+ results: list[int] = []
ait = self.get_async_iterable_with_delays(length)
dl = parallel_async(
ait,
@@ -240,7 +242,7 @@ class TestParallelAsync:
class TestDeferredFromCoro:
def test_deferred(self):
- d = Deferred()
+ d: Deferred[None] = Deferred()
result = deferred_from_coro(d)
assert isinstance(result, Deferred)
assert result is d
@@ -274,7 +276,7 @@ class TestDeferredFromCoro:
@pytest.mark.only_asyncio
@inline_callbacks_test
def test_future(self):
- future = Future()
+ future: Future[int] = Future()
result = deferred_from_coro(future)
assert isinstance(result, Deferred)
future.set_result(42)
@@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:
class TestDeferredToFuture:
@coroutine_test
async def test_deferred(self):
- d = Deferred()
+ d: Deferred[int] = Deferred()
result = deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@@ -359,7 +361,7 @@ class TestDeferredToFuture:
class TestMaybeDeferredToFutureAsyncio:
@coroutine_test
async def test_deferred(self):
- d = Deferred()
+ d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:
class TestMaybeDeferredToFutureNotAsyncio:
@coroutine_test
async def test_deferred(self):
- d = Deferred()
+ d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Deferred)
assert result is d
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 0706fec99..4c8585916 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import inspect
import warnings
from unittest import mock
@@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:
)
with pytest.warns(MyWarning, match=msg) as w:
- class UserClass(Deprecated):
+ class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
@@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
):
- class UserClass(Deprecated):
+ class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with pytest.warns(
@@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
- class UserClass(Deprecated):
+ class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
- class UserClass(Deprecated):
+ class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
- class FooClass(Deprecated):
+ class FooClass(Deprecated): # type: ignore[misc, valid-type]
pass
- class BarClass(Deprecated):
+ class BarClass(Deprecated): # type: ignore[misc, valid-type]
pass
def test_warning_on_instance(self):
@@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:
"Deprecated", NewName, warn_category=MyWarning
)
- with pytest.warns(MyWarning) as w:
- _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
-
- w = [x for x in w if x.category is MyWarning]
+ with pytest.warns(
+ MyWarning,
+ match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
+ r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
+ ) as w:
+ _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]
assert len(w) == 1
- assert (
- str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
- "instantiate tests.test_utils_deprecate.NewName instead."
- )
assert w[0].lineno == lineno
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", MyWarning)
- class UserClass(Deprecated):
+ class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
):
- class UserClass2(Deprecated):
+ class UserClass2(Deprecated): # type: ignore[misc, valid-type]
pass
def test_issubclass(self):
@@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass1a(NewName):
pass
- class OutdatedUserClass1(DeprecatedName):
+ class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]
pass
- class OutdatedUserClass1a(DeprecatedName):
+ class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
with pytest.raises(TypeError):
- issubclass(object(), DeprecatedName)
+ issubclass(object(), DeprecatedName) # type: ignore[arg-type]
def test_isinstance(self):
with warnings.catch_warnings():
@@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass2a(NewName):
pass
- class OutdatedUserClass2(DeprecatedName):
+ class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]
pass
- class OutdatedUserClass2a(DeprecatedName):
+ class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
- assert Deprecated.foo == "bar"
+ assert Deprecated.foo == "bar" # type: ignore[attr-defined]
def test_deprecate_a_class_with_custom_metaclass(self):
Meta1 = type("Meta1", (type,), {})
@@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
):
- class UserClass(AlsoDeprecated):
+ class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]
pass
def test_inspect_stack(self):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 9f9e24957..a87816c20 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -31,13 +31,13 @@ plain_string = "{'a': 1}"
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
-def test_pformat(isatty):
+def test_pformat(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value) in colorized_strings
@mock.patch("sys.stdout.isatty")
-def test_pformat_dont_colorize(isatty):
+def test_pformat_dont_colorize(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value, colorize=False) == plain_string
@@ -49,7 +49,7 @@ def test_pformat_not_tty():
@mock.patch("sys.platform", "win32")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
-def test_pformat_old_windows(isatty, version):
+def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
isatty.return_value = True
version.return_value = "10.0.14392"
assert pformat(value) in colorized_strings
@@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
-def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
+def test_pformat_windows_no_terminal_processing(
+ isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = False
@@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
-def test_pformat_windows(isatty, version, terminal_processing):
+def test_pformat_windows(
+ isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = True
@@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
-def test_pformat_no_pygments(isatty):
+def test_pformat_no_pygments(isatty: mock.Mock) -> None:
isatty.return_value = True
real_import = builtins.__import__
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 06fdf9cba..75f8be6f6 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from gzip import BadGzipFile
from pathlib import Path
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 0eb330461..610c463ec 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from urllib.parse import urlparse
from scrapy.http import Request
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index ab6965ed5..0775ec608 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -20,150 +20,157 @@ from scrapy.utils.misc import (
)
-class TestUtilsMisc:
- def test_load_object_class(self):
- obj = load_object(Field)
- assert obj is Field
- obj = load_object("scrapy.item.Field")
- assert obj is Field
-
- def test_load_object_function(self):
- obj = load_object(load_object)
- assert obj is load_object
- obj = load_object("scrapy.utils.misc.load_object")
- assert obj is load_object
-
- def test_load_object_exceptions(self):
- with pytest.raises(ImportError):
- load_object("nomodule999.mod.function")
- with pytest.raises(NameError):
- load_object("scrapy.utils.misc.load_object999")
- with pytest.raises(TypeError):
- load_object({}) # type: ignore[arg-type]
-
- def test_walk_modules(self):
- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+def test_load_object_class() -> None:
+ obj = load_object(Field)
+ assert obj is Field
+ obj = load_object("scrapy.item.Field")
+ assert obj is Field
+
+
+def test_load_object_function() -> None:
+ obj = load_object(load_object)
+ assert obj is load_object
+ obj = load_object("scrapy.utils.misc.load_object")
+ assert obj is load_object
+
+
+def test_load_object_exceptions() -> None:
+ with pytest.raises(ImportError):
+ load_object("nomodule999.mod.function")
+ with pytest.raises(NameError):
+ load_object("scrapy.utils.misc.load_object999")
+ with pytest.raises(TypeError):
+ load_object({}) # type: ignore[arg-type]
+
+
+def test_walk_modules() -> None:
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+ expected = [
+ "tests.test_utils_misc.test_walk_modules",
+ "tests.test_utils_misc.test_walk_modules.mod",
+ "tests.test_utils_misc.test_walk_modules.mod.mod0",
+ "tests.test_utils_misc.test_walk_modules.mod1",
+ ]
+ assert {m.__name__ for m in mods} == set(expected)
+
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
+ expected = [
+ "tests.test_utils_misc.test_walk_modules.mod",
+ "tests.test_utils_misc.test_walk_modules.mod.mod0",
+ ]
+ assert {m.__name__ for m in mods} == set(expected)
+
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
+ expected = [
+ "tests.test_utils_misc.test_walk_modules.mod1",
+ ]
+ assert {m.__name__ for m in mods} == set(expected)
+
+ with pytest.raises(ImportError):
+ for _ in walk_modules_iter("nomodule999"):
+ pass
+ with (
+ pytest.raises(ImportError),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
+ "removed in a future version of Scrapy. "
+ "Use scrapy.utils.misc.walk_modules_iter instead.",
+ ),
+ ):
+ walk_modules("nomodule999")
+
+
+def test_walk_modules_egg() -> None:
+ egg = str(Path(__file__).parent / "test.egg")
+ sys.path.append(egg)
+ try:
+ mods = walk_modules_iter("testegg")
expected = [
- "tests.test_utils_misc.test_walk_modules",
- "tests.test_utils_misc.test_walk_modules.mod",
- "tests.test_utils_misc.test_walk_modules.mod.mod0",
- "tests.test_utils_misc.test_walk_modules.mod1",
+ "testegg.spiders",
+ "testegg.spiders.a",
+ "testegg.spiders.b",
+ "testegg",
]
assert {m.__name__ for m in mods} == set(expected)
-
- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
- expected = [
- "tests.test_utils_misc.test_walk_modules.mod",
- "tests.test_utils_misc.test_walk_modules.mod.mod0",
- ]
- assert {m.__name__ for m in mods} == set(expected)
-
- mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
- expected = [
- "tests.test_utils_misc.test_walk_modules.mod1",
- ]
- assert {m.__name__ for m in mods} == set(expected)
-
- with pytest.raises(ImportError):
- for _ in walk_modules_iter("nomodule999"):
- pass
- with (
- pytest.raises(ImportError),
- pytest.warns(
- ScrapyDeprecationWarning,
- match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
- "removed in a future version of Scrapy. "
- "Use scrapy.utils.misc.walk_modules_iter instead.",
- ),
- ):
- walk_modules("nomodule999")
-
- def test_walk_modules_egg(self):
- egg = str(Path(__file__).parent / "test.egg")
- sys.path.append(egg)
- try:
- mods = walk_modules_iter("testegg")
- expected = [
- "testegg.spiders",
- "testegg.spiders.a",
- "testegg.spiders.b",
- "testegg",
- ]
- assert {m.__name__ for m in mods} == set(expected)
- finally:
- sys.path.remove(egg)
-
- def test_arg_to_iter(self):
- class TestItem(Item):
- name = Field()
-
- assert hasattr(arg_to_iter(None), "__iter__")
- assert hasattr(arg_to_iter(100), "__iter__")
- assert hasattr(arg_to_iter("lala"), "__iter__")
- assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
- assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
-
- assert not list(arg_to_iter(None))
- assert list(arg_to_iter("lala")) == ["lala"]
- assert list(arg_to_iter(100)) == [100]
- assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
- assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
- assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
- assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
-
- def test_build_from_crawler(self):
- crawler = mock.MagicMock(spec_set=["settings"])
- args = (True, 100.0)
- kwargs = {"key": "val"}
-
- def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
- build_from_crawler(mock, crawler, *args, **kwargs)
- if hasattr(mock, "from_crawler"):
- mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
- assert mock.call_count == 0
- else:
- mock.assert_called_once_with(*args, **kwargs)
-
- # Check usage of correct constructor using 2 mocks:
- # 1. with no alternative constructors
- # 2. with from_crawler() constructor
- spec_sets = (
- ["__qualname__"],
- ["__qualname__", "from_crawler"],
- )
- for specs in spec_sets:
- m = mock.MagicMock(spec_set=specs)
- _test_with_crawler(m, crawler)
- m.reset_mock()
-
- # Check adoption of crawler
- m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
- m.from_crawler.return_value = None
- with pytest.raises(TypeError):
- build_from_crawler(m, crawler, *args, **kwargs)
-
- def test_set_environ(self):
- assert os.environ.get("some_test_environ") is None
- with set_environ(some_test_environ="test_value"):
- assert os.environ.get("some_test_environ") == "test_value"
- assert os.environ.get("some_test_environ") is None
-
- os.environ["some_test_environ"] = "test"
- assert os.environ.get("some_test_environ") == "test"
- with set_environ(some_test_environ="test_value"):
- assert os.environ.get("some_test_environ") == "test_value"
- assert os.environ.get("some_test_environ") == "test"
-
- def test_rel_has_nofollow(self):
- assert rel_has_nofollow("ugc nofollow") is True
- assert rel_has_nofollow("ugc,nofollow") is True
- assert rel_has_nofollow("ugc") is False
- assert rel_has_nofollow("nofollow") is True
- assert rel_has_nofollow("nofollowfoo") is False
- assert rel_has_nofollow("foonofollow") is False
- assert rel_has_nofollow("ugc, , nofollow") is True
- # rel attribute values are ASCII case-insensitive per the HTML spec
- assert rel_has_nofollow("NoFollow") is True
- assert rel_has_nofollow("NOFOLLOW") is True
- assert rel_has_nofollow("UGC NoFollow") is True
- assert rel_has_nofollow("ugc,NoFollow") is True
+ finally:
+ sys.path.remove(egg)
+
+
+def test_arg_to_iter() -> None:
+ class TestItem(Item):
+ name = Field()
+
+ assert hasattr(arg_to_iter(None), "__iter__")
+ assert hasattr(arg_to_iter(100), "__iter__")
+ assert hasattr(arg_to_iter("lala"), "__iter__")
+ assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
+ assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
+
+ assert not list(arg_to_iter(None))
+ assert list(arg_to_iter("lala")) == ["lala"]
+ assert list(arg_to_iter(100)) == [100]
+ assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
+ assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
+ assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
+ assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
+
+
+def test_build_from_crawler() -> None:
+ crawler = mock.MagicMock(spec_set=["settings"])
+ args = (True, 100.0)
+ kwargs = {"key": "val"}
+
+ def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
+ build_from_crawler(mock, crawler, *args, **kwargs)
+ if hasattr(mock, "from_crawler"):
+ mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+ assert mock.call_count == 0
+ else:
+ mock.assert_called_once_with(*args, **kwargs)
+
+ # Check usage of correct constructor using 2 mocks:
+ # 1. with no alternative constructors
+ # 2. with from_crawler() constructor
+ spec_sets = (
+ ["__qualname__"],
+ ["__qualname__", "from_crawler"],
+ )
+ for specs in spec_sets:
+ m = mock.MagicMock(spec_set=specs)
+ _test_with_crawler(m, crawler)
+ m.reset_mock()
+
+ # Check adoption of crawler
+ m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
+ m.from_crawler.return_value = None
+ with pytest.raises(TypeError):
+ build_from_crawler(m, crawler, *args, **kwargs)
+
+
+def test_set_environ() -> None:
+ assert os.environ.get("some_test_environ") is None
+ with set_environ(some_test_environ="test_value"):
+ assert os.environ.get("some_test_environ") == "test_value"
+ assert os.environ.get("some_test_environ") is None
+
+ os.environ["some_test_environ"] = "test"
+ assert os.environ.get("some_test_environ") == "test"
+ with set_environ(some_test_environ="test_value"):
+ assert os.environ.get("some_test_environ") == "test_value"
+ assert os.environ.get("some_test_environ") == "test"
+
+
+def test_rel_has_nofollow() -> None:
+ assert rel_has_nofollow("ugc nofollow") is True
+ assert rel_has_nofollow("ugc,nofollow") is True
+ assert rel_has_nofollow("ugc") is False
+ assert rel_has_nofollow("nofollow") is True
+ assert rel_has_nofollow("nofollowfoo") is False
+ assert rel_has_nofollow("foonofollow") is False
+ assert rel_has_nofollow("ugc, , nofollow") is True
+ # rel attribute values are ASCII case-insensitive per the HTML spec
+ assert rel_has_nofollow("NoFollow") is True
+ assert rel_has_nofollow("NOFOLLOW") is True
+ assert rel_has_nofollow("UGC NoFollow") is True
+ assert rel_has_nofollow("ugc,NoFollow") is True
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 1acc3aac2..7343a135a 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -1,5 +1,8 @@
+from __future__ import annotations
+
import warnings
from functools import partial
+from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
@@ -9,6 +12,11 @@ from scrapy.utils.misc import (
warn_on_generator_with_return_value,
)
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from scrapy import Spider
+
def _indentation_error(*args, **kwargs):
raise IndentationError
@@ -35,244 +43,239 @@ https://example.org
yield url
-def generator_that_returns_stuff():
+def generator_that_returns_stuff() -> Generator[int, None, int]:
yield 1
yield 2
return 3
-class TestUtilsMisc:
- @pytest.fixture
- def mock_spider(self):
- class MockSettings:
- def __init__(self, settings_dict=None):
- self.settings_dict = settings_dict or {
- "WARN_ON_GENERATOR_RETURN_VALUE": True
- }
+def mock_spider() -> Spider:
+ class MockSettings:
+ def __init__(self, settings_dict: dict[str, Any] | None = None):
+ self.settings_dict = settings_dict or {
+ "WARN_ON_GENERATOR_RETURN_VALUE": True
+ }
- def getbool(self, name, default=False):
- return self.settings_dict.get(name, default)
+ def getbool(self, name, default=False):
+ return self.settings_dict.get(name, default)
- class MockSpider:
- def __init__(self):
- self.settings = MockSettings()
+ class MockSpider:
+ def __init__(self) -> None:
+ self.settings = MockSettings()
- return MockSpider()
+ return MockSpider() # type: ignore[return-value]
- def test_generators_return_something(self, mock_spider):
- def f1():
- yield 1
- return 2
- def g1():
- yield 1
- return "asdf"
+def test_generators_return_something(mock_spider):
+ def f1():
+ yield 1
+ return 2
- def h1():
- yield 1
+ def g1():
+ yield 1
+ return "asdf"
- def helper():
- return 0
+ def h1():
+ yield 1
- yield helper()
- return 2
+ def helper() -> int:
+ return 0
- def i1():
- """
- docstring
- """
- url = """
-https://example.org
+ yield helper()
+ return 2
+
+ def i1():
"""
- yield url
- return 1
-
- assert is_generator_with_return_value(top_level_return_something)
- assert is_generator_with_return_value(f1)
- assert is_generator_with_return_value(g1)
- assert is_generator_with_return_value(h1)
- assert is_generator_with_return_value(i1)
-
- with pytest.warns(
- UserWarning,
- match='The "MockSpider.top_level_return_something" method is a generator',
- ):
- warn_on_generator_with_return_value(mock_spider, top_level_return_something)
- with pytest.warns(
- UserWarning, match='The "MockSpider.f1" method is a generator'
- ):
- warn_on_generator_with_return_value(mock_spider, f1)
- with pytest.warns(
- UserWarning, match='The "MockSpider.g1" method is a generator'
- ):
- warn_on_generator_with_return_value(mock_spider, g1)
- with pytest.warns(
- UserWarning, match='The "MockSpider.h1" method is a generator'
- ):
- warn_on_generator_with_return_value(mock_spider, h1)
- with pytest.warns(
- UserWarning, match='The "MockSpider.i1" method is a generator'
- ):
- warn_on_generator_with_return_value(mock_spider, i1)
-
- def test_generators_return_none(self, mock_spider):
- def f2():
- yield 1
-
- def g2():
- yield 1
-
- def h2():
- yield 1
-
- def i2():
- yield 1
- yield from generator_that_returns_stuff()
-
- def j2():
- yield 1
-
- def helper():
- return 0
-
- yield helper()
-
- def k2():
- """
- docstring
- """
- url = """
+ docstring
+ """
+ url = """
https://example.org
+ """
+ yield url
+ return 1
+
+ assert is_generator_with_return_value(top_level_return_something)
+ assert is_generator_with_return_value(f1)
+ assert is_generator_with_return_value(g1)
+ assert is_generator_with_return_value(h1)
+ assert is_generator_with_return_value(i1)
+
+ with pytest.warns(
+ UserWarning,
+ match='The "MockSpider.top_level_return_something" method is a generator',
+ ):
+ warn_on_generator_with_return_value(mock_spider, top_level_return_something)
+ with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'):
+ warn_on_generator_with_return_value(mock_spider, f1)
+ with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'):
+ warn_on_generator_with_return_value(mock_spider, g1)
+ with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'):
+ warn_on_generator_with_return_value(mock_spider, h1)
+ with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'):
+ warn_on_generator_with_return_value(mock_spider, i1)
+
+
+def test_generators_return_none(mock_spider):
+ def f2():
+ yield 1
+
+ def g2():
+ yield 1
+
+ def h2():
+ yield 1
+
+ def i2():
+ yield 1
+ yield from generator_that_returns_stuff()
+
+ def j2():
+ yield 1
+
+ def helper() -> int:
+ return 0
+
+ yield helper()
+
+ def k2():
"""
- yield url
-
- def l2():
- return
-
- assert not is_generator_with_return_value(top_level_return_none)
- assert not is_generator_with_return_value(f2)
- assert not is_generator_with_return_value(g2)
- assert not is_generator_with_return_value(h2)
- assert not is_generator_with_return_value(i2)
- assert not is_generator_with_return_value(j2) # not recursive
- assert not is_generator_with_return_value(k2) # not recursive
- assert not is_generator_with_return_value(l2)
-
- with warnings.catch_warnings():
- warnings.simplefilter("error", UserWarning)
- warn_on_generator_with_return_value(mock_spider, top_level_return_none)
- warn_on_generator_with_return_value(mock_spider, f2)
- warn_on_generator_with_return_value(mock_spider, g2)
- warn_on_generator_with_return_value(mock_spider, h2)
- warn_on_generator_with_return_value(mock_spider, i2)
- warn_on_generator_with_return_value(mock_spider, j2)
- warn_on_generator_with_return_value(mock_spider, k2)
- warn_on_generator_with_return_value(mock_spider, l2)
-
- def test_generators_return_none_with_decorator(self, mock_spider):
- def decorator(func):
- def inner_func():
- func()
-
- return inner_func
-
- @decorator
- def f3():
- yield 1
-
- @decorator
- def g3():
- yield 1
-
- @decorator
- def h3():
- yield 1
-
- @decorator
- def i3():
- yield 1
- yield from generator_that_returns_stuff()
-
- @decorator
- def j3():
- yield 1
-
- def helper():
- return 0
-
- yield helper()
-
- @decorator
- def k3():
- """
- docstring
- """
- url = """
+ docstring
+ """
+ url = """
https://example.org
+ """
+ yield url
+
+ def l2():
+ return
+
+ assert not is_generator_with_return_value(top_level_return_none)
+ assert not is_generator_with_return_value(f2)
+ assert not is_generator_with_return_value(g2)
+ assert not is_generator_with_return_value(h2)
+ assert not is_generator_with_return_value(i2)
+ assert not is_generator_with_return_value(j2) # not recursive
+ assert not is_generator_with_return_value(k2) # not recursive
+ assert not is_generator_with_return_value(l2)
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", UserWarning)
+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+ warn_on_generator_with_return_value(mock_spider, f2)
+ warn_on_generator_with_return_value(mock_spider, g2)
+ warn_on_generator_with_return_value(mock_spider, h2)
+ warn_on_generator_with_return_value(mock_spider, i2)
+ warn_on_generator_with_return_value(mock_spider, j2)
+ warn_on_generator_with_return_value(mock_spider, k2)
+ warn_on_generator_with_return_value(mock_spider, l2)
+
+
+def test_generators_return_none_with_decorator(mock_spider):
+ def decorator(func):
+ def inner_func():
+ func()
+
+ return inner_func
+
+ @decorator
+ def f3():
+ yield 1
+
+ @decorator
+ def g3():
+ yield 1
+
+ @decorator
+ def h3():
+ yield 1
+
+ @decorator
+ def i3():
+ yield 1
+ yield from generator_that_returns_stuff()
+
+ @decorator
+ def j3():
+ yield 1
+
+ def helper() -> int:
+ return 0
+
+ yield helper()
+
+ @decorator
+ def k3():
"""
- yield url
-
- @decorator
- def l3():
- return
-
- assert not is_generator_with_return_value(top_level_return_none)
- assert not is_generator_with_return_value(f3)
- assert not is_generator_with_return_value(g3)
- assert not is_generator_with_return_value(h3)
- assert not is_generator_with_return_value(i3)
- assert not is_generator_with_return_value(j3) # not recursive
- assert not is_generator_with_return_value(k3) # not recursive
- assert not is_generator_with_return_value(l3)
-
- with warnings.catch_warnings():
- warnings.simplefilter("error", UserWarning)
- warn_on_generator_with_return_value(mock_spider, top_level_return_none)
- warn_on_generator_with_return_value(mock_spider, f3)
- warn_on_generator_with_return_value(mock_spider, g3)
- warn_on_generator_with_return_value(mock_spider, h3)
- warn_on_generator_with_return_value(mock_spider, i3)
- warn_on_generator_with_return_value(mock_spider, j3)
- warn_on_generator_with_return_value(mock_spider, k3)
- warn_on_generator_with_return_value(mock_spider, l3)
-
- @mock.patch(
- "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
- )
- def test_indentation_error(self, mock_spider):
- with pytest.warns(UserWarning, match="Unable to determine"):
- warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-
- def test_partial(self):
- def cb(arg1, arg2):
- yield {}
-
- partial_cb = partial(cb, arg1=42)
- assert not is_generator_with_return_value(partial_cb)
-
- def test_warn_on_generator_with_return_value_settings_disabled(self):
- class MockSettings:
- def __init__(self, settings_dict=None):
- self.settings_dict = settings_dict or {}
-
- def getbool(self, name, default=False):
- return self.settings_dict.get(name, default)
-
- class MockSpider:
- def __init__(self):
- self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
-
- spider = MockSpider()
-
- def gen_with_return():
- yield 1
- return "value"
-
- with warnings.catch_warnings():
- warnings.simplefilter("error", UserWarning)
- warn_on_generator_with_return_value(spider, gen_with_return)
-
- spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
-
- with pytest.warns(UserWarning, match="is a generator"):
- warn_on_generator_with_return_value(spider, gen_with_return)
+ docstring
+ """
+ url = """
+https://example.org
+ """
+ yield url
+
+ @decorator
+ def l3():
+ return
+
+ assert not is_generator_with_return_value(top_level_return_none)
+ assert not is_generator_with_return_value(f3)
+ assert not is_generator_with_return_value(g3)
+ assert not is_generator_with_return_value(h3)
+ assert not is_generator_with_return_value(i3)
+ assert not is_generator_with_return_value(j3) # not recursive
+ assert not is_generator_with_return_value(k3) # not recursive
+ assert not is_generator_with_return_value(l3)
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", UserWarning)
+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+ warn_on_generator_with_return_value(mock_spider, f3)
+ warn_on_generator_with_return_value(mock_spider, g3)
+ warn_on_generator_with_return_value(mock_spider, h3)
+ warn_on_generator_with_return_value(mock_spider, i3)
+ warn_on_generator_with_return_value(mock_spider, j3)
+ warn_on_generator_with_return_value(mock_spider, k3)
+ warn_on_generator_with_return_value(mock_spider, l3)
+
+
[email protected]("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+def test_indentation_error(mock_spider):
+ with pytest.warns(UserWarning, match="Unable to determine"):
+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+
+
+def test_partial() -> None:
+ def cb(arg1, arg2):
+ yield {}
+
+ partial_cb = partial(cb, arg1=42)
+ assert not is_generator_with_return_value(partial_cb)
+
+
+def test_warn_on_generator_with_return_value_settings_disabled() -> None:
+ class MockSettings:
+ def __init__(self, settings_dict: dict[str, Any] | None = None):
+ self.settings_dict = settings_dict or {}
+
+ def getbool(self, name, default=False):
+ return self.settings_dict.get(name, default)
+
+ class MockSpider:
+ def __init__(self) -> None:
+ self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
+
+ spider = MockSpider()
+
+ def gen_with_return():
+ yield 1
+ return "value"
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", UserWarning)
+ warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]
+
+ spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
+
+ with pytest.warns(UserWarning, match="is a generator"):
+ warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 5333a55cb..7eade6463 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,14 +1,20 @@
+from __future__ import annotations
+
import os
from pathlib import Path
+from typing import TYPE_CHECKING
import pytest
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
@pytest.fixture
-def proj_path(tmp_path):
+def proj_path(tmp_path: Path) -> Generator[Path]:
prev_dir = Path.cwd()
project_dir = tmp_path
@@ -21,7 +27,7 @@ def proj_path(tmp_path):
os.chdir(prev_dir)
-def test_data_path_outside_project():
+def test_data_path_outside_project() -> None:
assert str(Path(".scrapy", "somepath")) == data_path("somepath")
abspath = str(Path(os.path.sep, "absolute", "path"))
assert abspath == data_path(abspath)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 2e8047e2d..c3b5dfc99 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -22,8 +22,7 @@ from scrapy.utils.python import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from collections.abc import Iterable, Mapping
-
+ from collections.abc import AsyncIterator, Iterable, Mapping
_KT = TypeVar("_KT")
_VT = TypeVar("_VT")
@@ -31,22 +30,22 @@ _VT = TypeVar("_VT")
class TestMutableAsyncChain:
@staticmethod
- async def g1():
+ async def g1() -> AsyncIterator[int]:
for i in range(3):
yield i
@staticmethod
- async def g2():
+ async def g2() -> AsyncIterator[int]:
return
yield
@staticmethod
- async def g3():
+ async def g3() -> AsyncIterator[int]:
for i in range(7, 10):
yield i
@staticmethod
- async def g4():
+ async def g4() -> AsyncIterator[int]:
for i in range(3, 5):
yield i
1 / 0
@@ -85,7 +84,7 @@ class TestToUnicode:
def test_converting_a_strange_object_should_raise_type_error(self):
with pytest.raises(TypeError):
- to_unicode(423)
+ to_unicode(423) # type: ignore[arg-type]
def test_errors_argument(self):
assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb"
@@ -103,7 +102,7 @@ class TestToBytes:
def test_converting_a_strange_object_should_raise_type_error(self):
with pytest.raises(TypeError):
- to_bytes(pytest)
+ to_bytes(pytest) # type: ignore[arg-type]
def test_errors_argument(self):
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
@@ -112,10 +111,10 @@ class TestToBytes:
def test_memoizemethod_noargs():
class A:
@memoizemethod_noargs
- def cached(self):
+ def cached(self) -> object:
return object()
- def noncached(self):
+ def noncached(self) -> object:
return object()
a = A()
@@ -150,7 +149,7 @@ def test_get_func_args():
pass
class A:
- def __init__(self, a, b, c):
+ def __init__(self, a: int, b: int, c: int):
pass
def method(self, a, b, c):
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 7d39a478e..44cb5c306 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import asyncio
import pytest
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 1642a932b..eb95be06c 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
from hashlib import sha1
-from typing import Any
+from typing import TYPE_CHECKING, Any, Protocol
from weakref import WeakKeyDictionary
import pytest
@@ -17,6 +17,9 @@ from scrapy.utils.request import (
)
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
@pytest.mark.parametrize(
("r", "expected"),
@@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
request_httprepr(r)
+class _FingerprintFunction(Protocol):
+ def __call__(
+ self,
+ request: Request,
+ *,
+ include_headers: Iterable[bytes | str] | None = None,
+ keep_fragments: bool = False,
+ ) -> bytes: ...
+
+
class TestFingerprint:
- function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
+ function: _FingerprintFunction = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
@@ -261,6 +274,7 @@ class TestRequestFingerprinter:
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
+ assert crawler.request_fingerprinter
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
@@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
+ assert crawler.request_fingerprinter
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter:
def test_dont_canonicalize(self):
class RequestFingerprinter:
- cache = WeakKeyDictionary()
+ cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary()
- def fingerprint(self, request):
+ def fingerprint(self, request: Request) -> bytes:
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.url))
@@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
+ assert crawler.request_fingerprinter
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com?a=2&a=1")
@@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
+ assert crawler.request_fingerprinter
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
+ assert crawler.request_fingerprinter
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 146cdb802..608b2bbd9 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from pathlib import Path
from time import process_time
from urllib.parse import urlparse
@@ -15,7 +17,7 @@ from scrapy.utils.response import (
)
-def _read_browser_output(burl: str):
+def _read_browser_output(burl: str) -> bytes:
path = urlparse(burl).path
if not path or not Path(path).exists():
path = burl.replace("file://", "")
@@ -224,7 +226,7 @@ def test_open_in_browser_redos_head():
(b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
],
)
-def test_remove_html_comments(input_body, output_body):
+def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:
assert _remove_html_comments(input_body) == output_body
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 2e6a790f8..2702c2cce 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import dataclasses
import datetime
import json
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index dbb9caf2a..3109bc149 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Callable
-if TYPE_CHECKING:
- from collections.abc import Callable
-
class TestSendCatchLog:
# whether the function being tested returns exceptions or failures
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index ac57e1739..9f2dce6d6 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 4515ce36e..076edf3db 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,7 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
from scrapy.utils.template import render_templatefile
+if TYPE_CHECKING:
+ from pathlib import Path
+
-def test_simple_render(tmp_path):
+def test_simple_render(tmp_path: Path) -> None:
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 2334c76e9..5458aa603 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import sys
from io import StringIO
from unittest import mock
@@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago
@mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_empty(stdout):
+def test_print_live_refs_empty(stdout: StringIO) -> None:
trackref.print_live_refs()
assert stdout.getvalue() == "Live References\n\n\n"
@mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_with_objects(stdout):
+def test_print_live_refs_with_objects(stdout: StringIO) -> None:
o1 = Foo() # noqa: F841
trackref.print_live_refs()
assert (
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 5b98131a1..d9d162c23 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
@@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str):
),
],
)
-def test_guess_scheme_skipped(url: str, expected: str, reason: str):
+def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
pytest.skip(reason)
diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index a431091d5..4ff562dff 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -16,7 +16,7 @@ class MyRequest2(Request):
@pytest.mark.mypy_testing
def mypy_test_headers() -> None:
- Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+ Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
Request("data:,", headers=None)
Request("data:,", headers={})
Request("data:,", headers=[])
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index d497c2470..1c157328c 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
@pytest.mark.mypy_testing
def mypy_test_headers() -> None:
- Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+ Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
Response("data:,", headers=None)
Response("data:,", headers={})
Response("data:,", headers=[])