Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Disable mypy allow_any_generics. (#7532)

Andrey Rakhmatullin55c17a89858e
diff --git a/pyproject.toml b/pyproject.toml
index 76b8bbead..2f07dcd80 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,7 +86,6 @@ pattern = "^(?P<version>.+)$"
[tool.mypy]
strict = true
-allow_any_generics = true # 67 errors
extra_checks = false # weird addErrback() errors
untyped_calls_exclude = [
"twisted",
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 17bcf19b0..cc8624fa1 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -147,7 +147,7 @@ class Command(ScrapyCommand):
name: str,
url: str,
template_name: str,
- template_file: str | os.PathLike,
+ template_file: str | os.PathLike[str],
) -> None:
"""Generate the spider module, based on the given template"""
assert self.settings is not None
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 24f56b7a6..c0da7dfa4 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -27,7 +27,7 @@ class Contract:
request_cls: type[Request] | None = None
name: str
- def __init__(self, method: Callable, *args: Any):
+ def __init__(self, method: Callable[..., Any], *args: Any):
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args: tuple[Any, ...] = args
@@ -105,7 +105,7 @@ class ContractsManager:
return methods
- def extract_contracts(self, method: Callable) -> list[Contract]:
+ def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:
contracts: list[Contract] = []
assert method.__doc__ is not None
for line_ in method.__doc__.split("\n"):
@@ -134,7 +134,9 @@ class ContractsManager:
return requests
- def from_method(self, method: Callable, results: TestResult) -> Request | None:
+ def from_method(
+ self, method: Callable[..., Any], results: TestResult
+ ) -> Request | None:
contracts = self.extract_contracts(method)
if contracts:
request_cls = Request
@@ -170,7 +172,7 @@ class ContractsManager:
return None
def _clean_req(
- self, request: Request, method: Callable, results: TestResult
+ self, request: Request, method: Callable[..., Any], results: TestResult
) -> None:
"""stop the request from returning objects and records any errors"""
@@ -195,7 +197,7 @@ class ContractsManager:
request.errback = eb_wrapper
-def _create_testcase(method: Callable, desc: str) -> TestCase:
+def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase:
spider = method.__self__.name # type: ignore[attr-defined]
class ContractTestCase(TestCase):
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 19a1e8503..c6660ff48 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, cast
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import NotConfigured
@@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
+ from collections.abc import Mapping
+
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.http import Response
@@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler):
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
- headers=request.headers.to_unicode_dict(),
+ headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),
data=request.body,
)
assert self._signer
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 6685d90af..ab74e22a4 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -8,7 +8,7 @@ from __future__ import annotations
import warnings
from functools import wraps
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
@@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
result = await self._process_exception(ex, request)
return await self._process_response(result, request)
- def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+ def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:
if method in self._mw_methods_requiring_spider:
kwargs["spider"] = self._spider
@@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
download_func: Callable[[Request], Coroutine[Any, Any, Response]],
) -> Response | Request:
for method in self.methods["process_request"]:
- method = cast("Callable", method)
+ assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request),
_warn=global_object_name(method),
@@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
for method in self.methods["process_response"]:
- method = cast("Callable", method)
+ assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, response=response),
_warn=global_object_name(method),
@@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
self, exception: Exception, request: Request | Response
) -> Response | Request:
for method in self.methods["process_exception"]:
- method = cast("Callable", method)
+ assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, exception=exception),
_warn=global_object_name(method),
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index bcf38bb75..fbc6f2530 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from functools import wraps
from inspect import isasyncgenfunction
from itertools import islice
-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
from warnings import warn
from twisted.python.failure import Failure
@@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
request: Request,
) -> Iterable[_T] | AsyncIterator[_T]:
for method in self.methods["process_spider_input"]:
- method = cast("Callable", method)
+ assert method is not None
try:
if method in self._mw_methods_requiring_spider:
result = method(response=response, spider=self._spider)
@@ -248,9 +248,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
# This method is only needed until _async compatibility methods are removed.
@staticmethod
- def _get_process_spider_output(mw: Any) -> Callable | None:
- normal_method: Callable | None = getattr(mw, "process_spider_output", None)
- async_method: Callable | None = getattr(mw, "process_spider_output_async", None)
+ def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:
+ normal_method: Callable[..., Any] | None = getattr(
+ mw, "process_spider_output", None
+ )
+ async_method: Callable[..., Any] | None = getattr(
+ mw, "process_spider_output_async", None
+ )
if not async_method:
if normal_method and not isasyncgenfunction(normal_method):
raise TypeError(
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 86b066a38..dbb79b02d 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -313,7 +313,9 @@ class FilesystemCacheStorage:
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
# https://github.com/python/mypy/issues/10740
- self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = (
+ self._open: Callable[
+ Concatenate[str | os.PathLike[str], str, ...], IO[bytes]
+ ] = (
gzip.open if self.use_gzip else open # type: ignore[assignment]
)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 13853f64d..6876e35e8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -220,7 +220,7 @@ class TextResponse(Response):
def follow_all(
self,
- urls: Iterable[str | Link] | parsel.SelectorList | None = None,
+ urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 14b591a5f..37ec0f583 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -50,8 +50,8 @@ class MiddlewareManager(ABC):
)
self.middlewares: tuple[Any, ...] = middlewares
# Only process_spider_output and process_spider_exception can be None.
- self.methods: dict[str, deque[Callable | None]] = defaultdict(deque)
- self._mw_methods_requiring_spider: set[Callable] = set()
+ self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque)
+ self._mw_methods_requiring_spider: set[Callable[..., Any]] = set()
for mw in middlewares:
self._add_middleware(mw)
@@ -116,7 +116,7 @@ class MiddlewareManager(ABC):
def _add_middleware(self, mw: Any) -> None: # noqa: B027
pass
- def _check_mw_method_spider_arg(self, method: Callable) -> None:
+ def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None:
if argument_is_required(method, "spider"):
warnings.warn(
f"{method.__qualname__}() requires a spider argument,"
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f0d093c6e..e14279b64 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -46,7 +46,9 @@ def _identity_process_request(request: Request, response: Response) -> Request |
return request
-def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None:
+def _get_method(
+ method: Callable[..., Any] | str | None, spider: Spider
+) -> Callable[..., Any] | None:
if callable(method):
return method
if isinstance(method, str):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 7007cd4b8..e06e38e23 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -26,7 +26,7 @@ if TYPE_CHECKING:
def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]:
class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]
- def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any):
+ def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any):
dirname = Path(path).parent
if not dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index ecb4c1492..44604c0fe 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -148,7 +148,7 @@ class AsyncioLoopingCall:
self._func: Callable[_P, _T] = func
self._args: tuple[Any, ...] = args
self._kwargs: dict[str, Any] = kwargs
- self._task: asyncio.Task | None = None
+ self._task: asyncio.Task[None] | None = None
self.interval: float | None = None
self._start_time: float | None = None
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5869cf52e..4850b370b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
def closest_scrapy_cfg(
- path: str | os.PathLike = ".",
- prevpath: str | os.PathLike | None = None,
+ path: str | os.PathLike[str] = ".",
+ prevpath: str | os.PathLike[str] | None = None,
) -> str:
"""Return the path to the closest scrapy.cfg file by traversing the current
directory and its parents
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 146e3ae56..4e65c062e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -28,7 +28,7 @@ _KT = TypeVar("_KT")
_VT = TypeVar("_VT")
-class CaselessDict(dict):
+class CaselessDict(dict): # type: ignore[type-arg]
__slots__ = ()
def __new__(cls, *args: Any, **kwargs: Any) -> Self:
@@ -99,20 +99,20 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
"""A dict-like structure that accepts strings or bytes
as keys and allows case-insensitive lookups.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
- self._keys: dict = {}
+ self._keys: dict[str | bytes, Any] = {}
super().__init__(*args, **kwargs)
- def __getitem__(self, key: AnyStr) -> Any:
+ def __getitem__(self, key: str | bytes) -> Any:
normalized_key = self._normkey(key)
return super().__getitem__(self._keys[normalized_key.lower()])
- def __setitem__(self, key: AnyStr, value: Any) -> None:
+ def __setitem__(self, key: str | bytes, value: Any) -> None:
normalized_key = self._normkey(key)
try:
lower_key = self._keys[normalized_key.lower()]
@@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict):
super().__setitem__(normalized_key, self._normvalue(value))
self._keys[normalized_key.lower()] = normalized_key
- def __delitem__(self, key: AnyStr) -> None:
+ def __delitem__(self, key: str | bytes) -> None:
normalized_key = self._normkey(key)
stored_key = self._keys.pop(normalized_key.lower())
super().__delitem__(stored_key)
- def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
normalized_key = self._normkey(key)
return normalized_key.lower() in self._keys
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
- def _normkey(self, key: AnyStr) -> AnyStr:
+ def _normkey(self, key: str | bytes) -> str | bytes:
return key
def _normvalue(self, value: Any) -> Any:
@@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]):
super().__setitem__(key, value)
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
"""
A weakref.WeakKeyDictionary implementation that uses LocalCache as its
underlying data structure, making it ordered and capable of being size-limited.
@@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
def __init__(self, limit: int | None = None):
super().__init__()
- self.data: LocalCache = LocalCache(limit=limit)
+ self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
- def __setitem__(self, key: _KT, value: _VT) -> None:
+ def __setitem__(self, key: _KT, value: _VT | None) -> None:
# if raised, key is not weak-referenceable, skip caching
with contextlib.suppress(TypeError):
super().__setitem__(key, value)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 90fd04777..29a34d4ef 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -173,7 +173,7 @@ def parallel(
return DeferredList([coop.coiterate(work) for _ in range(count)])
-class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
+class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]):
"""A class that wraps an async iterable into a normal iterator suitable
for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
it calls the callable directly in the callback, instead of providing a more
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 102362506..ee65d4155 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -260,7 +260,8 @@ def logformatter_adapter(
return (level, message, args)
-class SpiderLoggerAdapter(logging.LoggerAdapter):
+# LoggerAdapter is only parameterized since Python 3.11
+class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg]
def process(
self, msg: str, kwargs: MutableMapping[str, Any]
) -> tuple[str, MutableMapping[str, Any]]:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 3baa02e2b..0b67eaa34 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
-_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
+_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes)
_T = TypeVar("_T")
_T_co = TypeVar("_T_co", covariant=True)
_P = ParamSpec("_P")
@@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
yield node
-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = (
+ LocalWeakReferencedCache(limit=128)
+)
def _returns_none(return_node: ast.Return) -> bool:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 37db6c25f..7ab58093a 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]):
self._a: tuple[Any, ...] = a
self._kw: dict[str, Any] = kw
self._call: CallLaterResult | None = None
- self._deferreds: list[Deferred] = []
+ self._deferreds: list[Deferred[None]] = []
def schedule(self, delay: float = 0) -> None:
# circular import
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index d9a72273a..ee391a49e 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -186,7 +186,9 @@ async def _send_catch_log_asyncio(
handlers: list[Awaitable[TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
- async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
+ async def handler(
+ receiver: Callable[..., Any],
+ ) -> tuple[Callable[..., Any], TypingAny]:
result: TypingAny
try:
result = await ensure_awaitable(
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 3e4dae5c8..977d5a42a 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
from os import PathLike
-def render_templatefile(path: str | PathLike, **kwargs: Any) -> None:
+def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
path_obj = Path(path)
raw = path_obj.read_text("utf8")
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 082aca4b1..87df10a02 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -30,7 +30,9 @@ if TYPE_CHECKING:
from typing_extensions import Self
-live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
+live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
+ WeakKeyDictionary
+)
class object_ref:
diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
index 93259f696..f49741441 100644
--- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
+++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
@@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider):
yield
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
try:
task.result()
except Exception:
diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
index f6fead718..b85f60f87 100644
--- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
@@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider):
}
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
try:
task.result()
except Exception:
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index e358eaca4..98fe58860 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -4,7 +4,7 @@ from collections import defaultdict
from typing import Any
-class DummyDB(dict):
+class DummyDB(dict): # type: ignore[type-arg]
"""Provide dummy DBM-like interface."""
def close(self):
diff --git a/tests/spiders.py b/tests/spiders.py
index 373dc00bd..065a0e6d7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -22,6 +22,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue
if TYPE_CHECKING:
+ from scrapy.settings import _SettingsKey
from tests.mockserver.http import MockServer
@@ -94,19 +95,19 @@ class DelaySpider(MetaSpider):
class LogSpider(MetaSpider):
name = "log_spider"
- def log_debug(self, message: str, extra: dict | None = None):
+ def log_debug(self, message: str, extra: dict[str, Any] | None = None):
self.logger.debug(message, extra=extra)
- def log_info(self, message: str, extra: dict | None = None):
+ def log_info(self, message: str, extra: dict[str, Any] | None = None):
self.logger.info(message, extra=extra)
- def log_warning(self, message: str, extra: dict | None = None):
+ def log_warning(self, message: str, extra: dict[str, Any] | None = None):
self.logger.warning(message, extra=extra)
- def log_error(self, message: str, extra: dict | None = None):
+ def log_error(self, message: str, extra: dict[str, Any] | None = None):
self.logger.error(message, extra=extra)
- def log_critical(self, message: str, extra: dict | None = None):
+ def log_critical(self, message: str, extra: dict[str, Any] | None = None):
self.logger.critical(message, extra=extra)
def parse(self, response):
@@ -417,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
"""
name = "crawl_spider_with_parse_method"
- custom_settings: dict = {
+ custom_settings: dict[_SettingsKey, Any] = {
"RETRY_HTTP_CODES": [], # no need to retry
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index fd9505060..67e3eb50a 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase
from tests.utils.cmdline import call, proc
-def find_in_file(filename: Path, regex: str) -> re.Match | None:
+def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
with filename.open("r", encoding="utf-8") as f:
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index b407211dd..1585835cc 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -4,7 +4,7 @@ import os
import sys
from io import BytesIO
from pathlib import Path
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
import pytest
from pexpect.popen_spawn import PopenSpawn
@@ -151,8 +151,7 @@ class TestInteractiveShell:
env = os.environ.copy()
env["SCRAPY_PYTHON_SHELL"] = "python"
logfile = BytesIO()
- # https://github.com/python/typeshed/issues/14915
- p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5)
+ p = PopenSpawn(args, env=env, timeout=5)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendline(f"fetch('{mockserver.url('/')}')")
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py
index 4eed09fcd..2a9d0ed57 100644
--- a/tests/test_command_startproject.py
+++ b/tests/test_command_startproject.py
@@ -79,7 +79,7 @@ class TestStartprojectCommand:
def get_permissions_dict(
- path: str | os.PathLike, renamings=None, ignore=None
+ path: str | os.PathLike[str], renamings=None, ignore=None
) -> dict[str, str]:
def get_permissions(path: Path) -> str:
return oct(path.stat().st_mode)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index cbf568524..27e0c6445 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -41,7 +41,7 @@ from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
- from collections.abc import Callable, Iterable
+ from collections.abc import Awaitable, Callable, Iterable
def path_to_url(path: str | Path) -> str:
@@ -1312,7 +1312,9 @@ class TestFeedExporterSignals:
self.feed_slot_closed_received = True
async def run_signaled_feed_exporter(
- self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable
+ self,
+ feed_exporter_signal_handler: Callable[[], Awaitable[None] | None],
+ feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
) -> None:
crawler = get_crawler(settings_dict=self.settings)
feed_exporter = FeedExporter.from_crawler(crawler)
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 3b70e896c..d855d0f74 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
from os import PathLike
-def build_url(path: str | PathLike) -> str:
+def build_url(path: str | PathLike[str]) -> str:
path_str = str(path)
if path_str[0] != "/":
path_str = "/" + path_str
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 6072e2f6d..cec5d728b 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes:
class DummySpider(Spider):
name = "dummy"
- start_urls: list = []
+ start_urls = []
def parse(self, response):
print(response)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 7cfc7ed26..c094d25d8 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -39,7 +39,7 @@ class AttrsNameItem:
@dataclasses.dataclass
class NameDataClass:
- name: list = dataclasses.field(default_factory=list)
+ name: list[str] = dataclasses.field(default_factory=list)
# test item loaders
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index dbe8d85a5..c6a41fa6e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
class FilesPipelineTestDataClass:
name: str
# default fields
- file_urls: list = dataclasses.field(default_factory=list)
- files: list = dataclasses.field(default_factory=list)
+ file_urls: list[str] = dataclasses.field(default_factory=list)
+ files: list[dict[str, str]] = dataclasses.field(default_factory=list)
# overridden fields
- custom_file_urls: list = dataclasses.field(default_factory=list)
- custom_files: list = dataclasses.field(default_factory=list)
+ custom_file_urls: list[str] = dataclasses.field(default_factory=list)
+ custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)
class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 199ec5afa..38662348f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):
class ImagesPipelineTestDataClass:
name: str
# default fields
- image_urls: list = dataclasses.field(default_factory=list)
- images: list = dataclasses.field(default_factory=list)
+ image_urls: list[str] = dataclasses.field(default_factory=list)
+ images: list[dict[str, str]] = dataclasses.field(default_factory=list)
# overridden fields
- custom_image_urls: list = dataclasses.field(default_factory=list)
- custom_images: list = dataclasses.field(default_factory=list)
+ custom_image_urls: list[str] = dataclasses.field(default_factory=list)
+ custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list)
class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 176fefbbf..db023e1f8 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -41,10 +41,10 @@ class MinimalScheduler:
class SimpleScheduler(MinimalScheduler):
- def open(self, spider: Spider) -> defer.Deferred:
+ def open(self, spider: Spider) -> defer.Deferred[str]:
return defer.succeed("open")
- def close(self, reason: str) -> defer.Deferred:
+ def close(self, reason: str) -> defer.Deferred[str]:
return defer.succeed("close")
def __len__(self) -> int:
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2e0436ae1..a0d296553 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -110,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
Should work for process_spider_output and, when it's supported, process_start.
"""
- ITEM_TYPE: type | tuple
+ ITEM_TYPE: type | tuple[type, ...]
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index c58b02ca5..f573993a1 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -2,6 +2,7 @@ import copy
import warnings
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
+from typing import Any
import pytest
@@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collect
class TestCaseInsensitiveDictBase(ABC):
@property
@abstractmethod
- def dict_class(self) -> type[MutableMapping]:
+ def dict_class(self) -> type[MutableMapping[str, Any]]:
raise NotImplementedError
def test_init_dict(self):
@@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC):
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
- dict_class = CaseInsensitiveDict
+ dict_class = CaseInsensitiveDict # type: ignore[assignment]
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index f40e424ff..318370d40 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -144,7 +144,9 @@ class TestStreamLogger:
],
)
def test_spider_logger_adapter_process(
- base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
+ base_extra: Mapping[str, Any],
+ log_extra: MutableMapping[str, Any],
+ expected_extra: dict[str, Any],
) -> None:
logger = logging.getLogger("test")
spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index b25129c87..70800dcec 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import json
from hashlib import sha1
+from typing import Any
from weakref import WeakKeyDictionary
import pytest
@@ -56,13 +57,13 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
class TestFingerprint:
- function: staticmethod = staticmethod(fingerprint)
+ function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
| WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
) = _fingerprint_cache
default_cache_key = (None, False)
- known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = (
+ known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
(
Request("http://example.org"),
b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",
diff --git a/tox.ini b/tox.ini
index da5a8a9a6..5fef37e5b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,13 +44,13 @@ commands =
[testenv:typing]
basepython = python3.10
deps =
- mypy==1.20.2
+ mypy==2.1.0
typing-extensions==4.15.0
Pillow==12.2.0
Protego==0.6.0
Twisted==26.4.0
attrs==26.1.0
- boto3-stubs[s3]==1.43.2
+ boto3-stubs[s3]==1.43.9
botocore-stubs==1.42.41
h2==4.3.0
httpx==0.28.1
@@ -58,12 +58,12 @@ deps =
ptpython==3.0.32
# newer ones require newer Python
ipython==8.39.0
- pyOpenSSL==26.1.0
+ pyOpenSSL==26.2.0
pytest==9.0.3
- types-Pygments==2.20.0.20260408
+ types-Pygments==2.20.0.20260508
types-defusedxml==0.7.0.20260504
types-lxml==2026.2.16
- types-pexpect==4.9.0.20260408
+ types-pexpect==4.9.0.20260508
uvloop==0.22.1
w3lib==2.4.1
zstandard==0.25.0