Scrapy, a fast high-level web crawling & scraping framework for Python.
Disable mypy allow_any_generics. (#7532)
Andrey Rakhmatullin55c17a89858e
diff --git a/pyproject.toml b/pyproject.tomlindex 76b8bbead..2f07dcd80 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -86,7 +86,6 @@ pattern = "^(?P<version>.+)$"[tool.mypy]strict = true-allow_any_generics = true # 67 errorsextra_checks = false # weird addErrback() errorsuntyped_calls_exclude = ["twisted",diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 17bcf19b0..cc8624fa1 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -147,7 +147,7 @@ class Command(ScrapyCommand):name: str,url: str,template_name: str,- template_file: str | os.PathLike,+ template_file: str | os.PathLike[str],) -> None:"""Generate the spider module, based on the given template"""assert self.settings is not Nonediff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 24f56b7a6..c0da7dfa4 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -27,7 +27,7 @@ class Contract:request_cls: type[Request] | None = Nonename: str- def __init__(self, method: Callable, *args: Any):+ def __init__(self, method: Callable[..., Any], *args: Any):self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")self.args: tuple[Any, ...] = args@@ -105,7 +105,7 @@ class ContractsManager:return methods- def extract_contracts(self, method: Callable) -> list[Contract]:+ def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:contracts: list[Contract] = []assert method.__doc__ is not Nonefor line_ in method.__doc__.split("\n"):@@ -134,7 +134,9 @@ class ContractsManager:return requests- def from_method(self, method: Callable, results: TestResult) -> Request | None:+ def from_method(+ self, method: Callable[..., Any], results: TestResult+ ) -> Request | None:contracts = self.extract_contracts(method)if contracts:request_cls = Request@@ -170,7 +172,7 @@ class ContractsManager:return Nonedef _clean_req(- self, request: Request, method: Callable, results: TestResult+ self, request: Request, method: Callable[..., Any], results: TestResult) -> None:"""stop the request from returning objects and records any errors"""@@ -195,7 +197,7 @@ class ContractsManager:request.errback = eb_wrapper-def _create_testcase(method: Callable, desc: str) -> TestCase:+def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase:spider = method.__self__.name # type: ignore[attr-defined]class ContractTestCase(TestCase):diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 19a1e8503..c6660ff48 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Any, castfrom scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import NotConfigured@@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:+ from collections.abc import Mapping+from scrapy import Requestfrom scrapy.crawler import Crawlerfrom scrapy.http import Response@@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler):awsrequest = botocore.awsrequest.AWSRequest(method=request.method,url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",- headers=request.headers.to_unicode_dict(),+ headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),data=request.body,)assert self._signerdiff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 6685d90af..ab74e22a4 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -8,7 +8,7 @@ from __future__ import annotationsimport warningsfrom functools import wraps-from typing import TYPE_CHECKING, Any, cast+from typing import TYPE_CHECKING, Anyfrom scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutputfrom scrapy.http import Request, Response@@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):result = await self._process_exception(ex, request)return await self._process_response(result, request)- def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:+ def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:if method in self._mw_methods_requiring_spider:kwargs["spider"] = self._spider@@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):download_func: Callable[[Request], Coroutine[Any, Any, Response]],) -> Response | Request:for method in self.methods["process_request"]:- method = cast("Callable", method)+ assert method is not Noneresponse = await ensure_awaitable(self._handle_mw_method(method, request=request),_warn=global_object_name(method),@@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):return responsefor method in self.methods["process_response"]:- method = cast("Callable", method)+ assert method is not Noneresponse = await ensure_awaitable(self._handle_mw_method(method, request=request, response=response),_warn=global_object_name(method),@@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):self, exception: Exception, request: Request | Response) -> Response | Request:for method in self.methods["process_exception"]:- method = cast("Callable", method)+ assert method is not Noneresponse = await ensure_awaitable(self._handle_mw_method(method, request=request, exception=exception),_warn=global_object_name(method),diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex bcf38bb75..fbc6f2530 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterablefrom functools import wrapsfrom inspect import isasyncgenfunctionfrom itertools import islice-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVarfrom warnings import warnfrom twisted.python.failure import Failure@@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):request: Request,) -> Iterable[_T] | AsyncIterator[_T]:for method in self.methods["process_spider_input"]:- method = cast("Callable", method)+ assert method is not Nonetry:if method in self._mw_methods_requiring_spider:result = method(response=response, spider=self._spider)@@ -248,9 +248,13 @@ class SpiderMiddlewareManager(MiddlewareManager):# This method is only needed until _async compatibility methods are removed.@staticmethod- def _get_process_spider_output(mw: Any) -> Callable | None:- normal_method: Callable | None = getattr(mw, "process_spider_output", None)- async_method: Callable | None = getattr(mw, "process_spider_output_async", None)+ def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:+ normal_method: Callable[..., Any] | None = getattr(+ mw, "process_spider_output", None+ )+ async_method: Callable[..., Any] | None = getattr(+ mw, "process_spider_output_async", None+ )if not async_method:if normal_method and not isasyncgenfunction(normal_method):raise TypeError(diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 86b066a38..dbb79b02d 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -313,7 +313,9 @@ class FilesystemCacheStorage:self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")# https://github.com/python/mypy/issues/10740- self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = (+ self._open: Callable[+ Concatenate[str | os.PathLike[str], str, ...], IO[bytes]+ ] = (gzip.open if self.use_gzip else open # type: ignore[assignment])diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 13853f64d..6876e35e8 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -220,7 +220,7 @@ class TextResponse(Response):def follow_all(self,- urls: Iterable[str | Link] | parsel.SelectorList | None = None,+ urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,callback: CallbackT | None = None,method: str = "GET",headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 14b591a5f..37ec0f583 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -50,8 +50,8 @@ class MiddlewareManager(ABC):)self.middlewares: tuple[Any, ...] = middlewares# Only process_spider_output and process_spider_exception can be None.- self.methods: dict[str, deque[Callable | None]] = defaultdict(deque)- self._mw_methods_requiring_spider: set[Callable] = set()+ self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque)+ self._mw_methods_requiring_spider: set[Callable[..., Any]] = set()for mw in middlewares:self._add_middleware(mw)@@ -116,7 +116,7 @@ class MiddlewareManager(ABC):def _add_middleware(self, mw: Any) -> None: # noqa: B027pass- def _check_mw_method_spider_arg(self, method: Callable) -> None:+ def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None:if argument_is_required(method, "spider"):warnings.warn(f"{method.__qualname__}() requires a spider argument,"diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex f0d093c6e..e14279b64 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -46,7 +46,9 @@ def _identity_process_request(request: Request, response: Response) -> Request |return request-def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None:+def _get_method(+ method: Callable[..., Any] | str | None, spider: Spider+) -> Callable[..., Any] | None:if callable(method):return methodif isinstance(method, str):diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 7007cd4b8..e06e38e23 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -26,7 +26,7 @@ if TYPE_CHECKING:def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]:class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]- def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any):+ def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any):dirname = Path(path).parentif not dirname.exists():dirname.mkdir(parents=True, exist_ok=True)diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex ecb4c1492..44604c0fe 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -148,7 +148,7 @@ class AsyncioLoopingCall:self._func: Callable[_P, _T] = funcself._args: tuple[Any, ...] = argsself._kwargs: dict[str, Any] = kwargs- self._task: asyncio.Task | None = None+ self._task: asyncio.Task[None] | None = Noneself.interval: float | None = Noneself._start_time: float | None = Nonediff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 5869cf52e..4850b370b 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:def closest_scrapy_cfg(- path: str | os.PathLike = ".",- prevpath: str | os.PathLike | None = None,+ path: str | os.PathLike[str] = ".",+ prevpath: str | os.PathLike[str] | None = None,) -> str:"""Return the path to the closest scrapy.cfg file by traversing the currentdirectory and its parentsdiff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 146e3ae56..4e65c062e 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -28,7 +28,7 @@ _KT = TypeVar("_KT")_VT = TypeVar("_VT")-class CaselessDict(dict):+class CaselessDict(dict): # type: ignore[type-arg]__slots__ = ()def __new__(cls, *args: Any, **kwargs: Any) -> Self:@@ -99,20 +99,20 @@ class CaselessDict(dict):return dict.pop(self, self.normkey(key), *args)-class CaseInsensitiveDict(collections.UserDict):+class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):"""A dict-like structure that accepts strings or bytesas keys and allows case-insensitive lookups."""def __init__(self, *args: Any, **kwargs: Any) -> None:- self._keys: dict = {}+ self._keys: dict[str | bytes, Any] = {}super().__init__(*args, **kwargs)- def __getitem__(self, key: AnyStr) -> Any:+ def __getitem__(self, key: str | bytes) -> Any:normalized_key = self._normkey(key)return super().__getitem__(self._keys[normalized_key.lower()])- def __setitem__(self, key: AnyStr, value: Any) -> None:+ def __setitem__(self, key: str | bytes, value: Any) -> None:normalized_key = self._normkey(key)try:lower_key = self._keys[normalized_key.lower()]@@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict):super().__setitem__(normalized_key, self._normvalue(value))self._keys[normalized_key.lower()] = normalized_key- def __delitem__(self, key: AnyStr) -> None:+ def __delitem__(self, key: str | bytes) -> None:normalized_key = self._normkey(key)stored_key = self._keys.pop(normalized_key.lower())super().__delitem__(stored_key)- def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]normalized_key = self._normkey(key)return normalized_key.lower() in self._keysdef __repr__(self) -> str:return f"<{self.__class__.__name__}: {super().__repr__()}>"- def _normkey(self, key: AnyStr) -> AnyStr:+ def _normkey(self, key: str | bytes) -> str | bytes:return keydef _normvalue(self, value: Any) -> Any:@@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]):super().__setitem__(key, value)-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):"""A weakref.WeakKeyDictionary implementation that uses LocalCache as itsunderlying data structure, making it ordered and capable of being size-limited.@@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):def __init__(self, limit: int | None = None):super().__init__()- self.data: LocalCache = LocalCache(limit=limit)+ self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)- def __setitem__(self, key: _KT, value: _VT) -> None:+ def __setitem__(self, key: _KT, value: _VT | None) -> None:# if raised, key is not weak-referenceable, skip cachingwith contextlib.suppress(TypeError):super().__setitem__(key, value)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 90fd04777..29a34d4ef 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -173,7 +173,7 @@ def parallel(return DeferredList([coop.coiterate(work) for _ in range(count)])-class _AsyncCooperatorAdapter(Iterator, Generic[_T]):+class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]):"""A class that wraps an async iterable into a normal iterator suitablefor using in Cooperator.coiterate(). As it's only needed for parallel_async(),it calls the callable directly in the callback, instead of providing a morediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 102362506..ee65d4155 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -260,7 +260,8 @@ def logformatter_adapter(return (level, message, args)-class SpiderLoggerAdapter(logging.LoggerAdapter):+# LoggerAdapter is only parameterized since Python 3.11+class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg]def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 3baa02e2b..0b67eaa34 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -28,7 +28,7 @@ if TYPE_CHECKING:_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes-_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)+_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes)_T = TypeVar("_T")_T_co = TypeVar("_T_co", covariant=True)_P = ParamSpec("_P")@@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:yield node-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)+_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = (+ LocalWeakReferencedCache(limit=128)+)def _returns_none(return_node: ast.Return) -> bool:diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 37db6c25f..7ab58093a 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]):self._a: tuple[Any, ...] = aself._kw: dict[str, Any] = kwself._call: CallLaterResult | None = None- self._deferreds: list[Deferred] = []+ self._deferreds: list[Deferred[None]] = []def schedule(self, delay: float = 0) -> None:# circular importdiff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex d9a72273a..ee391a49e 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -186,7 +186,9 @@ async def _send_catch_log_asyncio(handlers: list[Awaitable[TypingAny]] = []for receiver in liveReceivers(getAllReceivers(sender, signal)):- async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:+ async def handler(+ receiver: Callable[..., Any],+ ) -> tuple[Callable[..., Any], TypingAny]:result: TypingAnytry:result = await ensure_awaitable(diff --git a/scrapy/utils/template.py b/scrapy/utils/template.pyindex 3e4dae5c8..977d5a42a 100644--- a/scrapy/utils/template.py+++ b/scrapy/utils/template.py@@ -11,7 +11,7 @@ if TYPE_CHECKING:from os import PathLike-def render_templatefile(path: str | PathLike, **kwargs: Any) -> None:+def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:path_obj = Path(path)raw = path_obj.read_text("utf8")diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.pyindex 082aca4b1..87df10a02 100644--- a/scrapy/utils/trackref.py+++ b/scrapy/utils/trackref.py@@ -30,7 +30,9 @@ if TYPE_CHECKING:from typing_extensions import Self-live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)+live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(+ WeakKeyDictionary+)class object_ref:diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.pyindex 93259f696..f49741441 100644--- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py+++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py@@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider):yield-def log_task_exception(task: Task) -> None:+def log_task_exception(task: Task[None]) -> None:try:task.result()except Exception:diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.pyindex f6fead718..b85f60f87 100644--- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py@@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider):}-def log_task_exception(task: Task) -> None:+def log_task_exception(task: Task[None]) -> None:try:task.result()except Exception:diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.pyindex e358eaca4..98fe58860 100644--- a/tests/mocks/dummydbm.py+++ b/tests/mocks/dummydbm.py@@ -4,7 +4,7 @@ from collections import defaultdictfrom typing import Any-class DummyDB(dict):+class DummyDB(dict): # type: ignore[type-arg]"""Provide dummy DBM-like interface."""def close(self):diff --git a/tests/spiders.py b/tests/spiders.pyindex 373dc00bd..065a0e6d7 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -22,6 +22,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.test import get_from_asyncio_queueif TYPE_CHECKING:+ from scrapy.settings import _SettingsKeyfrom tests.mockserver.http import MockServer@@ -94,19 +95,19 @@ class DelaySpider(MetaSpider):class LogSpider(MetaSpider):name = "log_spider"- def log_debug(self, message: str, extra: dict | None = None):+ def log_debug(self, message: str, extra: dict[str, Any] | None = None):self.logger.debug(message, extra=extra)- def log_info(self, message: str, extra: dict | None = None):+ def log_info(self, message: str, extra: dict[str, Any] | None = None):self.logger.info(message, extra=extra)- def log_warning(self, message: str, extra: dict | None = None):+ def log_warning(self, message: str, extra: dict[str, Any] | None = None):self.logger.warning(message, extra=extra)- def log_error(self, message: str, extra: dict | None = None):+ def log_error(self, message: str, extra: dict[str, Any] | None = None):self.logger.error(message, extra=extra)- def log_critical(self, message: str, extra: dict | None = None):+ def log_critical(self, message: str, extra: dict[str, Any] | None = None):self.logger.critical(message, extra=extra)def parse(self, response):@@ -417,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):"""name = "crawl_spider_with_parse_method"- custom_settings: dict = {+ custom_settings: dict[_SettingsKey, Any] = {"RETRY_HTTP_CODES": [], # no need to retry}rules = (Rule(LinkExtractor(), callback="parse", follow=True),)diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.pyindex fd9505060..67e3eb50a 100644--- a/tests/test_command_genspider.py+++ b/tests/test_command_genspider.py@@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBasefrom tests.utils.cmdline import call, proc-def find_in_file(filename: Path, regex: str) -> re.Match | None:+def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:"""Find first pattern occurrence in file"""pattern = re.compile(regex)with filename.open("r", encoding="utf-8") as f:diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex b407211dd..1585835cc 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -4,7 +4,7 @@ import osimport sysfrom io import BytesIOfrom pathlib import Path-from typing import TYPE_CHECKING, cast+from typing import TYPE_CHECKINGimport pytestfrom pexpect.popen_spawn import PopenSpawn@@ -151,8 +151,7 @@ class TestInteractiveShell:env = os.environ.copy()env["SCRAPY_PYTHON_SHELL"] = "python"logfile = BytesIO()- # https://github.com/python/typeshed/issues/14915- p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5)+ p = PopenSpawn(args, env=env, timeout=5)p.logfile_read = logfilep.expect_exact("Available Scrapy objects")p.sendline(f"fetch('{mockserver.url('/')}')")diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.pyindex 4eed09fcd..2a9d0ed57 100644--- a/tests/test_command_startproject.py+++ b/tests/test_command_startproject.py@@ -79,7 +79,7 @@ class TestStartprojectCommand:def get_permissions_dict(- path: str | os.PathLike, renamings=None, ignore=None+ path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]:def get_permissions(path: Path) -> str:return oct(path.stat().st_mode)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex cbf568524..27e0c6445 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -41,7 +41,7 @@ from tests.spiders import ItemSpiderfrom tests.utils.decorators import coroutine_test, inline_callbacks_testif TYPE_CHECKING:- from collections.abc import Callable, Iterable+ from collections.abc import Awaitable, Callable, Iterabledef path_to_url(path: str | Path) -> str:@@ -1312,7 +1312,9 @@ class TestFeedExporterSignals:self.feed_slot_closed_received = Trueasync def run_signaled_feed_exporter(- self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable+ self,+ feed_exporter_signal_handler: Callable[[], Awaitable[None] | None],+ feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],) -> None:crawler = get_crawler(settings_dict=self.settings)feed_exporter = FeedExporter.from_crawler(crawler)diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 3b70e896c..d855d0f74 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -29,7 +29,7 @@ if TYPE_CHECKING:from os import PathLike-def build_url(path: str | PathLike) -> str:+def build_url(path: str | PathLike[str]) -> str:path_str = str(path)if path_str[0] != "/":path_str = "/" + path_strdiff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 6072e2f6d..cec5d728b 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes:class DummySpider(Spider):name = "dummy"- start_urls: list = []+ start_urls = []def parse(self, response):print(response)diff --git a/tests/test_loader.py b/tests/test_loader.pyindex 7cfc7ed26..c094d25d8 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -39,7 +39,7 @@ class AttrsNameItem:@dataclasses.dataclassclass NameDataClass:- name: list = dataclasses.field(default_factory=list)+ name: list[str] = dataclasses.field(default_factory=list)# test item loadersdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex dbe8d85a5..c6a41fa6e 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):class FilesPipelineTestDataClass:name: str# default fields- file_urls: list = dataclasses.field(default_factory=list)- files: list = dataclasses.field(default_factory=list)+ file_urls: list[str] = dataclasses.field(default_factory=list)+ files: list[dict[str, str]] = dataclasses.field(default_factory=list)# overridden fields- custom_file_urls: list = dataclasses.field(default_factory=list)- custom_files: list = dataclasses.field(default_factory=list)+ custom_file_urls: list[str] = dataclasses.field(default_factory=list)+ custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 199ec5afa..38662348f 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):class ImagesPipelineTestDataClass:name: str# default fields- image_urls: list = dataclasses.field(default_factory=list)- images: list = dataclasses.field(default_factory=list)+ image_urls: list[str] = dataclasses.field(default_factory=list)+ images: list[dict[str, str]] = dataclasses.field(default_factory=list)# overridden fields- custom_image_urls: list = dataclasses.field(default_factory=list)- custom_images: list = dataclasses.field(default_factory=list)+ custom_image_urls: list[str] = dataclasses.field(default_factory=list)+ custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list)class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 176fefbbf..db023e1f8 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -41,10 +41,10 @@ class MinimalScheduler:class SimpleScheduler(MinimalScheduler):- def open(self, spider: Spider) -> defer.Deferred:+ def open(self, spider: Spider) -> defer.Deferred[str]:return defer.succeed("open")- def close(self, reason: str) -> defer.Deferred:+ def close(self, reason: str) -> defer.Deferred[str]:return defer.succeed("close")def __len__(self) -> int:diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 2e0436ae1..a0d296553 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -110,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):Should work for process_spider_output and, when it's supported, process_start."""- ITEM_TYPE: type | tuple+ ITEM_TYPE: type | tuple[type, ...]RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects@staticmethoddiff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex c58b02ca5..f573993a1 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -2,6 +2,7 @@ import copyimport warningsfrom abc import ABC, abstractmethodfrom collections.abc import Iterator, Mapping, MutableMapping+from typing import Anyimport pytest@@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collectclass TestCaseInsensitiveDictBase(ABC):@property@abstractmethod- def dict_class(self) -> type[MutableMapping]:+ def dict_class(self) -> type[MutableMapping[str, Any]]:raise NotImplementedErrordef test_init_dict(self):@@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC):class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):- dict_class = CaseInsensitiveDict+ dict_class = CaseInsensitiveDict # type: ignore[assignment]def test_repr(self):d1 = self.dict_class({"foo": "bar"})diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex f40e424ff..318370d40 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -144,7 +144,9 @@ class TestStreamLogger:],)def test_spider_logger_adapter_process(- base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict+ base_extra: Mapping[str, Any],+ log_extra: MutableMapping[str, Any],+ expected_extra: dict[str, Any],) -> None:logger = logging.getLogger("test")spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex b25129c87..70800dcec 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport jsonfrom hashlib import sha1+from typing import Anyfrom weakref import WeakKeyDictionaryimport pytest@@ -56,13 +57,13 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:class TestFingerprint:- function: staticmethod = staticmethod(fingerprint)+ function: staticmethod[[Request], bytes] = staticmethod(fingerprint)cache: (WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]| WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]) = _fingerprint_cachedefault_cache_key = (None, False)- known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = (+ known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ((Request("http://example.org"),b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",diff --git a/tox.ini b/tox.iniindex da5a8a9a6..5fef37e5b 100644--- a/tox.ini+++ b/tox.ini@@ -44,13 +44,13 @@ commands =[testenv:typing]basepython = python3.10deps =- mypy==1.20.2+ mypy==2.1.0typing-extensions==4.15.0Pillow==12.2.0Protego==0.6.0Twisted==26.4.0attrs==26.1.0- boto3-stubs[s3]==1.43.2+ boto3-stubs[s3]==1.43.9botocore-stubs==1.42.41h2==4.3.0httpx==0.28.1@@ -58,12 +58,12 @@ deps =ptpython==3.0.32# newer ones require newer Pythonipython==8.39.0- pyOpenSSL==26.1.0+ pyOpenSSL==26.2.0pytest==9.0.3- types-Pygments==2.20.0.20260408+ types-Pygments==2.20.0.20260508types-defusedxml==0.7.0.20260504types-lxml==2026.2.16- types-pexpect==4.9.0.20260408+ types-pexpect==4.9.0.20260508uvloop==0.22.1w3lib==2.4.1zstandard==0.25.0