Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump tool versions. (#6941)
Andrey Rakhmatullindf342eee6e2f
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 6fbf74058..bcc10d5e8 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -1,8 +1,8 @@repos:- repo: https://github.com/astral-sh/ruff-pre-commit- rev: v0.9.3+ rev: v0.12.2hooks:- - id: ruff+ - id: ruff-checkargs: [ --fix ]- id: ruff-format- repo: https://github.com/adamchainz/blacken-docs@@ -10,7 +10,7 @@ repos:hooks:- id: blacken-docsadditional_dependencies:- - black==24.10.0+ - black==25.1.0- repo: https://github.com/pre-commit/pre-commit-hooksrev: v5.0.0hooks:diff --git a/conftest.py b/conftest.pyindex f952127b9..02697603d 100644--- a/conftest.py+++ b/conftest.py@@ -85,7 +85,7 @@ def requires_uvloop(request):if not request.node.get_closest_marker("requires_uvloop"):returntry:- import uvloop+ import uvloop # noqa: PLC0415del uvloopexcept ImportError:@@ -97,7 +97,7 @@ def requires_botocore(request):if not request.node.get_closest_marker("requires_botocore"):returntry:- import botocore+ import botocore # noqa: PLC0415del botocoreexcept ImportError:@@ -109,7 +109,7 @@ def requires_boto3(request):if not request.node.get_closest_marker("requires_boto3"):returntry:- import boto3+ import boto3 # noqa: PLC0415del boto3except ImportError:diff --git a/pyproject.toml b/pyproject.tomlindex a6b99ac5b..b25ccb894 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -106,10 +106,6 @@ follow_imports = "skip"module = "scrapy.settings.default_settings"ignore_errors = true-[[tool.mypy.overrides]]-module = "itemadapter"-implicit_reexport = true-[[tool.mypy.overrides]]module = "twisted"implicit_reexport = true@@ -394,6 +390,9 @@ banned-module-level-imports = ["twisted.internet.reactor",]+[tool.ruff.lint.isort]+split-on-trailing-comma = false+[tool.ruff.lint.per-file-ignores]# Circular import workarounds"scrapy/linkextractors/__init__.py" = ["E402"]diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 256504c9c..280a0f659 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -31,11 +31,11 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."def __getattr__(name: str):if name == "twisted_version":- import warnings # pylint: disable=reimported+ import warnings # noqa: PLC0415 # pylint: disable=reimported- from twisted import version as _txv+ from twisted import version as _txv # noqa: PLC0415- from scrapy.exceptions import ScrapyDeprecationWarning+ from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415warnings.warn("The scrapy.twisted_version attribute is deprecated, use twisted.version instead",diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex 96bb1ae84..c4e277a60 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -10,6 +10,7 @@ import scrapyfrom scrapy.commands import ScrapyCommandfrom scrapy.http import Response, TextResponsefrom scrapy.linkextractors import LinkExtractor+from scrapy.utils.test import get_testenvif TYPE_CHECKING:import argparse@@ -35,8 +36,6 @@ class Command(ScrapyCommand):class _BenchServer:def __enter__(self) -> None:- from scrapy.utils.test import get_testenv-pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]self.proc = subprocess.Popen( # noqa: S603pargs, stdout=subprocess.PIPE, env=get_testenv()diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 0e90c3188..9d2742afd 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -206,7 +206,7 @@ class Command(ScrapyCommand):# a file with the same name exists in the target directoryspiders_module = import_module(self.settings["NEWSPIDER_MODULE"])- spiders_dir = Path(cast(str, spiders_module.__file__)).parent+ spiders_dir = Path(cast("str", spiders_module.__file__)).parentspiders_dir_abs = spiders_dir.resolve()path = spiders_dir_abs / (name + ".py")if path.exists():diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex bdb68c4ad..f5ba5ba13 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -52,7 +52,7 @@ class Contract:cb_result = cb(response, **cb_kwargs)if isinstance(cb_result, (AsyncGenerator, CoroutineType)):raise TypeError("Contracts don't support async callbacks")- return list(cast(Iterable[Any], iterate_spider_output(cb_result)))+ return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))request.callback = wrapper@@ -68,7 +68,7 @@ class Contract:cb_result = cb(response, **cb_kwargs)if isinstance(cb_result, (AsyncGenerator, CoroutineType)):raise TypeError("Contracts don't support async callbacks")- output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))+ output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))try:results.startTest(self.testcase_post)self.post_process(output)@@ -181,7 +181,7 @@ class ContractsManager:def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:try:output = cb(response, **cb_kwargs)- output = list(cast(Iterable[Any], iterate_spider_output(output)))+ output = list(cast("Iterable[Any]", iterate_spider_output(output)))except Exception:case = _create_testcase(method, "callback")results.addError(case, sys.exc_info())diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex c0833c84d..4c9fed3bc 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -162,7 +162,7 @@ class Downloader:def get_slot_key(self, request: Request) -> str:if self.DOWNLOAD_SLOT in request.meta:- return cast(str, request.meta[self.DOWNLOAD_SLOT])+ return cast("str", request.meta[self.DOWNLOAD_SLOT])key = urlparse_cached(request).hostname or ""if self.ip_concurrency:diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 54fef48b6..50235846b 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -386,6 +386,7 @@ class ScrapyAgent:if not proxy_port:proxy_port = 443 if proxy_parsed.scheme == "https" else 80if urlparse_cached(request).scheme == "https":+ assert proxy_host is not NoneproxyAuth = request.headers.get(b"Proxy-Authorization", None)proxyConf = (proxy_host, proxy_port, proxyAuth)return self._TunnelingAgent(@@ -430,7 +431,7 @@ class ScrapyAgent:method,to_bytes(url, encoding="ascii"),headers,- cast(IBodyProducer, bodyproducer),+ cast("IBodyProducer", bodyproducer),)# set download latencyd.addCallback(self._cb_latency, request, start_time)diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 870a26f04..02beb2f8b 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -51,8 +51,8 @@ class S3DownloadHandler:self.anon = kw.get("anon")self._signer = None- import botocore.auth- import botocore.credentials+ import botocore.auth # noqa: PLC0415+ import botocore.credentials # noqa: PLC0415kw.pop("anon", None)if kw:@@ -87,7 +87,7 @@ class S3DownloadHandler:if self.anon:request = request.replace(url=url)else:- import botocore.awsrequest+ import botocore.awsrequest # noqa: PLC0415awsrequest = botocore.awsrequest.AWSRequest(method=request.method,diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex fa45d62ee..2e892cff7 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -6,7 +6,6 @@ See documentation in docs/topics/downloader-middleware.rstfrom __future__ import annotations-from collections.abc import Callablefrom typing import TYPE_CHECKING, Any, castfrom twisted.internet.defer import Deferred, inlineCallbacks@@ -18,7 +17,7 @@ from scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import _defer_sleep, deferred_from_coroif TYPE_CHECKING:- from collections.abc import Generator+ from collections.abc import Callable, Generatorfrom scrapy import Spiderfrom scrapy.settings import BaseSettings@@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):request: Request,) -> Generator[Deferred[Any], Any, Response | Request]:for method in self.methods["process_request"]:- method = cast(Callable, method)+ method = cast("Callable", method)response = yield deferred_from_coro(method(request=request, spider=spider))@@ -76,7 +75,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):return responsefor method in self.methods["process_response"]:- method = cast(Callable, method)+ method = cast("Callable", method)response = yield deferred_from_coro(method(request=request, response=response, spider=spider))@@ -94,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):exception: Exception,) -> Generator[Deferred[Any], Any, Response | Request]:for method in self.methods["process_exception"]:- method = cast(Callable, method)+ method = cast("Callable", method)response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex be76ad8c3..970a250ef 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -17,13 +17,11 @@ from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, sufrom twisted.python.failure import Failurefrom scrapy import signals+from scrapy.core.scheduler import BaseSchedulerfrom scrapy.core.scraper import Scraperfrom scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequestfrom scrapy.http import Request, Response-from scrapy.utils.asyncio import (- AsyncioLoopingCall,- create_looping_call,-)+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_callfrom scrapy.utils.defer import (deferred_f_from_coro_f,deferred_from_coro,@@ -39,7 +37,6 @@ if TYPE_CHECKING:from twisted.internet.task import LoopingCallfrom scrapy.core.downloader import Downloader- from scrapy.core.scheduler import BaseSchedulerfrom scrapy.crawler import Crawlerfrom scrapy.logformatter import LogFormatterfrom scrapy.settings import BaseSettings, Settings@@ -123,8 +120,6 @@ class ExecutionEngine:raisedef _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:- from scrapy.core.scheduler import BaseScheduler-scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"])if not issubclass(scheduler_cls, BaseScheduler):raise TypeError(@@ -506,7 +501,7 @@ class ExecutionEngine:dfd.addErrback(log_failure("Scraper close failure"))if hasattr(self._slot.scheduler, "close"):- dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))+ dfd.addBoth(lambda _: cast("_Slot", self._slot).scheduler.close(reason))dfd.addErrback(log_failure("Scheduler close failure"))dfd.addBoth(diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 9ac447289..9ae555300 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -491,7 +491,7 @@ class Scheduler(BaseScheduler):if not path.exists():return []with path.open(encoding="utf-8") as f:- return cast(list[int], json.load(f))+ return cast("list[int]", json.load(f))def _write_dqs_state(self, dqdir: str, state: list[int]) -> None:with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 77fcaa357..01e563e56 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -138,7 +138,7 @@ class SpiderMiddlewareManager(MiddlewareManager):spider: Spider,) -> Deferred[Iterable[_T] | AsyncIterator[_T]]:for method in self.methods["process_spider_input"]:- method = cast(Callable, method)+ method = cast("Callable", method)try:result = method(response=response, spider=spider)if result is not None:@@ -166,7 +166,7 @@ class SpiderMiddlewareManager(MiddlewareManager):yield from iterableexcept Exception as ex:exception_result = cast(- Union[Failure, MutableChain[_T]],+ "Union[Failure, MutableChain[_T]]",self._process_spider_exception(response, spider, ex, exception_processor_index),@@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager):yield rexcept Exception as ex:exception_result = cast(- Union[Failure, MutableAsyncChain[_T]],+ "Union[Failure, MutableAsyncChain[_T]]",self._process_spider_exception(response, spider, ex, exception_processor_index),@@ -212,7 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager):for method_index, method in enumerate(method_list, start=start_index):if method is None:continue- method = cast(Callable, method)+ method = cast("Callable", method)result = method(response=response, exception=exception, spider=spider)if _isiterable(result):# stop exception handling by handing control over to the@@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager):if dfd.called:# the result is available immediately if _process_spider_output didn't do downgradingreturn cast(- Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result+ "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result)# we forbid waiting here because otherwise we would need to return a deferred from# _process_spider_exception too, which complicates the architecturediff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex d6fb9972e..c6c65a993 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -8,11 +8,7 @@ import signalfrom abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, Any, TypeVar-from twisted.internet.defer import (- Deferred,- DeferredList,- inlineCallbacks,-)+from twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom scrapy import Spider, signalsfrom scrapy.addons import AddonManagerdiff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 612426371..530cccb53 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -202,6 +202,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):redirected = self._redirect_request_using_get(request, url)if urlparse_cached(redirected).scheme not in {"http", "https"}:return response- if cast(float, interval) < self._maxdelay:+ if cast("float", interval) < self._maxdelay:return self._redirect(redirected, request, spider, "meta refresh")return responsediff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 557ad99e1..0847c1e8b 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -218,7 +218,7 @@ class S3FeedStorage(BlockingFeedStorage):region_name: str | None = None,):try:- import boto3.session+ import boto3.session # noqa: PLC0415except ImportError:raise NotConfigured("missing boto3 library")u = urlparse(uri)@@ -317,7 +317,7 @@ class GCSFeedStorage(BlockingFeedStorage):def _store_in_thread(self, file: IO[bytes]) -> None:file.seek(0)- from google.cloud.storage import Client+ from google.cloud.storage import Client # noqa: PLC0415client = Client(project=self.project_id)bucket = client.get_bucket(self.bucket_name)@@ -413,7 +413,7 @@ class FeedSlot:self.file = self.storage.open(self.spider)if "postprocessing" in self.feed_options:self.file = cast(- IO[bytes],+ "IO[bytes]",PostProcessingManager(self.feed_options["postprocessing"],self.file,@@ -662,7 +662,7 @@ class FeedExporter:def _load_components(self, setting_prefix: str) -> dict[str, Any]:conf = without_none_values(- cast(dict[str, str], self.settings.getwithbase(setting_prefix))+ cast("dict[str, str]", self.settings.getwithbase(setting_prefix)))d = {}for k, v in conf.items():diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 0cd16d737..f704ae014 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -307,7 +307,7 @@ class DbmCacheStorage:if 0 < self.expiration_secs < time() - float(ts):return None # expired- return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301+ return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"])) # noqa: S301class FilesystemCacheStorage:@@ -389,7 +389,7 @@ class FilesystemCacheStorage:if 0 < self.expiration_secs < time() - mtime:return None # expiredwith self._open(metapath, "rb") as f:- return cast(dict[str, Any], pickle.load(f)) # noqa: S301+ return cast("dict[str, Any]", pickle.load(f)) # noqa: S301def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.pyindex 387cfddb3..3d7674905 100644--- a/scrapy/extensions/logstats.py+++ b/scrapy/extensions/logstats.py@@ -5,10 +5,7 @@ from typing import TYPE_CHECKINGfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured-from scrapy.utils.asyncio import (- AsyncioLoopingCall,- create_looping_call,-)+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_callif TYPE_CHECKING:from twisted.internet.task import LoopingCalldiff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex 2ef322f1a..e425749f7 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -16,10 +16,7 @@ from typing import TYPE_CHECKINGfrom scrapy import signalsfrom scrapy.exceptions import NotConfiguredfrom scrapy.mail import MailSender-from scrapy.utils.asyncio import (- AsyncioLoopingCall,- create_looping_call,-)+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_callfrom scrapy.utils.engine import get_engine_statusif TYPE_CHECKING:diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex 98210990a..860b97a55 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Anyfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured-from scrapy.utils.asyncio import (- AsyncioLoopingCall,- create_looping_call,-)+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_callfrom scrapy.utils.serialize import ScrapyJSONEncoderif TYPE_CHECKING:diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.pyindex b1fa160c8..01fd82830 100644--- a/scrapy/extensions/postprocessing.py+++ b/scrapy/extensions/postprocessing.py@@ -142,7 +142,7 @@ class PostProcessingManager(IOBase)::return: returns number of bytes written:rtype: int"""- return cast(int, self.head_plugin.write(data))+ return cast("int", self.head_plugin.write(data))def tell(self) -> int:return self.file.tell()diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex bacee8f0a..9fe71817a 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -12,6 +12,8 @@ import osimport pprintfrom typing import TYPE_CHECKING, Any+from twisted.conch import manhole, telnet+from twisted.conch.insults import insultsfrom twisted.internet import protocolfrom scrapy import signals@@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcpfrom scrapy.utils.trackref import print_live_refsif TYPE_CHECKING:- from twisted.conch import telnetfrom twisted.internet.tcp import Port# typing.Self requires Python 3.11@@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory):self.port.stopListening()def protocol(self) -> telnet.TelnetTransport:- # these import twisted.internet.reactor- from twisted.conch import manhole, telnet- from twisted.conch.insults import insults-class Portal:"""An implementation of IPortal"""diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex b7c3b9d37..e5b4b28a7 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -161,7 +161,7 @@ class WrappedRequest:HTML document, and the user had no option to approve the automaticfetching of the image, this should be true."""- return cast(bool, self.request.meta.get("is_unverifiable", False))+ return cast("bool", self.request.meta.get("is_unverifiable", False))@propertydef full_url(self) -> str:@@ -181,7 +181,7 @@ class WrappedRequest:@propertydef origin_req_host(self) -> str:- return cast(str, urlparse_cached(self.request).hostname)+ return cast("str", urlparse_cached(self.request).hostname)def has_header(self, name: str) -> bool:return name in self.request.headersdiff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 60b04753b..5498e1138 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -69,19 +69,19 @@ class Headers(CaselessDict):def __getitem__(self, key: AnyStr) -> bytes | None:try:- return cast(list[bytes], super().__getitem__(key))[-1]+ return cast("list[bytes]", super().__getitem__(key))[-1]except IndexError:return Nonedef get(self, key: AnyStr, def_val: Any = None) -> bytes | None:try:- return cast(list[bytes], super().get(key, def_val))[-1]+ return cast("list[bytes]", super().get(key, def_val))[-1]except IndexError:return Nonedef getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:try:- return cast(list[bytes], super().__getitem__(key))+ return cast("list[bytes]", super().__getitem__(key))except KeyError:if def_val is not None:return self.normvalue(def_val)diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 7681419c4..aa15a0222 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -11,13 +11,7 @@ from collections.abc import Iterablefrom typing import TYPE_CHECKING, Any, Optional, Union, castfrom urllib.parse import urlencode, urljoin, urlsplit, urlunsplit-from lxml.html import (- FormElement,- InputElement,- MultipleSelectOptions,- SelectElement,- TextareaElement,-)+from parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespacefrom scrapy.http.request import Request@@ -25,6 +19,13 @@ from scrapy.utils.python import is_listlike, to_bytesif TYPE_CHECKING:# typing.Self requires Python 3.11+ from lxml.html import (+ FormElement,+ InputElement,+ MultipleSelectOptions,+ SelectElement,+ TextareaElement,+ )from typing_extensions import Selffrom scrapy.http.response.text import TextResponse@@ -76,8 +77,6 @@ class FormRequest(Request):kwargs.setdefault("encoding", response.encoding)if formcss is not None:- from parsel.csstranslator import HTMLTranslator-formxpath = HTMLTranslator().css_to_xpath(formcss)form = _get_form(response, formname, formid, formnumber, formxpath)@@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:values = [(to_bytes(k, enc), to_bytes(v, enc))for k, vs in seq- for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])+ for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])]return urlencode(values, doseq=True)@@ -128,12 +127,12 @@ def _get_form(if formname is not None:f = root.xpath(f'//form[@name="{formname}"]')if f:- return cast(FormElement, f[0])+ return cast("FormElement", f[0])if formid is not None:f = root.xpath(f'//form[@id="{formid}"]')if f:- return cast(FormElement, f[0])+ return cast("FormElement", f[0])# Get form element from xpath, if not found, go upif formxpath is not None:@@ -142,7 +141,7 @@ def _get_form(el = nodes[0]while True:if el.tag == "form":- return cast(FormElement, el)+ return cast("FormElement", el)el = el.getparent()if el is None:break@@ -153,7 +152,7 @@ def _get_form(form = forms[formnumber]except IndexError:raise IndexError(f"Form number {formnumber} not found in {response}")- return cast(FormElement, form)+ return cast("FormElement", form)def _get_inputs(@@ -201,7 +200,7 @@ def _value(n = ele.namev = ele.valueif ele.tag == "select":- return _select_value(cast(SelectElement, ele), n, v)+ return _select_value(cast("SelectElement", ele), n, v)return n, v@@ -251,7 +250,7 @@ def _get_clickable(except IndexError:passelse:- return (el.get("name"), el.get("value") or "")+ return (cast("str", el.get("name")), el.get("value") or "")# We didn't find it, so now we build an XPath expression out of the other# arguments, because they can be used as suchdiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 081223882..9c4e4c151 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -104,13 +104,14 @@ class TextResponse(Response):@memoizemethod_noargsdef _headers_encoding(self) -> str | None:- content_type = cast(bytes, self.headers.get(b"Content-Type", b""))+ content_type = cast("bytes", self.headers.get(b"Content-Type", b""))return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))def _body_inferred_encoding(self) -> str:if self._cached_benc is None:content_type = to_unicode(- cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"+ cast("bytes", self.headers.get(b"Content-Type", b"")),+ encoding="latin-1",)benc, ubody = html_to_unicode(content_type,@@ -141,31 +142,25 @@ class TextResponse(Response):@propertydef selector(self) -> Selector:- from scrapy.selector import Selector+ # circular import+ from scrapy.selector import Selector # noqa: PLC0415if self._cached_selector is None:self._cached_selector = Selector(self)return self._cached_selectordef jmespath(self, query: str, **kwargs: Any) -> SelectorList:- from scrapy.selector import SelectorList-if not hasattr(self.selector, "jmespath"):raise AttributeError("Please install parsel >= 1.8.1 to get jmespath support")-- return cast(SelectorList, self.selector.jmespath(query, **kwargs))+ return cast("SelectorList", self.selector.jmespath(query, **kwargs))def xpath(self, query: str, **kwargs: Any) -> SelectorList:- from scrapy.selector import SelectorList-- return cast(SelectorList, self.selector.xpath(query, **kwargs))+ return cast("SelectorList", self.selector.xpath(query, **kwargs))def css(self, query: str) -> SelectorList:- from scrapy.selector import SelectorList-- return cast(SelectorList, self.selector.css(query))+ return cast("SelectorList", self.selector.css(query))def follow(self,diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 814e31fec..7c9b4d3e3 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -71,12 +71,12 @@ class LxmlParserLinkExtractor:self.scan_tag: Callable[[str], bool] = (tagif callable(tag)- else cast(Callable[[str], bool], partial(operator.eq, tag))+ else cast("Callable[[str], bool]", partial(operator.eq, tag)))self.scan_attr: Callable[[str], bool] = (attrif callable(attr)- else cast(Callable[[str], bool], partial(operator.eq, attr))+ else cast("Callable[[str], bool]", partial(operator.eq, attr)))self.process_attr: Callable[[Any], Any] = (process if callable(process) else _identity@@ -84,7 +84,7 @@ class LxmlParserLinkExtractor:self.unique: bool = uniqueself.strip: bool = stripself.link_key: Callable[[Link], str] = (- cast(Callable[[Link], str], operator.attrgetter("url"))+ cast("Callable[[Link], str]", operator.attrgetter("url"))if canonicalizedelse _canonicalize_link_url)diff --git a/scrapy/mail.py b/scrapy/mail.pyindex be2423965..88612daa9 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -223,7 +223,8 @@ class MailSender:def _create_sender_factory(self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any]) -> ESMTPSenderFactory:- from twisted.mail.smtp import ESMTPSenderFactory+ # imports twisted.internet.reactor+ from twisted.mail.smtp import ESMTPSenderFactory # noqa: PLC0415factory_keywords: dict[str, Any] = {"heloFallback": True,diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 25f2aa199..a4d7af7b6 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -63,7 +63,7 @@ class MiddlewareManager(ABC):method_name = "__new__"if instance is None:raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return cast(_T, instance)+ return cast("_T", instance)@classmethoddef from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 55d9cae6c..84d4104ed 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -169,7 +169,7 @@ class S3FilesStore:def __init__(self, uri: str):if not is_botocore_available():raise NotConfigured("missing botocore library")- import botocore.session+ import botocore.session # noqa: PLC0415session = botocore.session.get_session()self.s3_client = session.create_client(@@ -284,7 +284,7 @@ class GCSFilesStore:POLICY = Nonedef __init__(self, uri: str):- from google.cloud import storage+ from google.cloud import storage # noqa: PLC0415client = storage.Client(project=self.GCS_PROJECT_ID)bucket, prefix = uri[5:].split("/", 1)@@ -317,7 +317,7 @@ class GCSFilesStore:blob_path = self._get_blob_path(path)return cast(- Deferred[StatInfo],+ "Deferred[StatInfo]",deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),)@@ -531,7 +531,9 @@ class FilesPipeline(MediaPipeline):@classmethoddef _update_stores(cls, settings: BaseSettings) -> None:- s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])+ s3store: type[S3FilesStore] = cast(+ "type[S3FilesStore]", cls.STORE_SCHEMES["s3"]+ )s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]@@ -542,13 +544,13 @@ class FilesPipeline(MediaPipeline):s3store.POLICY = settings["FILES_STORE_S3_ACL"]gcs_store: type[GCSFilesStore] = cast(- type[GCSFilesStore], cls.STORE_SCHEMES["gs"]+ "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"])gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or Noneftp_store: type[FTPFilesStore] = cast(- type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]+ "type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"])ftp_store.FTP_USERNAME = settings["FTP_USER"]ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]@@ -742,5 +744,5 @@ class FilesPipeline(MediaPipeline):media_ext = ""media_type = mimetypes.guess_type(request.url)[0]if media_type:- media_ext = cast(str, mimetypes.guess_extension(media_type))+ media_ext = cast("str", mimetypes.guess_extension(media_type))return f"full/{media_guid}{media_ext}"diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 63c6908dc..19139b5d5 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -63,7 +63,7 @@ class ImagesPipeline(FilesPipeline):crawler: Crawler | None = None,):try:- from PIL import Image+ from PIL import Image # noqa: PLC0415self._Image = Imageexcept ImportError:diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 34b235d83..42c53a527 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -4,7 +4,6 @@ import hashlibimport loggingfrom typing import TYPE_CHECKING, Protocol, cast-from scrapy import Requestfrom scrapy.utils.misc import build_from_crawlerif TYPE_CHECKING:@@ -13,6 +12,7 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Requestfrom scrapy.core.downloader import Downloaderfrom scrapy.crawler import Crawler@@ -211,7 +211,7 @@ class ScrapyPriorityQueue:except KeyError:queue = self.queues[self.curprio]# Protocols can't declare optional members- return cast(Request, queue.peek()) # type: ignore[attr-defined]+ return cast("Request", queue.peek()) # type: ignore[attr-defined]def close(self) -> list[int]:active: set[int] = set()diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 417c9c142..e1a12be05 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -4,6 +4,9 @@ import loggingimport sysfrom abc import ABCMeta, abstractmethodfrom typing import TYPE_CHECKING+from urllib.robotparser import RobotFileParser++from protego import Protegofrom scrapy.utils.python import to_unicode@@ -67,8 +70,6 @@ class RobotParser(metaclass=ABCMeta):class PythonRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):- from urllib.robotparser import RobotFileParser-self.spider: Spider | None = spiderbody_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)self.rp: RobotFileParser = RobotFileParser()@@ -87,7 +88,7 @@ class PythonRobotParser(RobotParser):class RerpRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):- from robotexclusionrulesparser import RobotExclusionRulesParser+ from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: PLC0415self.spider: Spider | None = spiderself.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()@@ -107,8 +108,6 @@ class RerpRobotParser(RobotParser):class ProtegoRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):- from protego import Protego-self.spider: Spider | None = spiderbody_decoded = decode_robotstxt(robotstxt_body, spider)self.rp = Protego.parse(body_decoded)diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex 9363da492..334de6658 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -336,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):stored."""if len(self) > 0:- return max(cast(int, self.getpriority(name)) for name in self)+ return max(cast("int", self.getpriority(name)) for name in self)return get_settings_priority("default")def replace_in_component_priority_dict(@@ -519,11 +519,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):"""self._assert_mutability()if isinstance(values, str):- values = cast(dict[_SettingsKeyT, Any], json.loads(values))+ values = cast("dict[_SettingsKeyT, Any]", json.loads(values))if values is not None:if isinstance(values, BaseSettings):for name, value in values.items():- self.set(name, value, cast(int, values.getpriority(name)))+ self.set(name, value, cast("int", values.getpriority(name)))else:for name, value in values.items():self.set(name, value, priority)@@ -533,7 +533,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):raise KeyError(name)self._assert_mutability()priority = get_settings_priority(priority)- if priority >= cast(int, self.getpriority(name)):+ if priority >= cast("int", self.getpriority(name)):del self.attributes[name]def __delitem__(self, name: _SettingsKeyT) -> None:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 50c0088e7..f306569e4 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -535,9 +535,9 @@ WARN_ON_GENERATOR_RETURN_VALUE = Truedef __getattr__(name: str):if name == "CONCURRENT_REQUESTS_PER_IP":- import warnings+ import warnings # noqa: PLC0415- from scrapy.exceptions import ScrapyDeprecationWarning+ from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415warnings.warn("The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",diff --git a/scrapy/shell.py b/scrapy/shell.pyindex bb39eccc3..c3a274e0d 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -16,6 +16,7 @@ from twisted.internet import defer, threadsfrom twisted.python import threadablefrom w3lib.url import any_to_uri+import scrapyfrom scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequestfrom scrapy.http import Request, Response@@ -164,8 +165,6 @@ class Shell:request: Request | None = None,spider: Spider | None = None,) -> None:- import scrapy-self.vars["scrapy"] = scrapyself.vars["crawler"] = self.crawlerself.vars["item"] = self.item_class()diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 1d4d1527b..e4d0f4014 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -315,7 +315,7 @@ def _load_policy_class(from https://www.w3.org/TR/referrer-policy/#referrer-policies"""try:- return cast(type[ReferrerPolicy], load_object(policy))+ return cast("type[ReferrerPolicy]", load_object(policy))except ValueError:tokens = [token.strip() for token in policy.lower().split(",")]# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-headerdiff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex a722dd83b..8db00a1ca 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -55,7 +55,8 @@ class Spider(object_ref):@propertydef logger(self) -> SpiderLoggerAdapter:- from scrapy.utils.log import SpiderLoggerAdapter+ # circular import+ from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415logger = logging.getLogger(self.name)return SpiderLoggerAdapter(logger, {"spider": self})diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 98e7b23c0..0cbd6b3cc 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -12,8 +12,6 @@ import warningsfrom collections.abc import AsyncIterator, Awaitable, Callablefrom typing import TYPE_CHECKING, Any, Optional, TypeVar, cast-from twisted.python.failure import Failure-from scrapy.http import HtmlResponse, Request, Responsefrom scrapy.link import Linkfrom scrapy.linkextractors import LinkExtractor@@ -26,6 +24,8 @@ from scrapy.utils.spider import iterate_spider_outputif TYPE_CHECKING:from collections.abc import Iterable, Sequence+ from twisted.python.failure import Failure+# typing.Self requires Python 3.11from typing_extensions import Self@@ -81,12 +81,14 @@ class Rule:def _compile(self, spider: Spider) -> None:# this replaces method names with methods and we can't express this in type hintsself.callback = cast("CallbackT", _get_method(self.callback, spider))- self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))+ self.errback = cast(+ "Callable[[Failure], Any]", _get_method(self.errback, spider)+ )self.process_links = cast(- ProcessLinksT, _get_method(self.process_links, spider)+ "ProcessLinksT", _get_method(self.process_links, spider))self.process_request = cast(- ProcessRequestT, _get_method(self.process_request, spider)+ "ProcessRequestT", _get_method(self.process_request, spider))@@ -141,13 +143,13 @@ class CrawlSpider(Spider):for lnk in rule.link_extractor.extract_links(response)if lnk not in seen]- for link in cast(ProcessLinksT, rule.process_links)(links):+ for link in cast("ProcessLinksT", rule.process_links)(links):seen.add(link)request = self._build_request(rule_index, link)- yield cast(ProcessRequestT, rule.process_request)(request, response)+ yield cast("ProcessRequestT", rule.process_request)(request, response)def _callback(self, response: Response, **cb_kwargs: Any) -> Any:- rule = self._rules[cast(int, response.meta["rule"])]+ rule = self._rules[cast("int", response.meta["rule"])]return self.parse_with_rules(response,cast("CallbackT", rule.callback),@@ -156,9 +158,9 @@ class CrawlSpider(Spider):)def _errback(self, failure: Failure) -> Iterable[Any]:- rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined]+ rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined]return self._handle_failure(- failure, cast(Callable[[Failure], Any], rule.errback)+ failure, cast("Callable[[Failure], Any]", rule.errback))async def parse_with_rules(diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.pyindex e5548b9fa..957bfffd3 100644--- a/scrapy/spiders/init.py+++ b/scrapy/spiders/init.py@@ -1,15 +1,16 @@from __future__ import annotationsimport warnings-from collections.abc import AsyncIterator, Iterablefrom typing import TYPE_CHECKING, Any, cast-from scrapy import Requestfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.spiders import Spiderfrom scrapy.utils.spider import iterate_spider_outputif TYPE_CHECKING:+ from collections.abc import AsyncIterator, Iterable++ from scrapy import Requestfrom scrapy.http import Response@@ -39,7 +40,7 @@ class InitSpider(Spider):def start_requests(self) -> Iterable[Request]:self._postinit_reqs: Iterable[Request] = super().start_requests()- return cast(Iterable[Request], iterate_spider_output(self.init_request()))+ return cast("Iterable[Request]", iterate_spider_output(self.init_request()))def initialized(self, response: Response | None = None) -> Any:"""This method must be set as the callback of your last initializationdiff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.pyindex 53cfeddd0..73f86bc71 100644--- a/scrapy/utils/boto.py+++ b/scrapy/utils/boto.py@@ -3,7 +3,7 @@def is_botocore_available() -> bool:try:- import botocore # noqa: F401+ import botocore # noqa: F401,PLC0415return Trueexcept ImportError:diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 891cbb485..5a627fc83 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -153,7 +153,7 @@ def feed_process_params_from_cli(suitable to be used as the FEEDS setting."""valid_output_formats: Iterable[str] = without_none_values(- cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS"))+ cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS"))).keys()def check_valid_format(output_format: str) -> None:diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 7425543ff..644965cb5 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -1,5 +1,6 @@from __future__ import annotations+import codefrom collections.abc import Callablefrom functools import wrapsfrom typing import TYPE_CHECKING, Any@@ -16,13 +17,13 @@ def _embed_ipython_shell() -> EmbedFuncT:"""Start an IPython Shell"""try:- from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100- from IPython.terminal.ipapp import load_default_config+ from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415+ from IPython.terminal.ipapp import load_default_config # noqa: PLC0415except ImportError:- from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100+ from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100,PLC0415InteractiveShellEmbed,)- from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef]+ from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] # noqa: PLC0415load_default_config,)@@ -46,7 +47,7 @@ def _embed_bpython_shell(namespace: dict[str, Any] = {}, banner: str = "") -> EmbedFuncT:"""Start a bpython shell"""- import bpython+ import bpython # noqa: PLC0415@wraps(_embed_bpython_shell)def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:@@ -59,7 +60,7 @@ def _embed_ptpython_shell(namespace: dict[str, Any] = {}, banner: str = "") -> EmbedFuncT:"""Start a ptpython shell"""- import ptpython.repl # pylint: disable=import-error+ import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error@wraps(_embed_ptpython_shell)def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:@@ -73,14 +74,12 @@ def _embed_standard_shell(namespace: dict[str, Any] = {}, banner: str = "") -> EmbedFuncT:"""Start a standard python shell"""- import code-try: # readline module is only available on unix systems- import readline+ import readline # noqa: PLC0415except ImportError:passelse:- import rlcompleter # noqa: F401+ import rlcompleter # noqa: F401,PLC0415readline.parse_and_bind("tab:complete") # type: ignore[attr-defined]diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 3d0e0d3c7..ae7f627e1 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -32,7 +32,8 @@ class CaselessDict(dict):__slots__ = ()def __new__(cls, *args: Any, **kwargs: Any) -> Self:- from scrapy.http.headers import Headers+ # circular import+ from scrapy.http.headers import Headers # noqa: PLC0415if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):warnings.warn(diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 50f2af5b7..d161f30a6 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -12,12 +12,7 @@ from collections.abc import Awaitable, Coroutine, Iterable, Iteratorfrom functools import wrapsfrom typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload-from twisted.internet.defer import (- Deferred,- DeferredList,- fail,- succeed,-)+from twisted.internet.defer import Deferred, DeferredList, fail, succeedfrom twisted.internet.task import Cooperatorfrom twisted.python import failure@@ -414,7 +409,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2:if not is_asyncio_available():# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines# that use asyncio, e.g. "await asyncio.sleep(1)"- return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o))+ return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o))# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactorreturn Deferred.fromFuture(asyncio.ensure_future(o))return odiff --git a/scrapy/utils/display.py b/scrapy/utils/display.pyindex 20744a604..08e89dc0d 100644--- a/scrapy/utils/display.py+++ b/scrapy/utils/display.py@@ -34,11 +34,11 @@ def _colorize(text: str, colorize: bool = True) -> str:if not colorize or not sys.stdout.isatty() or not _tty_supports_color():return texttry:- from pygments import highlight+ from pygments import highlight # noqa: PLC0415except ImportError:return text- from pygments.formatters import TerminalFormatter- from pygments.lexers import PythonLexer+ from pygments.formatters import TerminalFormatter # noqa: PLC0415+ from pygments.lexers import PythonLexer # noqa: PLC0415return highlight(text, PythonLexer(), TerminalFormatter())diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex c646fc218..c70a0d327 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -147,12 +147,12 @@ class _StreamReader:def _read_string(self, n: int = 65535) -> bytes:s, e = self._ptr, self._ptr + nself._ptr = e- return cast(bytes, self._text)[s:e]+ return cast("bytes", self._text)[s:e]def _read_unicode(self, n: int = 65535) -> bytes:s, e = self._ptr, self._ptr + nself._ptr = e- return cast(str, self._text)[s:e].encode("utf-8")+ return cast("str", self._text)[s:e].encode("utf-8")def csviter(diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 24e17ecb6..533906003 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -5,9 +5,9 @@ import pprintimport sysfrom collections.abc import MutableMappingfrom logging.config import dictConfig-from types import TracebackTypefrom typing import TYPE_CHECKING, Any, Optional, cast+from twisted.internet import asyncioreactorfrom twisted.python import log as twisted_logfrom twisted.python.failure import Failure@@ -16,6 +16,8 @@ from scrapy.settings import Settings, _SettingsKeyTfrom scrapy.utils.versions import get_versionsif TYPE_CHECKING:+ from types import TracebackType+from scrapy.crawler import Crawlerfrom scrapy.logformatter import LogFormatterResult@@ -33,7 +35,7 @@ def failure_to_exc_info(return (failure.type,failure.value,- cast(Optional[TracebackType], failure.getTracebackObject()),+ cast("Optional[TracebackType]", failure.getTracebackObject()),)return None@@ -182,7 +184,7 @@ def log_scrapy_info(settings: Settings) -> None:def log_reactor_info() -> None:- from twisted.internet import asyncioreactor, reactor+ from twisted.internet import reactorlogger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):@@ -240,7 +242,7 @@ def logformatter_adapter(message = logkws.get("msg") or ""# NOTE: This also handles 'args' being an empty dict, that case doesn't# play well in logger.log calls- args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]+ args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"]return (level, message, args)diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex b7b436260..decd4473c 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -9,7 +9,6 @@ import osimport reimport warningsfrom collections import deque-from collections.abc import Iterablefrom contextlib import contextmanagerfrom functools import partialfrom importlib import import_module@@ -21,7 +20,7 @@ from scrapy.item import Itemfrom scrapy.utils.datatypes import LocalWeakReferencedCacheif TYPE_CHECKING:- from collections.abc import Callable, Iterator+ from collections.abc import Callable, Iterable, Iteratorfrom types import ModuleTypefrom scrapy import Spider@@ -41,7 +40,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:if arg is None:return []if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):- return cast(Iterable[Any], arg)+ return cast("Iterable[Any]", arg)return [arg]@@ -202,7 +201,7 @@ def build_from_crawler(method_name = "__new__"if instance is None:raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return cast(T, instance)+ return cast("T", instance)@contextmanagerdiff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex ce4c01731..373ad652c 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -29,7 +29,7 @@ if TYPE_CHECKING:_T = TypeVar("_T")-def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements+def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503"""Like reactor.listenTCP but tries different ports in a range."""from twisted.internet import reactor@@ -39,7 +39,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:return reactor.listenTCP(0, factory, interface=host)if len(portrange) == 1:return reactor.listenTCP(portrange[0], factory, interface=host)- for x in range(portrange[0], portrange[1] + 1): # noqa: RET503+ for x in range(portrange[0], portrange[1] + 1):try:return reactor.listenTCP(x, factory, interface=host)except error.CannotListenError:@@ -60,7 +60,8 @@ class CallLaterOnce(Generic[_T]):self._deferreds: list[Deferred] = []def schedule(self, delay: float = 0) -> None:- from scrapy.utils.asyncio import call_later+ # circular import+ from scrapy.utils.asyncio import call_later # noqa: PLC0415if self._call is None:self._call = call_later(delay, self)@@ -70,7 +71,8 @@ class CallLaterOnce(Generic[_T]):self._call.cancel()def __call__(self) -> _T:- from scrapy.utils.asyncio import call_later+ # circular import+ from scrapy.utils.asyncio import call_later # noqa: PLC0415self._call = Noneresult = self._func(*self._a, **self._kw)@@ -82,7 +84,8 @@ class CallLaterOnce(Generic[_T]):return resultasync def wait(self):- from scrapy.utils.defer import maybe_deferred_to_future+ # circular import+ from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415d = Deferred()self._deferreds.append(d)diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 76a6b7de6..b6550432c 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -91,7 +91,8 @@ def open_in_browser(if "item name" not in response.body:open_in_browser(response)"""- from scrapy.http import HtmlResponse, TextResponse+ # circular imports+ from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415# XXX: this implementation is a bit dirty and could be improvedbody = response.bodydiff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex e0d9f4595..c0100ea62 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -24,7 +24,7 @@ class Sitemap:xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)- self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320+ self._root = lxml.etree.fromstring(xmltext, parser=xmlp)rt = self._root.tagassert isinstance(rt, str)self.type = rt.split("}", 1)[1] if "}" in rt else rtdiff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 3780ad23e..4c0764396 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport asyncioimport osimport warnings+from ftplib import FTPfrom importlib import import_modulefrom pathlib import Pathfrom posixpath import split@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, castfrom unittest import TestCase, mockfrom twisted.trial.unittest import SkipTest+from twisted.web.client import Agent+from scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.deprecate import create_deprecated_class@@ -59,7 +62,7 @@ def skip_if_no_boto() -> None:def get_gcs_content_and_delete(bucket: Any, path: str) -> tuple[bytes, list[dict[str, str]], Any]:- from google.cloud import storage+ from google.cloud import storage # noqa: PLC0415warnings.warn("The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",@@ -83,8 +86,6 @@ def get_ftp_content_and_delete(password: str,use_active_mode: bool = False,) -> bytes:- from ftplib import FTP-warnings.warn("The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",category=ScrapyDeprecationWarning,@@ -137,8 +138,6 @@ def get_crawler(will be used to populate the crawler settings with a project levelpriority."""- from scrapy.crawler import CrawlerRunner-# When needed, useful settings can be added here, e.g. ones that prevent# deprecation warnings.settings: dict[str, Any] = {@@ -192,7 +191,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blobclasses and set their proper return values."""- from google.cloud.storage import Blob, Bucket, Client+ from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415warnings.warn("The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",@@ -213,7 +212,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:from twisted.internet import reactor- from twisted.web.client import Agent # imports twisted.internet.reactoragent = Agent(reactor)return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex 10f764ab8..ce26b7f92 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -6,7 +6,6 @@ import warningsfrom typing import TYPE_CHECKING, castfrom twisted.internet.defer import Deferred-from twisted.internet.error import ProcessTerminatedfrom twisted.internet.protocol import ProcessProtocolfrom scrapy.exceptions import ScrapyDeprecationWarning@@ -14,6 +13,7 @@ from scrapy.exceptions import ScrapyDeprecationWarningif TYPE_CHECKING:from collections.abc import Iterable+ from twisted.internet.error import ProcessTerminatedfrom twisted.python.failure import Failure@@ -55,7 +55,7 @@ class ProcessTest:msg += "\n"msg += f"\n>>> stderr <<<\n{pp.err.decode()}"raise RuntimeError(msg)- return cast(int, pp.exitcode), pp.out, pp.err+ return cast("int", pp.exitcode), pp.out, pp.errclass TestProcessProtocol(ProcessProtocol):@@ -72,5 +72,5 @@ class TestProcessProtocol(ProcessProtocol):self.err += datadef processEnded(self, status: Failure) -> None:- self.exitcode = cast(ProcessTerminated, status.value).exitCode+ self.exitcode = cast("ProcessTerminated", status.value).exitCodeself.deferred.callback(self)diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.pyindex 68239e651..f6fead718 100644--- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py@@ -21,7 +21,7 @@ def log_task_exception(task: Task) -> None:try:task.result()except Exception:- logging.exception("Crawl task failed")+ logging.exception("Crawl task failed") # noqa: LOG015process = AsyncCrawlerProcess()diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.pyindex 18ec81fed..c8c73ba15 100644--- a/tests/test_command_genspider.py+++ b/tests/test_command_genspider.py@@ -3,6 +3,8 @@ from __future__ import annotationsimport osfrom pathlib import Path+import pytest+from tests.test_commands import TestCommandBase, TestProjectBase@@ -15,7 +17,16 @@ class TestGenspiderCommand(TestCommandBase):assert self.call("genspider", "test_name", "test.com") == 0assert Path(self.proj_mod_path, "spiders", "test_name.py").exists()- def test_template(self, tplname="crawl"):+ @pytest.mark.parametrize(+ "tplname",+ [+ "basic",+ "crawl",+ "xmlfeed",+ "csvfeed",+ ],+ )+ def test_template(self, tplname: str) -> None:args = [f"--template={tplname}"] if tplname else []spname = "test_spider"spmodule = f"{self.project_name}.spiders.{spname}"@@ -35,15 +46,6 @@ class TestGenspiderCommand(TestCommandBase):)assert modify_time_after == modify_time_before- def test_template_basic(self):- self.test_template("basic")-- def test_template_csvfeed(self):- self.test_template("csvfeed")-- def test_template_xmlfeed(self):- self.test_template("xmlfeed")-def test_list(self):assert self.call("genspider", "--list") == 0@@ -57,7 +59,8 @@ class TestGenspiderCommand(TestCommandBase):self.proj_mod_path, "spiders", f"{self.project_name}.py").exists()- def test_same_filename_as_existing_spider(self, force=False):+ @pytest.mark.parametrize("force", [True, False])+ def test_same_filename_as_existing_spider(self, force: bool) -> None:file_name = "example"file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py")assert self.call("genspider", file_name, "example.com") == 0@@ -90,83 +93,60 @@ class TestGenspiderCommand(TestCommandBase):file_contents_after = file_path.read_text(encoding="utf-8")assert file_contents_after == file_contents_before- def test_same_filename_as_existing_spider_force(self):- self.test_same_filename_as_existing_spider(force=True)-- def test_url(self, url="test.com", domain="test.com"):+ @pytest.mark.parametrize(+ ("url", "domain"),+ [+ ("test.com", "test.com"),+ ("https://test.com", "test.com"),+ ],+ )+ def test_url(self, url: str, domain: str) -> None:assert self.call("genspider", "--force", "test_name", url) == 0- assert (- self.find_in_file(- Path(self.proj_mod_path, "spiders", "test_name.py"),- r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",- ).group(1)- == domain- )- assert (- self.find_in_file(- Path(self.proj_mod_path, "spiders", "test_name.py"),- r"start_urls\s*=\s*\[['\"](.+)['\"]\]",- ).group(1)- == f"https://{domain}"- )-- def test_url_schema(self):- self.test_url("https://test.com", "test.com")-- def test_template_start_urls(- self, url="test.com", expected="https://test.com", template="basic"- ):+ m = self.find_in_file(+ self.proj_mod_path / "spiders" / "test_name.py",+ r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",+ )+ assert m is not None+ assert m.group(1) == domain+ m = self.find_in_file(+ self.proj_mod_path / "spiders" / "test_name.py",+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",+ )+ assert m is not None+ assert m.group(1) == f"https://{domain}"++ @pytest.mark.parametrize(+ ("url", "expected", "template"),+ [+ # basic+ ("https://test.com", "https://test.com", "basic"),+ ("http://test.com", "http://test.com", "basic"),+ ("http://test.com/other/path", "http://test.com/other/path", "basic"),+ ("test.com/other/path", "https://test.com/other/path", "basic"),+ # crawl+ ("https://test.com", "https://test.com", "crawl"),+ ("http://test.com", "http://test.com", "crawl"),+ ("http://test.com/other/path", "http://test.com/other/path", "crawl"),+ ("test.com/other/path", "https://test.com/other/path", "crawl"),+ ("test.com", "https://test.com", "crawl"),+ # xmlfeed+ ("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),+ ("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"),+ ("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),+ # csvfeed+ ("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),+ ("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"),+ ("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),+ ],+ )+ def test_template_start_urls(self, url: str, expected: str, template: str) -> None:assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0- assert (- self.find_in_file(- Path(self.proj_mod_path, "spiders", "test_name.py"),- r"start_urls\s*=\s*\[['\"](.+)['\"]\]",- ).group(1)- == expected- )-- def test_genspider_basic_start_urls(self):- self.test_template_start_urls("https://test.com", "https://test.com", "basic")- self.test_template_start_urls("http://test.com", "http://test.com", "basic")- self.test_template_start_urls(- "http://test.com/other/path", "http://test.com/other/path", "basic"- )- self.test_template_start_urls(- "test.com/other/path", "https://test.com/other/path", "basic"- )-- def test_genspider_crawl_start_urls(self):- self.test_template_start_urls("https://test.com", "https://test.com", "crawl")- self.test_template_start_urls("http://test.com", "http://test.com", "crawl")- self.test_template_start_urls(- "http://test.com/other/path", "http://test.com/other/path", "crawl"- )- self.test_template_start_urls(- "test.com/other/path", "https://test.com/other/path", "crawl"- )- self.test_template_start_urls("test.com", "https://test.com", "crawl")-- def test_genspider_xmlfeed_start_urls(self):- self.test_template_start_urls(- "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"- )- self.test_template_start_urls(- "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"- )- self.test_template_start_urls(- "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"- )-- def test_genspider_csvfeed_start_urls(self):- self.test_template_start_urls(- "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"- )- self.test_template_start_urls(- "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"- )- self.test_template_start_urls(- "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"+ m = self.find_in_file(+ self.proj_mod_path / "spiders" / "test_name.py",+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",)+ assert m is not None+ assert m.group(1) == expectedclass TestGenspiderStandaloneCommand(TestProjectBase):@@ -174,7 +154,8 @@ class TestGenspiderStandaloneCommand(TestProjectBase):self.call("genspider", "example", "example.com")assert Path(self.temp_path, "example.py").exists()- def test_same_name_as_existing_file(self, force=False):+ @pytest.mark.parametrize("force", [True, False])+ def test_same_name_as_existing_file(self, force: bool) -> None:file_name = "example"file_path = Path(self.temp_path, file_name + ".py")p, out, err = self.proc("genspider", file_name, "example.com")@@ -203,6 +184,3 @@ class TestGenspiderStandaloneCommand(TestProjectBase):assert modify_time_after == modify_time_beforefile_contents_after = file_path.read_text(encoding="utf-8")assert file_contents_after == file_contents_before-- def test_same_name_as_existing_file_force(self):- self.test_same_name_as_existing_file(force=True)diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 0c12a59f7..89670feb3 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport inspectimport platformimport sys@@ -204,8 +205,6 @@ class MySpider(scrapy.Spider):"TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",],)- import asyncio-if sys.platform != "win32":loop = asyncio.new_event_loop()else:diff --git a/tests/test_commands.py b/tests/test_commands.pyindex a749de9fe..9ea893f05 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -9,7 +9,7 @@ from io import StringIOfrom pathlib import Pathfrom shutil import rmtreefrom tempfile import TemporaryFile, mkdtemp-from typing import TYPE_CHECKING, Any+from typing import Anyfrom unittest import mockimport pytest@@ -22,9 +22,6 @@ from scrapy.utils.python import to_unicodefrom scrapy.utils.reactor import _asyncio_reactor_pathfrom scrapy.utils.test import get_testenv-if TYPE_CHECKING:- import os-class EmptyCommand(ScrapyCommand):def short_desc(self) -> str:@@ -110,10 +107,11 @@ class TestProjectBase:return p, to_unicode(stdout), to_unicode(stderr)- def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None:+ @staticmethod+ def find_in_file(filename: Path, regex: str) -> re.Match | None:"""Find first pattern occurrence in file"""pattern = re.compile(regex)- with Path(filename).open("r", encoding="utf-8") as f:+ with filename.open("r", encoding="utf-8") as f:for line in f:match = pattern.search(line)if match is not None:diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 464320602..0a674d638 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -4,7 +4,7 @@ import shutilimport warningsfrom pathlib import Pathfrom tempfile import mkdtemp-from typing import Any, cast+from typing import TYPE_CHECKING, Any, castimport OpenSSL.SSLimport pytest@@ -14,7 +14,6 @@ from twisted.trial import unittestfrom twisted.web import server, staticfrom twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBodyfrom twisted.web.client import Response as TxResponse-from twisted.web.iweb import IBodyProducerfrom scrapy.core.downloader import Slotfrom scrapy.core.downloader.contextfactory import (@@ -29,6 +28,9 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import PayloadResource, ssl_context_factory+if TYPE_CHECKING:+ from twisted.web.iweb import IBodyProducer+class TestSlot:def test_repr(self):@@ -78,12 +80,12 @@ class TestContextFactoryBase(unittest.TestCase):agent = Agent(reactor, contextFactory=client_context_factory)body_producer = _RequestBodyProducer(body.encode()) if body else Noneresponse: TxResponse = cast(- TxResponse,+ "TxResponse",await maybe_deferred_to_future(agent.request(b"GET",url.encode(),- bodyProducer=cast(IBodyProducer, body_producer),+ bodyProducer=cast("IBodyProducer", body_producer),)),)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 42f9899f9..ef998ab4c 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -5,7 +5,7 @@ import loggingfrom ipaddress import IPv4Addressfrom socket import gethostbynamefrom typing import TYPE_CHECKING, Any-from urllib.parse import urlparse+from urllib.parse import urlencode, urlparseimport pytestfrom testfixtures import LogCapture@@ -20,6 +20,7 @@ from scrapy.exceptions import CloseSpider, StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.engine import format_engine_status, get_engine_statusfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests import NON_EXISTING_RESOLVABLE@@ -255,8 +256,6 @@ class TestCrawl(TestCase):def test_unbounded_response(self):# Completeness of responses without Content-Length or Transfer-Encoding# can not be determined, we treat them as valid but flagged as "partial"- from urllib.parse import urlencode-query = urlencode({"raw": """\@@ -339,8 +338,6 @@ with multiples lines@inlineCallbacksdef test_engine_status(self):- from scrapy.utils.engine import get_engine_status-est = []def cb(response):@@ -357,8 +354,6 @@ with multiples lines@inlineCallbacksdef test_format_engine_status(self):- from scrapy.utils.engine import format_engine_status-est = []def cb(response):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 56cb21650..e44bf1a46 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -526,10 +526,10 @@ class TestCrawlerLogging:crawler = get_crawler(MySpider)assert get_scrapy_root_handler().level == logging.INFOinfo_count = crawler.stats.get_value("log_count/INFO")- logging.debug("debug message")- logging.info("info message")- logging.warning("warning message")- logging.error("error message")+ logging.debug("debug message") # noqa: LOG015+ logging.info("info message") # noqa: LOG015+ logging.warning("warning message") # noqa: LOG015+ logging.error("error message") # noqa: LOG015logged = log_file.read_text(encoding="utf-8")@@ -556,7 +556,7 @@ class TestCrawlerLogging:configure_logging()get_crawler(MySpider)- logging.debug("debug message")+ logging.debug("debug message") # noqa: LOG015logged = log_file.read_text(encoding="utf-8")diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 77077e6e5..2dc6817a5 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -15,10 +15,7 @@ from twisted.web.http import H2_ENABLEDfrom scrapy.http import Requestfrom scrapy.spiders import Spider-from scrapy.utils.defer import (- deferred_f_from_coro_f,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import ssl_context_factory@@ -46,7 +43,9 @@ class H2DownloadHandlerMixin:@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:# the import can fail when H2_ENABLED is False- from scrapy.core.downloader.handlers.http2 import H2DownloadHandler+ from scrapy.core.downloader.handlers.http2 import ( # noqa: PLC0415+ H2DownloadHandler,+ )return H2DownloadHandlerdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 0775ae1ee..e73bfbccf 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -13,7 +13,7 @@ from unittest import mockimport pytestfrom twisted.cred import checkers, credentials, portalfrom twisted.internet.defer import inlineCallbacks-from twisted.protocols.ftp import FTPFactory, FTPRealm+from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealmfrom twisted.trial import unittestfrom w3lib.url import path_to_file_uri@@ -27,10 +27,7 @@ from scrapy.http import HtmlResponse, Request, Responsefrom scrapy.http.response.text import TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider-from scrapy.utils.defer import (- deferred_f_from_coro_f,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler@@ -184,7 +181,7 @@ class TestS3:@contextlib.contextmanagerdef _mocked_date(self, date):try:- import botocore.auth # noqa: F401+ import botocore.auth # noqa: F401,PLC0415except ImportError:yieldelse:@@ -443,7 +440,6 @@ class TestFTP(TestFTPBase):pytest.skip("This test produces DirtyReactorAggregateError on Windows with asyncio")- from twisted.protocols.ftp import ConnectionLostmeta = dict(self.req_meta)meta.update({"ftp_password": "invalid"})diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 3c26b242f..0b3941c09 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -50,6 +50,23 @@ FORMAT = {}+def _skip_if_no_br() -> None:+ try:+ try:+ import brotli # noqa: F401,PLC0415+ except ImportError:+ import brotlicffi # noqa: F401,PLC0415+ except ImportError:+ pytest.skip("no brotli support")+++def _skip_if_no_zstd() -> None:+ try:+ import zstandard # noqa: F401,PLC0415+ except ImportError:+ pytest.skip("no zstd support (zstandard)")++class TestHttpCompression:def setup_method(self):self.crawler = get_crawler(Spider)@@ -123,13 +140,8 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", 74837)def test_process_response_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+response = self._getresponse("br")request = response.requestassert response.headers["Content-Encoding"] == b"br"@@ -143,11 +155,11 @@ class TestHttpCompression:def test_process_response_br_unsupported(self):try:try:- import brotli # noqa: F401+ import brotli # noqa: F401,PLC0415pytest.skip("Requires not having brotli support")except ImportError:- import brotlicffi # noqa: F401+ import brotlicffi # noqa: F401,PLC0415pytest.skip("Requires not having brotli support")except ImportError:@@ -176,10 +188,8 @@ class TestHttpCompression:assert newresponse.headers.getlist("Content-Encoding") == [b"br"]def test_process_response_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+raw_content = Nonefor check_key in FORMAT:if not check_key.startswith("zstd-"):@@ -198,7 +208,7 @@ class TestHttpCompression:def test_process_response_zstd_unsupported(self):try:- import zstandard # noqa: F401+ import zstandard # noqa: F401,PLC0415pytest.skip("Requires not having zstandard support")except ImportError:@@ -513,13 +523,8 @@ class TestHttpCompression:mw.process_response(response.request, response, spider)def test_compression_bomb_setting_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_compression_bomb_setting("br")def test_compression_bomb_setting_deflate(self):@@ -529,10 +534,8 @@ class TestHttpCompression:self._test_compression_bomb_setting("gzip")def test_compression_bomb_setting_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_compression_bomb_setting("zstd")def _test_compression_bomb_spider_attr(self, compression_id):@@ -549,13 +552,8 @@ class TestHttpCompression:mw.process_response(response.request, response, spider)def test_compression_bomb_spider_attr_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_compression_bomb_spider_attr("br")def test_compression_bomb_spider_attr_deflate(self):@@ -565,10 +563,8 @@ class TestHttpCompression:self._test_compression_bomb_spider_attr("gzip")def test_compression_bomb_spider_attr_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_compression_bomb_spider_attr("zstd")def _test_compression_bomb_request_meta(self, compression_id):@@ -583,13 +579,8 @@ class TestHttpCompression:mw.process_response(response.request, response, spider)def test_compression_bomb_request_meta_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_compression_bomb_request_meta("br")def test_compression_bomb_request_meta_deflate(self):@@ -599,10 +590,8 @@ class TestHttpCompression:self._test_compression_bomb_request_meta("gzip")def test_compression_bomb_request_meta_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_compression_bomb_request_meta("zstd")def _test_download_warnsize_setting(self, compression_id):@@ -632,13 +621,8 @@ class TestHttpCompression:)def test_download_warnsize_setting_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_download_warnsize_setting("br")def test_download_warnsize_setting_deflate(self):@@ -648,10 +632,8 @@ class TestHttpCompression:self._test_download_warnsize_setting("gzip")def test_download_warnsize_setting_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_download_warnsize_setting("zstd")def _test_download_warnsize_spider_attr(self, compression_id):@@ -683,13 +665,8 @@ class TestHttpCompression:)def test_download_warnsize_spider_attr_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_download_warnsize_spider_attr("br")def test_download_warnsize_spider_attr_deflate(self):@@ -699,10 +676,8 @@ class TestHttpCompression:self._test_download_warnsize_spider_attr("gzip")def test_download_warnsize_spider_attr_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_download_warnsize_spider_attr("zstd")def _test_download_warnsize_request_meta(self, compression_id):@@ -732,13 +707,8 @@ class TestHttpCompression:)def test_download_warnsize_request_meta_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- pytest.skip("no brotli")+ _skip_if_no_br()+self._test_download_warnsize_request_meta("br")def test_download_warnsize_request_meta_deflate(self):@@ -748,8 +718,6 @@ class TestHttpCompression:self._test_download_warnsize_request_meta("gzip")def test_download_warnsize_request_meta_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- pytest.skip("no zstd support (zstandard)")+ _skip_if_no_zstd()+self._test_download_warnsize_request_meta("zstd")diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex ce1b7952e..9085af18e 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -5,7 +5,9 @@ import csvimport gzipimport jsonimport lzma+import marshalimport os+import pickleimport randomimport shutilimport string@@ -25,6 +27,7 @@ from urllib.request import pathname2urlimport lxml.etreeimport pytest+from packaging.version import Versionfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.internet.defer import inlineCallbacks@@ -79,7 +82,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blobclasses and set their proper return values."""- from google.cloud.storage import Blob, Bucket, Client+ from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415client_mock = mock.create_autospec(Client)@@ -507,7 +510,7 @@ class TestS3FeedStorage(unittest.TestCase):class TestGCSFeedStorage(unittest.TestCase):def test_parse_settings(self):try:- from google.cloud.storage import Client # noqa: F401+ from google.cloud.storage import Client # noqa: F401,PLC0415except ImportError:pytest.skip("GCSFeedStorage requires google-cloud-storage")@@ -521,7 +524,7 @@ class TestGCSFeedStorage(unittest.TestCase):def test_parse_empty_acl(self):try:- from google.cloud.storage import Client # noqa: F401+ from google.cloud.storage import Client # noqa: F401,PLC0415except ImportError:pytest.skip("GCSFeedStorage requires google-cloud-storage")@@ -538,7 +541,7 @@ class TestGCSFeedStorage(unittest.TestCase):@deferred_f_from_coro_fasync def test_store(self):try:- from google.cloud.storage import Client # noqa: F401+ from google.cloud.storage import Client # noqa: F401,PLC0415except ImportError:pytest.skip("GCSFeedStorage requires google-cloud-storage")@@ -976,7 +979,6 @@ class TestFeedExport(TestFeedExportBase):)data = await self.exported_data(items, settings)expected = [{k: v for k, v in row.items() if v} for row in rows]- import pickleresult = self._load_until_eof(data["pickle"], load_func=pickle.load)assert result == expected@@ -997,7 +999,6 @@ class TestFeedExport(TestFeedExportBase):)data = await self.exported_data(items, settings)expected = [{k: v for k, v in row.items() if v} for row in rows]- import marshalresult = self._load_until_eof(data["marshal"], load_func=marshal.load)assert result == expected@@ -2300,9 +2301,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):@deferred_f_from_coro_fasync def test_exports_compatibility_with_postproc(self):- import marshal- import pickle-filename_to_expected = {self._named_tempfile("csv"): b"foo\r\nbar\r\n",self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',@@ -2484,7 +2482,6 @@ class TestBatchDeliveries(TestFeedExportBase):batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")rows = [{k: v for k, v in row.items() if v} for row in rows]data = await self.exported_data(items, settings)- import picklefor batch in data["pickle"]:got_batch = self._load_until_eof(batch, load_func=pickle.load)@@ -2505,7 +2502,6 @@ class TestBatchDeliveries(TestFeedExportBase):batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")rows = [{k: v for k, v in row.items() if v} for row in rows]data = await self.exported_data(items, settings)- import marshalfor batch in data["marshal"]:got_batch = self._load_until_eof(batch, load_func=marshal.load)@@ -2714,9 +2710,8 @@ class TestBatchDeliveries(TestFeedExportBase):stubs = []def open(self, *args, **kwargs):- from botocore import __version__ as botocore_version- from botocore.stub import ANY, Stubber- from packaging.version import Version+ from botocore import __version__ as botocore_version # noqa: PLC0415+ from botocore.stub import ANY, Stubber # noqa: PLC0415expected_params = {"Body": ANY,@@ -2912,10 +2907,9 @@ class TestURIParams(ABC):match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",):crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)else:crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ feed_exporter = crawler.get_extension(FeedExporter)return crawler, feed_exporterdef test_default(self):diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 80edd50d6..eb9081e7e 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -243,7 +243,7 @@ class TestHttps2ClientProtocol(TestCase):self.conn_closed_deferred = Deferred()- from scrapy.core.http2.protocol import H2ClientFactory+ from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)client_endpoint = SSL4ClientEndpoint(@@ -445,7 +445,7 @@ class TestHttps2ClientProtocol(TestCase):def test_received_dataloss_response(self):"""In case when value of Header Content-Length != len(Received Data)ProtocolError is raised"""- from h2.exceptions import InvalidBodyLengthError+ from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415request = Request(url=self.get_url("/dataloss"))with pytest.raises(ResponseFailed) as exc_info:@@ -525,7 +525,7 @@ class TestHttps2ClientProtocol(TestCase):def assert_inactive_stream(failure):assert failure.check(ResponseFailed) is not None- from scrapy.core.http2.stream import InactiveStreamClosed+ from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415assert any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons@@ -593,7 +593,7 @@ class TestHttps2ClientProtocol(TestCase):assert str(response.ip_address) == "127.0.0.1"async def _check_invalid_netloc(self, url: str) -> None:- from scrapy.core.http2.stream import InvalidHostname+ from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415request = Request(url)with pytest.raises(InvalidHostname) as exc_info:@@ -630,7 +630,7 @@ class TestHttps2ClientProtocol(TestCase):yield self.make_request_dfd(request)for err in exc_info.value.reasons:- from scrapy.core.http2.protocol import H2ClientProtocol+ from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415if isinstance(err, TimeoutError):assert (diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 6bf0b8e3f..90a693bfa 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -1441,7 +1441,7 @@ class TestXmlRpcRequest(TestRequest):)assert r.method == "POST"assert r.encoding == kwargs.get("encoding", "utf-8")- assert r.dont_filter, True+ assert r.dont_filterdef test_xmlrpc_dumps(self):self._test_request(params=("value",))diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 047f8c610..682428954 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -110,7 +110,7 @@ class TestLogFormatter:assert logkws["level"] == unsupported_valuewith pytest.raises(TypeError):- logging.log(logkws["level"], "message")+ logging.log(logkws["level"], "message") # noqa: LOG015def test_dropitem_custom_log_level(self):item = {}diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 808fde23d..3d84c7e71 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -5,6 +5,7 @@ import timeimport warningsfrom abc import ABC, abstractmethodfrom datetime import datetime+from ftplib import FTPfrom io import BytesIOfrom pathlib import Pathfrom posixpath import split@@ -29,9 +30,7 @@ from scrapy.pipelines.files import (GCSFilesStore,S3FilesStore,)-from scrapy.utils.test import (- get_crawler,-)+from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockFTPServerfrom .test_pipeline_media import _mocked_download_func@@ -40,7 +39,7 @@ from .test_pipeline_media import _mocked_download_funcdef get_gcs_content_and_delete(bucket: Any, path: str) -> tuple[bytes, list[dict[str, str]], Any]:- from google.cloud import storage+ from google.cloud import storage # noqa: PLC0415client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))bucket = client.get_bucket(bucket)@@ -59,8 +58,6 @@ def get_ftp_content_and_delete(password: str,use_active_mode: bool = False,) -> bytes:- from ftplib import FTP-ftp = FTP()ftp.connect(host, port)ftp.login(username, password)@@ -553,7 +550,7 @@ class TestS3FilesStore(unittest.TestCase):content_type = "image/png"store = S3FilesStore(uri)- from botocore.stub import Stubber+ from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:stub.add_response(@@ -591,7 +588,7 @@ class TestS3FilesStore(unittest.TestCase):last_modified = datetime(2019, 12, 1)store = S3FilesStore(uri)- from botocore.stub import Stubber+ from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:stub.add_response(@@ -650,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase):already uploaded files."""try:- import google.cloud.storage # noqa: F401+ import google.cloud.storage # noqa: F401,PLC0415except ModuleNotFoundError:pytest.skip("google-cloud-storage is not installed")with (diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 801f88b5a..8875dc04f 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -74,7 +74,7 @@ class TestProxyConnect(TestCase):def setUp(self):try:- import mitmproxy # noqa: F401+ import mitmproxy # noqa: F401,PLC0415except ImportError:pytest.skip("mitmproxy is not installed")diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 221ccabe6..6a24d2e90 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -1,21 +1,19 @@import pytest-from scrapy.robotstxt import decode_robotstxt+from scrapy.robotstxt import (+ ProtegoRobotParser,+ PythonRobotParser,+ RerpRobotParser,+ decode_robotstxt,+)def rerp_available():# check if robotexclusionrulesparser is installedtry:- from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401- except ImportError:- return False- return True---def protego_available():- # check if protego parser is installed- try:- from protego import Protego # noqa: F401+ from robotexclusionrulesparser import ( # noqa: PLC0415+ RobotExclusionRulesParser, # noqa: F401+ )except ImportError:return Falsereturn True@@ -134,8 +132,6 @@ class TestDecodeRobotsTxt:class TestPythonRobotParser(BaseRobotParserTest):def setup_method(self):- from scrapy.robotstxt import PythonRobotParser-super()._setUp(PythonRobotParser)def test_length_based_precedence(self):@@ -150,19 +146,14 @@ class TestPythonRobotParser(BaseRobotParserTest):@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")class TestRerpRobotParser(BaseRobotParserTest):def setup_method(self):- from scrapy.robotstxt import RerpRobotParser-super()._setUp(RerpRobotParser)def test_length_based_precedence(self):pytest.skip("Rerp does not support length based directives precedence.")[email protected](not protego_available(), reason="Protego parser is not installed")class TestProtegoRobotParser(BaseRobotParserTest):def setup_method(self):- from scrapy.robotstxt import ProtegoRobotParser-super()._setUp(ProtegoRobotParser)def test_order_based_precedence(self):diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.pyindex b1956ee5c..fa6d11123 100644--- a/tests/test_scrapy__getattr__.py+++ b/tests/test_scrapy__getattr__.py@@ -3,7 +3,9 @@ import warningsdef test_deprecated_twisted_version():with warnings.catch_warnings(record=True) as warns:- from scrapy import twisted_version # pylint: disable=no-name-in-module+ from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module+ twisted_version,+ )assert twisted_version is not Noneassert isinstance(twisted_version, tuple)@@ -15,7 +17,7 @@ def test_deprecated_twisted_version():def test_deprecated_concurrent_requests_per_ip_attribute():with warnings.catch_warnings(record=True) as warns:- from scrapy.settings.default_settings import (+ from scrapy.settings.default_settings import ( # noqa: PLC0415CONCURRENT_REQUESTS_PER_IP,)diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 85dc0679e..76609ac52 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -6,6 +6,7 @@ from unittest import mockimport pytest+from scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.settings import (SETTINGS_PRIORITIES,BaseSettings,@@ -13,6 +14,8 @@ from scrapy.settings import (SettingsAttribute,get_settings_priority,)+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawlerfrom . import default_settings@@ -447,10 +450,6 @@ class TestSettings:assert mydict["key"] == "val"def test_passing_objects_as_values(self):- from scrapy.core.downloader.handlers.file import FileDownloadHandler- from scrapy.utils.misc import build_from_crawler- from scrapy.utils.test import get_crawler-class TestPipeline:def process_item(self, i, s):return idiff --git a/tests/test_spider.py b/tests/test_spider.pyindex 7c4b17144..e2843eca4 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -1,7 +1,9 @@from __future__ import annotationsimport gzip+import reimport warnings+from datetime import datetimefrom io import BytesIOfrom logging import ERROR, WARNINGfrom pathlib import Path@@ -297,8 +299,6 @@ class TestCrawlSpider(TestSpider):)class _CrawlSpider(self.spider_class):- import re-name = "test"allowed_domains = ["example.org"]rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)@@ -635,8 +635,6 @@ Sitemap: /sitemap-relative-url.xmlclass FilteredSitemapSpider(self.spider_class):def sitemap_filter(self, entries):- from datetime import datetime-for entry in entries:date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")if date_time.year > 2008:@@ -706,8 +704,6 @@ Sitemap: /sitemap-relative-url.xmlclass FilteredSitemapSpider(self.spider_class):def sitemap_filter(self, entries):- from datetime import datetime-for entry in entries:date_time = datetime.strptime(entry["lastmod"].split("T")[0], "%Y-%m-%d"diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex b17332182..895df1804 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -1,7 +1,7 @@from __future__ import annotationsimport warnings-from typing import Any, cast+from typing import TYPE_CHECKING, Any, castfrom urllib.parse import urlparseimport pytest@@ -33,6 +33,9 @@ from scrapy.spidermiddlewares.referer import ()from scrapy.spiders import Spider+if TYPE_CHECKING:+ from collections.abc import Sequence+@pytest.fixturedef spider() -> Spider:@@ -971,7 +974,7 @@ class TestPolicyHeaderPrecedence004(class TestReferrerOnRedirect(TestRefererMiddleware):settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}- scenarii: list[+ scenarii: Sequence[tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]] = [ # type: ignore[assignment](@@ -1041,7 +1044,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):request.url, headers={"Location": url}, status=status)request = cast(- Request, redirectmw.process_response(request, response, spider)+ "Request", redirectmw.process_response(request, response, spider))referrermw.request_scheduled(request, spider)diff --git a/tests/test_toplevel.py b/tests/test_toplevel.pyindex 66a6f5318..b7497c608 100644--- a/tests/test_toplevel.py+++ b/tests/test_toplevel.py@@ -10,26 +10,26 @@ def test_version_info():def test_request_shortcut():- from scrapy.http import FormRequest, Request+ from scrapy.http import FormRequest, Request # noqa: PLC0415assert scrapy.Request is Requestassert scrapy.FormRequest is FormRequestdef test_spider_shortcut():- from scrapy.spiders import Spider+ from scrapy.spiders import Spider # noqa: PLC0415assert scrapy.Spider is Spiderdef test_selector_shortcut():- from scrapy.selector import Selector+ from scrapy.selector import Selector # noqa: PLC0415assert scrapy.Selector is Selectordef test_item_shortcut():- from scrapy.item import Field, Item+ from scrapy.item import Field, Item # noqa: PLC0415assert scrapy.Item is Itemassert scrapy.Field is Fielddiff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex 20251ca59..3b1e626c9 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -1,3 +1,4 @@+import builtinsfrom io import StringIOfrom unittest import mock@@ -74,8 +75,6 @@ def test_pformat_windows(isatty, version, terminal_processing):def test_pformat_no_pygments(isatty):isatty.return_value = True- import builtins-real_import = builtins.__import__def mock_import(name, globals, locals, fromlist, level):diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 8600faa8d..5efb29a49 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -24,7 +24,7 @@ def test_iterate_spider_output():def test_iter_spider_classes():- import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self+ import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-selfit = iter_spider_classes(tests.test_utils_spider)assert set(it) == {MySpider1, MySpider2}diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex daf09f945..bcbbe74a3 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,4 +1,5 @@import warnings+from importlib import import_moduleimport pytest@@ -461,8 +462,6 @@ def test_deprecated_imports_from_w3lib(obj_name: str) -> None:obj_type = "attribute" if obj_name == "_safe_chars" else "function"message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."- from importlib import import_module-getattr(import_module("scrapy.utils.url"), obj_name)assert isinstance(warns[0].message, Warning)diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 569f4f639..dd6d7939e 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -18,11 +18,10 @@ from twisted.internet.testing import StringTransportfrom twisted.protocols.policies import WrappingFactoryfrom twisted.trial import unittestfrom twisted.web import resource, server, static, util+from twisted.web.client import _makeGetterFactoryfrom scrapy.core.downloader import webclient as client-from scrapy.core.downloader.contextfactory import (- ScrapyClientContextFactory,-)+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactoryfrom scrapy.http import Headers, Requestfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes, to_unicode@@ -51,8 +50,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):f.deferred.addCallback(response_transform or (lambda r: r.body))return f- from twisted.web.client import _makeGetterFactory-return _makeGetterFactory(to_bytes(url),_clientfactory,diff --git a/tox.ini b/tox.iniindex 85935b011..1c1d7c0ec 100644--- a/tox.ini+++ b/tox.ini@@ -46,17 +46,21 @@ install_command =[testenv:typing]basepython = python3.9deps =- mypy==1.14.0- typing-extensions==4.12.2- types-lxml==2024.12.13- types-Pygments==2.18.0.20240506- botocore-stubs==1.35.90- boto3-stubs[s3]==1.35.90+ mypy==1.16.1+ typing-extensions==4.14.1+ types-defusedxml==0.7.0.20250516+ types-lxml==2025.3.30+ types-pexpect==4.9.0.20250516+ types-Pygments==2.19.0.20250516+ botocore-stubs==1.38.46+ boto3-stubs[s3]==1.39.3+ itemadapter==0.11.0+ Protego==0.5.0+ w3lib==2.3.1attrs >= 18.2.0Pillow >= 10.3.0pyOpenSSL >= 24.2.1pytest >= 8.2.0- w3lib >= 2.2.0commands =mypy {posargs:scrapy tests}@@ -80,14 +84,14 @@ commands =basepython = python3deps ={[testenv:extra-deps]deps}- pylint==3.3.3+ pylint==3.3.7commands =pylint conftest.py docs extras scrapy tests[testenv:twinecheck]basepython = python3deps =- twine==6.0.1+ twine==6.1.0build==1.2.2.post1commands =python -m build --sdist