Scrapy, a fast high-level web crawling & scraping framework for Python.
Add flake8-type-checking. (#6413)
Andrey Rakhmatullind08f559600f0
diff --git a/.flake8 b/.flake8index be9d83eaf..c4814f13a 100644--- a/.flake8+++ b/.flake8@@ -1,6 +1,7 @@[flake8]max-line-length = 119+extend-select = TC, TC1ignore =# black disagrees with flake8 about theseE203, E501, E701, E704, W503@@ -58,6 +59,9 @@ ignore =D402# First word of the first line should be properly capitalizedD403++ # Annotation in typing.cast() should be a string literal+ TC006exclude =docs/conf.pydiff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex f70effc5d..38526d720 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -14,6 +14,7 @@ repos:- flake8-debugger- flake8-docstrings- flake8-string-format+ - flake8-type-checking- repo: https://github.com/psf/black.gitrev: 24.2.0hooks:diff --git a/scrapy/addons.py b/scrapy/addons.pyindex 65d7a0310..f9ec58cea 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -1,13 +1,16 @@+from __future__ import annotations+import loggingfrom typing import TYPE_CHECKING, Any, Listfrom scrapy.exceptions import NotConfigured-from scrapy.settings import Settingsfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:from scrapy.crawler import Crawler+ from scrapy.settings import Settings+logger = logging.getLogger(__name__)@@ -15,8 +18,8 @@ logger = logging.getLogger(__name__)class AddonManager:"""This class facilitates loading and storing :ref:`topics-addons`."""- def __init__(self, crawler: "Crawler") -> None:- self.crawler: "Crawler" = crawler+ def __init__(self, crawler: Crawler) -> None:+ self.crawler: Crawler = crawlerself.addons: List[Any] = []def load_settings(self, settings: Settings) -> None:diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex da0e51386..e010b159a 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -12,7 +12,6 @@ import scrapyfrom scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatterfrom scrapy.crawler import CrawlerProcessfrom scrapy.exceptions import UsageError-from scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.misc import walk_modulesfrom scrapy.utils.project import get_project_settings, inside_projectfrom scrapy.utils.python import garbage_collect@@ -21,6 +20,8 @@ if TYPE_CHECKING:# typing.ParamSpec requires Python 3.10from typing_extensions import ParamSpec+ from scrapy.settings import BaseSettings, Settings+_P = ParamSpec("_P")diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 9fe803d3c..0322390e5 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -2,18 +2,22 @@Base class for Scrapy commands"""+from __future__ import annotations+import argparseimport builtinsimport osfrom pathlib import Path-from typing import Any, Dict, Iterable, List, Optional+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optionalfrom twisted.python import failure-from scrapy.crawler import Crawler, CrawlerProcessfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli+if TYPE_CHECKING:+ from scrapy.crawler import Crawler, CrawlerProcess+class ScrapyCommand:requires_project: bool = Falsediff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex 0c4ebcd23..f91fec57e 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -1,16 +1,20 @@+from __future__ import annotations+import argparseimport subprocess # nosecimport sysimport time-from typing import Any, Iterable, List+from typing import TYPE_CHECKING, Any, Iterable, Listfrom urllib.parse import urlencodeimport scrapy-from scrapy import Requestfrom scrapy.commands import ScrapyCommandfrom scrapy.http import Response, TextResponsefrom scrapy.linkextractors import LinkExtractor+if TYPE_CHECKING:+ from scrapy import Request+class Command(ScrapyCommand):default_settings = {diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.pyindex 6e023af81..fe1864372 100644--- a/scrapy/commands/crawl.py+++ b/scrapy/commands/crawl.py@@ -1,11 +1,15 @@-import argparse-from typing import List, cast+from __future__ import annotations++from typing import TYPE_CHECKING, List, castfrom twisted.python.failure import Failurefrom scrapy.commands import BaseRunSpiderCommandfrom scrapy.exceptions import UsageError+if TYPE_CHECKING:+ import argparse+class Command(BaseRunSpiderCommand):requires_project = Truediff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex 1acf2d26f..0bdc429da 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -1,6 +1,7 @@+from __future__ import annotations+import sys-from argparse import ArgumentParser, Namespace-from typing import Dict, List, Type+from typing import TYPE_CHECKING, Dict, List, Typefrom w3lib.url import is_url@@ -11,6 +12,9 @@ from scrapy.http import Request, Responsefrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.spider import DefaultSpider, spidercls_for_request+if TYPE_CHECKING:+ from argparse import ArgumentParser, Namespace+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/commands/list.py b/scrapy/commands/list.pyindex dcc51a694..10330c92a 100644--- a/scrapy/commands/list.py+++ b/scrapy/commands/list.py@@ -1,8 +1,12 @@-import argparse-from typing import List+from __future__ import annotations++from typing import TYPE_CHECKING, Listfrom scrapy.commands import ScrapyCommand+if TYPE_CHECKING:+ import argparse+class Command(ScrapyCommand):requires_project = Truediff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 3320a1ee4..e6c5e2a47 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -6,6 +6,7 @@ import inspectimport jsonimport loggingfrom typing import (+ TYPE_CHECKING,Any,AsyncGenerator,Callable,@@ -22,13 +23,11 @@ from typing import (from itemadapter import ItemAdapter, is_itemfrom twisted.internet.defer import Deferred, maybeDeferred-from twisted.python.failure import Failurefrom w3lib.url import is_urlfrom scrapy.commands import BaseRunSpiderCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.http import Request, Response-from scrapy.spiders import Spiderfrom scrapy.utils import displayfrom scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.defer import aiter_errback, deferred_from_coro@@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.spider import spidercls_for_request+if TYPE_CHECKING:+ from twisted.python.failure import Failure++ from scrapy.spiders import Spider++logger = logging.getLogger(__name__)_T = TypeVar("_T")diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 77850e7b5..87acf9a01 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -1,17 +1,21 @@+from __future__ import annotations+import argparseimport sysfrom importlib import import_module-from os import PathLikefrom pathlib import Path-from types import ModuleType-from typing import List, Union+from typing import TYPE_CHECKING, List, Unionfrom scrapy.commands import BaseRunSpiderCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.spider import iter_spider_classes+if TYPE_CHECKING:+ from os import PathLike+ from types import ModuleType+-def _import_file(filepath: Union[str, PathLike]) -> ModuleType:+def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType:abspath = Path(filepath).resolve()if abspath.suffix not in (".py", ".pyw"):raise ValueError(f"Not a Python source file: {abspath}")diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 668c95a7b..f03cf997a 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -4,9 +4,10 @@ Scrapy ShellSee documentation in docs/topics/shell.rst"""-from argparse import ArgumentParser, Namespace+from __future__ import annotations+from threading import Thread-from typing import Any, Dict, List, Type+from typing import TYPE_CHECKING, Any, Dict, List, Typefrom scrapy import Spiderfrom scrapy.commands import ScrapyCommand@@ -15,6 +16,9 @@ from scrapy.shell import Shellfrom scrapy.utils.spider import DefaultSpider, spidercls_for_requestfrom scrapy.utils.url import guess_scheme+if TYPE_CHECKING:+ from argparse import ArgumentParser, Namespace+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 27bc2fcba..440e0dc44 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -1,9 +1,12 @@+from __future__ import annotations+import reimport sysfrom functools import wrapsfrom inspect import getmembersfrom types import CoroutineTypefrom typing import (+ TYPE_CHECKING,Any,AsyncGenerator,Callable,@@ -16,13 +19,15 @@ from typing import ()from unittest import TestCase, TestResult-from twisted.python.failure import Failure--from scrapy import Spiderfrom scrapy.http import Request, Responsefrom scrapy.utils.python import get_specfrom scrapy.utils.spider import iterate_spider_output+if TYPE_CHECKING:+ from twisted.python.failure import Failure++ from scrapy import Spider+class Contract:"""Abstract class for contracts"""diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 41f729ed9..6786d7acf 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -25,15 +25,16 @@ from scrapy import Request, Spider, signalsfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.http import Responsefrom scrapy.resolver import dnscache-from scrapy.settings import BaseSettingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.settings import BaseSettings+_T = TypeVar("_T")diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 9f6edb630..2b388a9f5 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import (ScrapyClientTLSOptions,openssl_methods,)-from scrapy.crawler import Crawler-from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:@@ -31,6 +29,9 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings+@implementer(IPolicyForHTTPS)class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex ebc4898b5..70d356b83 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -17,17 +17,19 @@ from typing import ()from twisted.internet import defer-from twisted.internet.defer import Deferredfrom scrapy import Request, Spider, signalsfrom scrapy.exceptions import NotConfigured, NotSupported-from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+from scrapy.crawler import Crawler+ from scrapy.http import Response+logger = logging.getLogger(__name__)diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.pyindex a7ae56a85..bf6879521 100644--- a/scrapy/core/downloader/handlers/datauri.py+++ b/scrapy/core/downloader/handlers/datauri.py@@ -1,12 +1,16 @@-from typing import Any, Dict+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Dictfrom w3lib.url import parse_data_uri-from scrapy import Request, Spiderfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.decorators import defers+if TYPE_CHECKING:+ from scrapy import Request, Spider+class DataURIDownloadHandler:lazy = Falsediff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.pyindex 17dd7483b..d55c516f0 100644--- a/scrapy/core/downloader/handlers/file.py+++ b/scrapy/core/downloader/handlers/file.py@@ -1,12 +1,17 @@+from __future__ import annotations+from pathlib import Path+from typing import TYPE_CHECKINGfrom w3lib.url import file_uri_to_path-from scrapy import Request, Spider-from scrapy.http import Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.decorators import defers+if TYPE_CHECKING:+ from scrapy import Request, Spider+ from scrapy.http import Response+class FileDownloadHandler:lazy = Falsediff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 724717ffd..69c2d88e1 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -35,23 +35,25 @@ from io import BytesIOfrom typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optionalfrom urllib.parse import unquote-from twisted.internet.defer import Deferredfrom twisted.internet.protocol import ClientCreator, Protocolfrom twisted.protocols.ftp import CommandFailed, FTPClient-from twisted.python.failure import Failure-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.http import Responsefrom scrapy.responsetypes import responsetypes-from scrapy.settings import BaseSettingsfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytesif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+ from twisted.python.failure import Failure+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings+class ReceivedDataProtocol(Protocol):def __init__(self, filename: Optional[str] = None):diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.pyindex 3c4e48abb..98f62efcf 100644--- a/scrapy/core/downloader/handlers/http10.py+++ b/scrapy/core/downloader/handlers/http10.py@@ -5,21 +5,21 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Type-from twisted.internet.defer import Deferred--from scrapy import Request, Spider-from scrapy.crawler import Crawler-from scrapy.http import Response-from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spiderfrom scrapy.core.downloader.contextfactory import ScrapyClientContextFactoryfrom scrapy.core.downloader.webclient import ScrapyHTTPClientFactory+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.settings import BaseSettingsclass HTTP10DownloadHandler:diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex e2ad8f59a..c06d90f01 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVarfrom urllib.parse import urldefrag, urlunparsefrom twisted.internet import ssl-from twisted.internet.base import ReactorBasefrom twisted.internet.defer import CancelledError, Deferred, succeedfrom twisted.internet.endpoints import TCP4ClientEndpointfrom twisted.internet.error import TimeoutError-from twisted.internet.interfaces import IConsumerfrom twisted.internet.protocol import Factory, Protocol, connectionDonefrom twisted.python.failure import Failurefrom twisted.web.client import URI, Agent, HTTPConnectionPool@@ -30,17 +28,22 @@ from zope.interface import implementerfrom scrapy import Request, Spider, signalsfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settingsfrom scrapy.core.downloader.webclient import _parse-from scrapy.crawler import Crawlerfrom scrapy.exceptions import StopDownloadfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypes-from scrapy.settings import BaseSettingsfrom scrapy.utils.python import to_bytes, to_unicodeif TYPE_CHECKING:+ from twisted.internet.base import ReactorBase+ from twisted.internet.interfaces import IConsumer+# typing.NotRequired and typing.Self require Python 3.11from typing_extensions import NotRequired, Self+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)_T = TypeVar("_T")diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 2ac4eca86..4722c612d 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -4,25 +4,27 @@ from time import timefrom typing import TYPE_CHECKING, Optionalfrom urllib.parse import urldefrag-from twisted.internet.base import DelayedCall-from twisted.internet.defer import Deferredfrom twisted.internet.error import TimeoutErrorfrom twisted.web.client import URI-from twisted.web.iweb import IPolicyForHTTPSfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settingsfrom scrapy.core.downloader.webclient import _parsefrom scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent-from scrapy.crawler import Crawler-from scrapy.http import Request, Response-from scrapy.settings import Settings-from scrapy.spiders import Spiderfrom scrapy.utils.python import to_bytesif TYPE_CHECKING:+ from twisted.internet.base import DelayedCall+ from twisted.internet.defer import Deferred+ from twisted.web.iweb import IPolicyForHTTPS+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Request, Response+ from scrapy.settings import Settings+ from scrapy.spiders import Spider+class H2DownloadHandler:def __init__(self, settings: Settings, crawler: Crawler):diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 0ad340721..edf370193 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -2,22 +2,23 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, Optional, Type-from twisted.internet.defer import Deferred--from scrapy import Request, Spiderfrom scrapy.core.downloader.handlers.http import HTTPDownloadHandler-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import Response-from scrapy.settings import BaseSettingsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawlerif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.settings import BaseSettings+class S3DownloadHandler:def __init__(diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 2d8af114f..0bdb756c8 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rstfrom __future__ import annotations-from typing import Any, Callable, Generator, List, Union, cast+from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, castfrom twisted.internet.defer import Deferred, inlineCallbacks-from twisted.python.failure import Failure-from scrapy import Spiderfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.middleware import MiddlewareManager-from scrapy.settings import BaseSettingsfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import deferred_from_coro, mustbe_deferred+if TYPE_CHECKING:+ from twisted.python.failure import Failure++ from scrapy import Spider+ from scrapy.settings import BaseSettings+class DownloaderMiddlewareManager(MiddlewareManager):component_name = "downloader middleware"diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 08a1d7c71..99502f0d2 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -1,18 +1,22 @@+from __future__ import annotations+import refrom time import time-from typing import Optional, Tuple+from typing import TYPE_CHECKING, Optional, Tuplefrom urllib.parse import ParseResult, urldefrag, urlparse, urlunparsefrom twisted.internet import deferfrom twisted.internet.protocol import ClientFactoryfrom twisted.web.http import HTTPClient-from scrapy import Requestfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode+if TYPE_CHECKING:+ from scrapy import Request+def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]:# Assume parsed is urlparse-d from Request.url,diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 4ffec78b9..5318cbd64 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraperfrom scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequestfrom scrapy.http import Request, Responsefrom scrapy.logformatter import LogFormatter-from scrapy.settings import BaseSettings, Settings+from scrapy.settings import Settingsfrom scrapy.signalmanager import SignalManager-from scrapy.spiders import Spiderfrom scrapy.utils.log import failure_to_exc_info, logformatter_adapterfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import global_object_name@@ -46,6 +45,9 @@ if TYPE_CHECKING:from scrapy.core.scheduler import BaseSchedulerfrom scrapy.core.scraper import _HandleOutputDeferredfrom scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings+ from scrapy.spiders import Spider+logger = logging.getLogger(__name__)diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pyindex 999764a6e..d291a5b8a 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/http2/agent.py@@ -1,10 +1,10 @@+from __future__ import annotations+from collections import deque-from typing import Deque, Dict, List, Optional, Tuple+from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuplefrom twisted.internet import defer-from twisted.internet.base import ReactorBasefrom twisted.internet.defer import Deferred-from twisted.internet.endpoints import HostnameEndpointfrom twisted.python.failure import Failurefrom twisted.web.client import (URI,@@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupportedfrom scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactoryfrom scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol-from scrapy.http.request import Request-from scrapy.settings import Settings-from scrapy.spiders import Spider++if TYPE_CHECKING:+ from twisted.internet.base import ReactorBase+ from twisted.internet.endpoints import HostnameEndpoint++ from scrapy.http.request import Request+ from scrapy.settings import Settings+ from scrapy.spiders import Spider+ConnectionKeyT = Tuple[bytes, bytes, int]diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex f2f1cb0b8..a6809102b 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -1,9 +1,10 @@+from __future__ import annotations+import ipaddressimport itertoolsimport loggingfrom collections import deque-from ipaddress import IPv4Address, IPv6Address-from typing import Any, Deque, Dict, List, Optional, Union+from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Unionfrom h2.config import H2Configurationfrom h2.connection import H2Connection@@ -20,7 +21,6 @@ from h2.events import (WindowUpdated,)from h2.exceptions import FrameTooLargeError, H2Error-from twisted.internet.defer import Deferredfrom twisted.internet.error import TimeoutErrorfrom twisted.internet.interfaces import (IAddress,@@ -30,14 +30,21 @@ from twisted.internet.interfaces import (from twisted.internet.protocol import Factory, Protocol, connectionDonefrom twisted.internet.ssl import Certificatefrom twisted.protocols.policies import TimeoutMixin-from twisted.python.failure import Failure-from twisted.web.client import URIfrom zope.interface import implementerfrom scrapy.core.http2.stream import Stream, StreamCloseReasonfrom scrapy.http import Request-from scrapy.settings import Settings-from scrapy.spiders import Spider++if TYPE_CHECKING:+ from ipaddress import IPv4Address, IPv6Address++ from twisted.internet.defer import Deferred+ from twisted.python.failure import Failure+ from twisted.web.client import URI++ from scrapy.settings import Settings+ from scrapy.spiders import Spider+logger = logging.getLogger(__name__)diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 7c70e86db..a02fbb328 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -1,3 +1,5 @@+from __future__ import annotations+import loggingfrom enum import Enumfrom io import BytesIO@@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuplefrom h2.errors import ErrorCodesfrom h2.exceptions import H2Error, ProtocolError, StreamClosedError-from hpack import HeaderTuplefrom twisted.internet.defer import CancelledError, Deferredfrom twisted.internet.error import ConnectionClosedfrom twisted.python.failure import Failurefrom twisted.web.client import ResponseFailed-from scrapy.http import Requestfrom scrapy.http.headers import Headersfrom scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:+ from hpack import HeaderTuple+from scrapy.core.http2.protocol import H2ClientProtocol+ from scrapy.http import Requestlogger = logging.getLogger(__name__)@@ -87,7 +90,7 @@ class Stream:self,stream_id: int,request: Request,- protocol: "H2ClientProtocol",+ protocol: H2ClientProtocol,download_maxsize: int = 0,download_warnsize: int = 0,) -> None:@@ -99,7 +102,7 @@ class Stream:"""self.stream_id: int = stream_idself._request: Request = request- self._protocol: "H2ClientProtocol" = protocol+ self._protocol: H2ClientProtocol = protocolself._download_maxsize = self._request.meta.get("download_maxsize", download_maxsizediff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 1e586c53a..d4286c874 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -6,14 +6,10 @@ from abc import abstractmethodfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, List, Optional, Type, cast-from twisted.internet.defer import Deferred--from scrapy.crawler import Crawler-from scrapy.dupefilters import BaseDupeFilter-from scrapy.http.request import Request-from scrapy.pqueues import ScrapyPriorityQueue-from scrapy.spiders import Spider-from scrapy.statscollectors import StatsCollector+# working around https://github.com/sphinx-doc/sphinx/issues/10400+from twisted.internet.defer import Deferred # noqa: TC002++from scrapy.spiders import Spider # noqa: TC001from scrapy.utils.job import job_dirfrom scrapy.utils.misc import build_from_crawler, load_object@@ -24,6 +20,12 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.dupefilters import BaseDupeFilter+ from scrapy.http.request import Request+ from scrapy.pqueues import ScrapyPriorityQueue+ from scrapy.statscollectors import StatsCollector+logger = logging.getLogger(__name__)diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex e792f8ca7..37a666605 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -10,6 +10,7 @@ import loggingfrom inspect import isasyncgenfunction, iscoroutinefrom itertools import islicefrom typing import (+ TYPE_CHECKING,Any,AsyncIterable,Callable,@@ -30,7 +31,6 @@ from scrapy import Request, Spiderfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Responsefrom scrapy.middleware import MiddlewareManager-from scrapy.settings import BaseSettingsfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import (@@ -41,6 +41,10 @@ from scrapy.utils.defer import ()from scrapy.utils.python import MutableAsyncChain, MutableChain+if TYPE_CHECKING:+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 0e757e4be..5fc7f31a3 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Unionfrom w3lib import html-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import HtmlResponse, Response-from scrapy.settings import BaseSettingsif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 73c2c57fe..23140d263 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -2,24 +2,26 @@ from __future__ import annotationsimport loggingfrom collections import defaultdict-from http.cookiejar import Cookiefrom typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Unionfrom tldextract import TLDExtract-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Responsefrom scrapy.http.cookies import CookieJar-from scrapy.http.request import VerboseCookiefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:+ from http.cookiejar import Cookie+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.http.request import VerboseCookie+logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.pyindex 58fd415b9..49b9fdc05 100644--- a/scrapy/downloadermiddlewares/defaultheaders.py+++ b/scrapy/downloadermiddlewares/defaultheaders.py@@ -8,15 +8,16 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Iterable, Tuple, Union-from scrapy import Request, Spider-from scrapy.crawler import Crawler-from scrapy.http import Responsefrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+class DefaultHeadersMiddleware:def __init__(self, headers: Iterable[Tuple[str, str]]):diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.pyindex fd7c03a38..ee7a24825 100644--- a/scrapy/downloadermiddlewares/downloadtimeout.py+++ b/scrapy/downloadermiddlewares/downloadtimeout.py@@ -9,13 +9,14 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Unionfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawler-from scrapy.http import Responseif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Response+class DownloadTimeoutMiddleware:def __init__(self, timeout: float = 180):diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex 63490a37a..39165e155 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Unionfrom w3lib.http import basic_auth_headerfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawler-from scrapy.http import Responsefrom scrapy.utils.url import url_is_from_any_domainif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Response+class HttpAuthMiddleware:"""Set Basic HTTP Authorization headerdiff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex 971473403..8377a3c1d 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -16,19 +16,20 @@ from twisted.internet.error import (from twisted.web.client import ResponseFailedfrom scrapy import signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfigured-from scrapy.http.request import Request-from scrapy.http.response import Response-from scrapy.settings import Settings-from scrapy.spiders import Spider-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.misc import load_objectif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http.request import Request+ from scrapy.http.response import Response+ from scrapy.settings import Settings+ from scrapy.spiders import Spider+ from scrapy.statscollectors import StatsCollector+class HttpCacheMiddleware:DOWNLOAD_EXCEPTIONS = (diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex f3647e05f..6b0a56f7f 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -6,11 +6,9 @@ from logging import getLoggerfrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Unionfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils._compression import (_DecompressionMaxSizeExceeded,_inflate,@@ -24,6 +22,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = getLogger(__name__)ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex 5b56ad449..a7af83f7d 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined]proxy_bypass,)-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes@@ -20,6 +17,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+class HttpProxyMiddleware:def __init__(self, auth_encoding: Optional[str] = "latin-1"):diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex bd8dbe329..6f67e3975 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -6,15 +6,17 @@ import warningsfrom typing import TYPE_CHECKING, Setfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 371e2fd3b..53081237c 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -6,11 +6,8 @@ from urllib.parse import urljoinfrom w3lib.url import safe_url_string-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import HtmlResponse, Response-from scrapy.settings import BaseSettingsfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.response import get_meta_refresh@@ -18,6 +15,11 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 0637f09d4..8d7b7293c 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -16,12 +16,8 @@ import warningsfrom logging import Logger, getLoggerfrom typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.http import Response-from scrapy.http.request import Requestfrom scrapy.settings import BaseSettings, Settings-from scrapy.spiders import Spiderfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import response_status_message@@ -30,6 +26,12 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.http.request import Request+ from scrapy.spiders import Spider++retry_logger = getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 6a0ecb7bf..70393576b 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -10,22 +10,24 @@ import loggingfrom typing import TYPE_CHECKING, Any, Dict, Optional, Unionfrom twisted.internet.defer import Deferred, maybeDeferred-from twisted.python.failure import Failure-from scrapy import Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK-from scrapy.robotstxt import RobotParserfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import load_objectif TYPE_CHECKING:+ from twisted.python.failure import Failure+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.crawler import Crawler+ from scrapy.robotstxt import RobotParser+logger = logging.getLogger(__name__)diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex 444702757..0faae7b5a 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Unionfrom twisted.web import http-from scrapy import Request, Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import Response-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.python import global_object_name, to_bytesfrom scrapy.utils.request import request_httprepr@@ -16,6 +12,11 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.statscollectors import StatsCollector+def get_header_size(headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.pyindex 92f1ec897..109f1a4d9 100644--- a/scrapy/downloadermiddlewares/useragent.py+++ b/scrapy/downloadermiddlewares/useragent.py@@ -5,13 +5,14 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Unionfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawler-from scrapy.http import Responseif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Response+class UserAgentMiddleware:"""This middleware allows spiders to override the user_agent"""diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex dd2420e98..ffaf783a7 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -4,11 +4,6 @@ import loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, Optional, Set-from twisted.internet.defer import Deferred--from scrapy.http.request import Request-from scrapy.settings import BaseSettings-from scrapy.spiders import Spiderfrom scrapy.utils.job import job_dirfrom scrapy.utils.request import (RequestFingerprinter,@@ -17,10 +12,15 @@ from scrapy.utils.request import ()if TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.http.request import Request+ from scrapy.settings import BaseSettings+ from scrapy.spiders import Spiderclass BaseDupeFilter:diff --git a/scrapy/extension.py b/scrapy/extension.pyindex 8221b675e..8c81ab356 100644--- a/scrapy/extension.py+++ b/scrapy/extension.py@@ -4,12 +4,16 @@ The Extension ManagerSee documentation in docs/topics/extensions.rst"""-from typing import Any, List+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Listfrom scrapy.middleware import MiddlewareManager-from scrapy.settings import Settingsfrom scrapy.utils.conf import build_component_list+if TYPE_CHECKING:+ from scrapy.settings import Settings+class ExtensionManager(MiddlewareManager):component_name = "extension"diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex 812b3553c..4627e7f98 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -10,17 +10,19 @@ import loggingfrom collections import defaultdictfrom typing import TYPE_CHECKING, Any, DefaultDict, Dict-from twisted.python.failure import Failure-from scrapy import Request, Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import Responseif TYPE_CHECKING:+ from twisted.python.failure import Failure+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.http import Response++logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex f3ac19623..6ef2d0382 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -8,13 +8,14 @@ from datetime import datetime, timezonefrom typing import TYPE_CHECKING, Any, Optionalfrom scrapy import Spider, signals-from scrapy.crawler import Crawler-from scrapy.statscollectors import StatsCollectorif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector+class CoreStats:def __init__(self, stats: StatsCollector):diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.pyindex b360ce48d..c54871e02 100644--- a/scrapy/extensions/debug.py+++ b/scrapy/extensions/debug.py@@ -12,17 +12,20 @@ import sysimport threadingimport tracebackfrom pdb import Pdb-from types import FrameTypefrom typing import TYPE_CHECKING, Optional-from scrapy.crawler import Crawlerfrom scrapy.utils.engine import format_engine_statusfrom scrapy.utils.trackref import format_live_refsif TYPE_CHECKING:+ from types import FrameType+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler++logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 941bd4b26..43c2d2815 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -31,18 +31,15 @@ from typing import ()from urllib.parse import unquote, urlparse-from twisted.internet import threadsfrom twisted.internet.defer import Deferred, DeferredList, maybeDeferred-from twisted.python.failure import Failure+from twisted.internet.threads import deferToThreadfrom w3lib.url import file_uri_to_pathfrom zope.interface import Interface, implementerfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.exporters import BaseItemExporterfrom scrapy.extensions.postprocessing import PostProcessingManager-from scrapy.settings import BaseSettings, Settings+from scrapy.settings import Settingsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.defer import maybe_deferred_to_future@@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_valuesif TYPE_CHECKING:from _typeshed import OpenBinaryMode+ from twisted.python.failure import Failure# typing.Self requires Python 3.11from typing_extensions import Self-logger = logging.getLogger(__name__)+ from scrapy.crawler import Crawler+ from scrapy.exporters import BaseItemExporter+ from scrapy.settings import BaseSettingstry:import boto3 # noqa: F401@@ -67,6 +67,9 @@ try:except ImportError:IS_BOTO3_AVAILABLE = False++logger = logging.getLogger(__name__)+UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")@@ -160,7 +163,7 @@ class BlockingFeedStorage:return NamedTemporaryFile(prefix="feed-", dir=path)def store(self, file: IO[bytes]) -> Optional[Deferred]:- return threads.deferToThread(self._store_in_thread, file)+ return deferToThread(self._store_in_thread, file)def _store_in_thread(self, file: IO[bytes]) -> None:raise NotImplementedErrordiff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex b7219bf07..448d5f1ab 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -1,3 +1,5 @@+from __future__ import annotations+import gzipimport loggingimport os@@ -13,10 +15,7 @@ from weakref import WeakKeyDictionaryfrom w3lib.http import headers_dict_to_raw, headers_raw_to_dictfrom scrapy.http import Headers, Response-from scrapy.http.request import Requestfrom scrapy.responsetypes import responsetypes-from scrapy.settings import BaseSettings-from scrapy.spiders import Spiderfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_pathfrom scrapy.utils.python import to_bytes, to_unicode@@ -26,6 +25,10 @@ if TYPE_CHECKING:# typing.Concatenate requires Python 3.10from typing_extensions import Concatenate+ from scrapy.http.request import Request+ from scrapy.settings import BaseSettings+ from scrapy.spiders import Spider+logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.pyindex 2388afa75..c4f43482d 100644--- a/scrapy/extensions/logstats.py+++ b/scrapy/extensions/logstats.py@@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Unionfrom twisted.internet import taskfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.statscollectors import StatsCollectorif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.pyindex f304e1bf2..3cbbb64e5 100644--- a/scrapy/extensions/memdebug.py+++ b/scrapy/extensions/memdebug.py@@ -10,15 +10,16 @@ import gcfrom typing import TYPE_CHECKINGfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.trackref import live_refsif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector+class MemoryDebugger:def __init__(self, stats: StatsCollector):diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex 9de06b24d..25f63ecc6 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, Listfrom twisted.internet import taskfrom scrapy import signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.mail import MailSenderfrom scrapy.utils.engine import get_engine_status@@ -25,6 +24,9 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler++logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex 9567f948a..80c0a3b26 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom twisted.internet import taskfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.serialize import ScrapyJSONEncoderif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = logging.getLogger(__name__)diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex c6eb20277..567efd7a1 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -5,7 +5,6 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Optionalfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.job import job_dir@@ -13,6 +12,8 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+class SpiderState:"""Store and load spider state during a scraping job"""diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.pyindex 20b8f910c..e43de6f5c 100644--- a/scrapy/extensions/statsmailer.py+++ b/scrapy/extensions/statsmailer.py@@ -8,18 +8,19 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, List, Optional-from twisted.internet.defer import Deferred-from scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.mail import MailSender-from scrapy.statscollectors import StatsCollectorif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector+class StatsMailer:def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 00c69434c..c4e01b3d9 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -26,7 +26,6 @@ except (ImportError, SyntaxError):TWISTED_CONCH_AVAILABLE = Falsefrom scrapy import signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.decorators import defersfrom scrapy.utils.engine import print_engine_status@@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refsif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self++ from scrapy.crawler import Crawler++logger = logging.getLogger(__name__)# signal to update telnet variablesdiff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex 217e61a81..6ce9ce63a 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -4,15 +4,17 @@ import loggingfrom typing import TYPE_CHECKING, Optional, Tuplefrom scrapy import Request, Spider, signals-from scrapy.core.downloader import Slot-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import Responseif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.core.downloader import Slot+ from scrapy.crawler import Crawler+ from scrapy.http import Response++logger = logging.getLogger(__name__)diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex 8af89c74f..cc88a9420 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -17,8 +17,6 @@ from typing import (cast,)-from scrapy import Request-from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicode@@ -26,6 +24,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request+ from scrapy.http import Response++# Defined in the http.cookiejar module, but undocumented:# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527IPV4_RE = re.compile(r"\.\d+$", re.ASCII)diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex ea98ed795..a8c242e8b 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -28,13 +28,14 @@ from lxml.html import TextareaElement # nosecfrom w3lib.html import strip_html5_whitespacefrom scrapy.http.request import Request-from scrapy.http.response.text import TextResponsefrom scrapy.utils.python import is_listlike, to_bytesif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.http.response.text import TextResponse+FormdataVType = Union[str, Iterable[str]]FormdataKVType = Tuple[str, FormdataVType]diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 684439097..ff3581abb 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rstfrom __future__ import annotations-from ipaddress import IPv4Address, IPv6Addressfrom typing import (TYPE_CHECKING,Any,@@ -26,8 +25,6 @@ from typing import ()from urllib.parse import urljoin-from twisted.internet.ssl import Certificate-from scrapy.exceptions import NotSupportedfrom scrapy.http.headers import Headersfrom scrapy.http.request import CookiesT, Request@@ -35,6 +32,10 @@ from scrapy.link import Linkfrom scrapy.utils.trackref import object_refif TYPE_CHECKING:+ from ipaddress import IPv4Address, IPv6Address++ from twisted.internet.ssl import Certificate+# typing.Self requires Python 3.11from typing_extensions import Selfdiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex df4d90829..0635f744f 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -35,15 +35,16 @@ from w3lib.encoding import ()from w3lib.html import strip_html5_whitespace-from scrapy.http.request import CookiesT, Requestfrom scrapy.http.response import Responsefrom scrapy.link import Linkfrom scrapy.utils.python import memoizemethod_noargs, to_unicodefrom scrapy.utils.response import get_base_urlif TYPE_CHECKING:+ from scrapy.http.request import CookiesT, Requestfrom scrapy.selector import Selector, SelectorList+_NONE = object()diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 33a10cd6c..d27a132b3 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -2,10 +2,13 @@Link extractor based on lxml.html"""+from __future__ import annotations+import loggingimport operatorfrom functools import partialfrom typing import (+ TYPE_CHECKING,Any,Callable,Iterable,@@ -20,13 +23,10 @@ from typing import (from urllib.parse import urljoin, urlparsefrom lxml import etree # nosec-from lxml.html import HtmlElement # nosecfrom parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespacefrom w3lib.url import canonicalize_url, safe_url_string-from scrapy import Selector-from scrapy.http import TextResponsefrom scrapy.link import Linkfrom scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, refrom scrapy.utils.misc import arg_to_iter, rel_has_nofollow@@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_listfrom scrapy.utils.response import get_base_urlfrom scrapy.utils.url import url_has_any_extension, url_is_from_any_domain+if TYPE_CHECKING:+ from lxml.html import HtmlElement # nosec++ from scrapy import Selector+ from scrapy.http import TextResponse++logger = logging.getLogger(__name__)# from lxml/src/lxml/html/__init__.pydiff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.pyindex db0b4820f..9644cc093 100644--- a/scrapy/loader/__init__.py+++ b/scrapy/loader/__init__.py@@ -4,14 +4,18 @@ Item LoaderSee documentation in docs/topics/loaders.rst"""-from typing import Any, Optional+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Optionalimport itemloaders-from scrapy.http import TextResponsefrom scrapy.item import Itemfrom scrapy.selector import Selector+if TYPE_CHECKING:+ from scrapy.http import TextResponse+class ItemLoader(itemloaders.ItemLoader):"""@@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader):selector: Optional[Selector] = None,response: Optional[TextResponse] = None,parent: Optional[itemloaders.ItemLoader] = None,- **context: Any+ **context: Any,):if selector is None and response is not None:try:diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 42a03b560..601209fb0 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Unionfrom twisted.python.failure import Failure-from scrapy import Request, Spider-from scrapy.http import Response+# working around https://github.com/sphinx-doc/sphinx/issues/10400+from scrapy import Request, Spider # noqa: TC001+from scrapy.http import Response # noqa: TC001from scrapy.utils.request import referer_strif TYPE_CHECKING:diff --git a/scrapy/mail.py b/scrapy/mail.pyindex f4ce2800c..3ea20e831 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -30,20 +30,22 @@ from typing import (from twisted import version as twisted_versionfrom twisted.internet import sslfrom twisted.internet.defer import Deferred-from twisted.python.failure import Failurefrom twisted.python.versions import Version-from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import to_bytesif TYPE_CHECKING:# imports twisted.internet.reactorfrom twisted.mail.smtp import ESMTPSenderFactory+ from twisted.python.failure import Failure# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex f60c726f9..ea5488ba1 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -17,19 +17,19 @@ from typing import (cast,)-from twisted.internet.defer import Deferred--from scrapy import Spiderfrom scrapy.exceptions import NotConfigured-from scrapy.settings import Settingsfrom scrapy.utils.defer import process_chain, process_parallelfrom scrapy.utils.misc import build_from_crawler, build_from_settings, load_objectif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spiderfrom scrapy.crawler import Crawler+ from scrapy.settings import Settingslogger = logging.getLogger(__name__)diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 21d649e3c..480a5a58c 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rstfrom __future__ import annotations-from typing import Any, List+from typing import TYPE_CHECKING, Any, List-from twisted.internet.defer import Deferred--from scrapy import Spiderfrom scrapy.middleware import MiddlewareManager-from scrapy.settings import Settingsfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import deferred_f_from_coro_f+if TYPE_CHECKING:+ from twisted.internet.defer import Deferred++ from scrapy import Spider+ from scrapy.settings import Settings+class ItemPipelineManager(MiddlewareManager):component_name = "item pipeline"diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 85a8c77da..1a13aeaf2 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -16,7 +16,6 @@ from collections import defaultdictfrom contextlib import suppressfrom ftplib import FTPfrom io import BytesIO-from os import PathLikefrom pathlib import Pathfrom typing import (IO,@@ -38,11 +37,9 @@ from typing import (from urllib.parse import urlparsefrom itemadapter import ItemAdapter-from twisted.internet import defer, threads-from twisted.internet.defer import Deferred-from twisted.python.failure import Failure+from twisted.internet.defer import Deferred, maybeDeferred+from twisted.internet.threads import deferToThread-from scrapy import Spiderfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK@@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.request import referer_strif TYPE_CHECKING:+ from os import PathLike++ from twisted.python.failure import Failure+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+logger = logging.getLogger(__name__)@@ -210,7 +213,7 @@ class S3FilesStore:key_name = f"{self.prefix}{path}"return cast("Deferred[Dict[str, Any]]",- threads.deferToThread(+ deferToThread(self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined]),)@@ -229,7 +232,7 @@ class S3FilesStore:extra = self._headers_to_botocore_kwargs(self.HEADERS)if headers:extra.update(self._headers_to_botocore_kwargs(headers))- return threads.deferToThread(+ return deferToThread(self.s3_client.put_object, # type: ignore[attr-defined]Bucket=self.bucket,Key=key_name,@@ -326,9 +329,7 @@ class GCSFilesStore:blob_path = self._get_blob_path(path)return cast(Deferred[StatInfo],- threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(- _onsuccess- ),+ deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),)def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:@@ -351,7 +352,7 @@ class GCSFilesStore:blob = self.bucket.blob(blob_path)blob.cache_control = self.CACHE_CONTROLblob.metadata = {k: str(v) for k, v in (meta or {}).items()}- return threads.deferToThread(+ return deferToThread(blob.upload_from_string,data=buf.getvalue(),content_type=self._get_content_type(headers),@@ -388,7 +389,7 @@ class FTPFilesStore:headers: Optional[Dict[str, str]] = None,) -> Deferred[Any]:path = f"{self.basedir}/{path}"- return threads.deferToThread(+ return deferToThread(ftp_store_file,path=path,file=buf,@@ -418,7 +419,7 @@ class FTPFilesStore:except Exception:return {}- return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))+ return cast("Deferred[StatInfo]", deferToThread(_stat_file, path))class FilesPipeline(MediaPipeline):@@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline):}path = self.file_path(request, info=info, item=item)- # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type- dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]+ # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type+ dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)dfd2.addErrback(lambda _: None)dfd2.addErrback(diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 27a57b17c..166f81314 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -11,7 +11,6 @@ import hashlibimport warningsfrom contextlib import suppressfrom io import BytesIO-from os import PathLikefrom typing import (TYPE_CHECKING,Any,@@ -28,7 +27,6 @@ from typing import (from itemadapter import ItemAdapter-from scrapy import Spiderfrom scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK@@ -40,15 +38,20 @@ from scrapy.pipelines.files import (S3FilesStore,_md5sum,)-from scrapy.pipelines.media import FileInfoOrError, MediaPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.python import get_func_args, to_bytesif TYPE_CHECKING:- # typing.Self requires Python 3.11+ from os import PathLike+from PIL import Image++ # typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.pipelines.media import FileInfoOrError, MediaPipeline+class NoimagesDrop(DropItem):"""Product with no images exception"""diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 09e95cf5d..ea36a9e8a 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -25,21 +25,23 @@ from typing import (from twisted.internet.defer import Deferred, DeferredListfrom twisted.python.failure import Failure-from scrapy import Spider-from scrapy.crawler import Crawler-from scrapy.http import Responsefrom scrapy.http.request import NO_CALLBACK, Requestfrom scrapy.settings import Settingsfrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.defer import defer_result, mustbe_deferredfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iter-from scrapy.utils.request import RequestFingerprinterif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.utils.request import RequestFingerprinter++_T = TypeVar("_T")diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex ba7cd716b..d5eedf9b1 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Typefrom twisted.internet import deferfrom twisted.internet.base import ReactorBase, ThreadedResolver-from twisted.internet.defer import Deferredfrom twisted.internet.interfaces import (IAddress,IHostnameResolver,@@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, providerfrom scrapy.utils.datatypes import LocalCacheif TYPE_CHECKING:+ from twisted.internet.defer import Deferred+# typing.Self requires Python 3.11from typing_extensions import Selfdiff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex a33f73306..0d282dc37 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethodfrom typing import TYPE_CHECKING, Optional, Unionfrom warnings import warn-from scrapy import Spiderfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import to_unicode@@ -14,8 +13,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spiderfrom scrapy.crawler import Crawler+logger = logging.getLogger(__name__)diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex ea1db03f1..6703c569f 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -4,7 +4,6 @@ import copyimport jsonfrom importlib import import_modulefrom pprint import pformat-from types import ModuleTypefrom typing import (TYPE_CHECKING,Any,@@ -27,6 +26,8 @@ from scrapy.settings import default_settings_SettingsKeyT = Union[bool, float, int, str, None]if TYPE_CHECKING:+ from types import ModuleType+# https://github.com/python/typing/issues/445#issuecomment-1131458824from _typeshed import SupportsItemsdiff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.pyindex f6df191d8..3d37b8235 100644--- a/scrapy/signalmanager.py+++ b/scrapy/signalmanager.py@@ -1,10 +1,14 @@-from typing import Any, List, Tuple+from __future__ import annotations++from typing import TYPE_CHECKING, Any, List, Tuplefrom pydispatch import dispatcher-from twisted.internet.defer import Deferredfrom scrapy.utils import signal as _signal+if TYPE_CHECKING:+ from twisted.internet.defer import Deferred+class SignalManager:def __init__(self, sender: Any = dispatcher.Anonymous):diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex d855c962c..b8fe65668 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -3,21 +3,23 @@ from __future__ import annotationsimport tracebackimport warningsfrom collections import defaultdict-from types import ModuleTypefrom typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Typefrom zope.interface import implementer-from scrapy import Request, Spiderfrom scrapy.interfaces import ISpiderLoader-from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import walk_modulesfrom scrapy.utils.spider import iter_spider_classesif TYPE_CHECKING:+ from types import ModuleType+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request, Spider+ from scrapy.settings import BaseSettings+@implementer(ISpiderLoader)class SpiderLoader:diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 1e96654e2..c5b7f0749 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -9,15 +9,17 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, AsyncIterable, Iterable-from scrapy import Spider-from scrapy.crawler import Crawlerfrom scrapy.http import Request, Response-from scrapy.statscollectors import StatsCollectorif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = logging.getLogger(__name__)diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 35c869a75..ea1686c25 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -9,16 +9,18 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, Iterable, List, Optional-from scrapy import Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest-from scrapy.http import Response-from scrapy.settings import BaseSettingsif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.crawler import Crawler+ from scrapy.http import Response+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 50c93ac9f..379c5d0a3 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -12,10 +12,8 @@ import warningsfrom typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Setfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Response-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.httpobj import urlparse_cachedwarnings.warn(@@ -28,6 +26,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector++logger = logging.getLogger(__name__)diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 8af0bdf5b..d35cf8f71 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -23,10 +23,8 @@ from urllib.parse import urlparsefrom w3lib.url import safe_url_stringfrom scrapy import Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Response-from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_unicodefrom scrapy.utils.url import strip_url@@ -35,6 +33,10 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings++LOCAL_SCHEMES: Tuple[str, ...] = ("about","blob",diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex e2aa554a7..34df54ca7 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -9,15 +9,17 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, AsyncIterable, Iterable-from scrapy import Spiderfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Response-from scrapy.settings import BaseSettingsif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Spider+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.pyindex a0898a0cf..ce0f1bbaa 100644--- a/scrapy/spiders/init.py+++ b/scrapy/spiders/init.py@@ -1,10 +1,14 @@-from typing import Any, Iterable, Optional, cast+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Iterable, Optional, castfrom scrapy import Request-from scrapy.http import Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.spider import iterate_spider_output+if TYPE_CHECKING:+ from scrapy.http import Response+class InitSpider(Spider):"""Base Spider with initialization facilities"""diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 6f80ee388..d3e7896c5 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -6,20 +6,22 @@ from __future__ import annotationsimport marshalimport pickle # nosec-from os import PathLikefrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Callable, Optional, Type, Unionfrom queuelib import queue-from scrapy import Request-from scrapy.crawler import Crawlerfrom scrapy.utils.request import request_from_dictif TYPE_CHECKING:+ from os import PathLike+# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy import Request+ from scrapy.crawler import Crawler+def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.pyindex ab571a3ab..88e72f366 100644--- a/scrapy/statscollectors.py+++ b/scrapy/statscollectors.py@@ -2,15 +2,17 @@Scrapy extension for collecting scraping stats"""+from __future__ import annotations+import loggingimport pprintfrom typing import TYPE_CHECKING, Any, Dict, Optional-from scrapy import Spider-if TYPE_CHECKING:+ from scrapy import Spiderfrom scrapy.crawler import Crawler+logger = logging.getLogger(__name__)@@ -18,7 +20,7 @@ StatsT = Dict[str, Any]class StatsCollector:- def __init__(self, crawler: "Crawler"):+ def __init__(self, crawler: Crawler):self._dump: bool = crawler.settings.getbool("STATS_DUMP")self._stats: StatsT = {}@@ -67,7 +69,7 @@ class StatsCollector:class MemoryStatsCollector(StatsCollector):- def __init__(self, crawler: "Crawler"):+ def __init__(self, crawler: Crawler):super().__init__(crawler)self.spider_stats: Dict[str, StatsT] = {}diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex 7e82dd519..2240f0b58 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -4,8 +4,8 @@ import warningsfrom functools import wrapsfrom typing import TYPE_CHECKING, Any, Callable, TypeVar-from twisted.internet import defer, threads-from twisted.internet.defer import Deferred+from twisted.internet.defer import Deferred, maybeDeferred+from twisted.internet.threads import deferToThreadfrom scrapy.exceptions import ScrapyDeprecationWarning@@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:@wraps(func)def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:- return defer.maybeDeferred(func, *a, **kw)+ return maybeDeferred(func, *a, **kw)return wrapped@@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:@wraps(func)def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:- return threads.deferToThread(func, *a, **kw)+ return deferToThread(func, *a, **kw)return wrappeddiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 877eb4388..1d578e8a3 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -34,12 +34,13 @@ from twisted.internet import deferfrom twisted.internet.defer import Deferred, DeferredList, ensureDeferredfrom twisted.internet.task import Cooperatorfrom twisted.python import failure-from twisted.python.failure import Failurefrom scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningfrom scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installedif TYPE_CHECKING:+ from twisted.python.failure import Failure+# typing.Concatenate and typing.ParamSpec require Python 3.10from typing_extensions import Concatenate, ParamSpecdiff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex fdcf484d4..770ee0b1b 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -4,9 +4,10 @@ from __future__ import annotations# used in global tests codefrom time import time # noqa: F401-from typing import Any, List, Tuple+from typing import TYPE_CHECKING, Any, List, Tuple-from scrapy.core.engine import ExecutionEngine+if TYPE_CHECKING:+ from scrapy.core.engine import ExecutionEnginedef get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex 2e487d88b..85324361c 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -1,11 +1,15 @@+from __future__ import annotations+import structfrom gzip import GzipFilefrom io import BytesIO--from scrapy.http import Response+from typing import TYPE_CHECKINGfrom ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded+if TYPE_CHECKING:+ from scrapy.http import Response+def gunzip(data: bytes, *, max_size: int = 0) -> bytes:"""Gunzip the given data and return as much data as possible.diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.pyindex d502e8910..3cf9585ec 100644--- a/scrapy/utils/httpobj.py+++ b/scrapy/utils/httpobj.py@@ -1,12 +1,16 @@"""Helper functions for scrapy.http objects (Request, Response)"""-from typing import Union+from __future__ import annotations++from typing import TYPE_CHECKING, Unionfrom urllib.parse import ParseResult, urlparsefrom weakref import WeakKeyDictionary-from scrapy.http import Request, Response+if TYPE_CHECKING:+ from scrapy.http import Request, Response+-_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = (+_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = (WeakKeyDictionary())diff --git a/scrapy/utils/job.py b/scrapy/utils/job.pyindex e230e4235..488c7994b 100644--- a/scrapy/utils/job.py+++ b/scrapy/utils/job.py@@ -1,7 +1,10 @@+from __future__ import annotations+from pathlib import Path-from typing import Optional+from typing import TYPE_CHECKING, Optional-from scrapy.settings import BaseSettings+if TYPE_CHECKING:+ from scrapy.settings import BaseSettingsdef job_dir(settings: BaseSettings) -> Optional[str]:diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex cbfd170ed..439b065a9 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -21,12 +21,13 @@ from twisted.python import log as twisted_logfrom twisted.python.failure import Failureimport scrapy-from scrapy.logformatter import LogFormatterResultfrom scrapy.settings import Settings, _SettingsKeyTfrom scrapy.utils.versions import scrapy_components_versionsif TYPE_CHECKING:from scrapy.crawler import Crawler+ from scrapy.logformatter import LogFormatterResult+logger = logging.getLogger(__name__)diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 3d11c1035..3c787e50f 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -13,7 +13,6 @@ from contextlib import contextmanagerfrom functools import partialfrom importlib import import_modulefrom pkgutil import iter_modules-from types import ModuleTypefrom typing import (IO,TYPE_CHECKING,@@ -35,10 +34,13 @@ from scrapy.item import Itemfrom scrapy.utils.datatypes import LocalWeakReferencedCacheif TYPE_CHECKING:+ from types import ModuleType+from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings+_ITERABLE_SINGLE_VALUES = dict, Item, str, bytesT = TypeVar("T")diff --git a/scrapy/utils/project.py b/scrapy/utils/project.pyindex de3c8eaf9..efb6af299 100644--- a/scrapy/utils/project.py+++ b/scrapy/utils/project.py@@ -1,7 +1,8 @@+from __future__ import annotations+import osimport warningsfrom importlib import import_module-from os import PathLikefrom pathlib import Pathfrom typing import Union@@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str:return str(d)-def data_path(path: Union[str, PathLike], createdir: bool = False) -> str:+def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str:"""Return the given path joined with the .scrapy data directory.If given an absolute path, return it unmodified.diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 5af6d22eb..a627db601 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport sys-from asyncio import AbstractEventLoop, AbstractEventLoopPolicyfrom contextlib import suppressfrom typing import (TYPE_CHECKING,@@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warnfrom twisted.internet import asyncioreactor, errorfrom twisted.internet.base import DelayedCall-from twisted.internet.protocol import ServerFactory-from twisted.internet.tcp import Portfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import load_objectif TYPE_CHECKING:+ from asyncio import AbstractEventLoop, AbstractEventLoopPolicy++ from twisted.internet.protocol import ServerFactory+ from twisted.internet.tcp import Port+# typing.ParamSpec requires Python 3.10from typing_extensions import ParamSpecdiff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex b05135c04..ce754fad3 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport inspectimport logging-from types import CoroutineType, ModuleTypefrom typing import (TYPE_CHECKING,Any,@@ -16,16 +15,19 @@ from typing import (overload,)-from twisted.internet.defer import Deferred--from scrapy import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.misc import arg_to_iterif TYPE_CHECKING:+ from types import CoroutineType, ModuleType++ from twisted.internet.defer import Deferred++ from scrapy import Requestfrom scrapy.spiderloader import SpiderLoader+logger = logging.getLogger(__name__)_T = TypeVar("_T")diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.pyindex d520ef809..95611ebd9 100644--- a/scrapy/utils/ssl.py+++ b/scrapy/utils/ssl.py@@ -1,12 +1,16 @@-from typing import Any, Optional+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Optionalimport OpenSSL._util as pyOpenSSLutilimport OpenSSL.SSLimport OpenSSL.version-from OpenSSL.crypto import X509Namefrom scrapy.utils.python import to_unicode+if TYPE_CHECKING:+ from OpenSSL.crypto import X509Name+def ffi_buf_to_string(buf: Any) -> str:return to_unicode(pyOpenSSLutil.ffi.string(buf))diff --git a/scrapy/utils/template.py b/scrapy/utils/template.pyindex 6b22f3bfa..08f3f2dc9 100644--- a/scrapy/utils/template.py+++ b/scrapy/utils/template.py@@ -1,10 +1,14 @@"""Helper functions for working with templates"""+from __future__ import annotations+import reimport string-from os import PathLikefrom pathlib import Path-from typing import Any, Union+from typing import TYPE_CHECKING, Any, Union++if TYPE_CHECKING:+ from os import PathLikedef render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None:diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 268d8d4be..fe2bfa042 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -2,21 +2,36 @@This module contains some assorted functions used in tests"""+from __future__ import annotations+import asyncioimport osfrom importlib import import_modulefrom pathlib import Pathfrom posixpath import split-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar+from typing import (+ TYPE_CHECKING,+ Any,+ Awaitable,+ Dict,+ List,+ Optional,+ Tuple,+ Type,+ TypeVar,+)from unittest import TestCase, mock-from twisted.internet.defer import Deferredfrom twisted.trial.unittest import SkipTestfrom scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.utils.boto import is_botocore_available+if TYPE_CHECKING:+ from twisted.internet.defer import Deferred++_T = TypeVar("_T")diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex 3bdffcaa7..8882bfc5f 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -2,12 +2,14 @@ from __future__ import annotationsimport osimport sys-from typing import Iterable, List, Optional, Tuple, cast+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, castfrom twisted.internet.defer import Deferredfrom twisted.internet.error import ProcessTerminatedfrom twisted.internet.protocol import ProcessProtocol-from twisted.python.failure import Failure++if TYPE_CHECKING:+ from twisted.python.failure import Failureclass ProcessTest:diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 233f6b934..6ec46aa3d 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -1,3 +1,5 @@+from __future__ import annotations+import argparseimport jsonimport os@@ -7,12 +9,11 @@ from pathlib import Pathfrom shutil import rmtreefrom subprocess import PIPE, Popenfrom tempfile import mkdtemp-from typing import Dict+from typing import TYPE_CHECKING, Dictfrom urllib.parse import urlencodefrom OpenSSL import SSLfrom twisted.internet import defer, reactor, ssl-from twisted.internet.protocol import ServerFactoryfrom twisted.internet.task import deferLaterfrom twisted.names import dns, errorfrom twisted.names.server import DNSServerFactory@@ -23,6 +24,9 @@ from twisted.web.util import redirectTofrom scrapy.utils.python import to_bytes, to_unicode+if TYPE_CHECKING:+ from twisted.internet.protocol import ServerFactory+def getarg(request, name, default=None, type=None):if name in request.args:diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 253987e15..ea3ed3b05 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1,3 +1,5 @@+from __future__ import annotations+import bz2import csvimport gzip@@ -14,10 +16,9 @@ from collections import defaultdictfrom contextlib import ExitStackfrom io import BytesIOfrom logging import getLogger-from os import PathLikefrom pathlib import Pathfrom string import ascii_letters, digits-from typing import Union+from typing import TYPE_CHECKING, Unionfrom unittest import mockfrom urllib.parse import quote, urljoinfrom urllib.request import pathname2url@@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_nofrom tests.mockserver import MockFTPServer, MockServerfrom tests.spiders import ItemSpider+if TYPE_CHECKING:+ from os import PathLike+def path_to_url(path):return urljoin("file:", pathname2url(str(path)))diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 995c02a1a..7ea3fe8c9 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -1,3 +1,5 @@+from __future__ import annotations+import jsonimport randomimport re@@ -6,7 +8,7 @@ import stringfrom ipaddress import IPv4Addressfrom pathlib import Pathfrom tempfile import mkdtemp-from typing import Dict+from typing import TYPE_CHECKING, Dictfrom unittest import mock, skipIffrom urllib.parse import urlencode@@ -20,7 +22,6 @@ from twisted.internet.defer import (from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpointfrom twisted.internet.error import TimeoutErrorfrom twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS-from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom twisted.web.client import URI, ResponseFailedfrom twisted.web.http import H2_ENABLED@@ -33,6 +34,9 @@ from scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom tests.mockserver import LeafResource, Status, ssl_context_factory+if TYPE_CHECKING:+ from twisted.python.failure import Failure+def generate_random_string(size):return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))