Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add flake8-type-checking. (#6413)

Andrey Rakhmatullind08f559600f0
diff --git a/.flake8 b/.flake8
index be9d83eaf..c4814f13a 100644
--- a/.flake8
+++ b/.flake8
@@ -1,6 +1,7 @@
[flake8]
max-line-length = 119
+extend-select = TC, TC1
ignore =
# black disagrees with flake8 about these
E203, E501, E701, E704, W503
@@ -58,6 +59,9 @@ ignore =
D402
# First word of the first line should be properly capitalized
D403
+
+ # Annotation in typing.cast() should be a string literal
+ TC006
exclude =
docs/conf.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f70effc5d..38526d720 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,6 +14,7 @@ repos:
- flake8-debugger
- flake8-docstrings
- flake8-string-format
+ - flake8-type-checking
- repo: https://github.com/psf/black.git
rev: 24.2.0
hooks:
diff --git a/scrapy/addons.py b/scrapy/addons.py
index 65d7a0310..f9ec58cea 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -1,13 +1,16 @@
+from __future__ import annotations
+
import logging
from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
-from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
+ from scrapy.settings import Settings
+
logger = logging.getLogger(__name__)
@@ -15,8 +18,8 @@ logger = logging.getLogger(__name__)
class AddonManager:
"""This class facilitates loading and storing :ref:`topics-addons`."""
- def __init__(self, crawler: "Crawler") -> None:
- self.crawler: "Crawler" = crawler
+ def __init__(self, crawler: Crawler) -> None:
+ self.crawler: Crawler = crawler
self.addons: List[Any] = []
def load_settings(self, settings: Settings) -> None:
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index da0e51386..e010b159a 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -12,7 +12,6 @@ import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
-from scrapy.settings import BaseSettings, Settings
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
@@ -21,6 +20,8 @@ if TYPE_CHECKING:
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
+ from scrapy.settings import BaseSettings, Settings
+
_P = ParamSpec("_P")
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9fe803d3c..0322390e5 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -2,18 +2,22 @@
Base class for Scrapy commands
"""
+from __future__ import annotations
+
import argparse
import builtins
import os
from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional
from twisted.python import failure
-from scrapy.crawler import Crawler, CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler, CrawlerProcess
+
class ScrapyCommand:
requires_project: bool = False
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 0c4ebcd23..f91fec57e 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,16 +1,20 @@
+from __future__ import annotations
+
import argparse
import subprocess # nosec
import sys
import time
-from typing import Any, Iterable, List
+from typing import TYPE_CHECKING, Any, Iterable, List
from urllib.parse import urlencode
import scrapy
-from scrapy import Request
from scrapy.commands import ScrapyCommand
from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
+if TYPE_CHECKING:
+ from scrapy import Request
+
class Command(ScrapyCommand):
default_settings = {
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 6e023af81..fe1864372 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,11 +1,15 @@
-import argparse
-from typing import List, cast
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List, cast
from twisted.python.failure import Failure
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
+if TYPE_CHECKING:
+ import argparse
+
class Command(BaseRunSpiderCommand):
requires_project = True
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 1acf2d26f..0bdc429da 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,6 +1,7 @@
+from __future__ import annotations
+
import sys
-from argparse import ArgumentParser, Namespace
-from typing import Dict, List, Type
+from typing import TYPE_CHECKING, Dict, List, Type
from w3lib.url import is_url
@@ -11,6 +12,9 @@ from scrapy.http import Request, Response
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
+if TYPE_CHECKING:
+ from argparse import ArgumentParser, Namespace
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index dcc51a694..10330c92a 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,8 +1,12 @@
-import argparse
-from typing import List
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
from scrapy.commands import ScrapyCommand
+if TYPE_CHECKING:
+ import argparse
+
class Command(ScrapyCommand):
requires_project = True
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 3320a1ee4..e6c5e2a47 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -6,6 +6,7 @@ import inspect
import json
import logging
from typing import (
+ TYPE_CHECKING,
Any,
AsyncGenerator,
Callable,
@@ -22,13 +23,11 @@ from typing import (
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.python.failure import Failure
from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
-from scrapy.spiders import Spider
from scrapy.utils import display
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
@@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
+ from scrapy.spiders import Spider
+
+
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 77850e7b5..87acf9a01 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,17 +1,21 @@
+from __future__ import annotations
+
import argparse
import sys
from importlib import import_module
-from os import PathLike
from pathlib import Path
-from types import ModuleType
-from typing import List, Union
+from typing import TYPE_CHECKING, List, Union
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
+if TYPE_CHECKING:
+ from os import PathLike
+ from types import ModuleType
+
-def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
+def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 668c95a7b..f03cf997a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -4,9 +4,10 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
-from argparse import ArgumentParser, Namespace
+from __future__ import annotations
+
from threading import Thread
-from typing import Any, Dict, List, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Type
from scrapy import Spider
from scrapy.commands import ScrapyCommand
@@ -15,6 +16,9 @@ from scrapy.shell import Shell
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
+if TYPE_CHECKING:
+ from argparse import ArgumentParser, Namespace
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 27bc2fcba..440e0dc44 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,9 +1,12 @@
+from __future__ import annotations
+
import re
import sys
from functools import wraps
from inspect import getmembers
from types import CoroutineType
from typing import (
+ TYPE_CHECKING,
Any,
AsyncGenerator,
Callable,
@@ -16,13 +19,15 @@ from typing import (
)
from unittest import TestCase, TestResult
-from twisted.python.failure import Failure
-
-from scrapy import Spider
from scrapy.http import Request, Response
from scrapy.utils.python import get_spec
from scrapy.utils.spider import iterate_spider_output
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
+ from scrapy import Spider
+
class Contract:
"""Abstract class for contracts"""
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 41f729ed9..6786d7acf 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -25,15 +25,16 @@ from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.http import Response
from scrapy.resolver import dnscache
-from scrapy.settings import BaseSettings
from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.settings import BaseSettings
+
_T = TypeVar("_T")
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 9f6edb630..2b388a9f5 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import (
ScrapyClientTLSOptions,
openssl_methods,
)
-from scrapy.crawler import Crawler
-from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
@@ -31,6 +29,9 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
@implementer(IPolicyForHTTPS)
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index ebc4898b5..70d356b83 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -17,17 +17,19 @@ from typing import (
)
from twisted.internet import defer
-from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
-from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index a7ae56a85..bf6879521 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -1,12 +1,16 @@
-from typing import Any, Dict
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Dict
from w3lib.url import parse_data_uri
-from scrapy import Request, Spider
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
+if TYPE_CHECKING:
+ from scrapy import Request, Spider
+
class DataURIDownloadHandler:
lazy = False
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 17dd7483b..d55c516f0 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,12 +1,17 @@
+from __future__ import annotations
+
from pathlib import Path
+from typing import TYPE_CHECKING
from w3lib.url import file_uri_to_path
-from scrapy import Request, Spider
-from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
+if TYPE_CHECKING:
+ from scrapy import Request, Spider
+ from scrapy.http import Response
+
class FileDownloadHandler:
lazy = False
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 724717ffd..69c2d88e1 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -35,23 +35,25 @@ from io import BytesIO
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional
from urllib.parse import unquote
-from twisted.internet.defer import Deferred
from twisted.internet.protocol import ClientCreator, Protocol
from twisted.protocols.ftp import CommandFailed, FTPClient
-from twisted.python.failure import Failure
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+ from twisted.python.failure import Failure
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
class ReceivedDataProtocol(Protocol):
def __init__(self, filename: Optional[str] = None):
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 3c4e48abb..98f62efcf 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -5,21 +5,21 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Type
-from twisted.internet.defer import Deferred
-
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.settings import BaseSettings
class HTTP10DownloadHandler:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e2ad8f59a..c06d90f01 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar
from urllib.parse import urldefrag, urlunparse
from twisted.internet import ssl
-from twisted.internet.base import ReactorBase
from twisted.internet.defer import CancelledError, Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
-from twisted.internet.interfaces import IConsumer
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web.client import URI, Agent, HTTPConnectionPool
@@ -30,17 +28,22 @@ from zope.interface import implementer
from scrapy import Request, Spider, signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
-from scrapy.crawler import Crawler
from scrapy.exceptions import StopDownload
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
+ from twisted.internet.base import ReactorBase
+ from twisted.internet.interfaces import IConsumer
+
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import NotRequired, Self
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 2ac4eca86..4722c612d 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -4,25 +4,27 @@ from time import time
from typing import TYPE_CHECKING, Optional
from urllib.parse import urldefrag
-from twisted.internet.base import DelayedCall
-from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
-from twisted.web.iweb import IPolicyForHTTPS
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
-from scrapy.crawler import Crawler
-from scrapy.http import Request, Response
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
+ from twisted.internet.base import DelayedCall
+ from twisted.internet.defer import Deferred
+ from twisted.web.iweb import IPolicyForHTTPS
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Request, Response
+ from scrapy.settings import Settings
+ from scrapy.spiders import Spider
+
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler: Crawler):
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 0ad340721..edf370193 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,22 +2,23 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Type
-from twisted.internet.defer import Deferred
-
-from scrapy import Request, Spider
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.settings import BaseSettings
+
class S3DownloadHandler:
def __init__(
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 2d8af114f..0bdb756c8 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rst
from __future__ import annotations
-from typing import Any, Callable, Generator, List, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast
from twisted.internet.defer import Deferred, inlineCallbacks
-from twisted.python.failure import Failure
-from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
-from scrapy.settings import BaseSettings
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
+ from scrapy import Spider
+ from scrapy.settings import BaseSettings
+
class DownloaderMiddlewareManager(MiddlewareManager):
component_name = "downloader middleware"
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 08a1d7c71..99502f0d2 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,18 +1,22 @@
+from __future__ import annotations
+
import re
from time import time
-from typing import Optional, Tuple
+from typing import TYPE_CHECKING, Optional, Tuple
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from twisted.web.http import HTTPClient
-from scrapy import Request
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
+if TYPE_CHECKING:
+ from scrapy import Request
+
def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]:
# Assume parsed is urlparse-d from Request.url,
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4ffec78b9..5318cbd64 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
-from scrapy.settings import BaseSettings, Settings
+from scrapy.settings import Settings
from scrapy.signalmanager import SignalManager
-from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name
@@ -46,6 +45,9 @@ if TYPE_CHECKING:
from scrapy.core.scheduler import BaseScheduler
from scrapy.core.scraper import _HandleOutputDeferred
from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+ from scrapy.spiders import Spider
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 999764a6e..d291a5b8a 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -1,10 +1,10 @@
+from __future__ import annotations
+
from collections import deque
-from typing import Deque, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple
from twisted.internet import defer
-from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
-from twisted.internet.endpoints import HostnameEndpoint
from twisted.python.failure import Failure
from twisted.web.client import (
URI,
@@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.http.request import Request
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
+
+if TYPE_CHECKING:
+ from twisted.internet.base import ReactorBase
+ from twisted.internet.endpoints import HostnameEndpoint
+
+ from scrapy.http.request import Request
+ from scrapy.settings import Settings
+ from scrapy.spiders import Spider
+
ConnectionKeyT = Tuple[bytes, bytes, int]
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index f2f1cb0b8..a6809102b 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,9 +1,10 @@
+from __future__ import annotations
+
import ipaddress
import itertools
import logging
from collections import deque
-from ipaddress import IPv4Address, IPv6Address
-from typing import Any, Deque, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union
from h2.config import H2Configuration
from h2.connection import H2Connection
@@ -20,7 +21,6 @@ from h2.events import (
WindowUpdated,
)
from h2.exceptions import FrameTooLargeError, H2Error
-from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import (
IAddress,
@@ -30,14 +30,21 @@ from twisted.internet.interfaces import (
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
-from twisted.python.failure import Failure
-from twisted.web.client import URI
from zope.interface import implementer
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.http import Request
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
+
+if TYPE_CHECKING:
+ from ipaddress import IPv4Address, IPv6Address
+
+ from twisted.internet.defer import Deferred
+ from twisted.python.failure import Failure
+ from twisted.web.client import URI
+
+ from scrapy.settings import Settings
+ from scrapy.spiders import Spider
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 7c70e86db..a02fbb328 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import logging
from enum import Enum
from io import BytesIO
@@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
-from hpack import HeaderTuple
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
-from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
+ from hpack import HeaderTuple
+
from scrapy.core.http2.protocol import H2ClientProtocol
+ from scrapy.http import Request
logger = logging.getLogger(__name__)
@@ -87,7 +90,7 @@ class Stream:
self,
stream_id: int,
request: Request,
- protocol: "H2ClientProtocol",
+ protocol: H2ClientProtocol,
download_maxsize: int = 0,
download_warnsize: int = 0,
) -> None:
@@ -99,7 +102,7 @@ class Stream:
"""
self.stream_id: int = stream_id
self._request: Request = request
- self._protocol: "H2ClientProtocol" = protocol
+ self._protocol: H2ClientProtocol = protocol
self._download_maxsize = self._request.meta.get(
"download_maxsize", download_maxsize
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 1e586c53a..d4286c874 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -6,14 +6,10 @@ from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any, List, Optional, Type, cast
-from twisted.internet.defer import Deferred
-
-from scrapy.crawler import Crawler
-from scrapy.dupefilters import BaseDupeFilter
-from scrapy.http.request import Request
-from scrapy.pqueues import ScrapyPriorityQueue
-from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from twisted.internet.defer import Deferred # noqa: TC002
+
+from scrapy.spiders import Spider # noqa: TC001
from scrapy.utils.job import job_dir
from scrapy.utils.misc import build_from_crawler, load_object
@@ -24,6 +20,12 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.dupefilters import BaseDupeFilter
+ from scrapy.http.request import Request
+ from scrapy.pqueues import ScrapyPriorityQueue
+ from scrapy.statscollectors import StatsCollector
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index e792f8ca7..37a666605 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -10,6 +10,7 @@ import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import (
+ TYPE_CHECKING,
Any,
AsyncIterable,
Callable,
@@ -30,7 +31,6 @@ from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
-from scrapy.settings import BaseSettings
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
@@ -41,6 +41,10 @@ from scrapy.utils.defer import (
)
from scrapy.utils.python import MutableAsyncChain, MutableChain
+if TYPE_CHECKING:
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 0e757e4be..5fc7f31a3 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union
from w3lib import html
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse, Response
-from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 73c2c57fe..23140d263 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -2,24 +2,26 @@ from __future__ import annotations
import logging
from collections import defaultdict
-from http.cookiejar import Cookie
from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
from tldextract import TLDExtract
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
-from scrapy.http.request import VerboseCookie
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
+ from http.cookiejar import Cookie
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http.request import VerboseCookie
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index 58fd415b9..49b9fdc05 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -8,15 +8,16 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, Tuple, Union
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
class DefaultHeadersMiddleware:
def __init__(self, headers: Iterable[Tuple[str, str]]):
diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
index fd7c03a38..ee7a24825 100644
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@@ -9,13 +9,14 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
class DownloadTimeoutMiddleware:
def __init__(self, timeout: float = 180):
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 63490a37a..39165e155 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union
from w3lib.http import basic_auth_header
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 971473403..8377a3c1d 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -16,19 +16,20 @@ from twisted.internet.error import (
from twisted.web.client import ResponseFailed
from scrapy import signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http.request import Request
-from scrapy.http.response import Response
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http.request import Request
+ from scrapy.http.response import Response
+ from scrapy.settings import Settings
+ from scrapy.spiders import Spider
+ from scrapy.statscollectors import StatsCollector
+
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index f3647e05f..6b0a56f7f 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -6,11 +6,9 @@ from logging import getLogger
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
-from scrapy.statscollectors import StatsCollector
from scrapy.utils._compression import (
_DecompressionMaxSizeExceeded,
_inflate,
@@ -24,6 +22,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = getLogger(__name__)
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 5b56ad449..a7af83f7d 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined]
proxy_bypass,
)
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
@@ -20,6 +17,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
class HttpProxyMiddleware:
def __init__(self, auth_encoding: Optional[str] = "latin-1"):
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index bd8dbe329..6f67e3975 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -6,15 +6,17 @@ import warnings
from typing import TYPE_CHECKING, Set
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 371e2fd3b..53081237c 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -6,11 +6,8 @@ from urllib.parse import urljoin
from w3lib.url import safe_url_string
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Response
-from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
@@ -18,6 +15,11 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 0637f09d4..8d7b7293c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -16,12 +16,8 @@ import warnings
from logging import Logger, getLogger
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Response
-from scrapy.http.request import Request
from scrapy.settings import BaseSettings, Settings
-from scrapy.spiders import Spider
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@@ -30,6 +26,12 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.http.request import Request
+ from scrapy.spiders import Spider
+
+
retry_logger = getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a0ecb7bf..70393576b 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -10,22 +10,24 @@ import logging
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.python.failure import Failure
-from scrapy import Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
-from scrapy.robotstxt import RobotParser
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.crawler import Crawler
+ from scrapy.robotstxt import RobotParser
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 444702757..0faae7b5a 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union
from twisted.web import http
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
@@ -16,6 +12,11 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.statscollectors import StatsCollector
+
def get_header_size(
headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]
diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index 92f1ec897..109f1a4d9 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -5,13 +5,14 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index dd2420e98..ffaf783a7 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -4,11 +4,6 @@ import logging
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Set
-from twisted.internet.defer import Deferred
-
-from scrapy.http.request import Request
-from scrapy.settings import BaseSettings
-from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.request import (
RequestFingerprinter,
@@ -17,10 +12,15 @@ from scrapy.utils.request import (
)
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.http.request import Request
+ from scrapy.settings import BaseSettings
+ from scrapy.spiders import Spider
class BaseDupeFilter:
diff --git a/scrapy/extension.py b/scrapy/extension.py
index 8221b675e..8c81ab356 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -4,12 +4,16 @@ The Extension Manager
See documentation in docs/topics/extensions.rst
"""
-from typing import Any, List
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List
from scrapy.middleware import MiddlewareManager
-from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
+if TYPE_CHECKING:
+ from scrapy.settings import Settings
+
class ExtensionManager(MiddlewareManager):
component_name = "extension"
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 812b3553c..4627e7f98 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -10,17 +10,19 @@ import logging
from collections import defaultdict
from typing import TYPE_CHECKING, Any, DefaultDict, Dict
-from twisted.python.failure import Failure
-
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index f3ac19623..6ef2d0382 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -8,13 +8,14 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Optional
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
class CoreStats:
def __init__(self, stats: StatsCollector):
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index b360ce48d..c54871e02 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -12,17 +12,20 @@ import sys
import threading
import traceback
from pdb import Pdb
-from types import FrameType
from typing import TYPE_CHECKING, Optional
-from scrapy.crawler import Crawler
from scrapy.utils.engine import format_engine_status
from scrapy.utils.trackref import format_live_refs
if TYPE_CHECKING:
+ from types import FrameType
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 941bd4b26..43c2d2815 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -31,18 +31,15 @@ from typing import (
)
from urllib.parse import unquote, urlparse
-from twisted.internet import threads
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
-from twisted.python.failure import Failure
+from twisted.internet.threads import deferToThread
from w3lib.url import file_uri_to_path
from zope.interface import Interface, implementer
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.exporters import BaseItemExporter
from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.settings import BaseSettings, Settings
+from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
@@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from _typeshed import OpenBinaryMode
+ from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
-logger = logging.getLogger(__name__)
+ from scrapy.crawler import Crawler
+ from scrapy.exporters import BaseItemExporter
+ from scrapy.settings import BaseSettings
try:
import boto3 # noqa: F401
@@ -67,6 +67,9 @@ try:
except ImportError:
IS_BOTO3_AVAILABLE = False
+
+logger = logging.getLogger(__name__)
+
UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
@@ -160,7 +163,7 @@ class BlockingFeedStorage:
return NamedTemporaryFile(prefix="feed-", dir=path)
def store(self, file: IO[bytes]) -> Optional[Deferred]:
- return threads.deferToThread(self._store_in_thread, file)
+ return deferToThread(self._store_in_thread, file)
def _store_in_thread(self, file: IO[bytes]) -> None:
raise NotImplementedError
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index b7219bf07..448d5f1ab 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import gzip
import logging
import os
@@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.http import Headers, Response
-from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
-from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
@@ -26,6 +25,10 @@ if TYPE_CHECKING:
# typing.Concatenate requires Python 3.10
from typing_extensions import Concatenate
+ from scrapy.http.request import Request
+ from scrapy.settings import BaseSettings
+ from scrapy.spiders import Spider
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 2388afa75..c4f43482d 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union
from twisted.internet import task
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index f304e1bf2..3cbbb64e5 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -10,15 +10,16 @@ import gc
from typing import TYPE_CHECKING
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.trackref import live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
class MemoryDebugger:
def __init__(self, stats: StatsCollector):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de06b24d..25f63ecc6 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List
from twisted.internet import task
from scrapy import signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.utils.engine import get_engine_status
@@ -25,6 +24,9 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 9567f948a..80c0a3b26 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from twisted.internet import task
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index c6eb20277..567efd7a1 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -5,7 +5,6 @@ from pathlib import Path
from typing import TYPE_CHECKING, Optional
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
@@ -13,6 +12,8 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+
class SpiderState:
"""Store and load spider state during a scraping job"""
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 20b8f910c..e43de6f5c 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,18 +8,19 @@ from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional
-from twisted.internet.defer import Deferred
-
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
-from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
class StatsMailer:
def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 00c69434c..c4e01b3d9 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -26,7 +26,6 @@ except (ImportError, SyntaxError):
TWISTED_CONCH_AVAILABLE = False
from scrapy import signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import defers
from scrapy.utils.engine import print_engine_status
@@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+
+ from scrapy.crawler import Crawler
+
+
logger = logging.getLogger(__name__)
# signal to update telnet variables
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 217e61a81..6ce9ce63a 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -4,15 +4,17 @@ import logging
from typing import TYPE_CHECKING, Optional, Tuple
from scrapy import Request, Spider, signals
-from scrapy.core.downloader import Slot
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.core.downloader import Slot
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 8af89c74f..cc88a9420 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -17,8 +17,6 @@ from typing import (
cast,
)
-from scrapy import Request
-from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@@ -26,6 +24,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request
+ from scrapy.http import Response
+
+
# Defined in the http.cookiejar module, but undocumented:
# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ea98ed795..a8c242e8b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -28,13 +28,14 @@ from lxml.html import TextareaElement # nosec
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
-from scrapy.http.response.text import TextResponse
from scrapy.utils.python import is_listlike, to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.http.response.text import TextResponse
+
FormdataVType = Union[str, Iterable[str]]
FormdataKVType = Tuple[str, FormdataVType]
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 684439097..ff3581abb 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
-from ipaddress import IPv4Address, IPv6Address
from typing import (
TYPE_CHECKING,
Any,
@@ -26,8 +25,6 @@ from typing import (
)
from urllib.parse import urljoin
-from twisted.internet.ssl import Certificate
-
from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
from scrapy.http.request import CookiesT, Request
@@ -35,6 +32,10 @@ from scrapy.link import Link
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
+ from ipaddress import IPv4Address, IPv6Address
+
+ from twisted.internet.ssl import Certificate
+
# typing.Self requires Python 3.11
from typing_extensions import Self
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index df4d90829..0635f744f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -35,15 +35,16 @@ from w3lib.encoding import (
)
from w3lib.html import strip_html5_whitespace
-from scrapy.http.request import CookiesT, Request
from scrapy.http.response import Response
from scrapy.link import Link
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
+ from scrapy.http.request import CookiesT, Request
from scrapy.selector import Selector, SelectorList
+
_NONE = object()
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 33a10cd6c..d27a132b3 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -2,10 +2,13 @@
Link extractor based on lxml.html
"""
+from __future__ import annotations
+
import logging
import operator
from functools import partial
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Iterable,
@@ -20,13 +23,10 @@ from typing import (
from urllib.parse import urljoin, urlparse
from lxml import etree # nosec
-from lxml.html import HtmlElement # nosec
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
-from scrapy import Selector
-from scrapy.http import TextResponse
from scrapy.link import Link
from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
+if TYPE_CHECKING:
+ from lxml.html import HtmlElement # nosec
+
+ from scrapy import Selector
+ from scrapy.http import TextResponse
+
+
logger = logging.getLogger(__name__)
# from lxml/src/lxml/html/__init__.py
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index db0b4820f..9644cc093 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -4,14 +4,18 @@ Item Loader
See documentation in docs/topics/loaders.rst
"""
-from typing import Any, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Optional
import itemloaders
-from scrapy.http import TextResponse
from scrapy.item import Item
from scrapy.selector import Selector
+if TYPE_CHECKING:
+ from scrapy.http import TextResponse
+
class ItemLoader(itemloaders.ItemLoader):
"""
@@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader):
selector: Optional[Selector] = None,
response: Optional[TextResponse] = None,
parent: Optional[itemloaders.ItemLoader] = None,
- **context: Any
+ **context: Any,
):
if selector is None and response is not None:
try:
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 42a03b560..601209fb0 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union
from twisted.python.failure import Failure
-from scrapy import Request, Spider
-from scrapy.http import Response
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from scrapy import Request, Spider # noqa: TC001
+from scrapy.http import Response # noqa: TC001
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
diff --git a/scrapy/mail.py b/scrapy/mail.py
index f4ce2800c..3ea20e831 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -30,20 +30,22 @@ from typing import (
from twisted import version as twisted_version
from twisted.internet import ssl
from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
from twisted.python.versions import Version
-from scrapy.settings import BaseSettings
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# imports twisted.internet.reactor
from twisted.mail.smtp import ESMTPSenderFactory
+ from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f60c726f9..ea5488ba1 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -17,19 +17,19 @@ from typing import (
cast,
)
-from twisted.internet.defer import Deferred
-
-from scrapy import Spider
from scrapy.exceptions import NotConfigured
-from scrapy.settings import Settings
from scrapy.utils.defer import process_chain, process_parallel
from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
from scrapy.crawler import Crawler
+ from scrapy.settings import Settings
logger = logging.getLogger(__name__)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 21d649e3c..480a5a58c 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rst
from __future__ import annotations
-from typing import Any, List
+from typing import TYPE_CHECKING, Any, List
-from twisted.internet.defer import Deferred
-
-from scrapy import Spider
from scrapy.middleware import MiddlewareManager
-from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_f_from_coro_f
+if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
+ from scrapy import Spider
+ from scrapy.settings import Settings
+
class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 85a8c77da..1a13aeaf2 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -16,7 +16,6 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
-from os import PathLike
from pathlib import Path
from typing import (
IO,
@@ -38,11 +37,9 @@ from typing import (
from urllib.parse import urlparse
from itemadapter import ItemAdapter
-from twisted.internet import defer, threads
-from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.threads import deferToThread
-from scrapy import Spider
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
+ from os import PathLike
+
+ from twisted.python.failure import Failure
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+
logger = logging.getLogger(__name__)
@@ -210,7 +213,7 @@ class S3FilesStore:
key_name = f"{self.prefix}{path}"
return cast(
"Deferred[Dict[str, Any]]",
- threads.deferToThread(
+ deferToThread(
self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined]
),
)
@@ -229,7 +232,7 @@ class S3FilesStore:
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
- return threads.deferToThread(
+ return deferToThread(
self.s3_client.put_object, # type: ignore[attr-defined]
Bucket=self.bucket,
Key=key_name,
@@ -326,9 +329,7 @@ class GCSFilesStore:
blob_path = self._get_blob_path(path)
return cast(
Deferred[StatInfo],
- threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
- _onsuccess
- ),
+ deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
)
def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:
@@ -351,7 +352,7 @@ class GCSFilesStore:
blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
- return threads.deferToThread(
+ return deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
content_type=self._get_content_type(headers),
@@ -388,7 +389,7 @@ class FTPFilesStore:
headers: Optional[Dict[str, str]] = None,
) -> Deferred[Any]:
path = f"{self.basedir}/{path}"
- return threads.deferToThread(
+ return deferToThread(
ftp_store_file,
path=path,
file=buf,
@@ -418,7 +419,7 @@ class FTPFilesStore:
except Exception:
return {}
- return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))
+ return cast("Deferred[StatInfo]", deferToThread(_stat_file, path))
class FilesPipeline(MediaPipeline):
@@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline):
}
path = self.file_path(request, info=info, item=item)
- # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
- dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
+ # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
+ dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
dfd2.addErrback(lambda _: None)
dfd2.addErrback(
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 27a57b17c..166f81314 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -11,7 +11,6 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
-from os import PathLike
from typing import (
TYPE_CHECKING,
Any,
@@ -28,7 +27,6 @@ from typing import (
from itemadapter import ItemAdapter
-from scrapy import Spider
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@@ -40,15 +38,20 @@ from scrapy.pipelines.files import (
S3FilesStore,
_md5sum,
)
-from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.python import get_func_args, to_bytes
if TYPE_CHECKING:
- # typing.Self requires Python 3.11
+ from os import PathLike
+
from PIL import Image
+
+ # typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
+
class NoimagesDrop(DropItem):
"""Product with no images exception"""
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 09e95cf5d..ea36a9e8a 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -25,21 +25,23 @@ from typing import (
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
-from scrapy import Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
from scrapy.http.request import NO_CALLBACK, Request
from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import defer_result, mustbe_deferred
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.request import RequestFingerprinter
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.utils.request import RequestFingerprinter
+
+
_T = TypeVar("_T")
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index ba7cd716b..d5eedf9b1 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type
from twisted.internet import defer
from twisted.internet.base import ReactorBase, ThreadedResolver
-from twisted.internet.defer import Deferred
from twisted.internet.interfaces import (
IAddress,
IHostnameResolver,
@@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
# typing.Self requires Python 3.11
from typing_extensions import Self
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index a33f73306..0d282dc37 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Optional, Union
from warnings import warn
-from scrapy import Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
@@ -14,8 +13,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
from scrapy.crawler import Crawler
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ea1db03f1..6703c569f 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -4,7 +4,6 @@ import copy
import json
from importlib import import_module
from pprint import pformat
-from types import ModuleType
from typing import (
TYPE_CHECKING,
Any,
@@ -27,6 +26,8 @@ from scrapy.settings import default_settings
_SettingsKeyT = Union[bool, float, int, str, None]
if TYPE_CHECKING:
+ from types import ModuleType
+
# https://github.com/python/typing/issues/445#issuecomment-1131458824
from _typeshed import SupportsItems
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index f6df191d8..3d37b8235 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,10 +1,14 @@
-from typing import Any, List, Tuple
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Tuple
from pydispatch import dispatcher
-from twisted.internet.defer import Deferred
from scrapy.utils import signal as _signal
+if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
class SignalManager:
def __init__(self, sender: Any = dispatcher.Anonymous):
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index d855c962c..b8fe65668 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -3,21 +3,23 @@ from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
-from types import ModuleType
from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type
from zope.interface import implementer
-from scrapy import Request, Spider
from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import BaseSettings
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
+ from types import ModuleType
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request, Spider
+ from scrapy.settings import BaseSettings
+
@implementer(ISpiderLoader)
class SpiderLoader:
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 1e96654e2..c5b7f0749 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -9,15 +9,17 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable
-from scrapy import Spider
-from scrapy.crawler import Crawler
from scrapy.http import Request, Response
-from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 35c869a75..ea1686c25 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -9,16 +9,18 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Iterable, List, Optional
-from scrapy import Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 50c93ac9f..379c5d0a3 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -12,10 +12,8 @@ import warnings
from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.httpobj import urlparse_cached
warnings.warn(
@@ -28,6 +26,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 8af0bdf5b..d35cf8f71 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -23,10 +23,8 @@ from urllib.parse import urlparse
from w3lib.url import safe_url_string
from scrapy import Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
@@ -35,6 +33,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
+
+
LOCAL_SCHEMES: Tuple[str, ...] = (
"about",
"blob",
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index e2aa554a7..34df54ca7 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -9,15 +9,17 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable
-from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Spider
+ from scrapy.settings import BaseSettings
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index a0898a0cf..ce0f1bbaa 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,10 +1,14 @@
-from typing import Any, Iterable, Optional, cast
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Iterable, Optional, cast
from scrapy import Request
-from scrapy.http import Response
from scrapy.spiders import Spider
from scrapy.utils.spider import iterate_spider_output
+if TYPE_CHECKING:
+ from scrapy.http import Response
+
class InitSpider(Spider):
"""Base Spider with initialization facilities"""
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 6f80ee388..d3e7896c5 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -6,20 +6,22 @@ from __future__ import annotations
import marshal
import pickle # nosec
-from os import PathLike
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
from queuelib import queue
-from scrapy import Request
-from scrapy.crawler import Crawler
from scrapy.utils.request import request_from_dict
if TYPE_CHECKING:
+ from os import PathLike
+
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request
+ from scrapy.crawler import Crawler
+
def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index ab571a3ab..88e72f366 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -2,15 +2,17 @@
Scrapy extension for collecting scraping stats
"""
+from __future__ import annotations
+
import logging
import pprint
from typing import TYPE_CHECKING, Any, Dict, Optional
-from scrapy import Spider
-
if TYPE_CHECKING:
+ from scrapy import Spider
from scrapy.crawler import Crawler
+
logger = logging.getLogger(__name__)
@@ -18,7 +20,7 @@ StatsT = Dict[str, Any]
class StatsCollector:
- def __init__(self, crawler: "Crawler"):
+ def __init__(self, crawler: Crawler):
self._dump: bool = crawler.settings.getbool("STATS_DUMP")
self._stats: StatsT = {}
@@ -67,7 +69,7 @@ class StatsCollector:
class MemoryStatsCollector(StatsCollector):
- def __init__(self, crawler: "Crawler"):
+ def __init__(self, crawler: Crawler):
super().__init__(crawler)
self.spider_stats: Dict[str, StatsT] = {}
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 7e82dd519..2240f0b58 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -4,8 +4,8 @@ import warnings
from functools import wraps
from typing import TYPE_CHECKING, Any, Callable, TypeVar
-from twisted.internet import defer, threads
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.threads import deferToThread
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
@wraps(func)
def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
- return defer.maybeDeferred(func, *a, **kw)
+ return maybeDeferred(func, *a, **kw)
return wrapped
@@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
@wraps(func)
def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
- return threads.deferToThread(func, *a, **kw)
+ return deferToThread(func, *a, **kw)
return wrapped
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 877eb4388..1d578e8a3 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -34,12 +34,13 @@ from twisted.internet import defer
from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
from twisted.internet.task import Cooperator
from twisted.python import failure
-from twisted.python.failure import Failure
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed
if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
# typing.Concatenate and typing.ParamSpec require Python 3.10
from typing_extensions import Concatenate, ParamSpec
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index fdcf484d4..770ee0b1b 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -4,9 +4,10 @@ from __future__ import annotations
# used in global tests code
from time import time # noqa: F401
-from typing import Any, List, Tuple
+from typing import TYPE_CHECKING, Any, List, Tuple
-from scrapy.core.engine import ExecutionEngine
+if TYPE_CHECKING:
+ from scrapy.core.engine import ExecutionEngine
def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 2e487d88b..85324361c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,11 +1,15 @@
+from __future__ import annotations
+
import struct
from gzip import GzipFile
from io import BytesIO
-
-from scrapy.http import Response
+from typing import TYPE_CHECKING
from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
+if TYPE_CHECKING:
+ from scrapy.http import Response
+
def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
"""Gunzip the given data and return as much data as possible.
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index d502e8910..3cf9585ec 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,12 +1,16 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
-from typing import Union
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
from urllib.parse import ParseResult, urlparse
from weakref import WeakKeyDictionary
-from scrapy.http import Request, Response
+if TYPE_CHECKING:
+ from scrapy.http import Request, Response
+
-_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = (
+_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = (
WeakKeyDictionary()
)
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index e230e4235..488c7994b 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,7 +1,10 @@
+from __future__ import annotations
+
from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
-from scrapy.settings import BaseSettings
+if TYPE_CHECKING:
+ from scrapy.settings import BaseSettings
def job_dir(settings: BaseSettings) -> Optional[str]:
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index cbfd170ed..439b065a9 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -21,12 +21,13 @@ from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
-from scrapy.logformatter import LogFormatterResult
from scrapy.settings import Settings, _SettingsKeyT
from scrapy.utils.versions import scrapy_components_versions
if TYPE_CHECKING:
from scrapy.crawler import Crawler
+ from scrapy.logformatter import LogFormatterResult
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 3d11c1035..3c787e50f 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,6 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
-from types import ModuleType
from typing import (
IO,
TYPE_CHECKING,
@@ -35,10 +34,13 @@ from scrapy.item import Item
from scrapy.utils.datatypes import LocalWeakReferencedCache
if TYPE_CHECKING:
+ from types import ModuleType
+
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
+
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
T = TypeVar("T")
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index de3c8eaf9..efb6af299 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,7 +1,8 @@
+from __future__ import annotations
+
import os
import warnings
from importlib import import_module
-from os import PathLike
from pathlib import Path
from typing import Union
@@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str:
return str(d)
-def data_path(path: Union[str, PathLike], createdir: bool = False) -> str:
+def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str:
"""
Return the given path joined with the .scrapy data directory.
If given an absolute path, return it unmodified.
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 5af6d22eb..a627db601 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import sys
-from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
from contextlib import suppress
from typing import (
TYPE_CHECKING,
@@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn
from twisted.internet import asyncioreactor, error
from twisted.internet.base import DelayedCall
-from twisted.internet.protocol import ServerFactory
-from twisted.internet.tcp import Port
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
+ from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
+
+ from twisted.internet.protocol import ServerFactory
+ from twisted.internet.tcp import Port
+
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index b05135c04..ce754fad3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import inspect
import logging
-from types import CoroutineType, ModuleType
from typing import (
TYPE_CHECKING,
Any,
@@ -16,16 +15,19 @@ from typing import (
overload,
)
-from twisted.internet.defer import Deferred
-
-from scrapy import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.misc import arg_to_iter
if TYPE_CHECKING:
+ from types import CoroutineType, ModuleType
+
+ from twisted.internet.defer import Deferred
+
+ from scrapy import Request
from scrapy.spiderloader import SpiderLoader
+
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index d520ef809..95611ebd9 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,12 +1,16 @@
-from typing import Any, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Optional
import OpenSSL._util as pyOpenSSLutil
import OpenSSL.SSL
import OpenSSL.version
-from OpenSSL.crypto import X509Name
from scrapy.utils.python import to_unicode
+if TYPE_CHECKING:
+ from OpenSSL.crypto import X509Name
+
def ffi_buf_to_string(buf: Any) -> str:
return to_unicode(pyOpenSSLutil.ffi.string(buf))
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 6b22f3bfa..08f3f2dc9 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -1,10 +1,14 @@
"""Helper functions for working with templates"""
+from __future__ import annotations
+
import re
import string
-from os import PathLike
from pathlib import Path
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
+
+if TYPE_CHECKING:
+ from os import PathLike
def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 268d8d4be..fe2bfa042 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,21 +2,36 @@
This module contains some assorted functions used in tests
"""
+from __future__ import annotations
+
import asyncio
import os
from importlib import import_module
from pathlib import Path
from posixpath import split
-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Awaitable,
+ Dict,
+ List,
+ Optional,
+ Tuple,
+ Type,
+ TypeVar,
+)
from unittest import TestCase, mock
-from twisted.internet.defer import Deferred
from twisted.trial.unittest import SkipTest
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.utils.boto import is_botocore_available
+if TYPE_CHECKING:
+ from twisted.internet.defer import Deferred
+
+
_T = TypeVar("_T")
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 3bdffcaa7..8882bfc5f 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -2,12 +2,14 @@ from __future__ import annotations
import os
import sys
-from typing import Iterable, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast
from twisted.internet.defer import Deferred
from twisted.internet.error import ProcessTerminated
from twisted.internet.protocol import ProcessProtocol
-from twisted.python.failure import Failure
+
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
class ProcessTest:
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 233f6b934..6ec46aa3d 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import argparse
import json
import os
@@ -7,12 +9,11 @@ from pathlib import Path
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
from urllib.parse import urlencode
from OpenSSL import SSL
from twisted.internet import defer, reactor, ssl
-from twisted.internet.protocol import ServerFactory
from twisted.internet.task import deferLater
from twisted.names import dns, error
from twisted.names.server import DNSServerFactory
@@ -23,6 +24,9 @@ from twisted.web.util import redirectTo
from scrapy.utils.python import to_bytes, to_unicode
+if TYPE_CHECKING:
+ from twisted.internet.protocol import ServerFactory
+
def getarg(request, name, default=None, type=None):
if name in request.args:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 253987e15..ea3ed3b05 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import bz2
import csv
import gzip
@@ -14,10 +16,9 @@ from collections import defaultdict
from contextlib import ExitStack
from io import BytesIO
from logging import getLogger
-from os import PathLike
from pathlib import Path
from string import ascii_letters, digits
-from typing import Union
+from typing import TYPE_CHECKING, Union
from unittest import mock
from urllib.parse import quote, urljoin
from urllib.request import pathname2url
@@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no
from tests.mockserver import MockFTPServer, MockServer
from tests.spiders import ItemSpider
+if TYPE_CHECKING:
+ from os import PathLike
+
def path_to_url(path):
return urljoin("file:", pathname2url(str(path)))
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 995c02a1a..7ea3fe8c9 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import json
import random
import re
@@ -6,7 +8,7 @@ import string
from ipaddress import IPv4Address
from pathlib import Path
from tempfile import mkdtemp
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
from unittest import mock, skipIf
from urllib.parse import urlencode
@@ -20,7 +22,6 @@ from twisted.internet.defer import (
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
from twisted.internet.error import TimeoutError
from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
-from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from twisted.web.client import URI, ResponseFailed
from twisted.web.http import H2_ENABLED
@@ -33,6 +34,9 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from tests.mockserver import LeafResource, Status, ssl_context_factory
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
def generate_random_string(size):
return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))