Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add flake8-type-checking rules to ruff.

Andrey Rakhmatullind47f142d0fff
diff --git a/pyproject.toml b/pyproject.toml
index 41ba77094..4d20e5c1c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -244,6 +244,8 @@ extend-select = [
"D",
# flake8-debugger
"T10",
+ # flake8-type-checking
+ "TC",
]
ignore = [
# Assigning to `os.environ` doesn't clear the environment.
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 4f6933006..b96c63eb7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import argparse
import subprocess # nosec
import sys
import time
@@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
if TYPE_CHECKING:
+ import argparse
from collections.abc import Iterable
from scrapy import Request
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index a1806f626..05e5e53e9 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
from w3lib.url import is_url
-from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
@@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
if TYPE_CHECKING:
from argparse import ArgumentParser, Namespace
+ from scrapy import Spider
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index b286e703e..2e70b2865 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,12 +1,11 @@
from __future__ import annotations
-import argparse
import os
import shutil
import string
from importlib import import_module
from pathlib import Path
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlparse
import scrapy
@@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
+if TYPE_CHECKING:
+ import argparse
+
def sanitize_module_name(module_name: str) -> str:
"""Sanitize the given module name, by replacing dashes and points
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index fba294851..fc16e46d1 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import argparse
import functools
import inspect
import json
@@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
if TYPE_CHECKING:
+ import argparse
from collections.abc import AsyncGenerator, Coroutine, Iterable
from twisted.python.failure import Failure
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 7ec56899c..55211f8d7 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import argparse
import sys
from importlib import import_module
from pathlib import Path
@@ -11,6 +10,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
+ import argparse
from os import PathLike
from types import ModuleType
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 27e6d68ee..4ca015f5e 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -9,7 +9,6 @@ from __future__ import annotations
from threading import Thread
from typing import TYPE_CHECKING, Any
-from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
@@ -19,6 +18,8 @@ from scrapy.utils.url import guess_scheme
if TYPE_CHECKING:
from argparse import ArgumentParser, Namespace
+ from scrapy import Spider
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index f54c02369..6da877610 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import argparse
import os
import re
import string
@@ -8,12 +7,16 @@ from importlib.util import find_spec
from pathlib import Path
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
+from typing import TYPE_CHECKING
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
+if TYPE_CHECKING:
+ import argparse
+
TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (
("scrapy.cfg",),
("${project_name}", "settings.py.tmpl"),
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 5040741e2..434b316e9 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.resolver import dnscache
-from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
@@ -23,6 +22,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import BaseSettings
+ from scrapy.signalmanager import SignalManager
_T = TypeVar("_T")
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 20377ac06..7f3da67eb 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import logging
-from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Protocol, cast
from twisted.internet import defer
@@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Callable, Generator
from twisted.internet.defer import Deferred
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 60cffae35..5480df72c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
from scrapy import signals
-from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper, _HandleOutputDeferred
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.settings import Settings
-from scrapy.signalmanager import SignalManager
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.reactor import CallLaterOnce
@@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnce
if TYPE_CHECKING:
from collections.abc import Callable, Generator, Iterable, Iterator
+ from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler
from scrapy.crawler import Crawler
- from scrapy.settings import BaseSettings
+ from scrapy.logformatter import LogFormatter
+ from scrapy.settings import BaseSettings, Settings
+ from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 83dad0c0b..03301717d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,9 +16,6 @@ from scrapy import Spider, signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.pipelines import ItemPipelineManager
-from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import (
aiter_errback,
defer_fail,
@@ -35,6 +32,9 @@ if TYPE_CHECKING:
from collections.abc import Generator, Iterable
from scrapy.crawler import Crawler
+ from scrapy.logformatter import LogFormatter
+ from scrapy.pipelines import ItemPipelineManager
+ from scrapy.signalmanager import SignalManager
logger = logging.getLogger(__name__)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1ad837a47..05af1bf8a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -18,10 +18,8 @@ from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
-from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.log import (
LogCounterHandler,
configure_logging,
@@ -42,7 +40,9 @@ from scrapy.utils.reactor import (
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
+ from scrapy.logformatter import LogFormatter
from scrapy.spiderloader import SpiderLoader
+ from scrapy.statscollectors import StatsCollector
from scrapy.utils.request import RequestFingerprinter
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c9350a956..9380b7e78 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -10,8 +10,7 @@ import pickle # nosec
import pprint
from collections.abc import Callable, Iterable, Mapping
from io import BytesIO, TextIOWrapper
-from json import JSONEncoder
-from typing import Any
+from typing import TYPE_CHECKING, Any
from xml.sax.saxutils import XMLGenerator # nosec
from xml.sax.xmlreader import AttributesImpl # nosec
@@ -21,6 +20,9 @@ from scrapy.item import Field, Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
+if TYPE_CHECKING:
+ from json import JSONEncoder
+
__all__ = [
"BaseItemExporter",
"PprintItemExporter",
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0cf44aed8..f6415ad8e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from zope.interface import Interface, implementer
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.settings import Settings
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.ftp import ftp_store_file
@@ -44,7 +43,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.exporters import BaseItemExporter
- from scrapy.settings import BaseSettings
+ from scrapy.settings import BaseSettings, Settings
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 0e6120c21..0edcce888 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -2,13 +2,11 @@ from __future__ import annotations
import gzip
import logging
-import os
import pickle # nosec
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
-from types import ModuleType
from typing import IO, TYPE_CHECKING, Any, cast
from weakref import WeakKeyDictionary
@@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import RequestFingerprinter
if TYPE_CHECKING:
+ import os
from collections.abc import Callable
+ from types import ModuleType
# typing.Concatenate requires Python 3.10
from typing_extensions import Concatenate
@@ -30,6 +29,7 @@ if TYPE_CHECKING:
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
+ from scrapy.utils.request import RequestFingerprinter
logger = logging.getLogger(__name__)
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index f2e3782a4..7cf08a1bb 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import logging
from datetime import datetime, timezone
-from json import JSONEncoder
from typing import TYPE_CHECKING, Any
from twisted.internet import task
@@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
# typing.Self requires Python 3.11
+ from json import JSONEncoder
+
from typing_extensions import Self
from scrapy.crawler import Crawler
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 07dc5880b..89c83d20d 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -13,7 +13,6 @@ import pprint
from typing import TYPE_CHECKING, Any
from twisted.internet import protocol
-from twisted.internet.tcp import Port
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
from twisted.conch import telnet
+ from twisted.internet.tcp import Port
# typing.Self requires Python 3.11
from typing_extensions import Self
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 9c29ea4d1..107469504 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -21,7 +21,6 @@ from typing import (
from w3lib.url import safe_url_string
-import scrapy
from scrapy.http.headers import Headers
from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
@@ -37,6 +36,7 @@ if TYPE_CHECKING:
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import Concatenate, NotRequired, Self
+ from scrapy import Spider
from scrapy.http import Response
CallbackT = Callable[Concatenate[Response, ...], Any]
@@ -252,7 +252,7 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
- def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
+ def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 28e2073a2..4dea5afea 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -5,7 +5,6 @@ import logging
from typing import TYPE_CHECKING, Protocol, cast
from scrapy import Request
-from scrapy.core.downloader import Downloader
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
@@ -14,6 +13,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.core.downloader import Downloader
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index e621525f2..a86aad51c 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import numbers
import os
import sys
-from collections.abc import Iterable
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
@@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
- from collections.abc import Collection, Mapping, MutableMapping
+ from collections.abc import Collection, Iterable, Mapping, MutableMapping
def build_component_list(
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ac4358410..2102ce798 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar
from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error
-from twisted.internet.base import DelayedCall
from scrapy.utils.misc import load_object
@@ -15,6 +14,7 @@ if TYPE_CHECKING:
from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
from collections.abc import Callable
+ from twisted.internet.base import DelayedCall
from twisted.internet.protocol import ServerFactory
from twisted.internet.tcp import Port
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 92b73a91a..a7b84baef 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -16,7 +16,6 @@ from unittest import TestCase, mock
from twisted.trial.unittest import SkipTest
from scrapy import Spider
-from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
@@ -26,6 +25,8 @@ if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.web.client import Response as TxResponse
+ from scrapy.crawler import Crawler
+
_T = TypeVar("_T")