Scrapy, a fast high-level web crawling & scraping framework for Python.
Add flake8-type-checking rules to ruff.
Andrey Rakhmatullind47f142d0fff
diff --git a/pyproject.toml b/pyproject.tomlindex 41ba77094..4d20e5c1c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -244,6 +244,8 @@ extend-select = ["D",# flake8-debugger"T10",+ # flake8-type-checking+ "TC",]ignore = [# Assigning to `os.environ` doesn't clear the environment.diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex 4f6933006..b96c63eb7 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -1,6 +1,5 @@from __future__ import annotations-import argparseimport subprocess # nosecimport sysimport time@@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponsefrom scrapy.linkextractors import LinkExtractorif TYPE_CHECKING:+ import argparsefrom collections.abc import Iterablefrom scrapy import Requestdiff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex a1806f626..05e5e53e9 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -5,7 +5,6 @@ from typing import TYPE_CHECKINGfrom w3lib.url import is_url-from scrapy import Spiderfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.http import Request, Response@@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_requestif TYPE_CHECKING:from argparse import ArgumentParser, Namespace+ from scrapy import Spider+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex b286e703e..2e70b2865 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -1,12 +1,11 @@from __future__ import annotations-import argparseimport osimport shutilimport stringfrom importlib import import_modulefrom pathlib import Path-from typing import Any, cast+from typing import TYPE_CHECKING, Any, castfrom urllib.parse import urlparseimport scrapy@@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.template import render_templatefile, string_camelcase+if TYPE_CHECKING:+ import argparse+def sanitize_module_name(module_name: str) -> str:"""Sanitize the given module name, by replacing dashes and pointsdiff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex fba294851..fc16e46d1 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -1,6 +1,5 @@from __future__ import annotations-import argparseimport functoolsimport inspectimport json@@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iterfrom scrapy.utils.spider import spidercls_for_requestif TYPE_CHECKING:+ import argparsefrom collections.abc import AsyncGenerator, Coroutine, Iterablefrom twisted.python.failure import Failurediff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 7ec56899c..55211f8d7 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -1,6 +1,5 @@from __future__ import annotations-import argparseimport sysfrom importlib import import_modulefrom pathlib import Path@@ -11,6 +10,7 @@ from scrapy.exceptions import UsageErrorfrom scrapy.utils.spider import iter_spider_classesif TYPE_CHECKING:+ import argparsefrom os import PathLikefrom types import ModuleTypediff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 27e6d68ee..4ca015f5e 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -9,7 +9,6 @@ from __future__ import annotationsfrom threading import Threadfrom typing import TYPE_CHECKING, Any-from scrapy import Spiderfrom scrapy.commands import ScrapyCommandfrom scrapy.http import Requestfrom scrapy.shell import Shell@@ -19,6 +18,8 @@ from scrapy.utils.url import guess_schemeif TYPE_CHECKING:from argparse import ArgumentParser, Namespace+ from scrapy import Spider+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex f54c02369..6da877610 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -1,6 +1,5 @@from __future__ import annotations-import argparseimport osimport reimport string@@ -8,12 +7,16 @@ from importlib.util import find_specfrom pathlib import Pathfrom shutil import copy2, copystat, ignore_patterns, movefrom stat import S_IWUSR as OWNER_WRITE_PERMISSION+from typing import TYPE_CHECKINGimport scrapyfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.template import render_templatefile, string_camelcase+if TYPE_CHECKING:+ import argparse+TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (("scrapy.cfg",),("${project_name}", "settings.py.tmpl"),diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 5040741e2..434b316e9 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.resolver import dnscache-from scrapy.signalmanager import SignalManagerfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.httpobj import urlparse_cached@@ -23,6 +22,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http import Responsefrom scrapy.settings import BaseSettings+ from scrapy.signalmanager import SignalManager_T = TypeVar("_T")diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 20377ac06..7f3da67eb 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -3,7 +3,6 @@from __future__ import annotationsimport logging-from collections.abc import Callablefrom typing import TYPE_CHECKING, Any, Protocol, castfrom twisted.internet import defer@@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:- from collections.abc import Generator+ from collections.abc import Callable, Generatorfrom twisted.internet.defer import Deferreddiff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 60cffae35..5480df72c 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCallfrom twisted.python.failure import Failurefrom scrapy import signals-from scrapy.core.downloader import Downloaderfrom scrapy.core.scraper import Scraper, _HandleOutputDeferredfrom scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequestfrom scrapy.http import Request, Response-from scrapy.logformatter import LogFormatter-from scrapy.settings import Settings-from scrapy.signalmanager import SignalManagerfrom scrapy.utils.log import failure_to_exc_info, logformatter_adapterfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.reactor import CallLaterOnce@@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnceif TYPE_CHECKING:from collections.abc import Callable, Generator, Iterable, Iterator+ from scrapy.core.downloader import Downloaderfrom scrapy.core.scheduler import BaseSchedulerfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettings+ from scrapy.logformatter import LogFormatter+ from scrapy.settings import BaseSettings, Settings+ from scrapy.signalmanager import SignalManagerfrom scrapy.spiders import Spiderdiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 83dad0c0b..03301717d 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -16,9 +16,6 @@ from scrapy import Spider, signalsfrom scrapy.core.spidermw import SpiderMiddlewareManagerfrom scrapy.exceptions import CloseSpider, DropItem, IgnoreRequestfrom scrapy.http import Request, Response-from scrapy.logformatter import LogFormatter-from scrapy.pipelines import ItemPipelineManager-from scrapy.signalmanager import SignalManagerfrom scrapy.utils.defer import (aiter_errback,defer_fail,@@ -35,6 +32,9 @@ if TYPE_CHECKING:from collections.abc import Generator, Iterablefrom scrapy.crawler import Crawler+ from scrapy.logformatter import LogFormatter+ from scrapy.pipelines import ItemPipelineManager+ from scrapy.signalmanager import SignalManagerlogger = logging.getLogger(__name__)diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 1ad837a47..05af1bf8a 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -18,10 +18,8 @@ from scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEnginefrom scrapy.extension import ExtensionManagerfrom scrapy.interfaces import ISpiderLoader-from scrapy.logformatter import LogFormatterfrom scrapy.settings import BaseSettings, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.log import (LogCounterHandler,configure_logging,@@ -42,7 +40,9 @@ from scrapy.utils.reactor import (if TYPE_CHECKING:from collections.abc import Generator, Iterable+ from scrapy.logformatter import LogFormatterfrom scrapy.spiderloader import SpiderLoader+ from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.request import RequestFingerprinterdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex c9350a956..9380b7e78 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -10,8 +10,7 @@ import pickle # nosecimport pprintfrom collections.abc import Callable, Iterable, Mappingfrom io import BytesIO, TextIOWrapper-from json import JSONEncoder-from typing import Any+from typing import TYPE_CHECKING, Anyfrom xml.sax.saxutils import XMLGenerator # nosecfrom xml.sax.xmlreader import AttributesImpl # nosec@@ -21,6 +20,9 @@ from scrapy.item import Field, Itemfrom scrapy.utils.python import is_listlike, to_bytes, to_unicodefrom scrapy.utils.serialize import ScrapyJSONEncoder+if TYPE_CHECKING:+ from json import JSONEncoder+__all__ = ["BaseItemExporter","PprintItemExporter",diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 0cf44aed8..f6415ad8e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -25,7 +25,6 @@ from zope.interface import Interface, implementerfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.extensions.postprocessing import PostProcessingManager-from scrapy.settings import Settingsfrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.ftp import ftp_store_file@@ -44,7 +43,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.exporters import BaseItemExporter- from scrapy.settings import BaseSettings+ from scrapy.settings import BaseSettings, Settingslogger = logging.getLogger(__name__)diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 0e6120c21..0edcce888 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -2,13 +2,11 @@ from __future__ import annotationsimport gzipimport logging-import osimport pickle # nosecfrom email.utils import mktime_tz, parsedate_tzfrom importlib import import_modulefrom pathlib import Pathfrom time import time-from types import ModuleTypefrom typing import IO, TYPE_CHECKING, Any, castfrom weakref import WeakKeyDictionary@@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_pathfrom scrapy.utils.python import to_bytes, to_unicode-from scrapy.utils.request import RequestFingerprinterif TYPE_CHECKING:+ import osfrom collections.abc import Callable+ from types import ModuleType# typing.Concatenate requires Python 3.10from typing_extensions import Concatenate@@ -30,6 +29,7 @@ if TYPE_CHECKING:from scrapy.http.request import Requestfrom scrapy.settings import BaseSettingsfrom scrapy.spiders import Spider+ from scrapy.utils.request import RequestFingerprinterlogger = logging.getLogger(__name__)diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex f2e3782a4..7cf08a1bb 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport loggingfrom datetime import datetime, timezone-from json import JSONEncoderfrom typing import TYPE_CHECKING, Anyfrom twisted.internet import task@@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoderif TYPE_CHECKING:# typing.Self requires Python 3.11+ from json import JSONEncoder+from typing_extensions import Selffrom scrapy.crawler import Crawlerdiff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 07dc5880b..89c83d20d 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -13,7 +13,6 @@ import pprintfrom typing import TYPE_CHECKING, Anyfrom twisted.internet import protocol-from twisted.internet.tcp import Portfrom scrapy import signalsfrom scrapy.exceptions import NotConfigured@@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refsif TYPE_CHECKING:from twisted.conch import telnet+ from twisted.internet.tcp import Port# typing.Self requires Python 3.11from typing_extensions import Selfdiff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 9c29ea4d1..107469504 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -21,7 +21,6 @@ from typing import (from w3lib.url import safe_url_string-import scrapyfrom scrapy.http.headers import Headersfrom scrapy.utils.curl import curl_to_request_kwargsfrom scrapy.utils.python import to_bytes@@ -37,6 +36,7 @@ if TYPE_CHECKING:# typing.NotRequired and typing.Self require Python 3.11from typing_extensions import Concatenate, NotRequired, Self+ from scrapy import Spiderfrom scrapy.http import ResponseCallbackT = Callable[Concatenate[Response, ...], Any]@@ -252,7 +252,7 @@ class Request(object_ref):request_kwargs.update(kwargs)return cls(**request_kwargs)- def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:+ def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]:"""Return a dictionary containing the Request's data.Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 28e2073a2..4dea5afea 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -5,7 +5,6 @@ import loggingfrom typing import TYPE_CHECKING, Protocol, castfrom scrapy import Request-from scrapy.core.downloader import Downloaderfrom scrapy.utils.misc import build_from_crawlerif TYPE_CHECKING:@@ -14,6 +13,7 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.core.downloader import Downloaderfrom scrapy.crawler import Crawlerlogger = logging.getLogger(__name__)diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex e621525f2..a86aad51c 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport numbersimport osimport sys-from collections.abc import Iterablefrom configparser import ConfigParserfrom operator import itemgetterfrom pathlib import Path@@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpathfrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:- from collections.abc import Collection, Mapping, MutableMapping+ from collections.abc import Collection, Iterable, Mapping, MutableMappingdef build_component_list(diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex ac4358410..2102ce798 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVarfrom warnings import catch_warnings, filterwarningsfrom twisted.internet import asyncioreactor, error-from twisted.internet.base import DelayedCallfrom scrapy.utils.misc import load_object@@ -15,6 +14,7 @@ if TYPE_CHECKING:from asyncio import AbstractEventLoop, AbstractEventLoopPolicyfrom collections.abc import Callable+ from twisted.internet.base import DelayedCallfrom twisted.internet.protocol import ServerFactoryfrom twisted.internet.tcp import Portdiff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 92b73a91a..a7b84baef 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -16,7 +16,6 @@ from unittest import TestCase, mockfrom twisted.trial.unittest import SkipTestfrom scrapy import Spider-from scrapy.crawler import Crawlerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_available@@ -26,6 +25,8 @@ if TYPE_CHECKING:from twisted.internet.defer import Deferredfrom twisted.web.client import Response as TxResponse+ from scrapy.crawler import Crawler+_T = TypeVar("_T")