Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump tool versions. (#6941)

Andrey Rakhmatullindf342eee6e2f
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6fbf74058..bcc10d5e8 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,8 +1,8 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.9.3
+ rev: v0.12.2
hooks:
- - id: ruff
+ - id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/adamchainz/blacken-docs
@@ -10,7 +10,7 @@ repos:
hooks:
- id: blacken-docs
additional_dependencies:
- - black==24.10.0
+ - black==25.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
diff --git a/conftest.py b/conftest.py
index f952127b9..02697603d 100644
--- a/conftest.py
+++ b/conftest.py
@@ -85,7 +85,7 @@ def requires_uvloop(request):
if not request.node.get_closest_marker("requires_uvloop"):
return
try:
- import uvloop
+ import uvloop # noqa: PLC0415
del uvloop
except ImportError:
@@ -97,7 +97,7 @@ def requires_botocore(request):
if not request.node.get_closest_marker("requires_botocore"):
return
try:
- import botocore
+ import botocore # noqa: PLC0415
del botocore
except ImportError:
@@ -109,7 +109,7 @@ def requires_boto3(request):
if not request.node.get_closest_marker("requires_boto3"):
return
try:
- import boto3
+ import boto3 # noqa: PLC0415
del boto3
except ImportError:
diff --git a/pyproject.toml b/pyproject.toml
index a6b99ac5b..b25ccb894 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -106,10 +106,6 @@ follow_imports = "skip"
module = "scrapy.settings.default_settings"
ignore_errors = true
-[[tool.mypy.overrides]]
-module = "itemadapter"
-implicit_reexport = true
-
[[tool.mypy.overrides]]
module = "twisted"
implicit_reexport = true
@@ -394,6 +390,9 @@ banned-module-level-imports = [
"twisted.internet.reactor",
]
+[tool.ruff.lint.isort]
+split-on-trailing-comma = false
+
[tool.ruff.lint.per-file-ignores]
# Circular import workarounds
"scrapy/linkextractors/__init__.py" = ["E402"]
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 256504c9c..280a0f659 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -31,11 +31,11 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
def __getattr__(name: str):
if name == "twisted_version":
- import warnings # pylint: disable=reimported
+ import warnings # noqa: PLC0415 # pylint: disable=reimported
- from twisted import version as _txv
+ from twisted import version as _txv # noqa: PLC0415
- from scrapy.exceptions import ScrapyDeprecationWarning
+ from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead",
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 96bb1ae84..c4e277a60 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -10,6 +10,7 @@ import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
+from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
import argparse
@@ -35,8 +36,6 @@ class Command(ScrapyCommand):
class _BenchServer:
def __enter__(self) -> None:
- from scrapy.utils.test import get_testenv
-
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen( # noqa: S603
pargs, stdout=subprocess.PIPE, env=get_testenv()
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 0e90c3188..9d2742afd 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -206,7 +206,7 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
- spiders_dir = Path(cast(str, spiders_module.__file__)).parent
+ spiders_dir = Path(cast("str", spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index bdb68c4ad..f5ba5ba13 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -52,7 +52,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
- return list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+ return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
request.callback = wrapper
@@ -68,7 +68,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
- output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+ output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@@ -181,7 +181,7 @@ class ContractsManager:
def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
try:
output = cb(response, **cb_kwargs)
- output = list(cast(Iterable[Any], iterate_spider_output(output)))
+ output = list(cast("Iterable[Any]", iterate_spider_output(output)))
except Exception:
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index c0833c84d..4c9fed3bc 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -162,7 +162,7 @@ class Downloader:
def get_slot_key(self, request: Request) -> str:
if self.DOWNLOAD_SLOT in request.meta:
- return cast(str, request.meta[self.DOWNLOAD_SLOT])
+ return cast("str", request.meta[self.DOWNLOAD_SLOT])
key = urlparse_cached(request).hostname or ""
if self.ip_concurrency:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 54fef48b6..50235846b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -386,6 +386,7 @@ class ScrapyAgent:
if not proxy_port:
proxy_port = 443 if proxy_parsed.scheme == "https" else 80
if urlparse_cached(request).scheme == "https":
+ assert proxy_host is not None
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxy_host, proxy_port, proxyAuth)
return self._TunnelingAgent(
@@ -430,7 +431,7 @@ class ScrapyAgent:
method,
to_bytes(url, encoding="ascii"),
headers,
- cast(IBodyProducer, bodyproducer),
+ cast("IBodyProducer", bodyproducer),
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 870a26f04..02beb2f8b 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -51,8 +51,8 @@ class S3DownloadHandler:
self.anon = kw.get("anon")
self._signer = None
- import botocore.auth
- import botocore.credentials
+ import botocore.auth # noqa: PLC0415
+ import botocore.credentials # noqa: PLC0415
kw.pop("anon", None)
if kw:
@@ -87,7 +87,7 @@ class S3DownloadHandler:
if self.anon:
request = request.replace(url=url)
else:
- import botocore.awsrequest
+ import botocore.awsrequest # noqa: PLC0415
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index fa45d62ee..2e892cff7 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -6,7 +6,6 @@ See documentation in docs/topics/downloader-middleware.rst
from __future__ import annotations
-from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast
from twisted.internet.defer import Deferred, inlineCallbacks
@@ -18,7 +17,7 @@ from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import _defer_sleep, deferred_from_coro
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Callable, Generator
from scrapy import Spider
from scrapy.settings import BaseSettings
@@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
request: Request,
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_request"]:
- method = cast(Callable, method)
+ method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, spider=spider)
)
@@ -76,7 +75,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
for method in self.methods["process_response"]:
- method = cast(Callable, method)
+ method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, response=response, spider=spider)
)
@@ -94,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
exception: Exception,
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_exception"]:
- method = cast(Callable, method)
+ method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, exception=exception, spider=spider)
)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index be76ad8c3..970a250ef 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -17,13 +17,11 @@ from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, su
from twisted.python.failure import Failure
from scrapy import signals
+from scrapy.core.scheduler import BaseScheduler
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
-from scrapy.utils.asyncio import (
- AsyncioLoopingCall,
- create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
@@ -39,7 +37,6 @@ if TYPE_CHECKING:
from twisted.internet.task import LoopingCall
from scrapy.core.downloader import Downloader
- from scrapy.core.scheduler import BaseScheduler
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
@@ -123,8 +120,6 @@ class ExecutionEngine:
raise
def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:
- from scrapy.core.scheduler import BaseScheduler
-
scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
@@ -506,7 +501,7 @@ class ExecutionEngine:
dfd.addErrback(log_failure("Scraper close failure"))
if hasattr(self._slot.scheduler, "close"):
- dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))
+ dfd.addBoth(lambda _: cast("_Slot", self._slot).scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 9ac447289..9ae555300 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -491,7 +491,7 @@ class Scheduler(BaseScheduler):
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
- return cast(list[int], json.load(f))
+ return cast("list[int]", json.load(f))
def _write_dqs_state(self, dqdir: str, state: list[int]) -> None:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 77fcaa357..01e563e56 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -138,7 +138,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
) -> Deferred[Iterable[_T] | AsyncIterator[_T]]:
for method in self.methods["process_spider_input"]:
- method = cast(Callable, method)
+ method = cast("Callable", method)
try:
result = method(response=response, spider=spider)
if result is not None:
@@ -166,7 +166,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield from iterable
except Exception as ex:
exception_result = cast(
- Union[Failure, MutableChain[_T]],
+ "Union[Failure, MutableChain[_T]]",
self._process_spider_exception(
response, spider, ex, exception_processor_index
),
@@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield r
except Exception as ex:
exception_result = cast(
- Union[Failure, MutableAsyncChain[_T]],
+ "Union[Failure, MutableAsyncChain[_T]]",
self._process_spider_exception(
response, spider, ex, exception_processor_index
),
@@ -212,7 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
- method = cast(Callable, method)
+ method = cast("Callable", method)
result = method(response=response, exception=exception, spider=spider)
if _isiterable(result):
# stop exception handling by handing control over to the
@@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(
- Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
+ "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result
)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d6fb9972e..c6c65a993 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -8,11 +8,7 @@ import signal
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, TypeVar
-from twisted.internet.defer import (
- Deferred,
- DeferredList,
- inlineCallbacks,
-)
+from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from scrapy import Spider, signals
from scrapy.addons import AddonManager
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 612426371..530cccb53 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -202,6 +202,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
redirected = self._redirect_request_using_get(request, url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
- if cast(float, interval) < self._maxdelay:
+ if cast("float", interval) < self._maxdelay:
return self._redirect(redirected, request, spider, "meta refresh")
return response
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 557ad99e1..0847c1e8b 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -218,7 +218,7 @@ class S3FeedStorage(BlockingFeedStorage):
region_name: str | None = None,
):
try:
- import boto3.session
+ import boto3.session # noqa: PLC0415
except ImportError:
raise NotConfigured("missing boto3 library")
u = urlparse(uri)
@@ -317,7 +317,7 @@ class GCSFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
- from google.cloud.storage import Client
+ from google.cloud.storage import Client # noqa: PLC0415
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
@@ -413,7 +413,7 @@ class FeedSlot:
self.file = self.storage.open(self.spider)
if "postprocessing" in self.feed_options:
self.file = cast(
- IO[bytes],
+ "IO[bytes]",
PostProcessingManager(
self.feed_options["postprocessing"],
self.file,
@@ -662,7 +662,7 @@ class FeedExporter:
def _load_components(self, setting_prefix: str) -> dict[str, Any]:
conf = without_none_values(
- cast(dict[str, str], self.settings.getwithbase(setting_prefix))
+ cast("dict[str, str]", self.settings.getwithbase(setting_prefix))
)
d = {}
for k, v in conf.items():
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 0cd16d737..f704ae014 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -307,7 +307,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return None # expired
- return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301
+ return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"])) # noqa: S301
class FilesystemCacheStorage:
@@ -389,7 +389,7 @@ class FilesystemCacheStorage:
if 0 < self.expiration_secs < time() - mtime:
return None # expired
with self._open(metapath, "rb") as f:
- return cast(dict[str, Any], pickle.load(f)) # noqa: S301
+ return cast("dict[str, Any]", pickle.load(f)) # noqa: S301
def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 387cfddb3..3d7674905 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -5,10 +5,7 @@ from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
- AsyncioLoopingCall,
- create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
if TYPE_CHECKING:
from twisted.internet.task import LoopingCall
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 2ef322f1a..e425749f7 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -16,10 +16,7 @@ from typing import TYPE_CHECKING
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
-from scrapy.utils.asyncio import (
- AsyncioLoopingCall,
- create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.engine import get_engine_status
if TYPE_CHECKING:
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 98210990a..860b97a55 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Any
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
- AsyncioLoopingCall,
- create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py
index b1fa160c8..01fd82830 100644
--- a/scrapy/extensions/postprocessing.py
+++ b/scrapy/extensions/postprocessing.py
@@ -142,7 +142,7 @@ class PostProcessingManager(IOBase):
:return: returns number of bytes written
:rtype: int
"""
- return cast(int, self.head_plugin.write(data))
+ return cast("int", self.head_plugin.write(data))
def tell(self) -> int:
return self.file.tell()
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index bacee8f0a..9fe71817a 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -12,6 +12,8 @@ import os
import pprint
from typing import TYPE_CHECKING, Any
+from twisted.conch import manhole, telnet
+from twisted.conch.insults import insults
from twisted.internet import protocol
from scrapy import signals
@@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcp
from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
- from twisted.conch import telnet
from twisted.internet.tcp import Port
# typing.Self requires Python 3.11
@@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory):
self.port.stopListening()
def protocol(self) -> telnet.TelnetTransport:
- # these import twisted.internet.reactor
- from twisted.conch import manhole, telnet
- from twisted.conch.insults import insults
-
class Portal:
"""An implementation of IPortal"""
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index b7c3b9d37..e5b4b28a7 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -161,7 +161,7 @@ class WrappedRequest:
HTML document, and the user had no option to approve the automatic
fetching of the image, this should be true.
"""
- return cast(bool, self.request.meta.get("is_unverifiable", False))
+ return cast("bool", self.request.meta.get("is_unverifiable", False))
@property
def full_url(self) -> str:
@@ -181,7 +181,7 @@ class WrappedRequest:
@property
def origin_req_host(self) -> str:
- return cast(str, urlparse_cached(self.request).hostname)
+ return cast("str", urlparse_cached(self.request).hostname)
def has_header(self, name: str) -> bool:
return name in self.request.headers
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 60b04753b..5498e1138 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -69,19 +69,19 @@ class Headers(CaselessDict):
def __getitem__(self, key: AnyStr) -> bytes | None:
try:
- return cast(list[bytes], super().__getitem__(key))[-1]
+ return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
try:
- return cast(list[bytes], super().get(key, def_val))[-1]
+ return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
try:
- return cast(list[bytes], super().__getitem__(key))
+ return cast("list[bytes]", super().__getitem__(key))
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 7681419c4..aa15a0222 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -11,13 +11,7 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
-from lxml.html import (
- FormElement,
- InputElement,
- MultipleSelectOptions,
- SelectElement,
- TextareaElement,
-)
+from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
@@ -25,6 +19,13 @@ from scrapy.utils.python import is_listlike, to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
+ from lxml.html import (
+ FormElement,
+ InputElement,
+ MultipleSelectOptions,
+ SelectElement,
+ TextareaElement,
+ )
from typing_extensions import Self
from scrapy.http.response.text import TextResponse
@@ -76,8 +77,6 @@ class FormRequest(Request):
kwargs.setdefault("encoding", response.encoding)
if formcss is not None:
- from parsel.csstranslator import HTMLTranslator
-
formxpath = HTMLTranslator().css_to_xpath(formcss)
form = _get_form(response, formname, formid, formnumber, formxpath)
@@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
values = [
(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
- for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
+ for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])
]
return urlencode(values, doseq=True)
@@ -128,12 +127,12 @@ def _get_form(
if formname is not None:
f = root.xpath(f'//form[@name="{formname}"]')
if f:
- return cast(FormElement, f[0])
+ return cast("FormElement", f[0])
if formid is not None:
f = root.xpath(f'//form[@id="{formid}"]')
if f:
- return cast(FormElement, f[0])
+ return cast("FormElement", f[0])
# Get form element from xpath, if not found, go up
if formxpath is not None:
@@ -142,7 +141,7 @@ def _get_form(
el = nodes[0]
while True:
if el.tag == "form":
- return cast(FormElement, el)
+ return cast("FormElement", el)
el = el.getparent()
if el is None:
break
@@ -153,7 +152,7 @@ def _get_form(
form = forms[formnumber]
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
- return cast(FormElement, form)
+ return cast("FormElement", form)
def _get_inputs(
@@ -201,7 +200,7 @@ def _value(
n = ele.name
v = ele.value
if ele.tag == "select":
- return _select_value(cast(SelectElement, ele), n, v)
+ return _select_value(cast("SelectElement", ele), n, v)
return n, v
@@ -251,7 +250,7 @@ def _get_clickable(
except IndexError:
pass
else:
- return (el.get("name"), el.get("value") or "")
+ return (cast("str", el.get("name")), el.get("value") or "")
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 081223882..9c4e4c151 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -104,13 +104,14 @@ class TextResponse(Response):
@memoizemethod_noargs
def _headers_encoding(self) -> str | None:
- content_type = cast(bytes, self.headers.get(b"Content-Type", b""))
+ content_type = cast("bytes", self.headers.get(b"Content-Type", b""))
return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
def _body_inferred_encoding(self) -> str:
if self._cached_benc is None:
content_type = to_unicode(
- cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"
+ cast("bytes", self.headers.get(b"Content-Type", b"")),
+ encoding="latin-1",
)
benc, ubody = html_to_unicode(
content_type,
@@ -141,31 +142,25 @@ class TextResponse(Response):
@property
def selector(self) -> Selector:
- from scrapy.selector import Selector
+ # circular import
+ from scrapy.selector import Selector # noqa: PLC0415
if self._cached_selector is None:
self._cached_selector = Selector(self)
return self._cached_selector
def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
- from scrapy.selector import SelectorList
-
if not hasattr(self.selector, "jmespath"):
raise AttributeError(
"Please install parsel >= 1.8.1 to get jmespath support"
)
-
- return cast(SelectorList, self.selector.jmespath(query, **kwargs))
+ return cast("SelectorList", self.selector.jmespath(query, **kwargs))
def xpath(self, query: str, **kwargs: Any) -> SelectorList:
- from scrapy.selector import SelectorList
-
- return cast(SelectorList, self.selector.xpath(query, **kwargs))
+ return cast("SelectorList", self.selector.xpath(query, **kwargs))
def css(self, query: str) -> SelectorList:
- from scrapy.selector import SelectorList
-
- return cast(SelectorList, self.selector.css(query))
+ return cast("SelectorList", self.selector.css(query))
def follow(
self,
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 814e31fec..7c9b4d3e3 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -71,12 +71,12 @@ class LxmlParserLinkExtractor:
self.scan_tag: Callable[[str], bool] = (
tag
if callable(tag)
- else cast(Callable[[str], bool], partial(operator.eq, tag))
+ else cast("Callable[[str], bool]", partial(operator.eq, tag))
)
self.scan_attr: Callable[[str], bool] = (
attr
if callable(attr)
- else cast(Callable[[str], bool], partial(operator.eq, attr))
+ else cast("Callable[[str], bool]", partial(operator.eq, attr))
)
self.process_attr: Callable[[Any], Any] = (
process if callable(process) else _identity
@@ -84,7 +84,7 @@ class LxmlParserLinkExtractor:
self.unique: bool = unique
self.strip: bool = strip
self.link_key: Callable[[Link], str] = (
- cast(Callable[[Link], str], operator.attrgetter("url"))
+ cast("Callable[[Link], str]", operator.attrgetter("url"))
if canonicalized
else _canonicalize_link_url
)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index be2423965..88612daa9 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -223,7 +223,8 @@ class MailSender:
def _create_sender_factory(
self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any]
) -> ESMTPSenderFactory:
- from twisted.mail.smtp import ESMTPSenderFactory
+ # imports twisted.internet.reactor
+ from twisted.mail.smtp import ESMTPSenderFactory # noqa: PLC0415
factory_keywords: dict[str, Any] = {
"heloFallback": True,
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 25f2aa199..a4d7af7b6 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -63,7 +63,7 @@ class MiddlewareManager(ABC):
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return cast(_T, instance)
+ return cast("_T", instance)
@classmethod
def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 55d9cae6c..84d4104ed 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -169,7 +169,7 @@ class S3FilesStore:
def __init__(self, uri: str):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
- import botocore.session
+ import botocore.session # noqa: PLC0415
session = botocore.session.get_session()
self.s3_client = session.create_client(
@@ -284,7 +284,7 @@ class GCSFilesStore:
POLICY = None
def __init__(self, uri: str):
- from google.cloud import storage
+ from google.cloud import storage # noqa: PLC0415
client = storage.Client(project=self.GCS_PROJECT_ID)
bucket, prefix = uri[5:].split("/", 1)
@@ -317,7 +317,7 @@ class GCSFilesStore:
blob_path = self._get_blob_path(path)
return cast(
- Deferred[StatInfo],
+ "Deferred[StatInfo]",
deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
)
@@ -531,7 +531,9 @@ class FilesPipeline(MediaPipeline):
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:
- s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
+ s3store: type[S3FilesStore] = cast(
+ "type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
+ )
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
@@ -542,13 +544,13 @@ class FilesPipeline(MediaPipeline):
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
gcs_store: type[GCSFilesStore] = cast(
- type[GCSFilesStore], cls.STORE_SCHEMES["gs"]
+ "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
)
gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
ftp_store: type[FTPFilesStore] = cast(
- type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]
+ "type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"]
)
ftp_store.FTP_USERNAME = settings["FTP_USER"]
ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
@@ -742,5 +744,5 @@ class FilesPipeline(MediaPipeline):
media_ext = ""
media_type = mimetypes.guess_type(request.url)[0]
if media_type:
- media_ext = cast(str, mimetypes.guess_extension(media_type))
+ media_ext = cast("str", mimetypes.guess_extension(media_type))
return f"full/{media_guid}{media_ext}"
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 63c6908dc..19139b5d5 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -63,7 +63,7 @@ class ImagesPipeline(FilesPipeline):
crawler: Crawler | None = None,
):
try:
- from PIL import Image
+ from PIL import Image # noqa: PLC0415
self._Image = Image
except ImportError:
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 34b235d83..42c53a527 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -4,7 +4,6 @@ import hashlib
import logging
from typing import TYPE_CHECKING, Protocol, cast
-from scrapy import Request
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
@@ -13,6 +12,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy import Request
from scrapy.core.downloader import Downloader
from scrapy.crawler import Crawler
@@ -211,7 +211,7 @@ class ScrapyPriorityQueue:
except KeyError:
queue = self.queues[self.curprio]
# Protocols can't declare optional members
- return cast(Request, queue.peek()) # type: ignore[attr-defined]
+ return cast("Request", queue.peek()) # type: ignore[attr-defined]
def close(self) -> list[int]:
active: set[int] = set()
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 417c9c142..e1a12be05 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -4,6 +4,9 @@ import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING
+from urllib.robotparser import RobotFileParser
+
+from protego import Protego
from scrapy.utils.python import to_unicode
@@ -67,8 +70,6 @@ class RobotParser(metaclass=ABCMeta):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
- from urllib.robotparser import RobotFileParser
-
self.spider: Spider | None = spider
body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp: RobotFileParser = RobotFileParser()
@@ -87,7 +88,7 @@ class PythonRobotParser(RobotParser):
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
- from robotexclusionrulesparser import RobotExclusionRulesParser
+ from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: PLC0415
self.spider: Spider | None = spider
self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()
@@ -107,8 +108,6 @@ class RerpRobotParser(RobotParser):
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
- from protego import Protego
-
self.spider: Spider | None = spider
body_decoded = decode_robotstxt(robotstxt_body, spider)
self.rp = Protego.parse(body_decoded)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 9363da492..334de6658 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -336,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
stored.
"""
if len(self) > 0:
- return max(cast(int, self.getpriority(name)) for name in self)
+ return max(cast("int", self.getpriority(name)) for name in self)
return get_settings_priority("default")
def replace_in_component_priority_dict(
@@ -519,11 +519,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
self._assert_mutability()
if isinstance(values, str):
- values = cast(dict[_SettingsKeyT, Any], json.loads(values))
+ values = cast("dict[_SettingsKeyT, Any]", json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
- self.set(name, value, cast(int, values.getpriority(name)))
+ self.set(name, value, cast("int", values.getpriority(name)))
else:
for name, value in values.items():
self.set(name, value, priority)
@@ -533,7 +533,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
raise KeyError(name)
self._assert_mutability()
priority = get_settings_priority(priority)
- if priority >= cast(int, self.getpriority(name)):
+ if priority >= cast("int", self.getpriority(name)):
del self.attributes[name]
def __delitem__(self, name: _SettingsKeyT) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 50c0088e7..f306569e4 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -535,9 +535,9 @@ WARN_ON_GENERATOR_RETURN_VALUE = True
def __getattr__(name: str):
if name == "CONCURRENT_REQUESTS_PER_IP":
- import warnings
+ import warnings # noqa: PLC0415
- from scrapy.exceptions import ScrapyDeprecationWarning
+ from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",
diff --git a/scrapy/shell.py b/scrapy/shell.py
index bb39eccc3..c3a274e0d 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -16,6 +16,7 @@ from twisted.internet import defer, threads
from twisted.python import threadable
from w3lib.url import any_to_uri
+import scrapy
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
@@ -164,8 +165,6 @@ class Shell:
request: Request | None = None,
spider: Spider | None = None,
) -> None:
- import scrapy
-
self.vars["scrapy"] = scrapy
self.vars["crawler"] = self.crawler
self.vars["item"] = self.item_class()
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1d4d1527b..e4d0f4014 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -315,7 +315,7 @@ def _load_policy_class(
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
- return cast(type[ReferrerPolicy], load_object(policy))
+ return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
tokens = [token.strip() for token in policy.lower().split(",")]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index a722dd83b..8db00a1ca 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -55,7 +55,8 @@ class Spider(object_ref):
@property
def logger(self) -> SpiderLoggerAdapter:
- from scrapy.utils.log import SpiderLoggerAdapter
+ # circular import
+ from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415
logger = logging.getLogger(self.name)
return SpiderLoggerAdapter(logger, {"spider": self})
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 98e7b23c0..0cbd6b3cc 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -12,8 +12,6 @@ import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
-from twisted.python.failure import Failure
-
from scrapy.http import HtmlResponse, Request, Response
from scrapy.link import Link
from scrapy.linkextractors import LinkExtractor
@@ -26,6 +24,8 @@ from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
+ from twisted.python.failure import Failure
+
# typing.Self requires Python 3.11
from typing_extensions import Self
@@ -81,12 +81,14 @@ class Rule:
def _compile(self, spider: Spider) -> None:
# this replaces method names with methods and we can't express this in type hints
self.callback = cast("CallbackT", _get_method(self.callback, spider))
- self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
+ self.errback = cast(
+ "Callable[[Failure], Any]", _get_method(self.errback, spider)
+ )
self.process_links = cast(
- ProcessLinksT, _get_method(self.process_links, spider)
+ "ProcessLinksT", _get_method(self.process_links, spider)
)
self.process_request = cast(
- ProcessRequestT, _get_method(self.process_request, spider)
+ "ProcessRequestT", _get_method(self.process_request, spider)
)
@@ -141,13 +143,13 @@ class CrawlSpider(Spider):
for lnk in rule.link_extractor.extract_links(response)
if lnk not in seen
]
- for link in cast(ProcessLinksT, rule.process_links)(links):
+ for link in cast("ProcessLinksT", rule.process_links)(links):
seen.add(link)
request = self._build_request(rule_index, link)
- yield cast(ProcessRequestT, rule.process_request)(request, response)
+ yield cast("ProcessRequestT", rule.process_request)(request, response)
def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
- rule = self._rules[cast(int, response.meta["rule"])]
+ rule = self._rules[cast("int", response.meta["rule"])]
return self.parse_with_rules(
response,
cast("CallbackT", rule.callback),
@@ -156,9 +158,9 @@ class CrawlSpider(Spider):
)
def _errback(self, failure: Failure) -> Iterable[Any]:
- rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined]
+ rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined]
return self._handle_failure(
- failure, cast(Callable[[Failure], Any], rule.errback)
+ failure, cast("Callable[[Failure], Any]", rule.errback)
)
async def parse_with_rules(
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index e5548b9fa..957bfffd3 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,15 +1,16 @@
from __future__ import annotations
import warnings
-from collections.abc import AsyncIterator, Iterable
from typing import TYPE_CHECKING, Any, cast
-from scrapy import Request
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spiders import Spider
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator, Iterable
+
+ from scrapy import Request
from scrapy.http import Response
@@ -39,7 +40,7 @@ class InitSpider(Spider):
def start_requests(self) -> Iterable[Request]:
self._postinit_reqs: Iterable[Request] = super().start_requests()
- return cast(Iterable[Request], iterate_spider_output(self.init_request()))
+ return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
def initialized(self, response: Response | None = None) -> Any:
"""This method must be set as the callback of your last initialization
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 53cfeddd0..73f86bc71 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -3,7 +3,7 @@
def is_botocore_available() -> bool:
try:
- import botocore # noqa: F401
+ import botocore # noqa: F401,PLC0415
return True
except ImportError:
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 891cbb485..5a627fc83 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -153,7 +153,7 @@ def feed_process_params_from_cli(
suitable to be used as the FEEDS setting.
"""
valid_output_formats: Iterable[str] = without_none_values(
- cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS"))
+ cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS"))
).keys()
def check_valid_format(output_format: str) -> None:
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 7425543ff..644965cb5 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import code
from collections.abc import Callable
from functools import wraps
from typing import TYPE_CHECKING, Any
@@ -16,13 +17,13 @@ def _embed_ipython_shell(
) -> EmbedFuncT:
"""Start an IPython Shell"""
try:
- from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100
- from IPython.terminal.ipapp import load_default_config
+ from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
+ from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
except ImportError:
- from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100
+ from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100,PLC0415
InteractiveShellEmbed,
)
- from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef]
+ from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] # noqa: PLC0415
load_default_config,
)
@@ -46,7 +47,7 @@ def _embed_bpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a bpython shell"""
- import bpython
+ import bpython # noqa: PLC0415
@wraps(_embed_bpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@@ -59,7 +60,7 @@ def _embed_ptpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a ptpython shell"""
- import ptpython.repl # pylint: disable=import-error
+ import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error
@wraps(_embed_ptpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@@ -73,14 +74,12 @@ def _embed_standard_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a standard python shell"""
- import code
-
try: # readline module is only available on unix systems
- import readline
+ import readline # noqa: PLC0415
except ImportError:
pass
else:
- import rlcompleter # noqa: F401
+ import rlcompleter # noqa: F401,PLC0415
readline.parse_and_bind("tab:complete") # type: ignore[attr-defined]
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 3d0e0d3c7..ae7f627e1 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -32,7 +32,8 @@ class CaselessDict(dict):
__slots__ = ()
def __new__(cls, *args: Any, **kwargs: Any) -> Self:
- from scrapy.http.headers import Headers
+ # circular import
+ from scrapy.http.headers import Headers # noqa: PLC0415
if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
warnings.warn(
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 50f2af5b7..d161f30a6 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -12,12 +12,7 @@ from collections.abc import Awaitable, Coroutine, Iterable, Iterator
from functools import wraps
from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload
-from twisted.internet.defer import (
- Deferred,
- DeferredList,
- fail,
- succeed,
-)
+from twisted.internet.defer import Deferred, DeferredList, fail, succeed
from twisted.internet.task import Cooperator
from twisted.python import failure
@@ -414,7 +409,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2:
if not is_asyncio_available():
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
# that use asyncio, e.g. "await asyncio.sleep(1)"
- return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o))
+ return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o))
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
return Deferred.fromFuture(asyncio.ensure_future(o))
return o
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 20744a604..08e89dc0d 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -34,11 +34,11 @@ def _colorize(text: str, colorize: bool = True) -> str:
if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
return text
try:
- from pygments import highlight
+ from pygments import highlight # noqa: PLC0415
except ImportError:
return text
- from pygments.formatters import TerminalFormatter
- from pygments.lexers import PythonLexer
+ from pygments.formatters import TerminalFormatter # noqa: PLC0415
+ from pygments.lexers import PythonLexer # noqa: PLC0415
return highlight(text, PythonLexer(), TerminalFormatter())
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c646fc218..c70a0d327 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -147,12 +147,12 @@ class _StreamReader:
def _read_string(self, n: int = 65535) -> bytes:
s, e = self._ptr, self._ptr + n
self._ptr = e
- return cast(bytes, self._text)[s:e]
+ return cast("bytes", self._text)[s:e]
def _read_unicode(self, n: int = 65535) -> bytes:
s, e = self._ptr, self._ptr + n
self._ptr = e
- return cast(str, self._text)[s:e].encode("utf-8")
+ return cast("str", self._text)[s:e].encode("utf-8")
def csviter(
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 24e17ecb6..533906003 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -5,9 +5,9 @@ import pprint
import sys
from collections.abc import MutableMapping
from logging.config import dictConfig
-from types import TracebackType
from typing import TYPE_CHECKING, Any, Optional, cast
+from twisted.internet import asyncioreactor
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
@@ -16,6 +16,8 @@ from scrapy.settings import Settings, _SettingsKeyT
from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
+ from types import TracebackType
+
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatterResult
@@ -33,7 +35,7 @@ def failure_to_exc_info(
return (
failure.type,
failure.value,
- cast(Optional[TracebackType], failure.getTracebackObject()),
+ cast("Optional[TracebackType]", failure.getTracebackObject()),
)
return None
@@ -182,7 +184,7 @@ def log_scrapy_info(settings: Settings) -> None:
def log_reactor_info() -> None:
- from twisted.internet import asyncioreactor, reactor
+ from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
@@ -240,7 +242,7 @@ def logformatter_adapter(
message = logkws.get("msg") or ""
# NOTE: This also handles 'args' being an empty dict, that case doesn't
# play well in logger.log calls
- args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]
+ args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"]
return (level, message, args)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b7b436260..decd4473c 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -9,7 +9,6 @@ import os
import re
import warnings
from collections import deque
-from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from importlib import import_module
@@ -21,7 +20,7 @@ from scrapy.item import Item
from scrapy.utils.datatypes import LocalWeakReferencedCache
if TYPE_CHECKING:
- from collections.abc import Callable, Iterator
+ from collections.abc import Callable, Iterable, Iterator
from types import ModuleType
from scrapy import Spider
@@ -41,7 +40,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
if arg is None:
return []
if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
- return cast(Iterable[Any], arg)
+ return cast("Iterable[Any]", arg)
return [arg]
@@ -202,7 +201,7 @@ def build_from_crawler(
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return cast(T, instance)
+ return cast("T", instance)
@contextmanager
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ce4c01731..373ad652c 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
_T = TypeVar("_T")
-def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements
+def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor
@@ -39,7 +39,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
return reactor.listenTCP(0, factory, interface=host)
if len(portrange) == 1:
return reactor.listenTCP(portrange[0], factory, interface=host)
- for x in range(portrange[0], portrange[1] + 1): # noqa: RET503
+ for x in range(portrange[0], portrange[1] + 1):
try:
return reactor.listenTCP(x, factory, interface=host)
except error.CannotListenError:
@@ -60,7 +60,8 @@ class CallLaterOnce(Generic[_T]):
self._deferreds: list[Deferred] = []
def schedule(self, delay: float = 0) -> None:
- from scrapy.utils.asyncio import call_later
+ # circular import
+ from scrapy.utils.asyncio import call_later # noqa: PLC0415
if self._call is None:
self._call = call_later(delay, self)
@@ -70,7 +71,8 @@ class CallLaterOnce(Generic[_T]):
self._call.cancel()
def __call__(self) -> _T:
- from scrapy.utils.asyncio import call_later
+ # circular import
+ from scrapy.utils.asyncio import call_later # noqa: PLC0415
self._call = None
result = self._func(*self._a, **self._kw)
@@ -82,7 +84,8 @@ class CallLaterOnce(Generic[_T]):
return result
async def wait(self):
- from scrapy.utils.defer import maybe_deferred_to_future
+ # circular import
+ from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
d = Deferred()
self._deferreds.append(d)
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 76a6b7de6..b6550432c 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -91,7 +91,8 @@ def open_in_browser(
if "item name" not in response.body:
open_in_browser(response)
"""
- from scrapy.http import HtmlResponse, TextResponse
+ # circular imports
+ from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415
# XXX: this implementation is a bit dirty and could be improved
body = response.body
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index e0d9f4595..c0100ea62 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -24,7 +24,7 @@ class Sitemap:
xmlp = lxml.etree.XMLParser(
recover=True, remove_comments=True, resolve_entities=False
)
- self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320
+ self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
rt = self._root.tag
assert isinstance(rt, str)
self.type = rt.split("}", 1)[1] if "}" in rt else rt
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 3780ad23e..4c0764396 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import asyncio
import os
import warnings
+from ftplib import FTP
from importlib import import_module
from pathlib import Path
from posixpath import split
@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast
from unittest import TestCase, mock
from twisted.trial.unittest import SkipTest
+from twisted.web.client import Agent
+from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.deprecate import create_deprecated_class
@@ -59,7 +62,7 @@ def skip_if_no_boto() -> None:
def get_gcs_content_and_delete(
bucket: Any, path: str
) -> tuple[bytes, list[dict[str, str]], Any]:
- from google.cloud import storage
+ from google.cloud import storage # noqa: PLC0415
warnings.warn(
"The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
@@ -83,8 +86,6 @@ def get_ftp_content_and_delete(
password: str,
use_active_mode: bool = False,
) -> bytes:
- from ftplib import FTP
-
warnings.warn(
"The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
@@ -137,8 +138,6 @@ def get_crawler(
will be used to populate the crawler settings with a project level
priority.
"""
- from scrapy.crawler import CrawlerRunner
-
# When needed, useful settings can be added here, e.g. ones that prevent
# deprecation warnings.
settings: dict[str, Any] = {
@@ -192,7 +191,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
- from google.cloud.storage import Blob, Bucket, Client
+ from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
warnings.warn(
"The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
@@ -213,7 +212,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
from twisted.internet import reactor
- from twisted.web.client import Agent # imports twisted.internet.reactor
agent = Agent(reactor)
return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 10f764ab8..ce26b7f92 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -6,7 +6,6 @@ import warnings
from typing import TYPE_CHECKING, cast
from twisted.internet.defer import Deferred
-from twisted.internet.error import ProcessTerminated
from twisted.internet.protocol import ProcessProtocol
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -14,6 +13,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable
+ from twisted.internet.error import ProcessTerminated
from twisted.python.failure import Failure
@@ -55,7 +55,7 @@ class ProcessTest:
msg += "\n"
msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
raise RuntimeError(msg)
- return cast(int, pp.exitcode), pp.out, pp.err
+ return cast("int", pp.exitcode), pp.out, pp.err
class TestProcessProtocol(ProcessProtocol):
@@ -72,5 +72,5 @@ class TestProcessProtocol(ProcessProtocol):
self.err += data
def processEnded(self, status: Failure) -> None:
- self.exitcode = cast(ProcessTerminated, status.value).exitCode
+ self.exitcode = cast("ProcessTerminated", status.value).exitCode
self.deferred.callback(self)
diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
index 68239e651..f6fead718 100644
--- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
@@ -21,7 +21,7 @@ def log_task_exception(task: Task) -> None:
try:
task.result()
except Exception:
- logging.exception("Crawl task failed")
+ logging.exception("Crawl task failed") # noqa: LOG015
process = AsyncCrawlerProcess()
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index 18ec81fed..c8c73ba15 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import os
from pathlib import Path
+import pytest
+
from tests.test_commands import TestCommandBase, TestProjectBase
@@ -15,7 +17,16 @@ class TestGenspiderCommand(TestCommandBase):
assert self.call("genspider", "test_name", "test.com") == 0
assert Path(self.proj_mod_path, "spiders", "test_name.py").exists()
- def test_template(self, tplname="crawl"):
+ @pytest.mark.parametrize(
+ "tplname",
+ [
+ "basic",
+ "crawl",
+ "xmlfeed",
+ "csvfeed",
+ ],
+ )
+ def test_template(self, tplname: str) -> None:
args = [f"--template={tplname}"] if tplname else []
spname = "test_spider"
spmodule = f"{self.project_name}.spiders.{spname}"
@@ -35,15 +46,6 @@ class TestGenspiderCommand(TestCommandBase):
)
assert modify_time_after == modify_time_before
- def test_template_basic(self):
- self.test_template("basic")
-
- def test_template_csvfeed(self):
- self.test_template("csvfeed")
-
- def test_template_xmlfeed(self):
- self.test_template("xmlfeed")
-
def test_list(self):
assert self.call("genspider", "--list") == 0
@@ -57,7 +59,8 @@ class TestGenspiderCommand(TestCommandBase):
self.proj_mod_path, "spiders", f"{self.project_name}.py"
).exists()
- def test_same_filename_as_existing_spider(self, force=False):
+ @pytest.mark.parametrize("force", [True, False])
+ def test_same_filename_as_existing_spider(self, force: bool) -> None:
file_name = "example"
file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py")
assert self.call("genspider", file_name, "example.com") == 0
@@ -90,83 +93,60 @@ class TestGenspiderCommand(TestCommandBase):
file_contents_after = file_path.read_text(encoding="utf-8")
assert file_contents_after == file_contents_before
- def test_same_filename_as_existing_spider_force(self):
- self.test_same_filename_as_existing_spider(force=True)
-
- def test_url(self, url="test.com", domain="test.com"):
+ @pytest.mark.parametrize(
+ ("url", "domain"),
+ [
+ ("test.com", "test.com"),
+ ("https://test.com", "test.com"),
+ ],
+ )
+ def test_url(self, url: str, domain: str) -> None:
assert self.call("genspider", "--force", "test_name", url) == 0
- assert (
- self.find_in_file(
- Path(self.proj_mod_path, "spiders", "test_name.py"),
- r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
- ).group(1)
- == domain
- )
- assert (
- self.find_in_file(
- Path(self.proj_mod_path, "spiders", "test_name.py"),
- r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
- ).group(1)
- == f"https://{domain}"
- )
-
- def test_url_schema(self):
- self.test_url("https://test.com", "test.com")
-
- def test_template_start_urls(
- self, url="test.com", expected="https://test.com", template="basic"
- ):
+ m = self.find_in_file(
+ self.proj_mod_path / "spiders" / "test_name.py",
+ r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
+ )
+ assert m is not None
+ assert m.group(1) == domain
+ m = self.find_in_file(
+ self.proj_mod_path / "spiders" / "test_name.py",
+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
+ )
+ assert m is not None
+ assert m.group(1) == f"https://{domain}"
+
+ @pytest.mark.parametrize(
+ ("url", "expected", "template"),
+ [
+ # basic
+ ("https://test.com", "https://test.com", "basic"),
+ ("http://test.com", "http://test.com", "basic"),
+ ("http://test.com/other/path", "http://test.com/other/path", "basic"),
+ ("test.com/other/path", "https://test.com/other/path", "basic"),
+ # crawl
+ ("https://test.com", "https://test.com", "crawl"),
+ ("http://test.com", "http://test.com", "crawl"),
+ ("http://test.com/other/path", "http://test.com/other/path", "crawl"),
+ ("test.com/other/path", "https://test.com/other/path", "crawl"),
+ ("test.com", "https://test.com", "crawl"),
+ # xmlfeed
+ ("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
+ ("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"),
+ ("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
+ # csvfeed
+ ("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
+ ("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"),
+ ("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
+ ],
+ )
+ def test_template_start_urls(self, url: str, expected: str, template: str) -> None:
assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0
- assert (
- self.find_in_file(
- Path(self.proj_mod_path, "spiders", "test_name.py"),
- r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
- ).group(1)
- == expected
- )
-
- def test_genspider_basic_start_urls(self):
- self.test_template_start_urls("https://test.com", "https://test.com", "basic")
- self.test_template_start_urls("http://test.com", "http://test.com", "basic")
- self.test_template_start_urls(
- "http://test.com/other/path", "http://test.com/other/path", "basic"
- )
- self.test_template_start_urls(
- "test.com/other/path", "https://test.com/other/path", "basic"
- )
-
- def test_genspider_crawl_start_urls(self):
- self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
- self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
- self.test_template_start_urls(
- "http://test.com/other/path", "http://test.com/other/path", "crawl"
- )
- self.test_template_start_urls(
- "test.com/other/path", "https://test.com/other/path", "crawl"
- )
- self.test_template_start_urls("test.com", "https://test.com", "crawl")
-
- def test_genspider_xmlfeed_start_urls(self):
- self.test_template_start_urls(
- "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
- )
- self.test_template_start_urls(
- "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
- )
- self.test_template_start_urls(
- "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
- )
-
- def test_genspider_csvfeed_start_urls(self):
- self.test_template_start_urls(
- "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
- )
- self.test_template_start_urls(
- "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
- )
- self.test_template_start_urls(
- "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
+ m = self.find_in_file(
+ self.proj_mod_path / "spiders" / "test_name.py",
+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
)
+ assert m is not None
+ assert m.group(1) == expected
class TestGenspiderStandaloneCommand(TestProjectBase):
@@ -174,7 +154,8 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
self.call("genspider", "example", "example.com")
assert Path(self.temp_path, "example.py").exists()
- def test_same_name_as_existing_file(self, force=False):
+ @pytest.mark.parametrize("force", [True, False])
+ def test_same_name_as_existing_file(self, force: bool) -> None:
file_name = "example"
file_path = Path(self.temp_path, file_name + ".py")
p, out, err = self.proc("genspider", file_name, "example.com")
@@ -203,6 +184,3 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
assert modify_time_after == modify_time_before
file_contents_after = file_path.read_text(encoding="utf-8")
assert file_contents_after == file_contents_before
-
- def test_same_name_as_existing_file_force(self):
- self.test_same_name_as_existing_file(force=True)
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 0c12a59f7..89670feb3 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import inspect
import platform
import sys
@@ -204,8 +205,6 @@ class MySpider(scrapy.Spider):
"TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
],
)
- import asyncio
-
if sys.platform != "win32":
loop = asyncio.new_event_loop()
else:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index a749de9fe..9ea893f05 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,7 +9,7 @@ from io import StringIO
from pathlib import Path
from shutil import rmtree
from tempfile import TemporaryFile, mkdtemp
-from typing import TYPE_CHECKING, Any
+from typing import Any
from unittest import mock
import pytest
@@ -22,9 +22,6 @@ from scrapy.utils.python import to_unicode
from scrapy.utils.reactor import _asyncio_reactor_path
from scrapy.utils.test import get_testenv
-if TYPE_CHECKING:
- import os
-
class EmptyCommand(ScrapyCommand):
def short_desc(self) -> str:
@@ -110,10 +107,11 @@ class TestProjectBase:
return p, to_unicode(stdout), to_unicode(stderr)
- def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None:
+ @staticmethod
+ def find_in_file(filename: Path, regex: str) -> re.Match | None:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
- with Path(filename).open("r", encoding="utf-8") as f:
+ with filename.open("r", encoding="utf-8") as f:
for line in f:
match = pattern.search(line)
if match is not None:
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 464320602..0a674d638 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -4,7 +4,7 @@ import shutil
import warnings
from pathlib import Path
from tempfile import mkdtemp
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
import OpenSSL.SSL
import pytest
@@ -14,7 +14,6 @@ from twisted.trial import unittest
from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
-from twisted.web.iweb import IBodyProducer
from scrapy.core.downloader import Slot
from scrapy.core.downloader.contextfactory import (
@@ -29,6 +28,9 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
from tests.mockserver import PayloadResource, ssl_context_factory
+if TYPE_CHECKING:
+ from twisted.web.iweb import IBodyProducer
+
class TestSlot:
def test_repr(self):
@@ -78,12 +80,12 @@ class TestContextFactoryBase(unittest.TestCase):
agent = Agent(reactor, contextFactory=client_context_factory)
body_producer = _RequestBodyProducer(body.encode()) if body else None
response: TxResponse = cast(
- TxResponse,
+ "TxResponse",
await maybe_deferred_to_future(
agent.request(
b"GET",
url.encode(),
- bodyProducer=cast(IBodyProducer, body_producer),
+ bodyProducer=cast("IBodyProducer", body_producer),
)
),
)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 42f9899f9..ef998ab4c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,7 +5,7 @@ import logging
from ipaddress import IPv4Address
from socket import gethostbyname
from typing import TYPE_CHECKING, Any
-from urllib.parse import urlparse
+from urllib.parse import urlencode, urlparse
import pytest
from testfixtures import LogCapture
@@ -20,6 +20,7 @@ from scrapy.exceptions import CloseSpider, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import NON_EXISTING_RESOLVABLE
@@ -255,8 +256,6 @@ class TestCrawl(TestCase):
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
- from urllib.parse import urlencode
-
query = urlencode(
{
"raw": """\
@@ -339,8 +338,6 @@ with multiples lines
@inlineCallbacks
def test_engine_status(self):
- from scrapy.utils.engine import get_engine_status
-
est = []
def cb(response):
@@ -357,8 +354,6 @@ with multiples lines
@inlineCallbacks
def test_format_engine_status(self):
- from scrapy.utils.engine import format_engine_status
-
est = []
def cb(response):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 56cb21650..e44bf1a46 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -526,10 +526,10 @@ class TestCrawlerLogging:
crawler = get_crawler(MySpider)
assert get_scrapy_root_handler().level == logging.INFO
info_count = crawler.stats.get_value("log_count/INFO")
- logging.debug("debug message")
- logging.info("info message")
- logging.warning("warning message")
- logging.error("error message")
+ logging.debug("debug message") # noqa: LOG015
+ logging.info("info message") # noqa: LOG015
+ logging.warning("warning message") # noqa: LOG015
+ logging.error("error message") # noqa: LOG015
logged = log_file.read_text(encoding="utf-8")
@@ -556,7 +556,7 @@ class TestCrawlerLogging:
configure_logging()
get_crawler(MySpider)
- logging.debug("debug message")
+ logging.debug("debug message") # noqa: LOG015
logged = log_file.read_text(encoding="utf-8")
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 77077e6e5..2dc6817a5 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -15,10 +15,7 @@ from twisted.web.http import H2_ENABLED
from scrapy.http import Request
from scrapy.spiders import Spider
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver import ssl_context_factory
@@ -46,7 +43,9 @@ class H2DownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
# the import can fail when H2_ENABLED is False
- from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+ from scrapy.core.downloader.handlers.http2 import ( # noqa: PLC0415
+ H2DownloadHandler,
+ )
return H2DownloadHandler
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0775ae1ee..e73bfbccf 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -13,7 +13,7 @@ from unittest import mock
import pytest
from twisted.cred import checkers, credentials, portal
from twisted.internet.defer import inlineCallbacks
-from twisted.protocols.ftp import FTPFactory, FTPRealm
+from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealm
from twisted.trial import unittest
from w3lib.url import path_to_file_uri
@@ -27,10 +27,7 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@@ -184,7 +181,7 @@ class TestS3:
@contextlib.contextmanager
def _mocked_date(self, date):
try:
- import botocore.auth # noqa: F401
+ import botocore.auth # noqa: F401,PLC0415
except ImportError:
yield
else:
@@ -443,7 +440,6 @@ class TestFTP(TestFTPBase):
pytest.skip(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
- from twisted.protocols.ftp import ConnectionLost
meta = dict(self.req_meta)
meta.update({"ftp_password": "invalid"})
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 3c26b242f..0b3941c09 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -50,6 +50,23 @@ FORMAT = {
}
+def _skip_if_no_br() -> None:
+ try:
+ try:
+ import brotli # noqa: F401,PLC0415
+ except ImportError:
+ import brotlicffi # noqa: F401,PLC0415
+ except ImportError:
+ pytest.skip("no brotli support")
+
+
+def _skip_if_no_zstd() -> None:
+ try:
+ import zstandard # noqa: F401,PLC0415
+ except ImportError:
+ pytest.skip("no zstd support (zstandard)")
+
+
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
@@ -123,13 +140,8 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
response = self._getresponse("br")
request = response.request
assert response.headers["Content-Encoding"] == b"br"
@@ -143,11 +155,11 @@ class TestHttpCompression:
def test_process_response_br_unsupported(self):
try:
try:
- import brotli # noqa: F401
+ import brotli # noqa: F401,PLC0415
pytest.skip("Requires not having brotli support")
except ImportError:
- import brotlicffi # noqa: F401
+ import brotlicffi # noqa: F401,PLC0415
pytest.skip("Requires not having brotli support")
except ImportError:
@@ -176,10 +188,8 @@ class TestHttpCompression:
assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
def test_process_response_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
@@ -198,7 +208,7 @@ class TestHttpCompression:
def test_process_response_zstd_unsupported(self):
try:
- import zstandard # noqa: F401
+ import zstandard # noqa: F401,PLC0415
pytest.skip("Requires not having zstandard support")
except ImportError:
@@ -513,13 +523,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_setting_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_compression_bomb_setting("br")
def test_compression_bomb_setting_deflate(self):
@@ -529,10 +534,8 @@ class TestHttpCompression:
self._test_compression_bomb_setting("gzip")
def test_compression_bomb_setting_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_compression_bomb_setting("zstd")
def _test_compression_bomb_spider_attr(self, compression_id):
@@ -549,13 +552,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_spider_attr_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_compression_bomb_spider_attr("br")
def test_compression_bomb_spider_attr_deflate(self):
@@ -565,10 +563,8 @@ class TestHttpCompression:
self._test_compression_bomb_spider_attr("gzip")
def test_compression_bomb_spider_attr_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
@@ -583,13 +579,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_request_meta_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_compression_bomb_request_meta("br")
def test_compression_bomb_request_meta_deflate(self):
@@ -599,10 +590,8 @@ class TestHttpCompression:
self._test_compression_bomb_request_meta("gzip")
def test_compression_bomb_request_meta_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_compression_bomb_request_meta("zstd")
def _test_download_warnsize_setting(self, compression_id):
@@ -632,13 +621,8 @@ class TestHttpCompression:
)
def test_download_warnsize_setting_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_download_warnsize_setting("br")
def test_download_warnsize_setting_deflate(self):
@@ -648,10 +632,8 @@ class TestHttpCompression:
self._test_download_warnsize_setting("gzip")
def test_download_warnsize_setting_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_download_warnsize_setting("zstd")
def _test_download_warnsize_spider_attr(self, compression_id):
@@ -683,13 +665,8 @@ class TestHttpCompression:
)
def test_download_warnsize_spider_attr_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_download_warnsize_spider_attr("br")
def test_download_warnsize_spider_attr_deflate(self):
@@ -699,10 +676,8 @@ class TestHttpCompression:
self._test_download_warnsize_spider_attr("gzip")
def test_download_warnsize_spider_attr_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_download_warnsize_spider_attr("zstd")
def _test_download_warnsize_request_meta(self, compression_id):
@@ -732,13 +707,8 @@ class TestHttpCompression:
)
def test_download_warnsize_request_meta_br(self):
- try:
- try:
- import brotli # noqa: F401
- except ImportError:
- import brotlicffi # noqa: F401
- except ImportError:
- pytest.skip("no brotli")
+ _skip_if_no_br()
+
self._test_download_warnsize_request_meta("br")
def test_download_warnsize_request_meta_deflate(self):
@@ -748,8 +718,6 @@ class TestHttpCompression:
self._test_download_warnsize_request_meta("gzip")
def test_download_warnsize_request_meta_zstd(self):
- try:
- import zstandard # noqa: F401
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ _skip_if_no_zstd()
+
self._test_download_warnsize_request_meta("zstd")
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ce1b7952e..9085af18e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,7 +5,9 @@ import csv
import gzip
import json
import lzma
+import marshal
import os
+import pickle
import random
import shutil
import string
@@ -25,6 +27,7 @@ from urllib.request import pathname2url
import lxml.etree
import pytest
+from packaging.version import Version
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
@@ -79,7 +82,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
- from google.cloud.storage import Blob, Bucket, Client
+ from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
client_mock = mock.create_autospec(Client)
@@ -507,7 +510,7 @@ class TestS3FeedStorage(unittest.TestCase):
class TestGCSFeedStorage(unittest.TestCase):
def test_parse_settings(self):
try:
- from google.cloud.storage import Client # noqa: F401
+ from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@@ -521,7 +524,7 @@ class TestGCSFeedStorage(unittest.TestCase):
def test_parse_empty_acl(self):
try:
- from google.cloud.storage import Client # noqa: F401
+ from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@@ -538,7 +541,7 @@ class TestGCSFeedStorage(unittest.TestCase):
@deferred_f_from_coro_f
async def test_store(self):
try:
- from google.cloud.storage import Client # noqa: F401
+ from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@@ -976,7 +979,6 @@ class TestFeedExport(TestFeedExportBase):
)
data = await self.exported_data(items, settings)
expected = [{k: v for k, v in row.items() if v} for row in rows]
- import pickle
result = self._load_until_eof(data["pickle"], load_func=pickle.load)
assert result == expected
@@ -997,7 +999,6 @@ class TestFeedExport(TestFeedExportBase):
)
data = await self.exported_data(items, settings)
expected = [{k: v for k, v in row.items() if v} for row in rows]
- import marshal
result = self._load_until_eof(data["marshal"], load_func=marshal.load)
assert result == expected
@@ -2300,9 +2301,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
@deferred_f_from_coro_f
async def test_exports_compatibility_with_postproc(self):
- import marshal
- import pickle
-
filename_to_expected = {
self._named_tempfile("csv"): b"foo\r\nbar\r\n",
self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
@@ -2484,7 +2482,6 @@ class TestBatchDeliveries(TestFeedExportBase):
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
- import pickle
for batch in data["pickle"]:
got_batch = self._load_until_eof(batch, load_func=pickle.load)
@@ -2505,7 +2502,6 @@ class TestBatchDeliveries(TestFeedExportBase):
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
- import marshal
for batch in data["marshal"]:
got_batch = self._load_until_eof(batch, load_func=marshal.load)
@@ -2714,9 +2710,8 @@ class TestBatchDeliveries(TestFeedExportBase):
stubs = []
def open(self, *args, **kwargs):
- from botocore import __version__ as botocore_version
- from botocore.stub import ANY, Stubber
- from packaging.version import Version
+ from botocore import __version__ as botocore_version # noqa: PLC0415
+ from botocore.stub import ANY, Stubber # noqa: PLC0415
expected_params = {
"Body": ANY,
@@ -2912,10 +2907,9 @@ class TestURIParams(ABC):
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
else:
crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ feed_exporter = crawler.get_extension(FeedExporter)
return crawler, feed_exporter
def test_default(self):
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 80edd50d6..eb9081e7e 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -243,7 +243,7 @@ class TestHttps2ClientProtocol(TestCase):
self.conn_closed_deferred = Deferred()
- from scrapy.core.http2.protocol import H2ClientFactory
+ from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
client_endpoint = SSL4ClientEndpoint(
@@ -445,7 +445,7 @@ class TestHttps2ClientProtocol(TestCase):
def test_received_dataloss_response(self):
"""In case when value of Header Content-Length != len(Received Data)
ProtocolError is raised"""
- from h2.exceptions import InvalidBodyLengthError
+ from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415
request = Request(url=self.get_url("/dataloss"))
with pytest.raises(ResponseFailed) as exc_info:
@@ -525,7 +525,7 @@ class TestHttps2ClientProtocol(TestCase):
def assert_inactive_stream(failure):
assert failure.check(ResponseFailed) is not None
- from scrapy.core.http2.stream import InactiveStreamClosed
+ from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415
assert any(
isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
@@ -593,7 +593,7 @@ class TestHttps2ClientProtocol(TestCase):
assert str(response.ip_address) == "127.0.0.1"
async def _check_invalid_netloc(self, url: str) -> None:
- from scrapy.core.http2.stream import InvalidHostname
+ from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415
request = Request(url)
with pytest.raises(InvalidHostname) as exc_info:
@@ -630,7 +630,7 @@ class TestHttps2ClientProtocol(TestCase):
yield self.make_request_dfd(request)
for err in exc_info.value.reasons:
- from scrapy.core.http2.protocol import H2ClientProtocol
+ from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415
if isinstance(err, TimeoutError):
assert (
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 6bf0b8e3f..90a693bfa 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1441,7 +1441,7 @@ class TestXmlRpcRequest(TestRequest):
)
assert r.method == "POST"
assert r.encoding == kwargs.get("encoding", "utf-8")
- assert r.dont_filter, True
+ assert r.dont_filter
def test_xmlrpc_dumps(self):
self._test_request(params=("value",))
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 047f8c610..682428954 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -110,7 +110,7 @@ class TestLogFormatter:
assert logkws["level"] == unsupported_value
with pytest.raises(TypeError):
- logging.log(logkws["level"], "message")
+ logging.log(logkws["level"], "message") # noqa: LOG015
def test_dropitem_custom_log_level(self):
item = {}
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 808fde23d..3d84c7e71 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -5,6 +5,7 @@ import time
import warnings
from abc import ABC, abstractmethod
from datetime import datetime
+from ftplib import FTP
from io import BytesIO
from pathlib import Path
from posixpath import split
@@ -29,9 +30,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
-from scrapy.utils.test import (
- get_crawler,
-)
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockFTPServer
from .test_pipeline_media import _mocked_download_func
@@ -40,7 +39,7 @@ from .test_pipeline_media import _mocked_download_func
def get_gcs_content_and_delete(
bucket: Any, path: str
) -> tuple[bytes, list[dict[str, str]], Any]:
- from google.cloud import storage
+ from google.cloud import storage # noqa: PLC0415
client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
bucket = client.get_bucket(bucket)
@@ -59,8 +58,6 @@ def get_ftp_content_and_delete(
password: str,
use_active_mode: bool = False,
) -> bytes:
- from ftplib import FTP
-
ftp = FTP()
ftp.connect(host, port)
ftp.login(username, password)
@@ -553,7 +550,7 @@ class TestS3FilesStore(unittest.TestCase):
content_type = "image/png"
store = S3FilesStore(uri)
- from botocore.stub import Stubber
+ from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
stub.add_response(
@@ -591,7 +588,7 @@ class TestS3FilesStore(unittest.TestCase):
last_modified = datetime(2019, 12, 1)
store = S3FilesStore(uri)
- from botocore.stub import Stubber
+ from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
stub.add_response(
@@ -650,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase):
already uploaded files.
"""
try:
- import google.cloud.storage # noqa: F401
+ import google.cloud.storage # noqa: F401,PLC0415
except ModuleNotFoundError:
pytest.skip("google-cloud-storage is not installed")
with (
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 801f88b5a..8875dc04f 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -74,7 +74,7 @@ class TestProxyConnect(TestCase):
def setUp(self):
try:
- import mitmproxy # noqa: F401
+ import mitmproxy # noqa: F401,PLC0415
except ImportError:
pytest.skip("mitmproxy is not installed")
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 221ccabe6..6a24d2e90 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,21 +1,19 @@
import pytest
-from scrapy.robotstxt import decode_robotstxt
+from scrapy.robotstxt import (
+ ProtegoRobotParser,
+ PythonRobotParser,
+ RerpRobotParser,
+ decode_robotstxt,
+)
def rerp_available():
# check if robotexclusionrulesparser is installed
try:
- from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401
- except ImportError:
- return False
- return True
-
-
-def protego_available():
- # check if protego parser is installed
- try:
- from protego import Protego # noqa: F401
+ from robotexclusionrulesparser import ( # noqa: PLC0415
+ RobotExclusionRulesParser, # noqa: F401
+ )
except ImportError:
return False
return True
@@ -134,8 +132,6 @@ class TestDecodeRobotsTxt:
class TestPythonRobotParser(BaseRobotParserTest):
def setup_method(self):
- from scrapy.robotstxt import PythonRobotParser
-
super()._setUp(PythonRobotParser)
def test_length_based_precedence(self):
@@ -150,19 +146,14 @@ class TestPythonRobotParser(BaseRobotParserTest):
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
class TestRerpRobotParser(BaseRobotParserTest):
def setup_method(self):
- from scrapy.robotstxt import RerpRobotParser
-
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
pytest.skip("Rerp does not support length based directives precedence.")
[email protected](not protego_available(), reason="Protego parser is not installed")
class TestProtegoRobotParser(BaseRobotParserTest):
def setup_method(self):
- from scrapy.robotstxt import ProtegoRobotParser
-
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py
index b1956ee5c..fa6d11123 100644
--- a/tests/test_scrapy__getattr__.py
+++ b/tests/test_scrapy__getattr__.py
@@ -3,7 +3,9 @@ import warnings
def test_deprecated_twisted_version():
with warnings.catch_warnings(record=True) as warns:
- from scrapy import twisted_version # pylint: disable=no-name-in-module
+ from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module
+ twisted_version,
+ )
assert twisted_version is not None
assert isinstance(twisted_version, tuple)
@@ -15,7 +17,7 @@ def test_deprecated_twisted_version():
def test_deprecated_concurrent_requests_per_ip_attribute():
with warnings.catch_warnings(record=True) as warns:
- from scrapy.settings.default_settings import (
+ from scrapy.settings.default_settings import ( # noqa: PLC0415
CONCURRENT_REQUESTS_PER_IP,
)
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 85dc0679e..76609ac52 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -6,6 +6,7 @@ from unittest import mock
import pytest
+from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.settings import (
SETTINGS_PRIORITIES,
BaseSettings,
@@ -13,6 +14,8 @@ from scrapy.settings import (
SettingsAttribute,
get_settings_priority,
)
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
from . import default_settings
@@ -447,10 +450,6 @@ class TestSettings:
assert mydict["key"] == "val"
def test_passing_objects_as_values(self):
- from scrapy.core.downloader.handlers.file import FileDownloadHandler
- from scrapy.utils.misc import build_from_crawler
- from scrapy.utils.test import get_crawler
-
class TestPipeline:
def process_item(self, i, s):
return i
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 7c4b17144..e2843eca4 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,7 +1,9 @@
from __future__ import annotations
import gzip
+import re
import warnings
+from datetime import datetime
from io import BytesIO
from logging import ERROR, WARNING
from pathlib import Path
@@ -297,8 +299,6 @@ class TestCrawlSpider(TestSpider):
)
class _CrawlSpider(self.spider_class):
- import re
-
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@@ -635,8 +635,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
- from datetime import datetime
-
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
@@ -706,8 +704,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
- from datetime import datetime
-
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b17332182..895df1804 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlparse
import pytest
@@ -33,6 +33,9 @@ from scrapy.spidermiddlewares.referer import (
)
from scrapy.spiders import Spider
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
@pytest.fixture
def spider() -> Spider:
@@ -971,7 +974,7 @@ class TestPolicyHeaderPrecedence004(
class TestReferrerOnRedirect(TestRefererMiddleware):
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
- scenarii: list[
+ scenarii: Sequence[
tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]
] = [ # type: ignore[assignment]
(
@@ -1041,7 +1044,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
request.url, headers={"Location": url}, status=status
)
request = cast(
- Request, redirectmw.process_response(request, response, spider)
+ "Request", redirectmw.process_response(request, response, spider)
)
referrermw.request_scheduled(request, spider)
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 66a6f5318..b7497c608 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -10,26 +10,26 @@ def test_version_info():
def test_request_shortcut():
- from scrapy.http import FormRequest, Request
+ from scrapy.http import FormRequest, Request # noqa: PLC0415
assert scrapy.Request is Request
assert scrapy.FormRequest is FormRequest
def test_spider_shortcut():
- from scrapy.spiders import Spider
+ from scrapy.spiders import Spider # noqa: PLC0415
assert scrapy.Spider is Spider
def test_selector_shortcut():
- from scrapy.selector import Selector
+ from scrapy.selector import Selector # noqa: PLC0415
assert scrapy.Selector is Selector
def test_item_shortcut():
- from scrapy.item import Field, Item
+ from scrapy.item import Field, Item # noqa: PLC0415
assert scrapy.Item is Item
assert scrapy.Field is Field
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 20251ca59..3b1e626c9 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -1,3 +1,4 @@
+import builtins
from io import StringIO
from unittest import mock
@@ -74,8 +75,6 @@ def test_pformat_windows(isatty, version, terminal_processing):
def test_pformat_no_pygments(isatty):
isatty.return_value = True
- import builtins
-
real_import = builtins.__import__
def mock_import(name, globals, locals, fromlist, level):
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 8600faa8d..5efb29a49 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -24,7 +24,7 @@ def test_iterate_spider_output():
def test_iter_spider_classes():
- import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
+ import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-self
it = iter_spider_classes(tests.test_utils_spider)
assert set(it) == {MySpider1, MySpider2}
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index daf09f945..bcbbe74a3 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,4 +1,5 @@
import warnings
+from importlib import import_module
import pytest
@@ -461,8 +462,6 @@ def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
- from importlib import import_module
-
getattr(import_module("scrapy.utils.url"), obj_name)
assert isinstance(warns[0].message, Warning)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 569f4f639..dd6d7939e 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -18,11 +18,10 @@ from twisted.internet.testing import StringTransport
from twisted.protocols.policies import WrappingFactory
from twisted.trial import unittest
from twisted.web import resource, server, static, util
+from twisted.web.client import _makeGetterFactory
from scrapy.core.downloader import webclient as client
-from scrapy.core.downloader.contextfactory import (
- ScrapyClientContextFactory,
-)
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.http import Headers, Request
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes, to_unicode
@@ -51,8 +50,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
f.deferred.addCallback(response_transform or (lambda r: r.body))
return f
- from twisted.web.client import _makeGetterFactory
-
return _makeGetterFactory(
to_bytes(url),
_clientfactory,
diff --git a/tox.ini b/tox.ini
index 85935b011..1c1d7c0ec 100644
--- a/tox.ini
+++ b/tox.ini
@@ -46,17 +46,21 @@ install_command =
[testenv:typing]
basepython = python3.9
deps =
- mypy==1.14.0
- typing-extensions==4.12.2
- types-lxml==2024.12.13
- types-Pygments==2.18.0.20240506
- botocore-stubs==1.35.90
- boto3-stubs[s3]==1.35.90
+ mypy==1.16.1
+ typing-extensions==4.14.1
+ types-defusedxml==0.7.0.20250516
+ types-lxml==2025.3.30
+ types-pexpect==4.9.0.20250516
+ types-Pygments==2.19.0.20250516
+ botocore-stubs==1.38.46
+ boto3-stubs[s3]==1.39.3
+ itemadapter==0.11.0
+ Protego==0.5.0
+ w3lib==2.3.1
attrs >= 18.2.0
Pillow >= 10.3.0
pyOpenSSL >= 24.2.1
pytest >= 8.2.0
- w3lib >= 2.2.0
commands =
mypy {posargs:scrapy tests}
@@ -80,14 +84,14 @@ commands =
basepython = python3
deps =
{[testenv:extra-deps]deps}
- pylint==3.3.3
+ pylint==3.3.7
commands =
pylint conftest.py docs extras scrapy tests
[testenv:twinecheck]
basepython = python3
deps =
- twine==6.0.1
+ twine==6.1.0
build==1.2.2.post1
commands =
python -m build --sdist