Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add flake8-pyi rules to ruff.

Andrey Rakhmatullinc2832ed1316b
diff --git a/pyproject.toml b/pyproject.toml
index b3dd9f057..973d43162 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -244,6 +244,8 @@ extend-select = [
"PGH",
# flake8-pie
"PIE",
+ # flake8-pyi
+ "PYI",
# flake8-quotes
"Q",
# flake8-return
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index ea9f47d69..9411cff14 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING, TypeVar
+from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred, maybeDeferred
@@ -31,8 +31,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-_T = TypeVar("_T")
-
class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY: int = 1000
diff --git a/scrapy/link.py b/scrapy/link.py
index 4bdbc1823..1a569f892 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -5,8 +5,6 @@ For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
-from typing import Any
-
class Link:
"""Link objects represent an extracted link by the LinkExtractor.
@@ -39,7 +37,7 @@ class Link:
self.fragment: str = fragment
self.nofollow: bool = nofollow
- def __eq__(self, other: Any) -> bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, Link):
raise NotImplementedError
return (
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index b16f1cb84..5438b8522 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -5,16 +5,7 @@ import logging
import warnings
from abc import ABC, abstractmethod
from collections import defaultdict
-from typing import (
- TYPE_CHECKING,
- Any,
- Literal,
- NoReturn,
- TypedDict,
- TypeVar,
- Union,
- cast,
-)
+from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypedDict, Union, cast
from twisted import version as twisted_version
from twisted.internet.defer import Deferred, DeferredList
@@ -41,8 +32,6 @@ if TYPE_CHECKING:
from scrapy.http import Response
from scrapy.utils.request import RequestFingerprinter
-_T = TypeVar("_T")
-
class FileInfo(TypedDict):
url: str
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index a7ad4544d..76a6b7de6 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -53,7 +53,7 @@ def get_meta_refresh(
return _metaref_cache[response]
-def response_status_message(status: bytes | float | int | str) -> str:
+def response_status_message(status: bytes | float | str) -> str:
"""Return status code plus status text descriptive message"""
status_int = int(status)
message = http.RESPONSES.get(status_int, "Unknown Status")
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 387bc7c20..8bd1480ad 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1,9 +1,9 @@
from __future__ import annotations
-import collections
import shutil
import tempfile
import unittest
+from typing import Any, NamedTuple
from twisted.internet import defer
from twisted.trial.unittest import TestCase
@@ -18,8 +18,13 @@ from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-MockEngine = collections.namedtuple("MockEngine", ["downloader"])
-MockSlot = collections.namedtuple("MockSlot", ["active"])
+
+class MockEngine(NamedTuple):
+ downloader: MockDownloader
+
+
+class MockSlot(NamedTuple):
+ active: list[Any]
class MockDownloader: