Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add flake8-pie rules to ruff.

Andrey Rakhmatullin93644f2c30ee
diff --git a/pyproject.toml b/pyproject.toml
index 131684724..b3dd9f057 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -242,6 +242,8 @@ extend-select = [
"LOG",
# pygrep-hooks
"PGH",
+ # flake8-pie
+ "PIE",
# flake8-quotes
"Q",
# flake8-return
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 4ca015f5e..3047ae396 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -61,7 +61,6 @@ class Command(ScrapyCommand):
"""You can use this function to update the Scrapy objects that will be
available in the shell
"""
- pass
def run(self, args: list[str], opts: Namespace) -> None:
url = args[0] if args else None
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 823677175..fcc94879a 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
:param spider: the spider object for the current crawl
:type spider: :class:`~scrapy.spiders.Spider`
"""
- pass
def close(self, reason: str) -> Deferred[None] | None:
"""
@@ -91,7 +90,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
:param reason: a string which describes the reason why the spider was closed
:type reason: :class:`str`
"""
- pass
@abstractmethod
def has_pending_requests(self) -> bool:
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 7b8eea135..caf69daf4 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -50,7 +50,6 @@ class BaseDupeFilter:
def log(self, request: Request, spider: Spider) -> None:
"""Log that a request has been filtered"""
- pass
class RFPDupeFilter(BaseDupeFilter):
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index e7ecdbe0c..96566ba86 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -13,8 +13,6 @@ from typing import Any
class NotConfigured(Exception):
"""Indicates a missing configuration situation"""
- pass
-
class _InvalidOutput(TypeError):
"""
@@ -22,8 +20,6 @@ class _InvalidOutput(TypeError):
Internal and undocumented, it should not be raised or caught by user code.
"""
- pass
-
# HTTP and crawling
@@ -35,8 +31,6 @@ class IgnoreRequest(Exception):
class DontCloseSpider(Exception):
"""Request the spider not to be closed yet"""
- pass
-
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
@@ -64,14 +58,10 @@ class StopDownload(Exception):
class DropItem(Exception):
"""Drop item from the item pipeline"""
- pass
-
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
- pass
-
# Commands
@@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning):
DeprecationWarning is silenced on Python 2.7+
"""
- pass
-
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
-
- pass
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 844969c6d..417c9c142 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta):
:param robotstxt_body: content of a robots.txt_ file.
:type robotstxt_body: bytes
"""
- pass
@abstractmethod
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
@@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta):
:param user_agent: User agent
:type user_agent: str or bytes
"""
- pass
class PythonRobotParser(RobotParser):
diff --git a/tests/spiders.py b/tests/spiders.py
index 0180cf757..3c44d7da5 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider):
dupe_factor = 3
def start_requests(self):
- for i in range(0, self.distinct_urls):
- for j in range(0, self.dupe_factor):
+ for i in range(self.distinct_urls):
+ for j in range(self.dupe_factor):
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b0cb92d12..743889234 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -178,27 +178,23 @@ class TestSpider(Spider):
"""method with no url
@returns items 1 1
"""
- pass
def custom_form(self, response):
"""
@url http://scrapy.org
@custom_form
"""
- pass
def invalid_regex(self, response):
"""method with invalid regex
@ Scrapy is awsome
"""
- pass
def invalid_regex_with_valid_contract(self, response):
"""method with invalid regex
@ scrapy is awsome
@url http://scrapy.org
"""
- pass
def returns_request_meta(self, response):
"""method which returns request
@@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider):
"""
@custom_success_contract
"""
- pass
class CustomContractFailSpider(Spider):
@@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider):
"""
@custom_fail_contract
"""
- pass
class InheritsTestSpider(TestSpider):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index a010865ef..c99f19b03 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase):
spider = spider or self.spider
middleware = middleware or self.mw
- for i in range(0, max_retry_times):
+ for i in range(max_retry_times):
req = middleware.process_exception(req, exception, spider)
assert isinstance(req, Request)
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 9fd680e9f..8c897c223 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase):
console = TelnetConsole(crawler)
# This function has some side effects we don't need for this test
- console._get_telnet_vars = lambda: {}
+ console._get_telnet_vars = dict
console.start_listening()
protocol = console.protocol()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 2be5e09bc..a6c5f0a94 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass(
class FilesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
- file_urls: list[str] = attr.ib(default=lambda: [])
- files: list[dict[str, str]] = attr.ib(default=lambda: [])
+ file_urls: list[str] = attr.ib(default=list)
+ files: list[dict[str, str]] = attr.ib(default=list)
# overridden fields
- custom_file_urls: list[str] = attr.ib(default=lambda: [])
- custom_files: list[dict[str, str]] = attr.ib(default=lambda: [])
+ custom_file_urls: list[str] = attr.ib(default=list)
+ custom_files: list[dict[str, str]] = attr.ib(default=list)
class FilesPipelineTestCaseFieldsAttrsItem(
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 3ffef4102..3d049843a 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass(
class ImagesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
- image_urls: list[str] = attr.ib(default=lambda: [])
- images: list[dict[str, str]] = attr.ib(default=lambda: [])
+ image_urls: list[str] = attr.ib(default=list)
+ images: list[dict[str, str]] = attr.ib(default=list)
# overridden fields
- custom_image_urls: list[str] = attr.ib(default=lambda: [])
- custom_images: list[dict[str, str]] = attr.ib(default=lambda: [])
+ custom_image_urls: list[str] = attr.ib(default=list)
+ custom_images: list[dict[str, str]] = attr.ib(default=list)
class ImagesPipelineTestCaseFieldsAttrsItem(