Scrapy, a fast high-level web crawling & scraping framework for Python.
Add flake8-pie rules to ruff.
Andrey Rakhmatullin93644f2c30ee
diff --git a/pyproject.toml b/pyproject.tomlindex 131684724..b3dd9f057 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -242,6 +242,8 @@ extend-select = ["LOG",# pygrep-hooks"PGH",+ # flake8-pie+ "PIE",# flake8-quotes"Q",# flake8-returndiff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 4ca015f5e..3047ae396 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -61,7 +61,6 @@ class Command(ScrapyCommand):"""You can use this function to update the Scrapy objects that will beavailable in the shell"""- passdef run(self, args: list[str], opts: Namespace) -> None:url = args[0] if args else Nonediff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 823677175..fcc94879a 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta)::param spider: the spider object for the current crawl:type spider: :class:`~scrapy.spiders.Spider`"""- passdef close(self, reason: str) -> Deferred[None] | None:"""@@ -91,7 +90,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta)::param reason: a string which describes the reason why the spider was closed:type reason: :class:`str`"""- pass@abstractmethoddef has_pending_requests(self) -> bool:diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 7b8eea135..caf69daf4 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -50,7 +50,6 @@ class BaseDupeFilter:def log(self, request: Request, spider: Spider) -> None:"""Log that a request has been filtered"""- passclass RFPDupeFilter(BaseDupeFilter):diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex e7ecdbe0c..96566ba86 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -13,8 +13,6 @@ from typing import Anyclass NotConfigured(Exception):"""Indicates a missing configuration situation"""- pass-class _InvalidOutput(TypeError):"""@@ -22,8 +20,6 @@ class _InvalidOutput(TypeError):Internal and undocumented, it should not be raised or caught by user code."""- pass-# HTTP and crawling@@ -35,8 +31,6 @@ class IgnoreRequest(Exception):class DontCloseSpider(Exception):"""Request the spider not to be closed yet"""- pass-class CloseSpider(Exception):"""Raise this from callbacks to request the spider to be closed"""@@ -64,14 +58,10 @@ class StopDownload(Exception):class DropItem(Exception):"""Drop item from the item pipeline"""- pass-class NotSupported(Exception):"""Indicates a feature or method is not supported"""- pass-# Commands@@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning):DeprecationWarning is silenced on Python 2.7+"""- pass-class ContractFail(AssertionError):"""Error raised in case of a failing contract"""-- passdiff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 844969c6d..417c9c142 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta)::param robotstxt_body: content of a robots.txt_ file.:type robotstxt_body: bytes"""- pass@abstractmethoddef allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:@@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta)::param user_agent: User agent:type user_agent: str or bytes"""- passclass PythonRobotParser(RobotParser):diff --git a/tests/spiders.py b/tests/spiders.pyindex 0180cf757..3c44d7da5 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider):dupe_factor = 3def start_requests(self):- for i in range(0, self.distinct_urls):- for j in range(0, self.dupe_factor):+ for i in range(self.distinct_urls):+ for j in range(self.dupe_factor):url = self.mockserver.url(f"/echo?headers=1&body=test{i}")yield Request(url, dont_filter=self.dont_filter)diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex b0cb92d12..743889234 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -178,27 +178,23 @@ class TestSpider(Spider):"""method with no url@returns items 1 1"""- passdef custom_form(self, response):"""@url http://scrapy.org@custom_form"""- passdef invalid_regex(self, response):"""method with invalid regex@ Scrapy is awsome"""- passdef invalid_regex_with_valid_contract(self, response):"""method with invalid regex@ scrapy is awsome@url http://scrapy.org"""- passdef returns_request_meta(self, response):"""method which returns request@@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider):"""@custom_success_contract"""- passclass CustomContractFailSpider(Spider):@@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider):"""@custom_fail_contract"""- passclass InheritsTestSpider(TestSpider):diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex a010865ef..c99f19b03 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase):spider = spider or self.spidermiddleware = middleware or self.mw- for i in range(0, max_retry_times):+ for i in range(max_retry_times):req = middleware.process_exception(req, exception, spider)assert isinstance(req, Request)diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex 9fd680e9f..8c897c223 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase):console = TelnetConsole(crawler)# This function has some side effects we don't need for this test- console._get_telnet_vars = lambda: {}+ console._get_telnet_vars = dictconsole.start_listening()protocol = console.protocol()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 2be5e09bc..a6c5f0a94 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass(class FilesPipelineTestAttrsItem:name = attr.ib(default="")# default fields- file_urls: list[str] = attr.ib(default=lambda: [])- files: list[dict[str, str]] = attr.ib(default=lambda: [])+ file_urls: list[str] = attr.ib(default=list)+ files: list[dict[str, str]] = attr.ib(default=list)# overridden fields- custom_file_urls: list[str] = attr.ib(default=lambda: [])- custom_files: list[dict[str, str]] = attr.ib(default=lambda: [])+ custom_file_urls: list[str] = attr.ib(default=list)+ custom_files: list[dict[str, str]] = attr.ib(default=list)class FilesPipelineTestCaseFieldsAttrsItem(diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 3ffef4102..3d049843a 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass(class ImagesPipelineTestAttrsItem:name = attr.ib(default="")# default fields- image_urls: list[str] = attr.ib(default=lambda: [])- images: list[dict[str, str]] = attr.ib(default=lambda: [])+ image_urls: list[str] = attr.ib(default=list)+ images: list[dict[str, str]] = attr.ib(default=list)# overridden fields- custom_image_urls: list[str] = attr.ib(default=lambda: [])- custom_images: list[dict[str, str]] = attr.ib(default=lambda: [])+ custom_image_urls: list[str] = attr.ib(default=list)+ custom_images: list[dict[str, str]] = attr.ib(default=list)class ImagesPipelineTestCaseFieldsAttrsItem(