Scrapy, a fast high-level web crawling & scraping framework for Python.
Add pylint rules to ruff, refresh the ignore list of pylint itself (#6584)
Andrey Rakhmatullin7dd92e6e4341
diff --git a/pyproject.toml b/pyproject.tomlindex c0297e192..7dc1f6ec3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -122,6 +122,9 @@ profile = "black"[tool.pylint.MASTER]persistent = "no"jobs = 1 # >1 hides results+extension-pkg-allow-list=[+ "lxml",+][tool.pylint."MESSAGES CONTROL"]disable = [@@ -129,11 +132,7 @@ disable = ["arguments-differ","arguments-renamed","attribute-defined-outside-init",- "bad-classmethod-argument",- "bare-except",- "broad-except",- "broad-exception-raised",- "c-extension-no-member",+ "broad-exception-caught","consider-using-with","cyclic-import","dangerous-default-value",@@ -141,9 +140,6 @@ disable = ["duplicate-code", # https://github.com/PyCQA/pylint/issues/214"eval-used","fixme",- "function-redefined",- "global-statement",- "implicit-str-concat","import-error","import-outside-toplevel","inherit-non-class",@@ -155,7 +151,6 @@ disable = ["logging-format-interpolation","logging-fstring-interpolation","logging-not-lazy",- "lost-exception","missing-docstring","no-member","no-method-argument",@@ -169,13 +164,11 @@ disable = ["raise-missing-from","redefined-builtin","redefined-outer-name",- "reimported","signature-differs","too-few-public-methods","too-many-ancestors","too-many-arguments","too-many-branches",- "too-many-format-args","too-many-function-args","too-many-instance-attributes","too-many-lines",@@ -184,14 +177,11 @@ disable = ["too-many-return-statements","unbalanced-tuple-unpacking","unnecessary-dunder-call",- "unnecessary-pass",- "unreachable","unused-argument","unused-import","unused-variable","used-before-assignment","useless-return",- "wildcard-import","wrong-import-position",]@@ -246,6 +236,8 @@ extend-select = ["PGH",# flake8-pie"PIE",+ # pylint+ "PL",# flake8-pyi"PYI",# flake8-quotes@@ -320,6 +312,18 @@ ignore = ["D403",# `try`-`except` within a loop incurs performance overhead"PERF203",+ # Too many return statements+ "PLR0911",+ # Too many branches+ "PLR0912",+ # Too many arguments in function definition+ "PLR0913",+ # Too many statements+ "PLR0915",+ # Magic value used in comparison+ "PLR2004",+ # `for` loop variable overwritten by assignment target+ "PLW2901",# Use of `assert` detected; needed for mypy"S101",# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 921296502..c19710a6a 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -31,7 +31,7 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."def __getattr__(name: str):if name == "twisted_version":- import warnings+ import warnings # pylint: disable=reimportedfrom twisted import version as _txvdiff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex b6f19a37f..9a24871de 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -74,7 +74,7 @@ def _get_commands_from_entry_points(if inspect.isclass(obj):cmds[entry_point.name] = obj()else:- raise Exception(f"Invalid entry point {entry_point.name}")+ raise ValueError(f"Invalid entry point {entry_point.name}")return cmdsdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex b6997ef67..cdb7ac159 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -92,11 +92,10 @@ class BaseItemExporter:field_iter = ((x, y) for x, y in self.fields_to_export.items() if x in item)+ elif include_empty:+ field_iter = self.fields_to_exportelse:- if include_empty:- field_iter = self.fields_to_export- else:- field_iter = (x for x in self.fields_to_export if x in item)+ field_iter = (x for x in self.fields_to_export if x in item)for field_name in field_iter:if isinstance(field_name, str):diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 9f1b816c8..000ab5c65 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -398,7 +398,7 @@ def maybeDeferred_coro("""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""try:result = f(*args, **kw)- except: # noqa: E722,B001+ except: # noqa: E722 # pylint: disable=bare-exceptreturn defer.fail(failure.Failure(captureVars=Deferred.debug))if isinstance(result, Deferred):diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 32430cd6c..0a0acc742 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -60,7 +60,7 @@ def create_deprecated_class(deprecated_class: type | None = Nonewarned_on_subclass: bool = False- def __new__(+ def __new__( # pylint: disable=bad-classmethod-argumentmetacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]) -> type:cls = super().__new__(metacls, name, bases, clsdict_)diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex a40b835cd..6165d1f72 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -130,7 +130,7 @@ _scrapy_root_handler: logging.Handler | None = Nonedef install_scrapy_root_handler(settings: Settings) -> None:- global _scrapy_root_handler+ global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statementif (_scrapy_root_handler is not Nonediff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 2d781cc27..66a06a9f0 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -149,12 +149,11 @@ def verify_installed_reactor(reactor_path: str) -> None:reactor_class = load_object(reactor_path)if not reactor.__class__ == reactor_class:- msg = (+ raise RuntimeError("The installed reactor "f"({reactor.__module__}.{reactor.__class__.__name__}) does not "f"match the requested one ({reactor_path})")- raise Exception(msg)def verify_installed_asyncio_event_loop(loop_path: str) -> None:@@ -168,7 +167,7 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None:f".{reactor._asyncioEventloop.__class__.__qualname__}")specified = f"{loop_class.__module__}.{loop_class.__qualname__}"- raise Exception(+ raise RuntimeError("Scrapy found an asyncio Twisted reactor already "f"installed, and its event loop class ({installed}) does ""not match the one specified in the ASYNCIO_EVENT_LOOP "diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex e58eb8134..5277a292c 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -52,10 +52,6 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:"""Return an iterator over all spider classes defined in the given modulethat can be instantiated (i.e. which have name)"""- # this needs to be imported here until get rid of the spider manager- # singleton in scrapy.spider.spiders- from scrapy.spiders import Spider-for obj in vars(module).values():if (inspect.isclass(obj)diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 2539f30c7..1cbfbfd99 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -14,7 +14,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse# scrapy.utils.url was moved to w3lib.url and import * ensures this# move doesn't break old code-from w3lib.url import * # pylint: disable=unused-wildcard-import+from w3lib.url import * # pylint: disable=unused-wildcard-import,wildcard-importfrom w3lib.url import _safe_chars, _unquotepath # noqa: F401from scrapy.utils.python import to_unicode@@ -50,7 +50,9 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:return any(lowercase_path.endswith(ext) for ext in extensions)-def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult:+def parse_url( # pylint: disable=function-redefined+ url: UrlT, encoding: str | None = None+) -> ParseResult:"""Return urlparsed url from the given argument (which could be an alreadyparsed url)"""diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.pyindex af15cac68..3e29c70ed 100644--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py@@ -8,7 +8,7 @@ class TestSpiderPipeline:class TestSpiderExceptionPipeline:def open_spider(self, spider):- raise Exception("exception")+ raise RuntimeError("exception")def process_item(self, item, spider):return itemdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 3fcba4ef2..8ecba41bf 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -349,11 +349,6 @@ class HttpTestCase(unittest.TestCase):request = Request(self.getURL("host"), headers={"Host": host})return self.download_request(request, Spider("foo")).addCallback(_test)- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"localhost")- return d-def test_content_length_zero_bodyless_post_request_headers(self):"""Tests if "Content-Length: 0" is sent for bodyless POST requests.diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex e650b4936..c581e7596 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):class InvalidProcessExceptionMiddleware:def process_request(self, request, spider):- raise Exception+ raise RuntimeErrordef process_exception(self, request, exception, spider):return 1diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex b7312bbcd..15129e31f 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -192,4 +192,3 @@ class TestPeriodicLog(unittest.TestCase):{"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},lambda k, v: "downloader/" in k and "bytes" not in k,)- #diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 2debbe0d7..b087aaab1 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -727,7 +727,7 @@ class ExceptionJsonItemExporter(JsonItemExporter):"""JsonItemExporter that throws an exception every time export_item is called."""def export_item(self, _):- raise Exception("foo")+ raise RuntimeError("foo")class FeedExportTest(FeedExportTestBase):diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex f9b841a61..4bf22f6a0 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -330,10 +330,10 @@ class BasicItemLoaderTest(unittest.TestCase):il.add_value("name", ["mar", "ta"])self.assertEqual(il.get_output_value("name"), "Mar Ta")- class TakeFirstItemLoader(TestItemLoader):+ class TakeFirstItemLoader2(TestItemLoader):name_out = Join("<br>")- il = TakeFirstItemLoader()+ il = TakeFirstItemLoader2()il.add_value("name", ["mar", "ta"])self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex f2a57bd88..ba64ba721 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -78,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):class RaiseExceptionProcessSpiderOutputMiddleware:def process_spider_output(self, response, result, spider):- raise Exception+ raise RuntimeErrorself.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 4c19d167f..e51957497 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -247,7 +247,6 @@ class NotGeneratorFailMiddleware:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")out.append(r)raise ReferenceError- return outdef process_spider_exception(self, response, exception, spider):method = f"{self.__class__.__name__}.process_spider_exception"diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 1c93829e9..e00f69573 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -26,7 +26,7 @@ class AsyncioTest(TestCase):with warnings.catch_warnings(record=True) as w:install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")self.assertEqual(len(w), 0)- from twisted.internet import reactor+ from twisted.internet import reactor # pylint: disable=reimportedassert original_reactor == reactordiff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 3f153bdc0..e4ab97e5d 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -153,7 +153,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):@mark.xfail(reason="Checks that the test is actually executed", strict=True)@deferred_f_from_coro_fasync def test_deferred_f_from_coro_f_xfail(self):- raise Exception("This is expected to be raised")+ raise RuntimeError("This is expected to be raised")class AsyncCooperatorTest(unittest.TestCase):diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.pyindex c7774751e..480729d11 100644--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py@@ -32,7 +32,6 @@ def top_level_return_none():https://example.org"""yield url- returndef generator_that_returns_stuff():@@ -103,11 +102,9 @@ https://example.orgdef test_generators_return_none(self):def f2():yield 1- return Nonedef g2():yield 1- returndef h2():yield 1@@ -132,7 +129,6 @@ https://example.orghttps://example.org"""yield url- returndef l2():return@@ -181,12 +177,10 @@ https://example.org@decoratordef f3():yield 1- return None@decoratordef g3():yield 1- return@decoratordef h3():@@ -215,7 +209,6 @@ https://example.orghttps://example.org"""yield url- return@decoratordef l3():diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex dd1d26448..ae59d0137 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])def test_iter_spider_classes(self):- import tests.test_utils_spider # pylint: disable=import-self+ import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-selfit = iter_spider_classes(tests.test_utils_spider)self.assertEqual(set(it), {MySpider1, MySpider2})diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 59a95b0e2..a15ad749d 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -327,8 +327,6 @@ def create_guess_scheme_t(args):def create_skipped_scheme_t(args):def do_expected(self):raise unittest.SkipTest(args[2])- url = guess_scheme(args[0])- assert url.startswith(args[1])return do_expected