Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump pylint, cleanup the ignored tags.

Andrey Rakhmatullin4f9dd998dcf0
diff --git a/docs/conf.py b/docs/conf.py
index 9ca0f817a..399078010 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -227,7 +227,7 @@ latex_documents = [
# A list of regular expressions that match URIs that should not be checked when
# doing a linkcheck build.
linkcheck_ignore = [
- "http://localhost:\d+",
+ r"http://localhost:\d+",
"http://hg.scrapy.org",
"http://directory.google.com/",
]
diff --git a/pylintrc b/pylintrc
index c8654b8d3..78004e78a 100644
--- a/pylintrc
+++ b/pylintrc
@@ -4,21 +4,14 @@ jobs=1 # >1 hides results
[MESSAGES CONTROL]
disable=abstract-method,
- anomalous-backslash-in-string,
arguments-differ,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
- bad-mcs-classmethod-argument,
bare-except,
broad-except,
broad-exception-raised,
c-extension-no-member,
- catching-non-exception,
- cell-var-from-loop,
- comparison-with-callable,
- consider-using-dict-items,
- consider-using-in,
consider-using-with,
cyclic-import,
dangerous-default-value,
@@ -32,7 +25,6 @@ disable=abstract-method,
implicit-str-concat,
import-error,
import-outside-toplevel,
- import-self,
inconsistent-return-statements,
inherit-non-class,
invalid-name,
@@ -44,7 +36,6 @@ disable=abstract-method,
logging-fstring-interpolation,
logging-not-lazy,
lost-exception,
- method-hidden,
missing-docstring,
no-else-raise,
no-else-return,
@@ -52,7 +43,7 @@ disable=abstract-method,
no-method-argument,
no-name-in-module,
no-self-argument,
- no-value-for-parameter,
+ no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268
not-callable,
pointless-exception-statement,
pointless-statement,
@@ -77,14 +68,10 @@ disable=abstract-method,
too-many-public-methods,
too-many-return-statements,
unbalanced-tuple-unpacking,
- undefined-variable,
- undefined-loop-variable,
- unexpected-special-method-signature,
unnecessary-comprehension,
unnecessary-dunder-call,
unnecessary-pass,
unreachable,
- unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
@@ -92,8 +79,6 @@ disable=abstract-method,
unused-wildcard-import,
use-dict-literal,
used-before-assignment,
- useless-object-inheritance, # Required for Python 2 support
useless-return,
- useless-super-delegation,
wildcard-import,
wrong-import-position
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 1e340abb6..f0ad24f72 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -169,7 +169,7 @@ class HttpCompressionMiddleware:
return to_decode, to_keep
def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
- if encoding == b"gzip" or encoding == b"x-gzip":
+ if encoding in {b"gzip", b"x-gzip"}:
return gunzip(body, max_size=max_size)
if encoding == b"deflate":
return _inflate(body, max_size=max_size)
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 21eb9fb73..73aee7178 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -113,7 +113,9 @@ class Headers(CaselessDict):
return ((k, self.getlist(k)) for k in self.keys())
def values(self) -> List[Optional[bytes]]: # type: ignore[override]
- return [self[k] for k in self.keys()]
+ return [
+ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items
+ ]
def to_string(self) -> bytes:
# cast() can be removed if the headers_dict_to_raw() hint is improved
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index db9a71273..5985a847e 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -24,7 +24,11 @@ def install_shutdown_handlers(
(e.g. Pdb)
"""
signal.signal(signal.SIGTERM, function)
- if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
+ if (
+ signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable
+ == signal.default_int_handler
+ or override_sigint
+ ):
signal.signal(signal.SIGINT, function)
# Catch Ctrl-Break in windows
if hasattr(signal, "SIGBREAK"):
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index a25100c03..89cfbd2ec 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -98,7 +98,10 @@ def send_catch_log_deferred(
robustApply, receiver, signal=signal, sender=sender, *arguments, **named
)
d.addErrback(logerror, receiver)
- d.addBoth(lambda result: (receiver, result))
+ # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
+ d.addBoth(
+ lambda result: (receiver, result) # pylint: disable=cell-var-from-loop
+ )
dfds.append(d)
d = DeferredList(dfds)
d.addCallback(lambda out: [x[1] for x in out])
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2f36baa87..febad21da 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -991,38 +991,11 @@ class MySpider(scrapy.Spider):
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
spider_filename = "myspider.pyw"
- def setUp(self):
- super().setUp()
-
def test_start_requests_errors(self):
log = self.get_log(self.badspider, name="badspider.pyw")
self.assertIn("start_requests", log)
self.assertIn("badspider.pyw", log)
- def test_run_good_spider(self):
- super().test_run_good_spider()
-
- def test_runspider(self):
- super().test_runspider()
-
- def test_runspider_dnscache_disabled(self):
- super().test_runspider_dnscache_disabled()
-
- def test_runspider_log_level(self):
- super().test_runspider_log_level()
-
- def test_runspider_log_short_names(self):
- super().test_runspider_log_short_names()
-
- def test_runspider_no_spider_found(self):
- super().test_runspider_no_spider_found()
-
- def test_output(self):
- super().test_output()
-
- def test_overwrite_output(self):
- super().test_overwrite_output()
-
def test_runspider_unable_to_load(self):
raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
diff --git a/tests/test_item.py b/tests/test_item.py
index ce2b4fd15..daf5d4f59 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase):
class ItemMetaClassCellRegression(unittest.TestCase):
def test_item_meta_classcell_regression(self):
class MyItem(Item, metaclass=ItemMeta):
- def __init__(self, *args, **kwargs):
+ def __init__(
+ self, *args, **kwargs
+ ): # pylint: disable=useless-parent-delegation
# This call to super() trigger the __classcell__ propagation
# requirement. When not done properly raises an error:
# TypeError: __class__ set to <class '__main__.MyItem'>
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 55ea9eed2..6b4df90d8 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
],
)
- def test_restrict_xpaths_with_html_entities(self):
- super().test_restrict_xpaths_with_html_entities()
-
@mark.skipif(
Version(w3lib_version) < Version("2.0.0"),
reason=(
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d7f773d5c..99cdf88d9 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase):
}
def test_output(self):
- for tl in self.test_list_equals:
- expr, test_list, expected = self.test_list_equals[tl]
+ for k, v in self.test_list_equals.items():
+ expr, test_list, expected = v
test = SelectJmes(expr)(test_list)
self.assertEqual(
- test, expected, msg=f'test "{tl}" got {test} expected {expected}'
+ test, expected, msg=f'test "{k}" got {test} expected {expected}'
)
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 3fde5e8c5..9ee248538 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase):
mydict = settings.get("TEST_DICT")
self.assertIsInstance(mydict, BaseSettings)
self.assertIn("key", mydict)
- self.assertEqual(mydict["key"], "val")
+ self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object
self.assertEqual(mydict.getpriority("key"), 0)
@mock.patch("scrapy.settings.default_settings", default_settings)
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 9e5f88f48..be5c6de81 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
for i, r in enumerate(refs):
self.assertIn(r, cache)
self.assertEqual(cache[r], i)
- del r # delete reference to the last object in the list
+ del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable
# delete half of the objects, make sure that is reflected in the cache
for _ in range(max // 2):
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 65b99e0c4..60232f10b 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
await defer.succeed(42)
return "OK"
- def test_send_catch_log(self):
- return super().test_send_catch_log()
-
@mark.only_asyncio()
class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
@@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
await asyncio.sleep(0.2)
return await get_from_asyncio_queue("OK")
- def test_send_catch_log(self):
- return super().test_send_catch_log()
-
class SendCatchLogTest2(unittest.TestCase):
def test_error_logged_if_deferred_not_supported(self):
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 460ae40c3..dd1d26448 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):
self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
def test_iter_spider_classes(self):
- import tests.test_utils_spider
+ import tests.test_utils_spider # pylint: disable=import-self
it = iter_spider_classes(tests.test_utils_spider)
self.assertEqual(set(it), {MySpider1, MySpider2})
diff --git a/tox.ini b/tox.ini
index e787c7bf3..4ed9b3bd7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -53,7 +53,7 @@ commands =
basepython = python3
deps =
{[testenv:extra-deps]deps}
- pylint==3.0.1
+ pylint==3.1.0
commands =
pylint conftest.py docs extras scrapy setup.py tests