Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump pylint, cleanup the ignored tags.
Andrey Rakhmatullin4f9dd998dcf0
diff --git a/docs/conf.py b/docs/conf.pyindex 9ca0f817a..399078010 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -227,7 +227,7 @@ latex_documents = [# A list of regular expressions that match URIs that should not be checked when# doing a linkcheck build.linkcheck_ignore = [- "http://localhost:\d+",+ r"http://localhost:\d+","http://hg.scrapy.org","http://directory.google.com/",]diff --git a/pylintrc b/pylintrcindex c8654b8d3..78004e78a 100644--- a/pylintrc+++ b/pylintrc@@ -4,21 +4,14 @@ jobs=1 # >1 hides results[MESSAGES CONTROL]disable=abstract-method,- anomalous-backslash-in-string,arguments-differ,arguments-renamed,attribute-defined-outside-init,bad-classmethod-argument,- bad-mcs-classmethod-argument,bare-except,broad-except,broad-exception-raised,c-extension-no-member,- catching-non-exception,- cell-var-from-loop,- comparison-with-callable,- consider-using-dict-items,- consider-using-in,consider-using-with,cyclic-import,dangerous-default-value,@@ -32,7 +25,6 @@ disable=abstract-method,implicit-str-concat,import-error,import-outside-toplevel,- import-self,inconsistent-return-statements,inherit-non-class,invalid-name,@@ -44,7 +36,6 @@ disable=abstract-method,logging-fstring-interpolation,logging-not-lazy,lost-exception,- method-hidden,missing-docstring,no-else-raise,no-else-return,@@ -52,7 +43,7 @@ disable=abstract-method,no-method-argument,no-name-in-module,no-self-argument,- no-value-for-parameter,+ no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268not-callable,pointless-exception-statement,pointless-statement,@@ -77,14 +68,10 @@ disable=abstract-method,too-many-public-methods,too-many-return-statements,unbalanced-tuple-unpacking,- undefined-variable,- undefined-loop-variable,- unexpected-special-method-signature,unnecessary-comprehension,unnecessary-dunder-call,unnecessary-pass,unreachable,- unsubscriptable-object,unused-argument,unused-import,unused-private-member,@@ -92,8 +79,6 @@ disable=abstract-method,unused-wildcard-import,use-dict-literal,used-before-assignment,- useless-object-inheritance, # Required for Python 2 supportuseless-return,- useless-super-delegation,wildcard-import,wrong-import-positiondiff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 1e340abb6..f0ad24f72 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -169,7 +169,7 @@ class HttpCompressionMiddleware:return to_decode, to_keepdef _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:- if encoding == b"gzip" or encoding == b"x-gzip":+ if encoding in {b"gzip", b"x-gzip"}:return gunzip(body, max_size=max_size)if encoding == b"deflate":return _inflate(body, max_size=max_size)diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 21eb9fb73..73aee7178 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -113,7 +113,9 @@ class Headers(CaselessDict):return ((k, self.getlist(k)) for k in self.keys())def values(self) -> List[Optional[bytes]]: # type: ignore[override]- return [self[k] for k in self.keys()]+ return [+ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items+ ]def to_string(self) -> bytes:# cast() can be removed if the headers_dict_to_raw() hint is improveddiff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.pyindex db9a71273..5985a847e 100644--- a/scrapy/utils/ossignal.py+++ b/scrapy/utils/ossignal.py@@ -24,7 +24,11 @@ def install_shutdown_handlers((e.g. Pdb)"""signal.signal(signal.SIGTERM, function)- if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:+ if (+ signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable+ == signal.default_int_handler+ or override_sigint+ ):signal.signal(signal.SIGINT, function)# Catch Ctrl-Break in windowsif hasattr(signal, "SIGBREAK"):diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex a25100c03..89cfbd2ec 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -98,7 +98,10 @@ def send_catch_log_deferred(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)d.addErrback(logerror, receiver)- d.addBoth(lambda result: (receiver, result))+ # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html+ d.addBoth(+ lambda result: (receiver, result) # pylint: disable=cell-var-from-loop+ )dfds.append(d)d = DeferredList(dfds)d.addCallback(lambda out: [x[1] for x in out])diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 2f36baa87..febad21da 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -991,38 +991,11 @@ class MySpider(scrapy.Spider):class WindowsRunSpiderCommandTest(RunSpiderCommandTest):spider_filename = "myspider.pyw"- def setUp(self):- super().setUp()-def test_start_requests_errors(self):log = self.get_log(self.badspider, name="badspider.pyw")self.assertIn("start_requests", log)self.assertIn("badspider.pyw", log)- def test_run_good_spider(self):- super().test_run_good_spider()-- def test_runspider(self):- super().test_runspider()-- def test_runspider_dnscache_disabled(self):- super().test_runspider_dnscache_disabled()-- def test_runspider_log_level(self):- super().test_runspider_log_level()-- def test_runspider_log_short_names(self):- super().test_runspider_log_short_names()-- def test_runspider_no_spider_found(self):- super().test_runspider_no_spider_found()-- def test_output(self):- super().test_output()-- def test_overwrite_output(self):- super().test_overwrite_output()-def test_runspider_unable_to_load(self):raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")diff --git a/tests/test_item.py b/tests/test_item.pyindex ce2b4fd15..daf5d4f59 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase):class ItemMetaClassCellRegression(unittest.TestCase):def test_item_meta_classcell_regression(self):class MyItem(Item, metaclass=ItemMeta):- def __init__(self, *args, **kwargs):+ def __init__(+ self, *args, **kwargs+ ): # pylint: disable=useless-parent-delegation# This call to super() trigger the __classcell__ propagation# requirement. When not done properly raises an error:# TypeError: __class__ set to <class '__main__.MyItem'>diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 55ea9eed2..6b4df90d8 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):],)- def test_restrict_xpaths_with_html_entities(self):- super().test_restrict_xpaths_with_html_entities()-@mark.skipif(Version(w3lib_version) < Version("2.0.0"),reason=(diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex d7f773d5c..99cdf88d9 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase):}def test_output(self):- for tl in self.test_list_equals:- expr, test_list, expected = self.test_list_equals[tl]+ for k, v in self.test_list_equals.items():+ expr, test_list, expected = vtest = SelectJmes(expr)(test_list)self.assertEqual(- test, expected, msg=f'test "{tl}" got {test} expected {expected}'+ test, expected, msg=f'test "{k}" got {test} expected {expected}')diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 3fde5e8c5..9ee248538 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase):mydict = settings.get("TEST_DICT")self.assertIsInstance(mydict, BaseSettings)self.assertIn("key", mydict)- self.assertEqual(mydict["key"], "val")+ self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-objectself.assertEqual(mydict.getpriority("key"), 0)@mock.patch("scrapy.settings.default_settings", default_settings)diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 9e5f88f48..be5c6de81 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):for i, r in enumerate(refs):self.assertIn(r, cache)self.assertEqual(cache[r], i)- del r # delete reference to the last object in the list+ del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable# delete half of the objects, make sure that is reflected in the cachefor _ in range(max // 2):diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 65b99e0c4..60232f10b 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):await defer.succeed(42)return "OK"- def test_send_catch_log(self):- return super().test_send_catch_log()-@mark.only_asyncio()class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):@@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):await asyncio.sleep(0.2)return await get_from_asyncio_queue("OK")- def test_send_catch_log(self):- return super().test_send_catch_log()-class SendCatchLogTest2(unittest.TestCase):def test_error_logged_if_deferred_not_supported(self):diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 460ae40c3..dd1d26448 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])def test_iter_spider_classes(self):- import tests.test_utils_spider+ import tests.test_utils_spider # pylint: disable=import-selfit = iter_spider_classes(tests.test_utils_spider)self.assertEqual(set(it), {MySpider1, MySpider2})diff --git a/tox.ini b/tox.iniindex e787c7bf3..4ed9b3bd7 100644--- a/tox.ini+++ b/tox.ini@@ -53,7 +53,7 @@ commands =basepython = python3deps ={[testenv:extra-deps]deps}- pylint==3.0.1+ pylint==3.1.0commands =pylint conftest.py docs extras scrapy setup.py tests