Scrapy, a fast high-level web crawling & scraping framework for Python.
Flake8: remove W504 code (#4525)
Eugenio Lacuesta17c0cf64aee1
diff --git a/pytest.ini b/pytest.iniindex e8911ee3f..4f3494e0e 100644--- a/pytest.ini+++ b/pytest.ini@@ -44,12 +44,12 @@ flake8-ignore =scrapy/commands/startproject.py E127 E501 E128scrapy/commands/version.py E501 E128# scrapy/contracts- scrapy/contracts/__init__.py E501 W504+ scrapy/contracts/__init__.py E501scrapy/contracts/default.py E128# scrapy/corescrapy/core/engine.py E501 E128 E127scrapy/core/scheduler.py E501- scrapy/core/scraper.py E501 E128 W504+ scrapy/core/scraper.py E501 E128scrapy/core/spidermw.py E501 E126scrapy/core/downloader/__init__.py E501scrapy/core/downloader/contextfactory.py E501 E128 E126@@ -68,7 +68,7 @@ flake8-ignore =scrapy/downloadermiddlewares/httpcache.py E501 E126scrapy/downloadermiddlewares/httpcompression.py E501 E128scrapy/downloadermiddlewares/httpproxy.py E501- scrapy/downloadermiddlewares/redirect.py E501 W504+ scrapy/downloadermiddlewares/redirect.py E501scrapy/downloadermiddlewares/retry.py E501 E126scrapy/downloadermiddlewares/robotstxt.py E501scrapy/downloadermiddlewares/stats.py E501@@ -79,7 +79,7 @@ flake8-ignore =scrapy/extensions/httpcache.py E128 E501scrapy/extensions/memdebug.py E501scrapy/extensions/spiderstate.py E501- scrapy/extensions/telnet.py E501 W504+ scrapy/extensions/telnet.py E501scrapy/extensions/throttle.py E501# scrapy/httpscrapy/http/common.py E501@@ -90,7 +90,7 @@ flake8-ignore =scrapy/http/response/__init__.py E501 E128scrapy/http/response/text.py E501 E128 E124# scrapy/linkextractors- scrapy/linkextractors/__init__.py E501 E402 W504+ scrapy/linkextractors/__init__.py E501 E402scrapy/linkextractors/lxmlhtml.py E501# scrapy/loaderscrapy/loader/__init__.py E501 E128@@ -110,7 +110,7 @@ flake8-ignore =# scrapy/spidermiddlewaresscrapy/spidermiddlewares/httperror.py E501scrapy/spidermiddlewares/offsite.py E501- scrapy/spidermiddlewares/referer.py E501 E129 W504+ scrapy/spidermiddlewares/referer.py E501 E129scrapy/spidermiddlewares/urllength.py E501# scrapy/spidersscrapy/spiders/__init__.py E501 E402@@ -125,7 +125,7 @@ flake8-ignore =scrapy/utils/decorators.py E501scrapy/utils/defer.py E501 E128scrapy/utils/deprecate.py E128 E501 E127- scrapy/utils/gz.py E501 W504+ scrapy/utils/gz.py E501scrapy/utils/http.py F403scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501@@ -234,7 +234,7 @@ flake8-ignore =tests/test_utils_datatypes.py E402 E501tests/test_utils_defer.py E501 F841tests/test_utils_deprecate.py F841 E501- tests/test_utils_http.py E501 E128 W504+ tests/test_utils_http.py E501 E128tests/test_utils_iterators.py E501 E128 E129tests/test_utils_log.py E741tests/test_utils_python.py E501diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 41d4f25b2..5af3831a2 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -17,10 +17,10 @@ class ContractsManager:self.contracts[contract.name] = contractdef tested_methods_from_spidercls(self, spidercls):+ is_method = re.compile(r"^\s*@", re.MULTILINE).searchmethods = []for key, value in getmembers(spidercls):- if (callable(value) and value.__doc__ and- re.search(r'^\s*@', value.__doc__, re.MULTILINE)):+ if callable(value) and value.__doc__ and is_method(value.__doc__):methods.append(key)return methodsdiff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 09ee8377e..b32afb8e4 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):Handle redirection of requests based on response statusand meta-refresh html tag."""+def process_response(self, request, response, spider):- if (request.meta.get('dont_redirect', False) or- response.status in getattr(spider, 'handle_httpstatus_list', []) or- response.status in request.meta.get('handle_httpstatus_list', []) or- request.meta.get('handle_httpstatus_all', False)):+ if (+ request.meta.get('dont_redirect', False)+ or response.status in getattr(spider, 'handle_httpstatus_list', [])+ or response.status in request.meta.get('handle_httpstatus_list', [])+ or request.meta.get('handle_httpstatus_all', False)+ ):return responseallowed_status = (301, 302, 303, 307, 308)diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 04ffd7235..1663604e7 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):"""An implementation of IPortal"""@defersdef login(self_, credentials, mind, *interfaces):- if not (credentials.username == self.username.encode('utf8') and- credentials.checkPassword(self.password.encode('utf8'))):+ if not (+ credentials.username == self.username.encode('utf8')+ and credentials.checkPassword(self.password.encode('utf8'))+ ):raise ValueError("Invalid credentials")protocol = telnet.TelnetBootstrapProtocol(diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex d0b5066b6..ae019c70f 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -61,8 +61,7 @@ class FilteringLinkExtractor:def __new__(cls, *args, **kwargs):from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor- if (issubclass(cls, FilteringLinkExtractor) and- not issubclass(cls, LxmlLinkExtractor)):+ if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ''please use scrapy.linkextractors.LinkExtractor instead',ScrapyDeprecationWarning, stacklevel=2)diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 3784de885..434067b00 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):name = POLICY_STRICT_ORIGINdef referrer(self, response_url, request_url):- if ((self.tls_protected(response_url) and- self.potentially_trustworthy(request_url))- or not self.tls_protected(response_url)):+ if (+ self.tls_protected(response_url) and self.potentially_trustworthy(request_url)+ or not self.tls_protected(response_url)+ ):return self.origin_referrer(response_url)@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):origin = self.origin(response_url)if origin == self.origin(request_url):return self.stripped_referrer(response_url)- elif ((self.tls_protected(response_url) and- self.potentially_trustworthy(request_url))- or not self.tls_protected(response_url)):+ elif (+ self.tls_protected(response_url) and self.potentially_trustworthy(request_url)+ or not self.tls_protected(response_url)+ ):return self.origin_referrer(response_url)diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex c291ae237..fbd7bd18f 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -52,8 +52,7 @@ def is_gzipped(response):"""Return True if the response is gzipped, or False otherwise"""ctype = response.headers.get('Content-Type', b'')cenc = response.headers.get('Content-Encoding', b'').lower()- return (_is_gzipped(ctype) or- (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))+ return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')def gzip_magic_number(response):diff --git a/tests/test_utils_http.py b/tests/test_utils_http.pyindex 2fac3da1f..363b015a8 100644--- a/tests/test_utils_http.py+++ b/tests/test_utils_http.py@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):chunked_body += "8\r\n" + "sequence\r\n"chunked_body += "0\r\n\r\n"body = decode_chunked_transfer(chunked_body)- self.assertEqual(body,- "This is the data in the first chunk\r\n" +- "and this is the second one\r\n" +- "consequence")+ self.assertEqual(+ body,+ "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"+ )