Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Flake8: remove W504 code (#4525)

Eugenio Lacuesta17c0cf64aee1
diff --git a/pytest.ini b/pytest.ini
index e8911ee3f..4f3494e0e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -44,12 +44,12 @@ flake8-ignore =
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
- scrapy/contracts/__init__.py E501 W504
+ scrapy/contracts/__init__.py E501
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127
scrapy/core/scheduler.py E501
- scrapy/core/scraper.py E501 E128 W504
+ scrapy/core/scraper.py E501 E128
scrapy/core/spidermw.py E501 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
@@ -68,7 +68,7 @@ flake8-ignore =
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
- scrapy/downloadermiddlewares/redirect.py E501 W504
+ scrapy/downloadermiddlewares/redirect.py E501
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
@@ -79,7 +79,7 @@ flake8-ignore =
scrapy/extensions/httpcache.py E128 E501
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
- scrapy/extensions/telnet.py E501 W504
+ scrapy/extensions/telnet.py E501
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
@@ -90,7 +90,7 @@ flake8-ignore =
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
- scrapy/linkextractors/__init__.py E501 E402 W504
+ scrapy/linkextractors/__init__.py E501 E402
scrapy/linkextractors/lxmlhtml.py E501
# scrapy/loader
scrapy/loader/__init__.py E501 E128
@@ -110,7 +110,7 @@ flake8-ignore =
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
- scrapy/spidermiddlewares/referer.py E501 E129 W504
+ scrapy/spidermiddlewares/referer.py E501 E129
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
@@ -125,7 +125,7 @@ flake8-ignore =
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127
- scrapy/utils/gz.py E501 W504
+ scrapy/utils/gz.py E501
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501
@@ -234,7 +234,7 @@ flake8-ignore =
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
- tests/test_utils_http.py E501 E128 W504
+ tests/test_utils_http.py E501 E128
tests/test_utils_iterators.py E501 E128 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 41d4f25b2..5af3831a2 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -17,10 +17,10 @@ class ContractsManager:
self.contracts[contract.name] = contract
def tested_methods_from_spidercls(self, spidercls):
+ is_method = re.compile(r"^\s*@", re.MULTILINE).search
methods = []
for key, value in getmembers(spidercls):
- if (callable(value) and value.__doc__ and
- re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+ if callable(value) and value.__doc__ and is_method(value.__doc__):
methods.append(key)
return methods
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 09ee8377e..b32afb8e4 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
Handle redirection of requests based on response status
and meta-refresh html tag.
"""
+
def process_response(self, request, response, spider):
- if (request.meta.get('dont_redirect', False) or
- response.status in getattr(spider, 'handle_httpstatus_list', []) or
- response.status in request.meta.get('handle_httpstatus_list', []) or
- request.meta.get('handle_httpstatus_all', False)):
+ if (
+ request.meta.get('dont_redirect', False)
+ or response.status in getattr(spider, 'handle_httpstatus_list', [])
+ or response.status in request.meta.get('handle_httpstatus_list', [])
+ or request.meta.get('handle_httpstatus_all', False)
+ ):
return response
allowed_status = (301, 302, 303, 307, 308)
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 04ffd7235..1663604e7 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
"""An implementation of IPortal"""
@defers
def login(self_, credentials, mind, *interfaces):
- if not (credentials.username == self.username.encode('utf8') and
- credentials.checkPassword(self.password.encode('utf8'))):
+ if not (
+ credentials.username == self.username.encode('utf8')
+ and credentials.checkPassword(self.password.encode('utf8'))
+ ):
raise ValueError("Invalid credentials")
protocol = telnet.TelnetBootstrapProtocol(
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index d0b5066b6..ae019c70f 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -61,8 +61,7 @@ class FilteringLinkExtractor:
def __new__(cls, *args, **kwargs):
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
- if (issubclass(cls, FilteringLinkExtractor) and
- not issubclass(cls, LxmlLinkExtractor)):
+ if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 3784de885..434067b00 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
name = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
- if ((self.tls_protected(response_url) and
- self.potentially_trustworthy(request_url))
- or not self.tls_protected(response_url)):
+ if (
+ self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+ or not self.tls_protected(response_url)
+ ):
return self.origin_referrer(response_url)
@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
- elif ((self.tls_protected(response_url) and
- self.potentially_trustworthy(request_url))
- or not self.tls_protected(response_url)):
+ elif (
+ self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+ or not self.tls_protected(response_url)
+ ):
return self.origin_referrer(response_url)
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index c291ae237..fbd7bd18f 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -52,8 +52,7 @@ def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
ctype = response.headers.get('Content-Type', b'')
cenc = response.headers.get('Content-Encoding', b'').lower()
- return (_is_gzipped(ctype) or
- (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+ return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
def gzip_magic_number(response):
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 2fac3da1f..363b015a8 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
chunked_body += "8\r\n" + "sequence\r\n"
chunked_body += "0\r\n\r\n"
body = decode_chunked_transfer(chunked_body)
- self.assertEqual(body,
- "This is the data in the first chunk\r\n" +
- "and this is the second one\r\n" +
- "consequence")
+ self.assertEqual(
+ body,
+ "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
+ )