Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

E501 compliance

Eugenio Lacuesta36a3913a41a0
diff --git a/pytest.ini b/pytest.ini
index 141a13a4f..781a45541 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,6 +20,7 @@ addopts =
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-max-line-length = 119
flake8-ignore =
W503
# Files that are only meant to provide top-level imports are expected not
@@ -27,146 +28,85 @@ flake8-ignore =
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
# Issues pending a review:
- # extras
- extras/qps-bench-server.py E501
- extras/qpsclient.py E501 E501
# scrapy/commands
- scrapy/commands/__init__.py E128 E501
- scrapy/commands/check.py E501
- scrapy/commands/crawl.py E501
- scrapy/commands/edit.py E501
- scrapy/commands/fetch.py E401 E501 E128 E731
- scrapy/commands/genspider.py E128 E501 E502
- scrapy/commands/parse.py E128 E501 E731
- scrapy/commands/runspider.py E501
+ scrapy/commands/__init__.py E128
+ scrapy/commands/fetch.py E401 E128 E731
+ scrapy/commands/genspider.py E128 E502
+ scrapy/commands/parse.py E128 E731
+ scrapy/commands/runspider.py
scrapy/commands/settings.py E128
- scrapy/commands/shell.py E128 E501 E502
- scrapy/commands/startproject.py E127 E501 E128
- scrapy/commands/version.py E501 E128
+ scrapy/commands/shell.py E128 E502
+ scrapy/commands/startproject.py E127 E128
+ scrapy/commands/version.py E128
# scrapy/contracts
- scrapy/contracts/__init__.py E501 W504
+ scrapy/contracts/__init__.py W504
scrapy/contracts/default.py E128
# scrapy/core
- scrapy/core/engine.py E501 E128 E127 E502
- scrapy/core/scheduler.py E501
- scrapy/core/scraper.py E501 E128 W504
- scrapy/core/spidermw.py E501 E731 E126
- scrapy/core/downloader/__init__.py E501
- scrapy/core/downloader/contextfactory.py E501 E128 E126
- scrapy/core/downloader/middleware.py E501 E502
- scrapy/core/downloader/tls.py E501 E241
- scrapy/core/downloader/webclient.py E731 E501 E128 E126
- scrapy/core/downloader/handlers/__init__.py E501
- scrapy/core/downloader/handlers/ftp.py E501 E128 E127
- scrapy/core/downloader/handlers/http10.py E501
- scrapy/core/downloader/handlers/http11.py E501
- scrapy/core/downloader/handlers/s3.py E501 E128 E126
+ scrapy/core/engine.py E128 E127 E502
+ scrapy/core/scheduler.py
+ scrapy/core/scraper.py E128 W504
+ scrapy/core/spidermw.py E731 E126
+ scrapy/core/downloader/__init__.py
+ scrapy/core/downloader/contextfactory.py E128 E126
+ scrapy/core/downloader/middleware.py E502
+ scrapy/core/downloader/tls.py E241
+ scrapy/core/downloader/webclient.py E731 E128 E126
+ scrapy/core/downloader/handlers/ftp.py E128 E127
+ scrapy/core/downloader/handlers/s3.py E128 E126
# scrapy/downloadermiddlewares
- scrapy/downloadermiddlewares/ajaxcrawl.py E501
- scrapy/downloadermiddlewares/decompression.py E501
- scrapy/downloadermiddlewares/defaultheaders.py E501
- scrapy/downloadermiddlewares/httpcache.py E501 E126
- scrapy/downloadermiddlewares/httpcompression.py E501 E128
- scrapy/downloadermiddlewares/httpproxy.py E501
- scrapy/downloadermiddlewares/redirect.py E501 W504
- scrapy/downloadermiddlewares/retry.py E501 E126
- scrapy/downloadermiddlewares/robotstxt.py E501
- scrapy/downloadermiddlewares/stats.py E501
+ scrapy/downloadermiddlewares/httpcache.py E126
+ scrapy/downloadermiddlewares/httpcompression.py E128
+ scrapy/downloadermiddlewares/redirect.py W504
+ scrapy/downloadermiddlewares/retry.py E126
# scrapy/extensions
- scrapy/extensions/closespider.py E501 E128 E123
- scrapy/extensions/corestats.py E501
- scrapy/extensions/feedexport.py E128 E501
- scrapy/extensions/httpcache.py E128 E501
- scrapy/extensions/memdebug.py E501
- scrapy/extensions/spiderstate.py E501
- scrapy/extensions/telnet.py E501 W504
- scrapy/extensions/throttle.py E501
+ scrapy/extensions/closespider.py E128 E123
+ scrapy/extensions/feedexport.py E128
+ scrapy/extensions/httpcache.py E128
+ scrapy/extensions/telnet.py W504
# scrapy/http
- scrapy/http/common.py E501
- scrapy/http/cookies.py E501
- scrapy/http/request/__init__.py E501
- scrapy/http/request/form.py E501 E123
- scrapy/http/request/json_request.py E501
- scrapy/http/response/__init__.py E501 E128
- scrapy/http/response/text.py E501 E128 E124
+ scrapy/http/request/form.py E123
+ scrapy/http/response/__init__.py E128
+ scrapy/http/response/text.py E128 E124
# scrapy/linkextractors
- scrapy/linkextractors/__init__.py E731 E501 E402 W504
- scrapy/linkextractors/lxmlhtml.py E501 E731
+ scrapy/linkextractors/__init__.py E731 E402 W504
+ scrapy/linkextractors/lxmlhtml.py E731
# scrapy/loader
- scrapy/loader/__init__.py E501 E128
- scrapy/loader/processors.py E501
+ scrapy/loader/__init__.py E128
+ scrapy/loader/processors.py
# scrapy/pipelines
- scrapy/pipelines/__init__.py E501
- scrapy/pipelines/files.py E116 E501 E266
- scrapy/pipelines/images.py E265 E501
- scrapy/pipelines/media.py E125 E501 E266
+ scrapy/pipelines/files.py E116 E266
+ scrapy/pipelines/images.py E265
+ scrapy/pipelines/media.py E125 E266
# scrapy/selector
scrapy/selector/__init__.py F403
- scrapy/selector/unified.py E501 E111
+ scrapy/selector/unified.py E111
# scrapy/settings
- scrapy/settings/__init__.py E501
- scrapy/settings/default_settings.py E501 E114 E116
- scrapy/settings/deprecated.py E501
+ scrapy/settings/default_settings.py E114 E116
# scrapy/spidermiddlewares
- scrapy/spidermiddlewares/httperror.py E501
- scrapy/spidermiddlewares/offsite.py E501
- scrapy/spidermiddlewares/referer.py E501 E129 W504
- scrapy/spidermiddlewares/urllength.py E501
+ scrapy/spidermiddlewares/referer.py E129 W504
# scrapy/spiders
- scrapy/spiders/__init__.py E501 E402
- scrapy/spiders/crawl.py E501
- scrapy/spiders/feed.py E501
- scrapy/spiders/sitemap.py E501
+ scrapy/spiders/__init__.py E402
# scrapy/utils
- scrapy/utils/asyncio.py E501
- scrapy/utils/benchserver.py E501
- scrapy/utils/conf.py E402 E501
- scrapy/utils/datatypes.py E501
- scrapy/utils/decorators.py E501
- scrapy/utils/defer.py E501 E128
- scrapy/utils/deprecate.py E128 E501 E127 E502
- scrapy/utils/gz.py E501 W504
+ scrapy/utils/defer.py E128
+ scrapy/utils/deprecate.py E128 E127 E502
+ scrapy/utils/gz.py W504
scrapy/utils/http.py F403
- scrapy/utils/httpobj.py E501
- scrapy/utils/iterators.py E501
- scrapy/utils/log.py E128 E501
+ scrapy/utils/log.py E128
scrapy/utils/markup.py F403
- scrapy/utils/misc.py E501
scrapy/utils/multipart.py F403
- scrapy/utils/project.py E501
- scrapy/utils/python.py E501
- scrapy/utils/reactor.py E501
- scrapy/utils/reqser.py E501
- scrapy/utils/request.py E127 E501
- scrapy/utils/response.py E501 E128
- scrapy/utils/signal.py E501 E128
- scrapy/utils/sitemap.py E501
- scrapy/utils/spider.py E501
- scrapy/utils/ssl.py E501
- scrapy/utils/test.py E501
- scrapy/utils/url.py E501 F403 E128 F405
+ scrapy/utils/request.py E127
+ scrapy/utils/response.py E128
+ scrapy/utils/signal.py E128
+ scrapy/utils/url.py F403 E128 F405
# scrapy
- scrapy/__init__.py E402 E501
- scrapy/cmdline.py E501
- scrapy/crawler.py E501
- scrapy/dupefilters.py E501 E202
- scrapy/exceptions.py E501
- scrapy/exporters.py E501
- scrapy/interfaces.py E501
- scrapy/item.py E501 E128
- scrapy/link.py E501
- scrapy/logformatter.py E501
- scrapy/mail.py E402 E128 E501 E502
- scrapy/middleware.py E128 E501
- scrapy/pqueues.py E501
- scrapy/resolver.py E501
- scrapy/responsetypes.py E128 E501
- scrapy/robotstxt.py E501
- scrapy/shell.py E501
- scrapy/signalmanager.py E501
- scrapy/spiderloader.py F841 E501 E126
+ scrapy/__init__.py E402
+ scrapy/dupefilters.py E202
+ scrapy/item.py E128
+ scrapy/mail.py E402 E128 E502
+ scrapy/middleware.py E128
+ scrapy/responsetypes.py E128
+ scrapy/spiderloader.py F841 E126
scrapy/squeues.py E128
- scrapy/statscollectors.py E501
# tests
tests/__init__.py E402 E501
tests/mockserver.py E401 E501 E126 E123
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index a4ec7c8ae..b189e016b 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -165,6 +165,7 @@ if __name__ == '__main__':
try:
execute()
finally:
- # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
- # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+ # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
+ # http://doc.pypy.org/en/latest/cpython_differences.html
+ # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
garbage_collect()
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 9c0014206..83c7b1f19 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
- raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
- (method.__self__.__class__.__name__, response.__class__.__name__))
+ raise _InvalidOutput(
+ "Middleware %s.process_request must return None, Response or Request, got %s"
+ % (method.__self__.__class__.__name__, response.__class__.__name__)
+ )
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_response']:
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
- raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
- (method.__self__.__class__.__name__, type(response)))
+ raise _InvalidOutput(
+ "Middleware %s.process_response must return Response or Request, got %s"
+ % (method.__self__.__class__.__name__, type(response))
+ )
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
- raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (method.__self__.__class__.__name__, type(response)))
+ raise _InvalidOutput(
+ "Middleware %s.process_exception must return None, Response or Request, got %s"
+ % (method.__self__.__class__.__name__, type(response))
+ )
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 0a9af3a62..52cf09844 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
except UnicodeDecodeError:
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
# Switch to 'allow all' state.
- logger.warning("Failure while parsing robots.txt. "
- "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
- exc_info=sys.exc_info(),
- extra={'spider': spider})
+ logger.warning(
+ "Failure while parsing robots.txt. File either contains garbage or "
+ "is in an encoding other than UTF-8, treating it as an empty file.",
+ exc_info=sys.exc_info(),
+ extra={'spider': spider},
+ )
robotstxt_body = ''
return robotstxt_body
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index d76a96451..3d62224bf 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -54,8 +54,12 @@ class Rule:
self.process_request = _get_method(self.process_request, spider)
self.process_request_argcount = len(get_func_args(self.process_request))
if self.process_request_argcount == 1:
- msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
- warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+ warnings.warn(
+ "Rule.process_request should accept two arguments"
+ " (request, response), accepting only one is deprecated",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
def _process_request(self, request, response):
"""