Scrapy, a fast high-level web crawling & scraping framework for Python.
E501 compliance
Eugenio Lacuesta36a3913a41a0
diff --git a/pytest.ini b/pytest.iniindex 141a13a4f..781a45541 100644--- a/pytest.ini+++ b/pytest.ini@@ -20,6 +20,7 @@ addopts =twisted = 1markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passed+flake8-max-line-length = 119flake8-ignore =W503# Files that are only meant to provide top-level imports are expected not@@ -27,146 +28,85 @@ flake8-ignore =scrapy/core/downloader/handlers/http.py F401scrapy/http/__init__.py F401# Issues pending a review:- # extras- extras/qps-bench-server.py E501- extras/qpsclient.py E501 E501# scrapy/commands- scrapy/commands/__init__.py E128 E501- scrapy/commands/check.py E501- scrapy/commands/crawl.py E501- scrapy/commands/edit.py E501- scrapy/commands/fetch.py E401 E501 E128 E731- scrapy/commands/genspider.py E128 E501 E502- scrapy/commands/parse.py E128 E501 E731- scrapy/commands/runspider.py E501+ scrapy/commands/__init__.py E128+ scrapy/commands/fetch.py E401 E128 E731+ scrapy/commands/genspider.py E128 E502+ scrapy/commands/parse.py E128 E731+ scrapy/commands/runspider.pyscrapy/commands/settings.py E128- scrapy/commands/shell.py E128 E501 E502- scrapy/commands/startproject.py E127 E501 E128- scrapy/commands/version.py E501 E128+ scrapy/commands/shell.py E128 E502+ scrapy/commands/startproject.py E127 E128+ scrapy/commands/version.py E128# scrapy/contracts- scrapy/contracts/__init__.py E501 W504+ scrapy/contracts/__init__.py W504scrapy/contracts/default.py E128# scrapy/core- scrapy/core/engine.py E501 E128 E127 E502- scrapy/core/scheduler.py E501- scrapy/core/scraper.py E501 E128 W504- scrapy/core/spidermw.py E501 E731 E126- scrapy/core/downloader/__init__.py E501- scrapy/core/downloader/contextfactory.py E501 E128 E126- scrapy/core/downloader/middleware.py E501 E502- scrapy/core/downloader/tls.py E501 E241- scrapy/core/downloader/webclient.py E731 E501 E128 E126- scrapy/core/downloader/handlers/__init__.py E501- scrapy/core/downloader/handlers/ftp.py E501 E128 E127- scrapy/core/downloader/handlers/http10.py E501- scrapy/core/downloader/handlers/http11.py E501- scrapy/core/downloader/handlers/s3.py E501 E128 E126+ scrapy/core/engine.py E128 E127 E502+ scrapy/core/scheduler.py+ scrapy/core/scraper.py E128 W504+ scrapy/core/spidermw.py E731 E126+ scrapy/core/downloader/__init__.py+ scrapy/core/downloader/contextfactory.py E128 E126+ scrapy/core/downloader/middleware.py E502+ scrapy/core/downloader/tls.py E241+ scrapy/core/downloader/webclient.py E731 E128 E126+ scrapy/core/downloader/handlers/ftp.py E128 E127+ scrapy/core/downloader/handlers/s3.py E128 E126# scrapy/downloadermiddlewares- scrapy/downloadermiddlewares/ajaxcrawl.py E501- scrapy/downloadermiddlewares/decompression.py E501- scrapy/downloadermiddlewares/defaultheaders.py E501- scrapy/downloadermiddlewares/httpcache.py E501 E126- scrapy/downloadermiddlewares/httpcompression.py E501 E128- scrapy/downloadermiddlewares/httpproxy.py E501- scrapy/downloadermiddlewares/redirect.py E501 W504- scrapy/downloadermiddlewares/retry.py E501 E126- scrapy/downloadermiddlewares/robotstxt.py E501- scrapy/downloadermiddlewares/stats.py E501+ scrapy/downloadermiddlewares/httpcache.py E126+ scrapy/downloadermiddlewares/httpcompression.py E128+ scrapy/downloadermiddlewares/redirect.py W504+ scrapy/downloadermiddlewares/retry.py E126# scrapy/extensions- scrapy/extensions/closespider.py E501 E128 E123- scrapy/extensions/corestats.py E501- scrapy/extensions/feedexport.py E128 E501- scrapy/extensions/httpcache.py E128 E501- scrapy/extensions/memdebug.py E501- scrapy/extensions/spiderstate.py E501- scrapy/extensions/telnet.py E501 W504- scrapy/extensions/throttle.py E501+ scrapy/extensions/closespider.py E128 E123+ scrapy/extensions/feedexport.py E128+ scrapy/extensions/httpcache.py E128+ scrapy/extensions/telnet.py W504# scrapy/http- scrapy/http/common.py E501- scrapy/http/cookies.py E501- scrapy/http/request/__init__.py E501- scrapy/http/request/form.py E501 E123- scrapy/http/request/json_request.py E501- scrapy/http/response/__init__.py E501 E128- scrapy/http/response/text.py E501 E128 E124+ scrapy/http/request/form.py E123+ scrapy/http/response/__init__.py E128+ scrapy/http/response/text.py E128 E124# scrapy/linkextractors- scrapy/linkextractors/__init__.py E731 E501 E402 W504- scrapy/linkextractors/lxmlhtml.py E501 E731+ scrapy/linkextractors/__init__.py E731 E402 W504+ scrapy/linkextractors/lxmlhtml.py E731# scrapy/loader- scrapy/loader/__init__.py E501 E128- scrapy/loader/processors.py E501+ scrapy/loader/__init__.py E128+ scrapy/loader/processors.py# scrapy/pipelines- scrapy/pipelines/__init__.py E501- scrapy/pipelines/files.py E116 E501 E266- scrapy/pipelines/images.py E265 E501- scrapy/pipelines/media.py E125 E501 E266+ scrapy/pipelines/files.py E116 E266+ scrapy/pipelines/images.py E265+ scrapy/pipelines/media.py E125 E266# scrapy/selectorscrapy/selector/__init__.py F403- scrapy/selector/unified.py E501 E111+ scrapy/selector/unified.py E111# scrapy/settings- scrapy/settings/__init__.py E501- scrapy/settings/default_settings.py E501 E114 E116- scrapy/settings/deprecated.py E501+ scrapy/settings/default_settings.py E114 E116# scrapy/spidermiddlewares- scrapy/spidermiddlewares/httperror.py E501- scrapy/spidermiddlewares/offsite.py E501- scrapy/spidermiddlewares/referer.py E501 E129 W504- scrapy/spidermiddlewares/urllength.py E501+ scrapy/spidermiddlewares/referer.py E129 W504# scrapy/spiders- scrapy/spiders/__init__.py E501 E402- scrapy/spiders/crawl.py E501- scrapy/spiders/feed.py E501- scrapy/spiders/sitemap.py E501+ scrapy/spiders/__init__.py E402# scrapy/utils- scrapy/utils/asyncio.py E501- scrapy/utils/benchserver.py E501- scrapy/utils/conf.py E402 E501- scrapy/utils/datatypes.py E501- scrapy/utils/decorators.py E501- scrapy/utils/defer.py E501 E128- scrapy/utils/deprecate.py E128 E501 E127 E502- scrapy/utils/gz.py E501 W504+ scrapy/utils/defer.py E128+ scrapy/utils/deprecate.py E128 E127 E502+ scrapy/utils/gz.py W504scrapy/utils/http.py F403- scrapy/utils/httpobj.py E501- scrapy/utils/iterators.py E501- scrapy/utils/log.py E128 E501+ scrapy/utils/log.py E128scrapy/utils/markup.py F403- scrapy/utils/misc.py E501scrapy/utils/multipart.py F403- scrapy/utils/project.py E501- scrapy/utils/python.py E501- scrapy/utils/reactor.py E501- scrapy/utils/reqser.py E501- scrapy/utils/request.py E127 E501- scrapy/utils/response.py E501 E128- scrapy/utils/signal.py E501 E128- scrapy/utils/sitemap.py E501- scrapy/utils/spider.py E501- scrapy/utils/ssl.py E501- scrapy/utils/test.py E501- scrapy/utils/url.py E501 F403 E128 F405+ scrapy/utils/request.py E127+ scrapy/utils/response.py E128+ scrapy/utils/signal.py E128+ scrapy/utils/url.py F403 E128 F405# scrapy- scrapy/__init__.py E402 E501- scrapy/cmdline.py E501- scrapy/crawler.py E501- scrapy/dupefilters.py E501 E202- scrapy/exceptions.py E501- scrapy/exporters.py E501- scrapy/interfaces.py E501- scrapy/item.py E501 E128- scrapy/link.py E501- scrapy/logformatter.py E501- scrapy/mail.py E402 E128 E501 E502- scrapy/middleware.py E128 E501- scrapy/pqueues.py E501- scrapy/resolver.py E501- scrapy/responsetypes.py E128 E501- scrapy/robotstxt.py E501- scrapy/shell.py E501- scrapy/signalmanager.py E501- scrapy/spiderloader.py F841 E501 E126+ scrapy/__init__.py E402+ scrapy/dupefilters.py E202+ scrapy/item.py E128+ scrapy/mail.py E402 E128 E502+ scrapy/middleware.py E128+ scrapy/responsetypes.py E128+ scrapy/spiderloader.py F841 E126scrapy/squeues.py E128- scrapy/statscollectors.py E501# teststests/__init__.py E402 E501tests/mockserver.py E401 E501 E126 E123diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex a4ec7c8ae..b189e016b 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -165,6 +165,7 @@ if __name__ == '__main__':try:execute()finally:- # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()- # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies+ # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:+ # http://doc.pypy.org/en/latest/cpython_differences.html+ # ?highlight=gc.collect#differences-related-to-garbage-collection-strategiesgarbage_collect()diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 9c0014206..83c7b1f19 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):for method in self.methods['process_request']:response = yield deferred_from_coro(method(request=request, spider=spider))if response is not None and not isinstance(response, (Response, Request)):- raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \- (method.__self__.__class__.__name__, response.__class__.__name__))+ raise _InvalidOutput(+ "Middleware %s.process_request must return None, Response or Request, got %s"+ % (method.__self__.__class__.__name__, response.__class__.__name__)+ )if response:defer.returnValue(response)defer.returnValue((yield download_func(request=request, spider=spider)))@@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):for method in self.methods['process_response']:response = yield deferred_from_coro(method(request=request, response=response, spider=spider))if not isinstance(response, (Response, Request)):- raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \- (method.__self__.__class__.__name__, type(response)))+ raise _InvalidOutput(+ "Middleware %s.process_response must return Response or Request, got %s"+ % (method.__self__.__class__.__name__, type(response))+ )if isinstance(response, Request):defer.returnValue(response)defer.returnValue(response)@@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):for method in self.methods['process_exception']:response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))if response is not None and not isinstance(response, (Response, Request)):- raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \- (method.__self__.__class__.__name__, type(response)))+ raise _InvalidOutput(+ "Middleware %s.process_exception must return None, Response or Request, got %s"+ % (method.__self__.__class__.__name__, type(response))+ )if response:defer.returnValue(response)defer.returnValue(_failure)diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 0a9af3a62..52cf09844 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):except UnicodeDecodeError:# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.# Switch to 'allow all' state.- logger.warning("Failure while parsing robots.txt. "- "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",- exc_info=sys.exc_info(),- extra={'spider': spider})+ logger.warning(+ "Failure while parsing robots.txt. File either contains garbage or "+ "is in an encoding other than UTF-8, treating it as an empty file.",+ exc_info=sys.exc_info(),+ extra={'spider': spider},+ )robotstxt_body = ''return robotstxt_bodydiff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex d76a96451..3d62224bf 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -54,8 +54,12 @@ class Rule:self.process_request = _get_method(self.process_request, spider)self.process_request_argcount = len(get_func_args(self.process_request))if self.process_request_argcount == 1:- msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'- warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)+ warnings.warn(+ "Rule.process_request should accept two arguments"+ " (request, response), accepting only one is deprecated",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )def _process_request(self, request, response):"""