Scrapy, a fast high-level web crawling & scraping framework for Python.
Flake8: Remove E126
Eugenio Lacuesta4c12a234ae65
diff --git a/pytest.ini b/pytest.iniindex 998633d54..1570a3a75 100644--- a/pytest.ini+++ b/pytest.ini@@ -50,26 +50,26 @@ flake8-ignore =scrapy/core/engine.py E501 E128scrapy/core/scheduler.py E501scrapy/core/scraper.py E501 E128- scrapy/core/spidermw.py E501 E126+ scrapy/core/spidermw.py E501scrapy/core/downloader/__init__.py E501- scrapy/core/downloader/contextfactory.py E501 E128 E126+ scrapy/core/downloader/contextfactory.py E501 E128scrapy/core/downloader/middleware.py E501scrapy/core/downloader/tls.py E501- scrapy/core/downloader/webclient.py E501 E128 E126+ scrapy/core/downloader/webclient.py E501 E128scrapy/core/downloader/handlers/__init__.py E501scrapy/core/downloader/handlers/ftp.py E501 E128scrapy/core/downloader/handlers/http10.py E501scrapy/core/downloader/handlers/http11.py E501- scrapy/core/downloader/handlers/s3.py E501 E128 E126+ scrapy/core/downloader/handlers/s3.py E501 E128# scrapy/downloadermiddlewaresscrapy/downloadermiddlewares/ajaxcrawl.py E501scrapy/downloadermiddlewares/decompression.py E501scrapy/downloadermiddlewares/defaultheaders.py E501- scrapy/downloadermiddlewares/httpcache.py E501 E126+ scrapy/downloadermiddlewares/httpcache.py E501scrapy/downloadermiddlewares/httpcompression.py E501 E128scrapy/downloadermiddlewares/httpproxy.py E501scrapy/downloadermiddlewares/redirect.py E501- scrapy/downloadermiddlewares/retry.py E501 E126+ scrapy/downloadermiddlewares/retry.py E501scrapy/downloadermiddlewares/robotstxt.py E501scrapy/downloadermiddlewares/stats.py E501# scrapy/extensions@@ -164,12 +164,12 @@ flake8-ignore =scrapy/robotstxt.py E501scrapy/shell.py E501scrapy/signalmanager.py E501- scrapy/spiderloader.py F841 E501 E126+ scrapy/spiderloader.py F841 E501scrapy/squeues.py E128scrapy/statscollectors.py E501# teststests/__init__.py E402 E501- tests/mockserver.py E501 E126+ tests/mockserver.py E501tests/pipelines.py F841tests/spiders.py E501tests/test_closespider.py E501@@ -181,18 +181,18 @@ flake8-ignore =tests/test_crawl.py E501 E741tests/test_crawler.py F841 E501tests/test_dependencies.py F841 E501- tests/test_downloader_handlers.py E128 E501 E126+ tests/test_downloader_handlers.py E128 E501tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501- tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126+ tests/test_downloadermiddleware_cookies.py E741 E501 E128tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501tests/test_downloadermiddleware_httpcache.py E501- tests/test_downloadermiddleware_httpcompression.py E501 E126+ tests/test_downloadermiddleware_httpcompression.py E501tests/test_downloadermiddleware_decompression.py E501tests/test_downloadermiddleware_httpproxy.py E501 E128tests/test_downloadermiddleware_redirect.py E501 E128- tests/test_downloadermiddleware_retry.py E501 E128 E126+ tests/test_downloadermiddleware_retry.py E501 E128tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501tests/test_dupefilters.py E501 E741 E128@@ -202,7 +202,7 @@ flake8-ignore =tests/test_feedexport.py E501 F841tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_http_request.py E402 E501 E128 E128 E126+ tests/test_http_request.py E402 E501 E128 E128tests/test_http_response.py E501 E128tests/test_item.py E128 F841tests/test_link.py E501@@ -211,7 +211,7 @@ flake8-ignore =tests/test_logformatter.py E128 E501tests/test_mail.py E128 E501tests/test_middleware.py E501 E128- tests/test_pipeline_crawl.py E501 E128 E126+ tests/test_pipeline_crawl.py E501 E128tests/test_pipeline_files.py E501tests/test_pipeline_images.py F841 E501tests/test_pipeline_media.py E501 E741 E128@@ -219,7 +219,7 @@ flake8-ignore =tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501tests/test_robotstxt_interface.py E501 E501- tests/test_scheduler.py E501 E126+ tests/test_scheduler.py E501tests/test_selector.py E501tests/test_spider.py E501tests/test_spidermiddleware.py E501@@ -243,8 +243,8 @@ flake8-ignore =tests/test_utils_response.py E501tests/test_utils_signal.py E741 F841tests/test_utils_sitemap.py E128 E501- tests/test_utils_url.py E501 E501 E126- tests/test_webclient.py E501 E128 E402 E126+ tests/test_utils_url.py E501 E501+ tests/test_webclient.py E501 E128 E402tests/test_cmdline/__init__.py E501tests/test_settings/__init__.py E501 E128tests/test_spiderloader/__init__.py E128 E501diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 6e023ebcc..ab73e12c8 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):## This means that a website like https://www.cacert.org will be rejected# by default, since CAcert.org CA certificate is seldom shipped.- return optionsForClientTLS(hostname.decode("ascii"),- trustRoot=platformTrust(),- extraCertificateOptions={- 'method': self._ssl_method,- })+ return optionsForClientTLS(+ hostname=hostname.decode("ascii"),+ trustRoot=platformTrust(),+ extraCertificateOptions={'method': self._ssl_method},+ )diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 40a1fa48e..8f63ad974 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -100,11 +100,12 @@ class S3DownloadHandler:url=url, headers=awsrequest.headers.items())else:signed_headers = self.conn.make_request(- method=request.method,- bucket=bucket,- key=unquote(p.path),- query_args=unquote(p.query),- headers=request.headers,- data=request.body)+ method=request.method,+ bucket=bucket,+ key=unquote(p.path),+ query_args=unquote(p.query),+ headers=request.headers,+ data=request.body,+ )request = request.replace(url=url, headers=signed_headers)return self._download_http(request, spider)diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex a90a77b2b..355045d74 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):self.transport.stopProducing()self.factory.noPage(- defer.TimeoutError("Getting %s took longer than %s seconds." %- (self.factory.url, self.factory.timeout)))+ defer.TimeoutError("Getting %s took longer than %s seconds."+ % (self.factory.url, self.factory.timeout)))class ScrapyHTTPClientFactory(HTTPClientFactory):diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex bbf5fca05..6d11af5b2 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.import loggingfrom twisted.internet import defer-from twisted.internet.error import TimeoutError, DNSLookupError, \- ConnectionRefusedError, ConnectionDone, ConnectError, \- ConnectionLost, TCPTimedOutError+from twisted.internet.error import (+ ConnectError,+ ConnectionDone,+ ConnectionLost,+ ConnectionRefusedError,+ DNSLookupError,+ TCPTimedOutError,+ TimeoutError,+)from twisted.web.client import ResponseFailedfrom scrapy.exceptions import NotConfigureddiff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex 3be5aaec5..8dc89c2e9 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -24,15 +24,17 @@ class SpiderLoader:self._load_all_spiders()def _check_name_duplicates(self):- dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(- module=mod, cls=cls, name=name)- for (mod, cls) in locations)- for name, locations in self._found.items()- if len(locations) > 1]+ dupes = []+ for name, locations in self._found.items():+ dupes.extend([+ " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)+ for mod, cls in locations+ ])+if dupes:+ dupes_string = "\n\n".join(dupes)msg = ("There are several spiders with the same name:\n\n"- "{}\n\n This can cause unexpected behavior.".format(- "\n\n".join(dupes)))+ "{}\n\n This can cause unexpected behavior.".format(dupes_string))warnings.warn(msg, UserWarning)def _load_spiders(self, module):@@ -45,11 +47,12 @@ class SpiderLoader:try:for module in walk_modules(name):self._load_spiders(module)- except ImportError as e:+ except ImportError:if self.warn_only:- msg = ("\n{tb}Could not load spiders from module '{modname}'. "- "See above traceback for details.".format(- modname=name, tb=traceback.format_exc()))+ msg = (+ "\n{tb}Could not load spiders from module '{modname}'. "+ "See above traceback for details.".format(modname=name, tb=traceback.format_exc())+ )warnings.warn(msg, RuntimeWarning)else:raisediff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex b686a14d6..f86c50f50 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):def test_complex_cookies(self):# merge some cookies into jar- cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},- {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},- {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},- {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]+ cookies = [+ {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},+ {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},+ {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},+ {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},+ ]req = Request('http://scrapytest.org/', cookies=cookies)self.mw.process_request(req, self.spider)diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 9c989977e..e118750e3 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -1,8 +1,14 @@import unittestfrom twisted.internet import defer-from twisted.internet.error import TimeoutError, DNSLookupError, \- ConnectionRefusedError, ConnectionDone, ConnectError, \- ConnectionLost, TCPTimedOutError+from twisted.internet.error import (+ ConnectError,+ ConnectionDone,+ ConnectionLost,+ ConnectionRefusedError,+ DNSLookupError,+ TCPTimedOutError,+ TimeoutError,+)from twisted.web.client import ResponseFailedfrom scrapy.downloadermiddlewares.retry import RetryMiddlewarediff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 3b6d119a9..77da15ce6 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -502,11 +502,13 @@ class FormRequestTest(RequestTest):def test_from_response_duplicate_form_key(self):response = _buildresponse(- '<form></form>',- url='http://www.example.com')- req = self.request_class.from_response(response,- method='GET',- formdata=(('foo', 'bar'), ('foo', 'baz')))+ '<form></form>',+ url='http://www.example.com')+ req = self.request_class.from_response(+ response=response,+ method='GET',+ formdata=(('foo', 'bar'), ('foo', 'baz')),+ )self.assertEqual(urlparse(req.url).hostname, 'www.example.com')self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')@@ -530,9 +532,11 @@ class FormRequestTest(RequestTest):<input type="hidden" name="test" value="val2"><input type="hidden" name="test2" value="xxx"></form>""")- req = self.request_class.from_response(response,- formdata={'one': ['two', 'three'], 'six': 'seven'},- headers={"Accept-Encoding": "gzip,deflate"})+ req = self.request_class.from_response(+ response=response,+ formdata={'one': ['two', 'three'], 'six': 'seven'},+ headers={"Accept-Encoding": "gzip,deflate"},+ )self.assertEqual(req.method, 'POST')self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')@@ -580,9 +584,9 @@ class FormRequestTest(RequestTest):def test_from_response_override_method(self):response = _buildresponse(- '''<html><body>- <form action="/app"></form>- </body></html>''')+ '''<html><body>+ <form action="/app"></form>+ </body></html>''')request = FormRequest.from_response(response)self.assertEqual(request.method, 'GET')request = FormRequest.from_response(response, method='POST')@@ -590,9 +594,9 @@ class FormRequestTest(RequestTest):def test_from_response_override_url(self):response = _buildresponse(- '''<html><body>- <form action="/app"></form>- </body></html>''')+ '''<html><body>+ <form action="/app"></form>+ </body></html>''')request = FormRequest.from_response(response)self.assertEqual(request.url, 'http://example.com/app')request = FormRequest.from_response(response, url='http://foo.bar/absolute')diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 962c33144..24c516473 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):name = 'redirectedmedia'def _process_url(self, url):- return add_or_replace_parameter(- self.mockserver.url('/redirect-to'),- 'goto', url)+ return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)class FileDownloadCrawlTestCase(TestCase):diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex b657c7ab6..307fadb5c 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -18,6 +18,14 @@ except ImportError:from twisted.python.filepath import FilePathfrom twisted.protocols.policies import WrappingFactoryfrom twisted.internet.defer import inlineCallbacks+from twisted.web.test.test_webclient import (+ ForeverTakingResource,+ ErrorResource,+ NoLengthResource,+ HostHeaderResource,+ PayloadResource,+ BrokenDownloadResource,+)from scrapy.core.downloader import webclient as clientfrom scrapy.core.downloader.contextfactory import ScrapyClientContextFactory@@ -202,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):Headers({'Hello': ['World'], 'Foo': ['Bar']}))-from twisted.web.test.test_webclient import ForeverTakingResource, \- ErrorResource, NoLengthResource, HostHeaderResource, \- PayloadResource, BrokenDownloadResource--class EncodingResource(resource.Resource):out_encoding = 'cp1251'