Scrapy, a fast high-level web crawling & scraping framework for Python.
Flake8: remove E127 in tests (continuation line over-indented for visual indent)
Eugenio Lacuestafe0c582ee083
diff --git a/pytest.ini b/pytest.iniindex fa65a0da2..3eefe70f1 100644--- a/pytest.ini+++ b/pytest.ini@@ -171,8 +171,8 @@ flake8-ignore =tests/__init__.py E402 E501tests/mockserver.py E401 E501 E126 E123tests/pipelines.py F841- tests/spiders.py E501 E127- tests/test_closespider.py E501 E127+ tests/spiders.py E501+ tests/test_closespider.py E501tests/test_command_fetch.py E501tests/test_command_parse.py E501 E128tests/test_command_shell.py E501 E128@@ -181,17 +181,17 @@ flake8-ignore =tests/test_crawl.py E501 E741tests/test_crawler.py F841 E501tests/test_dependencies.py F841 E501- tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123+ tests/test_downloader_handlers.py E124 E128 E501 E126 E123tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126- tests/test_downloadermiddleware_decompression.py E127tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501tests/test_downloadermiddleware_httpcache.py E501tests/test_downloadermiddleware_httpcompression.py E501 E126 E123+ tests/test_downloadermiddleware_decompression.py E501tests/test_downloadermiddleware_httpproxy.py E501 E128- tests/test_downloadermiddleware_redirect.py E501 E128 E127+ tests/test_downloadermiddleware_redirect.py E501 E128tests/test_downloadermiddleware_retry.py E501 E128 E126tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501@@ -202,7 +202,7 @@ flake8-ignore =tests/test_feedexport.py E501 F841tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123+ tests/test_http_request.py E402 E501 E128 E128 E126 E123tests/test_http_response.py E501 E128tests/test_item.py E128 F841tests/test_link.py E501@@ -220,10 +220,10 @@ flake8-ignore =tests/test_responsetypes.py E501tests/test_robotstxt_interface.py E501 E501tests/test_scheduler.py E501 E126 E123- tests/test_selector.py E501 E127+ tests/test_selector.py E501tests/test_spider.py E501tests/test_spidermiddleware.py E501- tests/test_spidermiddleware_httperror.py E128 E501 E127 E121+ tests/test_spidermiddleware_httperror.py E128 E501 E121tests/test_spidermiddleware_offsite.py E501 E128 E111tests/test_spidermiddleware_output_chain.py E501tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121@@ -243,7 +243,7 @@ flake8-ignore =tests/test_utils_response.py E501tests/test_utils_signal.py E741 F841tests/test_utils_sitemap.py E128 E501 E124- tests/test_utils_url.py E501 E127 E125 E501 E126 E123+ tests/test_utils_url.py E501 E125 E501 E126 E123tests/test_webclient.py E501 E128 E122 E402 E123 E126tests/test_cmdline/__init__.py E501tests/test_settings/__init__.py E501 E128diff --git a/tests/spiders.py b/tests/spiders.pyindex 284c77829..33d5d02e1 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):if self.fail_yielding:2 / 0- assert self.seedsseen, \- 'All start requests consumed before any download happened'+ assert self.seedsseen, 'All start requests consumed before any download happened'def parse(self, response):self.seedsseen.append(response.meta.get('seed'))diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 4a56425b7..5ec5e2989 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):yield crawler.crawl(total=1000000, mockserver=self.mockserver)reason = crawler.spider.meta['close_reason']self.assertEqual(reason, 'closespider_errorcount')- key = 'spider_exceptions/{name}'\- .format(name=crawler.spider.exception_cls.__name__)+ key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)errorcount = crawler.stats.get_value(key)self.assertTrue(errorcount >= close_on)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 29d06bab4..24ef560c1 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -1090,8 +1090,7 @@ class DataURITestCase(unittest.TestCase):def test_default_mediatype_encoding(self):def _test(response):self.assertEqual(response.text, 'A brief note')- self.assertEqual(type(response),- responsetypes.from_mimetype("text/plain"))+ self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))self.assertEqual(response.encoding, "US-ASCII")request = Request("data:,A%20brief%20note")@@ -1100,8 +1099,7 @@ class DataURITestCase(unittest.TestCase):def test_default_mediatype(self):def _test(response):self.assertEqual(response.text, u'\u038e\u03a3\u038e')- self.assertEqual(type(response),- responsetypes.from_mimetype("text/plain"))+ self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))self.assertEqual(response.encoding, "iso-8859-7")request = Request("data:;charset=iso-8859-7,%be%d3%be")@@ -1119,8 +1117,7 @@ class DataURITestCase(unittest.TestCase):def test_mediatype_parameters(self):def _test(response):self.assertEqual(response.text, u'\u038e\u03a3\u038e')- self.assertEqual(type(response),- responsetypes.from_mimetype("text/plain"))+ self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))self.assertEqual(response.encoding, "utf-8")request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.pyindex 77b35a8c3..dbae4d3ae 100644--- a/tests/test_downloadermiddleware_decompression.py+++ b/tests/test_downloadermiddleware_decompression.py@@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):for fmt in self.test_formats:rsp = self.test_responses[fmt]new = self.mw.process_response(None, rsp, self.spider)- assert isinstance(new, XmlResponse), \- 'Failed %s, response type %s' % (fmt, type(new).__name__)+ error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)+ assert isinstance(new, XmlResponse), error_msgassert_samelines(self, new.body, self.uncompressed_body, fmt)def test_plain_response(self):diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 053e26fc3..551e124ab 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):rsp = Response(url, headers={'Location': url2}, status=301, request=req)r = self.mw.process_response(req, rsp, self.spider)self.assertIs(r, rsp)- _test_passthrough(Request(url, meta={'handle_httpstatus_list':- [404, 301, 302]}))+ _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))def test_latin1_location(self):diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex cc2cddda4..b12841ba2 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):def test_custom_encoding_bytes(self):data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}- r2 = self.request_class("http://www.example.com", formdata=data,- encoding='latin1')+ r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')self.assertEqual(r2.method, 'POST')self.assertEqual(r2.encoding, 'latin1')self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')@@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):def test_custom_encoding_textual_data(self):data = {'price': u'£ 100'}- r3 = self.request_class("http://www.example.com", formdata=data,- encoding='latin1')+ r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')self.assertEqual(r3.encoding, 'latin1')self.assertEqual(r3.body, b'price=%A3+100')diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 09c2546fb..65b0f5860 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):headers = {'Content-Type': ['text/html; charset=utf-8']}response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)x = Selector(response)- self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),- [u'\xa3'])+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])def test_badly_encoded_body(self):# \xe9 alone isn't valid utf8 sequencediff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex dacd0147f..6b61df56f 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):self.mw.process_spider_input(self.res402, self.spider))def test_meta_overrides_settings(self):- request = Request('http://scrapytest.org',- meta={'handle_httpstatus_list': [404]})+ request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})res404 = self.res404.copy()res404.request = requestres402 = self.res402.copy()@@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):self.mw.process_spider_input(self.res404, self.spider))def test_meta_overrides_settings(self):- request = Request('http://scrapytest.org',- meta={'handle_httpstatus_list': [404]})+ request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})res404 = self.res404.copy()res404.request = requestres402 = self.res402.copy()diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 72a16e9b1..3bb6d40db 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):class AddHttpIfNoScheme(unittest.TestCase):def test_add_scheme(self):- self.assertEqual(add_http_if_no_scheme('www.example.com'),- 'http://www.example.com')+ self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')def test_without_subdomain(self):- self.assertEqual(add_http_if_no_scheme('example.com'),- 'http://example.com')+ self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')def test_path(self):- self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),- 'http://www.example.com/some/page.html')+ self.assertEqual(+ add_http_if_no_scheme('www.example.com/some/page.html'),+ 'http://www.example.com/some/page.html')def test_port(self):- self.assertEqual(add_http_if_no_scheme('www.example.com:80'),- 'http://www.example.com:80')+ self.assertEqual(+ add_http_if_no_scheme('www.example.com:80'),+ 'http://www.example.com:80')def test_fragment(self):- self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),- 'http://www.example.com/some/page#frag')+ self.assertEqual(+ add_http_if_no_scheme('www.example.com/some/page#frag'),+ 'http://www.example.com/some/page#frag')def test_query(self):- self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),- 'http://www.example.com/do?a=1&b=2&c=3')+ self.assertEqual(+ add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),+ 'http://www.example.com/do?a=1&b=2&c=3')def test_username_password(self):- self.assertEqual(add_http_if_no_scheme('username:[email protected]'),- 'http://username:[email protected]')+ self.assertEqual(+ add_http_if_no_scheme('username:[email protected]'),+ 'http://username:[email protected]')def test_complete_url(self):- self.assertEqual(add_http_if_no_scheme('username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),- 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')+ self.assertEqual(+ add_http_if_no_scheme('username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),+ 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')def test_preserve_http(self):- self.assertEqual(add_http_if_no_scheme('http://www.example.com'),- 'http://www.example.com')+ self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')def test_preserve_http_without_subdomain(self):- self.assertEqual(add_http_if_no_scheme('http://example.com'),- 'http://example.com')+ self.assertEqual(+ add_http_if_no_scheme('http://example.com'),+ 'http://example.com')def test_preserve_http_path(self):- self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),- 'http://www.example.com/some/page.html')+ self.assertEqual(+ add_http_if_no_scheme('http://www.example.com/some/page.html'),+ 'http://www.example.com/some/page.html')def test_preserve_http_port(self):- self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),- 'http://www.example.com:80')+ self.assertEqual(+ add_http_if_no_scheme('http://www.example.com:80'),+ 'http://www.example.com:80')def test_preserve_http_fragment(self):- self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),- 'http://www.example.com/some/page#frag')+ self.assertEqual(+ add_http_if_no_scheme('http://www.example.com/some/page#frag'),+ 'http://www.example.com/some/page#frag')def test_preserve_http_query(self):- self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),- 'http://www.example.com/do?a=1&b=2&c=3')+ self.assertEqual(+ add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),+ 'http://www.example.com/do?a=1&b=2&c=3')def test_preserve_http_username_password(self):- self.assertEqual(add_http_if_no_scheme('http://username:[email protected]'),- 'http://username:[email protected]')+ self.assertEqual(+ add_http_if_no_scheme('http://username:[email protected]'),+ 'http://username:[email protected]')def test_preserve_http_complete_url(self):- self.assertEqual(add_http_if_no_scheme('http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),- 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')+ self.assertEqual(+ add_http_if_no_scheme('http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),+ 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')def test_protocol_relative(self):- self.assertEqual(add_http_if_no_scheme('//www.example.com'),- 'http://www.example.com')+ self.assertEqual(+ add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')def test_protocol_relative_without_subdomain(self):- self.assertEqual(add_http_if_no_scheme('//example.com'),- 'http://example.com')+ self.assertEqual(+ add_http_if_no_scheme('//example.com'), 'http://example.com')def test_protocol_relative_path(self):- self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),- 'http://www.example.com/some/page.html')+ self.assertEqual(+ add_http_if_no_scheme('//www.example.com/some/page.html'),+ 'http://www.example.com/some/page.html')def test_protocol_relative_port(self):- self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),- 'http://www.example.com:80')+ self.assertEqual(+ add_http_if_no_scheme('//www.example.com:80'),+ 'http://www.example.com:80')def test_protocol_relative_fragment(self):- self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),- 'http://www.example.com/some/page#frag')+ self.assertEqual(+ add_http_if_no_scheme('//www.example.com/some/page#frag'),+ 'http://www.example.com/some/page#frag')def test_protocol_relative_query(self):- self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),- 'http://www.example.com/do?a=1&b=2&c=3')+ self.assertEqual(+ add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),+ 'http://www.example.com/do?a=1&b=2&c=3')def test_protocol_relative_username_password(self):- self.assertEqual(add_http_if_no_scheme('//username:[email protected]'),- 'http://username:[email protected]')+ self.assertEqual(+ add_http_if_no_scheme('//username:[email protected]'),+ 'http://username:[email protected]')def test_protocol_relative_complete_url(self):- self.assertEqual(add_http_if_no_scheme('//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),- 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')+ self.assertEqual(+ add_http_if_no_scheme('//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag'),+ 'http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag')def test_preserve_https(self):- self.assertEqual(add_http_if_no_scheme('https://www.example.com'),- 'https://www.example.com')+ self.assertEqual(+ add_http_if_no_scheme('https://www.example.com'),+ 'https://www.example.com')def test_preserve_ftp(self):- self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),- 'ftp://www.example.com')+ self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')class GuessSchemeTest(unittest.TestCase):