Scrapy, a fast high-level web crawling & scraping framework for Python.
Flake8: Remove E128 (tests, part 2)
Eugenio Lacuestabf56517abfc3
diff --git a/pytest.ini b/pytest.iniindex d8eefea7e..0b2766031 100644--- a/pytest.ini+++ b/pytest.ini@@ -200,19 +200,19 @@ flake8-ignore =tests/test_feedexport.py E501 F841tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_item.py E128 F841tests/test_http_request.py E402 E501tests/test_http_response.py E501+ tests/test_item.py E501 F841tests/test_link.py E501- tests/test_linkextractors.py E501 E128- tests/test_loader.py E501 E741 E128 E117- tests/test_logformatter.py E128 E501- tests/test_mail.py E128 E501- tests/test_middleware.py E501 E128- tests/test_pipeline_crawl.py E501 E128+ tests/test_linkextractors.py E501+ tests/test_loader.py E501 E741 E117+ tests/test_logformatter.py E501+ tests/test_mail.py E501+ tests/test_middleware.py E501+ tests/test_pipeline_crawl.py E501tests/test_pipeline_files.py E501tests/test_pipeline_images.py F841 E501- tests/test_pipeline_media.py E501 E741 E128+ tests/test_pipeline_media.py E501 E741tests/test_proxy_connect.py E501 E741tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501@@ -221,8 +221,8 @@ flake8-ignore =tests/test_selector.py E501tests/test_spider.py E501tests/test_spidermiddleware.py E501- tests/test_spidermiddleware_httperror.py E128 E501 E121- tests/test_spidermiddleware_offsite.py E501 E128 E111+ tests/test_spidermiddleware_httperror.py E501 E121+ tests/test_spidermiddleware_offsite.py E501 E111tests/test_spidermiddleware_output_chain.py E501tests/test_spidermiddleware_referer.py E501 F841 E501 E121tests/test_squeues.py E501 E741diff --git a/tests/test_item.py b/tests/test_item.pyindex 4017f6e84..4fe247cda 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):item = D(save='X', load='Y')self.assertEqual(item['save'], 'X')self.assertEqual(item['load'], 'Y')- self.assertEqual(D.fields, {'load': {'default': 'A'},- 'save': {'default': 'A'}})+ self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})# D class invertedclass E(C, B):@@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):self.assertEqual(E(save='X')['save'], 'X')self.assertEqual(E(load='X')['load'], 'X')- self.assertEqual(E.fields, {'load': {'default': 'C'},- 'save': {'default': 'C'}})+ self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})def test_metaclass_multiple_inheritance_diamond(self):class A(Item):@@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):self.assertEqual(D(save='X')['save'], 'X')self.assertEqual(D(load='X')['load'], 'X')- self.assertEqual(D.fields, {'save': {'default': 'C'},- 'load': {'default': 'D'}, 'update': {'default': 'D'}})+ self.assertEqual(+ D.fields,+ {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})# D class invertedclass E(C, B):@@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):self.assertEqual(E(save='X')['save'], 'X')self.assertEqual(E(load='X')['load'], 'X')- self.assertEqual(E.fields, {'save': {'default': 'C'},- 'load': {'default': 'E'}, 'update': {'default': 'C'}})+ self.assertEqual(+ E.fields,+ {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})def test_metaclass_multiple_inheritance_without_metaclass(self):class A(Item):@@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):self.assertRaises(KeyError, D, not_allowed='value')self.assertEqual(D(save='X')['save'], 'X')- self.assertEqual(D.fields, {'save': {'default': 'A'},- 'load': {'default': 'A'}})+ self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})# D class invertedclass E(C, B):@@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):self.assertRaises(KeyError, E, not_allowed='value')self.assertEqual(E(save='X')['save'], 'X')- self.assertEqual(E.fields, {'save': {'default': 'A'},- 'load': {'default': 'A'}})+ self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})def test_to_dict(self):class TestItem(Item):diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 68e8514ba..7e1b62b7f 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -171,9 +171,9 @@ class Base:self.assertEqual(lx.matches(url1), False)self.assertEqual(lx.matches(url2), True)- lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),- allow_domains=('blah1.com',),- deny_domains=('blah2.com',))+ lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],+ allow_domains=['blah1.com'],+ deny_domains=['blah2.com'])self.assertEqual(lx.matches('http://blah1.com/blah1'), True)self.assertEqual(lx.matches('http://blah1.com/blah2'), False)self.assertEqual(lx.matches('http://blah2.com/blah1'), False)diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex cd6cb8016..7064337ad 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline,- "Crawled (200) <GET http://www.example.com> (referer: None)")+ self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")def test_crawled_without_referer(self):req = Request("http://www.example.com", headers={'referer': 'http://example.com'})res = Response("http://www.example.com", flags=['cached'])logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline,+ self.assertEqual(+ logline,"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")def test_flags_in_request(self):@@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline,- "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")+ self.assertEqual(+ logline,+ "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")def test_dropped(self):item = {}@@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline,+ self.assertEqual(+ logline,"Crawled (200) <GET http://www.example.com> (referer: None) []")def test_crawled_without_referer(self):@@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline,+ self.assertEqual(+ logline,"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")def test_flags_in_request(self):@@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")+ self.assertEqual(+ logline,+ "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")class SkipMessagesLogFormatter(LogFormatter):diff --git a/tests/test_mail.py b/tests/test_mail.pyindex f5cb81a8b..53dbc0686 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):mailsender = MailSender(debug=True)mailsender.send(to=['[email protected]'], subject='subject', body='body',- attachs=attachs, _callback=self._catch_mail_sent)+ attachs=attachs, _callback=self._catch_mail_sent)assert self.catched_msgself.assertEqual(self.catched_msg['to'], ['[email protected]'])diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex 3af514bb0..3364d2258 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):def test_methods(self):mwman = TestMiddlewareManager(M1(), M2(), M3())- self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],+ self.assertEqual(+ [x.__self__.__class__ for x in mwman.methods['open_spider']],[M1, M2])- self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],+ self.assertEqual(+ [x.__self__.__class__ for x in mwman.methods['close_spider']],[M2, M1])- self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],+ self.assertEqual(+ [x.__self__.__class__ for x in mwman.methods['process']],[M1, M3])def test_enabled(self):diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 24c516473..188c5c3cf 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase):def test_download_media(self):crawler = self._create_crawler(MediaDownloadSpider)with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/files/images/"),+ yield crawler.crawl(+ self.mockserver.url("/files/images/"),media_key=self.media_key,media_urls_key=self.media_urls_key)self._assert_files_downloaded(self.items, str(log))@@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase):def test_download_media_wrong_urls(self):crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/files/images/"),+ yield crawler.crawl(+ self.mockserver.url("/files/images/"),media_key=self.media_key,media_urls_key=self.media_urls_key)self._assert_files_download_failure(crawler, self.items, 404, str(log))@@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase):def test_download_media_redirected_default_failure(self):crawler = self._create_crawler(RedirectedMediaDownloadSpider)with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/files/images/"),+ yield crawler.crawl(+ self.mockserver.url("/files/images/"),media_key=self.media_key,media_urls_key=self.media_urls_key,mockserver=self.mockserver)@@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase):crawler = self._create_crawler(RedirectedMediaDownloadSpider)with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/files/images/"),+ yield crawler.crawl(+ self.mockserver.url("/files/images/"),media_key=self.media_key,media_urls_key=self.media_urls_key,mockserver=self.mockserver)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 949f0dea1..b51f82422 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):item = dict(requests=req)new_item = yield self.pipe.process_item(item, self.spider)self.assertEqual(new_item['results'], [(True, rsp)])- self.assertEqual(self.pipe._mockcalled,- ['get_media_requests', 'media_to_download',- 'media_downloaded', 'request_callback', 'item_completed'])+ self.assertEqual(+ self.pipe._mockcalled,+ ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])@inlineCallbacksdef test_result_failure(self):@@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):item = dict(requests=req)new_item = yield self.pipe.process_item(item, self.spider)self.assertEqual(new_item['results'], [(False, fail)])- self.assertEqual(self.pipe._mockcalled,- ['get_media_requests', 'media_to_download',- 'media_failed', 'request_errback', 'item_completed'])+ self.assertEqual(+ self.pipe._mockcalled,+ ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])@inlineCallbacksdef test_mix_of_success_and_failure(self):diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 6b61df56f..714279ae0 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):self.res200, self.res404 = _responses(self.req, [200, 404])def test_process_spider_input(self):- self.assertEqual(None,- self.mw.process_spider_input(self.res200, self.spider))- self.assertRaises(HttpError,- self.mw.process_spider_input, self.res404, self.spider)+ self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))+ self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)def test_process_spider_exception(self):- self.assertEqual([],- self.mw.process_spider_exception(self.res404,- HttpError(self.res404), self.spider))- self.assertEqual(None,- self.mw.process_spider_exception(self.res404,- Exception(), self.spider))+ self.assertEqual(+ [],+ self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))+ self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))def test_handle_httpstatus_list(self):res = self.res404.copy()res.request = Request('http://scrapytest.org',meta={'handle_httpstatus_list': [404]})- self.assertEqual(None,- self.mw.process_spider_input(res, self.spider))+ self.assertIsNone(self.mw.process_spider_input(res, self.spider))self.spider.handle_httpstatus_list = [404]- self.assertEqual(None,- self.mw.process_spider_input(self.res404, self.spider))+ self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))class TestHttpErrorMiddlewareSettings(TestCase):@@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])def test_process_spider_input(self):- self.assertEqual(None,- self.mw.process_spider_input(self.res200, self.spider))- self.assertRaises(HttpError,- self.mw.process_spider_input, self.res404, self.spider)- self.assertEqual(None,- self.mw.process_spider_input(self.res402, self.spider))+ self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))+ self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)+ self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))def test_meta_overrides_settings(self):request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})@@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):res402 = self.res402.copy()res402.request = request- self.assertEqual(None,- self.mw.process_spider_input(res404, self.spider))- self.assertRaises(HttpError,- self.mw.process_spider_input, res402, self.spider)+ self.assertIsNone(self.mw.process_spider_input(res404, self.spider))+ self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)def test_spider_override_settings(self):self.spider.handle_httpstatus_list = [404]- self.assertEqual(None,- self.mw.process_spider_input(self.res404, self.spider))- self.assertRaises(HttpError,- self.mw.process_spider_input, self.res402, self.spider)+ self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))+ self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)class TestHttpErrorMiddlewareHandleAll(TestCase):@@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])def test_process_spider_input(self):- self.assertEqual(None,- self.mw.process_spider_input(self.res200, self.spider))- self.assertEqual(None,- self.mw.process_spider_input(self.res404, self.spider))+ self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))+ self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))def test_meta_overrides_settings(self):request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})@@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):res402 = self.res402.copy()res402.request = request- self.assertEqual(None,- self.mw.process_spider_input(res404, self.spider))- self.assertRaises(HttpError,- self.mw.process_spider_input, res402, self.spider)+ self.assertIsNone(self.mw.process_spider_input(res404, self.spider))+ self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)class TestHttpErrorMiddlewareIntegrational(TrialTestCase):diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex b96807bc2..0f4b98a07 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):def test_process_spider_output(self):res = Response('http://scrapytest.org')- onsite_reqs = [Request('http://scrapytest.org/1'),- Request('http://scrapy.org/1'),- Request('http://sub.scrapy.org/1'),- Request('http://offsite.tld/letmepass', dont_filter=True),- Request('http://scrapy.test.org/'),- Request('http://scrapy.test.org:8000/')]- offsite_reqs = [Request('http://scrapy2.org'),- Request('http://offsite.tld/'),- Request('http://offsite.tld/scrapytest.org'),- Request('http://offsite.tld/rogue.scrapytest.org'),- Request('http://rogue.scrapytest.org.haha.com'),- Request('http://roguescrapytest.org'),- Request('http://test.org/'),- Request('http://notscrapy.test.org/')]+ onsite_reqs = [+ Request('http://scrapytest.org/1'),+ Request('http://scrapy.org/1'),+ Request('http://sub.scrapy.org/1'),+ Request('http://offsite.tld/letmepass', dont_filter=True),+ Request('http://scrapy.test.org/'),+ Request('http://scrapy.test.org:8000/'),+ ]+ offsite_reqs = [+ Request('http://scrapy2.org'),+ Request('http://offsite.tld/'),+ Request('http://offsite.tld/scrapytest.org'),+ Request('http://offsite.tld/rogue.scrapytest.org'),+ Request('http://rogue.scrapytest.org.haha.com'),+ Request('http://roguescrapytest.org'),+ Request('http://test.org/'),+ Request('http://notscrapy.test.org/'),+ ]reqs = onsite_reqs + offsite_reqsout = list(self.mw.process_spider_output(res, reqs, self.spider))