Scrapy, a fast high-level web crawling & scraping framework for Python.
fix E30X flake8 (#4355)
Marc Hernández91bbc70bc10c
diff --git a/pytest.ini b/pytest.iniindex 552829d4e..0758d2f8b 100644--- a/pytest.ini+++ b/pytest.ini@@ -47,17 +47,17 @@ flake8-ignore =scrapy/contracts/__init__.py E501 W504scrapy/contracts/default.py E128# scrapy/core- scrapy/core/engine.py E501 E128 E127 E306 E502+ scrapy/core/engine.py E501 E128 E127 E502scrapy/core/scheduler.py E501- scrapy/core/scraper.py E501 E306 E128 W504+ scrapy/core/scraper.py E501 E128 W504scrapy/core/spidermw.py E501 E731 E126 E226scrapy/core/downloader/__init__.py E501scrapy/core/downloader/contextfactory.py E501 E128 E126scrapy/core/downloader/middleware.py E501 E502- scrapy/core/downloader/tls.py E501 E305 E241+ scrapy/core/downloader/tls.py E501 E241scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226scrapy/core/downloader/handlers/__init__.py E501- scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127+ scrapy/core/downloader/handlers/ftp.py E501 E128 E127scrapy/core/downloader/handlers/http10.py E501scrapy/core/downloader/handlers/http11.py E501scrapy/core/downloader/handlers/s3.py E501 E128 E126@@ -76,7 +76,7 @@ flake8-ignore =scrapy/extensions/closespider.py E501 E128 E123scrapy/extensions/corestats.py E501scrapy/extensions/feedexport.py E128 E501- scrapy/extensions/httpcache.py E128 E501 E303+ scrapy/extensions/httpcache.py E128 E501scrapy/extensions/memdebug.py E501scrapy/extensions/spiderstate.py E501scrapy/extensions/telnet.py E501 W504@@ -121,12 +121,11 @@ flake8-ignore =scrapy/utils/asyncio.py E501scrapy/utils/benchserver.py E501scrapy/utils/conf.py E402 E501- scrapy/utils/console.py E306 E305scrapy/utils/datatypes.py E501 E226scrapy/utils/decorators.py E501scrapy/utils/defer.py E501 E128scrapy/utils/deprecate.py E128 E501 E127 E502- scrapy/utils/gz.py E305 E501 W504+ scrapy/utils/gz.py E501 W504scrapy/utils/http.py F403 E226scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501 E701@@ -161,7 +160,7 @@ flake8-ignore =scrapy/middleware.py E128 E501scrapy/pqueues.py E501scrapy/resolver.py E501- scrapy/responsetypes.py E128 E501 E305+ scrapy/responsetypes.py E128 E501scrapy/robotstxt.py E501scrapy/shell.py E501scrapy/signalmanager.py E501@@ -175,50 +174,50 @@ flake8-ignore =tests/spiders.py E501 E127tests/test_closespider.py E501 E127tests/test_command_fetch.py E501- tests/test_command_parse.py E501 E128 E303 E226+ tests/test_command_parse.py E501 E128 E226tests/test_command_shell.py E501 E128tests/test_commands.py E128 E501tests/test_contracts.py E501 E128tests/test_crawl.py E501 E741 E265- tests/test_crawler.py F841 E306 E501- tests/test_dependencies.py F841 E501 E305+ tests/test_crawler.py F841 E501+ tests/test_dependencies.py F841 E501tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501- tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126+ tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126tests/test_downloadermiddleware_decompression.py E127tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501- tests/test_downloadermiddleware_httpcache.py E501 E305+ tests/test_downloadermiddleware_httpcache.py E501tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123tests/test_downloadermiddleware_httpproxy.py E501 E128- tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305- tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126+ tests/test_downloadermiddleware_redirect.py E501 E128 E127+ tests/test_downloadermiddleware_retry.py E501 E128 E251 E126tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501tests/test_dupefilters.py E221 E501 E741 E128 E124tests/test_engine.py E401 E501 E128- tests/test_exporters.py E501 E731 E306 E128 E124+ tests/test_exporters.py E501 E731 E128 E124tests/test_extension_telnet.py F841tests/test_feedexport.py E501 F841 E241tests/test_http_cookies.py E501tests/test_http_headers.py E501tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123- tests/test_http_response.py E501 E301 E128 E265- tests/test_item.py E701 E128 F841 E306+ tests/test_http_response.py E501 E128 E265+ tests/test_item.py E701 E128 F841tests/test_link.py E501tests/test_linkextractors.py E501 E128 E124- tests/test_loader.py E501 E731 E303 E741 E128 E117 E241+ tests/test_loader.py E501 E731 E741 E128 E117 E241tests/test_logformatter.py E128 E501 E122- tests/test_mail.py E128 E501 E305+ tests/test_mail.py E128 E501tests/test_middleware.py E501 E128tests/test_pipeline_crawl.py E131 E501 E128 E126- tests/test_pipeline_files.py E501 E303 E272 E226- tests/test_pipeline_images.py F841 E501 E303- tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502+ tests/test_pipeline_files.py E501 E272 E226+ tests/test_pipeline_images.py F841 E501+ tests/test_pipeline_media.py E501 E741 E731 E128 E502tests/test_proxy_connect.py E501 E741tests/test_request_cb_kwargs.py E501- tests/test_responsetypes.py E501 E305+ tests/test_responsetypes.py E501tests/test_robotstxt_interface.py E501 E501tests/test_scheduler.py E501 E126 E123tests/test_selector.py E501 E127@@ -230,24 +229,22 @@ flake8-ignore =tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121tests/test_squeues.py E501 E701 E741tests/test_utils_asyncio.py E501- tests/test_utils_conf.py E501 E303 E128+ tests/test_utils_conf.py E501 E128tests/test_utils_curl.py E501- tests/test_utils_datatypes.py E402 E501 E305- tests/test_utils_defer.py E306 E501 F841 E226- tests/test_utils_deprecate.py F841 E306 E501+ tests/test_utils_datatypes.py E402 E501+ tests/test_utils_defer.py E501 F841 E226+ tests/test_utils_deprecate.py F841 E501tests/test_utils_http.py E501 E128 W504- tests/test_utils_iterators.py E501 E128 E129 E303 E241+ tests/test_utils_iterators.py E501 E128 E129 E241tests/test_utils_log.py E741 E226- tests/test_utils_python.py E501 E303 E731 E701 E305+ tests/test_utils_python.py E501 E731 E701tests/test_utils_reqser.py E501 E128- tests/test_utils_request.py E501 E128 E305+ tests/test_utils_request.py E501 E128tests/test_utils_response.py E501tests/test_utils_signal.py E741 F841 E731 E226tests/test_utils_sitemap.py E128 E501 E124- tests/test_utils_spider.py E305- tests/test_utils_template.py E305- tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123- tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126+ tests/test_utils_url.py E501 E127 E211 E125 E501 E226 E241 E126 E123+ tests/test_webclient.py E501 E128 E122 E402 E226 E241 E123 E126tests/test_cmdline/__init__.py E501tests/test_settings/__init__.py E501 E128tests/test_spiderloader/__init__.py E128 E501diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex 4ed482058..a1c881d5e 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions):'from host "{}" (exception: {})'.format(self._hostnameASCII, repr(e)))+DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 829e69993..6ab8cde6b 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -230,6 +230,7 @@ class ExecutionEngine(object):def _download(self, request, spider):slot = self.slotslot.add_request(request)+def _on_success(response):assert isinstance(response, (Response, Request))if isinstance(response, Response):diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.pyindex 91d309147..64bf93e86 100644--- a/scrapy/responsetypes.py+++ b/scrapy/responsetypes.py@@ -116,4 +116,5 @@ class ResponseTypes(object):cls = self.from_body(body)return cls+responsetypes = ResponseTypes()diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 7eb40f0ce..c7a2ace88 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -54,6 +54,7 @@ def _embed_standard_shell(namespace={}, banner=''):else:import rlcompleter # noqa: F401readline.parse_and_bind("tab:complete")+@wraps(_embed_standard_shell)def wrapper(namespace=namespace, banner=''):code.interact(banner=banner, local=namespace)diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex 9672e28da..c291ae237 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -42,6 +42,7 @@ def gunzip(data):raisereturn b''.join(output_list)+_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).searchdiff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex b7035fdff..8a54d2c74 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -147,7 +147,6 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))-@defer.inlineCallbacksdef test_pipelines(self):_, _, stderr = yield self.execute(['--spider', self.spider_name,diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex f8fa26def..7bd76601d 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):def test_spider_custom_settings_log_level(self):log_file = self.mktemp()+class MySpider(scrapy.Spider):name = 'spider'custom_settings = {diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex e31ccd9b5..a169acbe6 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -13,5 +13,6 @@ class ScrapyUtilsTest(unittest.TestCase):installed_version = [int(x) for x in module.__version__.split('.')[:2]]assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"+if __name__ == "__main__":unittest.main()diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 04884fb78..051f66680 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -145,7 +145,6 @@ class CookiesMiddlewareTest(TestCase):{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]-req = Request('http://scrapytest.org/', cookies=cookies)self.mw.process_request(req, self.spider)diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 9401dd66d..9b77c97a8 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -501,5 +501,6 @@ class RFC2616PolicyTest(DefaultStorageTest):self.assertEqualResponse(res1, res2)assert 'cached' in res2.flags+if __name__ == '__main__':unittest.main()diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex e0f145d0e..053e26fc3 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -68,7 +68,6 @@ class RedirectMiddlewareTest(unittest.TestCase):assert isinstance(r, Response)assert r is rsp-def test_redirect_302(self):url = 'http://www.example.com/302'url2 = 'http://www.example.com/redirected2'@@ -122,7 +121,6 @@ class RedirectMiddlewareTest(unittest.TestCase):del rsp.headers['Location']assert self.mw.process_response(req, rsp, self.spider) is rsp-def test_max_redirect_times(self):self.mw.max_redirect_times = 1req = Request('http://scrapytest.org/302')@@ -178,6 +176,7 @@ class RedirectMiddlewareTest(unittest.TestCase):def test_request_meta_handling(self):url = 'http://www.example.com/301'url2 = 'http://www.example.com/redirected'+def _test_passthrough(req):rsp = Response(url, headers={'Location': url2}, status=301, request=req)r = self.mw.process_response(req, rsp, self.spider)@@ -316,5 +315,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):response = mw.process_response(req, rsp, self.spider)assert isinstance(response, Response)+if __name__ == "__main__":unittest.main()diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 5d1f5c182..6e2507508 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -312,6 +312,7 @@ class XmlItemExporterTest(BaseItemExporterTest):for child in children]else:return [(elem.tag, [(elem.text, ())])]+def xmlsplit(xmlcontent):doc = lxml.etree.fromstring(xmlcontent)return xmltuple(doc)diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 0dc603923..be17dfd6b 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -182,6 +182,7 @@ class BaseResponseTest(unittest.TestCase):def test_follow_whitespace_link(self):self._assert_followed_url(Link('http://example.com/foo '),'http://example.com/foo%20')+def test_follow_flags(self):res = self.response_class('http://example.com/')fol = res.follow('http://example.com/', flags=['cached', 'allowed'])diff --git a/tests/test_item.py b/tests/test_item.pyindex 30463a0f5..823bf1ced 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -259,6 +259,7 @@ class ItemTest(unittest.TestCase):with catch_warnings(record=True) as warnings:item = Item()self.assertEqual(len(warnings), 0)+class SubclassedItem(Item):passsubclassed_item = SubclassedItem()diff --git a/tests/test_mail.py b/tests/test_mail.pyindex ddb0f1e70..f5cb81a8b 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -121,5 +121,6 @@ class MailSenderTest(unittest.TestCase):self.assertEqual(text.get_charset(), Charset('utf-8'))self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)+if __name__ == "__main__":unittest.main()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex e5bad2ed0..88ce1cf18 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -286,7 +286,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):self.assertEqual(pipeline.files_result_field, "this")self.assertEqual(pipeline.files_urls_field, "that")-def test_user_defined_subclass_default_key_names(self):"""Test situation when user defines subclass of FilesPipeline,but uses attribute names for default pipeline (without prefixingdiff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 7f1cb4a11..5018d6802 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -177,7 +177,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):IMAGES_RESULT_FIELD='images')-def setUp(self):self.tempdir = mkdtemp()diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 1fcc5799e..d369e147d 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -304,6 +304,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):return responsersp1 = Response('http://url')+def rsp1_func():dfd = Deferred().addCallback(_check_downloading)reactor.callLater(.1, dfd.callback, rsp1)diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex d5a3371ab..8cdf7a176 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -90,5 +90,6 @@ class ResponseTypesTest(unittest.TestCase):# check that mime.types files shipped with scrapy are loadedself.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test')+if __name__ == "__main__":unittest.main()diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 02d8ba51e..61e110845 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -83,7 +83,6 @@ class BuildComponentListTest(unittest.TestCase):self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)-class UtilsConfTestCase(unittest.TestCase):def test_arglist_to_dict(self):diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex dfbe71ae2..89b5fb4fb 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -9,6 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, process_chain, \class MustbeDeferredTest(unittest.TestCase):def test_success_function(self):steps = []+def _append(v):steps.append(v)return steps@@ -20,6 +21,7 @@ class MustbeDeferredTest(unittest.TestCase):def test_unfired_deferred(self):steps = []+def _append(v):steps.append(v)dfd = defer.Deferred()diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 159ef8f25..b3a90d314 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -110,6 +110,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):# ignore subclassing warningswith warnings.catch_warnings():warnings.simplefilter('ignore', ScrapyDeprecationWarning)+class UserClass(Deprecated):pass@@ -233,6 +234,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):with warnings.catch_warnings(record=True) as w:AlsoDeprecated()+class UserClass(AlsoDeprecated):pass@@ -247,6 +249,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):with mock.patch('inspect.stack', side_effect=IndexError):with warnings.catch_warnings(record=True) as w:DeprecatedName = create_deprecated_class('DeprecatedName', NewName)+class SubClass(DeprecatedName):passdiff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex 9776dfb2a..33fc4d570 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -387,7 +387,6 @@ class TestHelper(unittest.TestCase):self.assertTrue(type(r1) is type(r2))self.assertTrue(type(r1) is not type(r3))-def _assert_type_and_value(self, a, b, obj):self.assertTrue(type(a) is type(b),'Got {}, expected {} for {!r}'.format(type(a), type(b), obj))diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex b79e0ac1c..4202e8c89 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -104,7 +104,6 @@ class BinaryIsTextTest(unittest.TestCase):assert not binary_is_text(b"\x02\xa3")-class UtilsPythonTestCase(unittest.TestCase):def test_equal_attributes(self):@@ -215,7 +214,6 @@ class UtilsPythonTestCase(unittest.TestCase):self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])-def test_without_none_values(self):self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))@@ -223,5 +221,6 @@ class UtilsPythonTestCase(unittest.TestCase):without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),{'one': 1, 'three': 3, 'four': 4})+if __name__ == "__main__":unittest.main()diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 3e664fc74..45f0f59e4 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -83,5 +83,6 @@ class UtilsRequestTest(unittest.TestCase):request_httprepr(Request("file:///tmp/foo.txt"))request_httprepr(Request("ftp://localhost/tmp/foo.txt"))+if __name__ == "__main__":unittest.main()diff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 40b733233..5a52dd695 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -38,5 +38,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):os.remove(render_path)assert not os.path.exists(render_path) # Failure of test iself+if '__main__' == __name__:unittest.main()diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 21e9a056a..9f1acbc75 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -201,6 +201,7 @@ def create_skipped_scheme_t(args):assert url.startswith(args[1])return do_expected+for k, args in enumerate ([('/index', 'file://'),('/index.html', 'file://'),diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 746367b41..b602a3ea0 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -294,6 +294,7 @@ class WebClientTestCase(unittest.TestCase):finished = self.assertFailure(getPage(self.getURL("wait"), timeout=0.000001),defer.TimeoutError)+def cleanup(passthrough):# Clean up the server which is hanging around not doing# anything.