Scrapy, a fast high-level web crawling & scraping framework for Python.
Flake8: Remove E741
Eugenio Lacuestadf8a1d1c0108
diff --git a/pytest.ini b/pytest.iniindex fee54dcbd..c141d49a4 100644--- a/pytest.ini+++ b/pytest.ini@@ -178,13 +178,13 @@ flake8-ignore =tests/test_command_shell.py E501 E128tests/test_commands.py E128 E501tests/test_contracts.py E501 E128- tests/test_crawl.py E501 E741+ tests/test_crawl.py E501tests/test_crawler.py E501tests/test_dependencies.py E501tests/test_downloader_handlers.py E128 E501tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501- tests/test_downloadermiddleware_cookies.py E741 E501 E128+ tests/test_downloadermiddleware_cookies.py E501 E128tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501tests/test_downloadermiddleware_httpcache.py E501@@ -195,7 +195,7 @@ flake8-ignore =tests/test_downloadermiddleware_retry.py E501 E128tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501- tests/test_dupefilters.py E501 E741 E128+ tests/test_dupefilters.py E501 E128tests/test_engine.py E501 E128tests/test_exporters.py E501 E128tests/test_feedexport.py E501@@ -206,15 +206,15 @@ flake8-ignore =tests/test_item.py E128tests/test_link.py E501tests/test_linkextractors.py E501 E128- tests/test_loader.py E501 E741 E128+ tests/test_loader.py E128 E501 E741tests/test_logformatter.py E128 E501tests/test_mail.py E128 E501tests/test_middleware.py E501 E128tests/test_pipeline_crawl.py E501 E128tests/test_pipeline_files.py E501tests/test_pipeline_images.py E501- tests/test_pipeline_media.py E501 E741 E128- tests/test_proxy_connect.py E501 E741+ tests/test_pipeline_media.py E501 E128+ tests/test_proxy_connect.py E501tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501tests/test_robotstxt_interface.py E501 E501@@ -225,8 +225,8 @@ flake8-ignore =tests/test_spidermiddleware_httperror.py E128 E501tests/test_spidermiddleware_offsite.py E501 E128tests/test_spidermiddleware_output_chain.py E501- tests/test_spidermiddleware_referer.py E501 E501- tests/test_squeues.py E501 E741+ tests/test_spidermiddleware_referer.py E501+ tests/test_squeues.py E501tests/test_utils_asyncio.py E501tests/test_utils_conf.py E501 E128tests/test_utils_curl.py E501@@ -235,12 +235,10 @@ flake8-ignore =tests/test_utils_deprecate.py E501tests/test_utils_http.py E501 E128tests/test_utils_iterators.py E501 E128- tests/test_utils_log.py E741tests/test_utils_python.py E501tests/test_utils_reqser.py E501 E128tests/test_utils_request.py E501 E128tests/test_utils_response.py E501- tests/test_utils_signal.py E741tests/test_utils_sitemap.py E128 E501tests/test_utils_url.py E501 E501tests/test_webclient.py E501 E128 E402diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 08ce89481..2a3e13ddd 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -146,14 +146,13 @@ class Shell:b.append("Useful shortcuts:")if self.inthread:b.append(" fetch(url[, redirect=True]) "- "Fetch URL and update local objects "- "(by default, redirects are followed)")+ "Fetch URL and update local objects (by default, redirects are followed)")b.append(" fetch(req) ""Fetch a scrapy.Request and update local objects ")b.append(" shelp() Shell help (print this help)")b.append(" view(response) View response in a browser")- return "\n".join("[s] %s" % l for l in b)+ return "\n".join("[s] %s" % line for line in b)def _is_relevant(self, value):return isinstance(value, self.relevant_classes)diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex d368c7108..c5360bfa7 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -96,5 +96,4 @@ def iterloc(it, alt=False):# Also consider alternate URLs (xhtml:link rel="alternate")if alt and 'alternate' in d:- for l in d['alternate']:- yield l+ yield from d['alternate']diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 203842fc8..6392862ee 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter):self.loggers = loggers or []def filter(self, record):- if any(record.name.startswith(l + '.') for l in self.loggers):+ if any(record.name.startswith(logger + '.') for logger in self.loggers):record.name = record.name.split('.', 1)[0]return Truediff --git a/tests/test_command_version.py b/tests/test_command_version.pyindex 4ac7fb786..99c01c2b7 100644--- a/tests/test_command_version.py+++ b/tests/test_command_version.py@@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase):def test_verbose_output(self):encoding = getattr(sys.stdout, 'encoding') or 'utf-8'_, out, _ = yield self.execute(['-v'])- headers = [l.partition(":")[0].strip()- for l in out.strip().decode(encoding).splitlines()]+ headers = [+ line.partition(":")[0].strip()+ for line in out.strip().decode(encoding).splitlines()+ ]self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2','cssselect', 'parsel', 'w3lib','Twisted', 'Python', 'pyOpenSSL',diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 4215ca56c..84f80d103 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -104,44 +104,44 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_retry_503(self):crawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)- self._assert_retried(l)+ self._assert_retried(log)@defer.inlineCallbacksdef test_retry_conn_failed(self):crawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)- self._assert_retried(l)+ self._assert_retried(log)@defer.inlineCallbacksdef test_retry_dns_error(self):crawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:# try to fetch the homepage of a non-existent domainyield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)- self._assert_retried(l)+ self._assert_retried(log)@defer.inlineCallbacksdef test_start_requests_bug_before_yield(self):- with LogCapture('scrapy', level=logging.ERROR) as l:+ with LogCapture('scrapy', level=logging.ERROR) as log:crawler = self.runner.create_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)- self.assertEqual(len(l.records), 1)- record = l.records[0]+ self.assertEqual(len(log.records), 1)+ record = log.records[0]self.assertIsNotNone(record.exc_info)self.assertIs(record.exc_info[0], ZeroDivisionError)@defer.inlineCallbacksdef test_start_requests_bug_yielding(self):- with LogCapture('scrapy', level=logging.ERROR) as l:+ with LogCapture('scrapy', level=logging.ERROR) as log:crawler = self.runner.create_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)- self.assertEqual(len(l.records), 1)- record = l.records[0]+ self.assertEqual(len(log.records), 1)+ record = log.records[0]self.assertIsNotNone(record.exc_info)self.assertIs(record.exc_info[0], ZeroDivisionError)@@ -187,25 +187,25 @@ foo bodywith multiples lines'''})crawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)- self.assertEqual(str(l).count("Got response 200"), 1)+ self.assertEqual(str(log).count("Got response 200"), 1)@defer.inlineCallbacksdef test_retry_conn_lost(self):# connection lost after receiving datacrawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)- self._assert_retried(l)+ self._assert_retried(log)@defer.inlineCallbacksdef test_retry_conn_aborted(self):# connection lost before receiving datacrawler = self.runner.create_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)- self._assert_retried(l)+ self._assert_retried(log)def _assert_retried(self, log):self.assertEqual(str(log).count("Retrying"), 2)diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex f86c50f50..d54434c8f 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase):mw = CookiesMiddleware.from_crawler(crawler)with LogCapture('scrapy.downloadermiddlewares.cookies',propagate=False,- level=logging.DEBUG) as l:+ level=logging.DEBUG) as log:req = Request('http://scrapytest.org/')res = Response('http://scrapytest.org/',headers={'Set-Cookie': 'C1=value1; path=/'})@@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase):req2 = Request('http://scrapytest.org/sub1/')mw.process_request(req2, crawler.spider)- l.check(+ log.check(('scrapy.downloadermiddlewares.cookies','DEBUG','Received cookies from: <200 http://scrapytest.org/>\n'@@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase):mw = CookiesMiddleware.from_crawler(crawler)with LogCapture('scrapy.downloadermiddlewares.cookies',propagate=False,- level=logging.DEBUG) as l:+ level=logging.DEBUG) as log:req = Request('http://scrapytest.org/')res = Response('http://scrapytest.org/',headers={'Set-Cookie': 'C1=value1; path=/'})@@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase):req2 = Request('http://scrapytest.org/sub1/')mw.process_request(req2, crawler.spider)- l.check()+ log.check()def test_do_not_break_on_non_utf8_header(self):req = Request('http://scrapytest.org/')diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 7426107c1..ee1794f4f 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase):shutil.rmtree(path)def test_log(self):- with LogCapture() as l:+ with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': False,'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}crawler = get_crawler(SimpleSpider, settings_dict=settings)@@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.log(r2, spider)assert crawler.stats.get_value('dupefilter/filtered') == 2- l.check_present(('scrapy.dupefilters', 'DEBUG',+ log.check_present(('scrapy.dupefilters', 'DEBUG',('Filtered duplicate request: <GET http://scrapytest.org/index.html>'' - no more duplicates will be shown'' (see DUPEFILTER_DEBUG to show all duplicates)')))@@ -185,7 +185,7 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.close('finished')def test_log_debug(self):- with LogCapture() as l:+ with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': True,'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}crawler = get_crawler(SimpleSpider, settings_dict=settings)@@ -203,10 +203,10 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.log(r2, spider)assert crawler.stats.get_value('dupefilter/filtered') == 2- l.check_present(('scrapy.dupefilters', 'DEBUG',+ log.check_present(('scrapy.dupefilters', 'DEBUG',('Filtered duplicate request: <GET http://scrapytest.org/index.html>'' (referer: None)')))- l.check_present(('scrapy.dupefilters', 'DEBUG',+ log.check_present(('scrapy.dupefilters', 'DEBUG',('Filtered duplicate request: <GET http://scrapytest.org/index.html>'' (referer: http://scrapytest.org/INDEX.html)')))diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 949f0dea1..d7b0d32b2 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):fail = Failure(Exception())results = [(True, 1), (False, fail)]- with LogCapture() as l:+ with LogCapture() as log:new_item = self.pipe.item_completed(results, item, self.info)assert new_item is item- assert len(l.records) == 1- record = l.records[0]+ assert len(log.records) == 1+ record = log.records[0]assert record.levelname == 'ERROR'self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))# disable failure logging and check againself.pipe.LOG_FAILED_RESULTS = False- with LogCapture() as l:+ with LogCapture() as log:new_item = self.pipe.item_completed(results, item, self.info)assert new_item is item- assert len(l.records) == 0+ assert len(log.records) == 0@inlineCallbacksdef test_default_process_item(self):diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 188ec68dd..4763a5417 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase):@defer.inlineCallbacksdef test_https_connect_tunnel(self):crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, l)+ self._assert_got_response_code(200, log)@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)@defer.inlineCallbacksdef test_https_connect_tunnel_error(self):crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl("https://localhost:99999/status?n=200")- self._assert_got_tunnel_error(l)+ self._assert_got_tunnel_error(log)@defer.inlineCallbacksdef test_https_tunnel_auth_error(self):os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))# The proxy returns a 407 error code but it does not reach the client;# he just sees a TunnelError.- self._assert_got_tunnel_error(l)+ self._assert_got_tunnel_error(log)@defer.inlineCallbacksdef test_https_tunnel_without_leak_proxy_authorization_header(self):request = Request(self.mockserver.url("/echo", is_secure=True))crawler = get_crawler(SingleRequestSpider)- with LogCapture() as l:+ with LogCapture() as log:yield crawler.crawl(seed=request)- self._assert_got_response_code(200, l)+ self._assert_got_response_code(200, log)echo = json.loads(crawler.spider.meta['responses'][0].text)self.assertTrue('Proxy-Authorization' not in echo['headers'])diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex d2cf9135f..becacce62 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):def test_serialize_loader(self):q = self.queue()- l = TestLoader()- q.push(l)- l2 = q.pop()- assert isinstance(l2, TestLoader)- assert l2.default_item_class is TestItem- self.assertEqual(l2.name_out('x'), 'xx')+ loader = TestLoader()+ q.push(loader)+ loader2 = q.pop()+ assert isinstance(loader2, TestLoader)+ assert loader2.default_item_class is TestItem+ self.assertEqual(loader2.name_out('x'), 'xx')def test_serialize_request_recursive(self):q = self.queue()@@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):def test_serialize_loader(self):q = self.queue()- l = TestLoader()- q.push(l)- l2 = q.pop()- assert isinstance(l2, TestLoader)- assert l2.default_item_class is TestItem- self.assertEqual(l2.name_out('x'), 'xx')+ loader = TestLoader()+ q.push(loader)+ loader2 = q.pop()+ assert isinstance(loader2, TestLoader)+ assert loader2.default_item_class is TestItem+ self.assertEqual(loader2.name_out('x'), 'xx')def test_serialize_request_recursive(self):q = self.queue()diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 25cd904bc..535f56691 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase):def test_top_level_logger(self):logger = logging.getLogger('test')- with self.handler as l:+ with self.handler as log:logger.warning('test log msg')-- l.check(('test', 'WARNING', 'test log msg'))+ log.check(('test', 'WARNING', 'test log msg'))def test_children_logger(self):logger = logging.getLogger('test.test1')- with self.handler as l:+ with self.handler as log:logger.warning('test log msg')-- l.check(('test', 'WARNING', 'test log msg'))+ log.check(('test', 'WARNING', 'test log msg'))def test_overlapping_name_logger(self):logger = logging.getLogger('test2')- with self.handler as l:+ with self.handler as log:logger.warning('test log msg')-- l.check(('test2', 'WARNING', 'test log msg'))+ log.check(('test2', 'WARNING', 'test log msg'))def test_different_name_logger(self):logger = logging.getLogger('different')- with self.handler as l:+ with self.handler as log:logger.warning('test log msg')-- l.check(('different', 'WARNING', 'test log msg'))+ log.check(('different', 'WARNING', 'test log msg'))class LogCounterHandlerTest(unittest.TestCase):@@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase):sys.stdout = self.stdoutdef test_redirect(self):- with LogCapture() as l:+ with LogCapture() as log:print('test log msg')- l.check(('test', 'ERROR', 'test log msg'))+ log.check(('test', 'ERROR', 'test log msg'))diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex 015a0e5a2..28205e0d9 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase):assert hasattr(arg_to_iter(100), '__iter__')assert hasattr(arg_to_iter('lala'), '__iter__')assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')- assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')+ assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')self.assertEqual(list(arg_to_iter(None)), [])self.assertEqual(list(arg_to_iter('lala')), ['lala'])self.assertEqual(list(arg_to_iter(100)), [100])- self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])+ self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex c83c9398c..b66588efb 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase):dispatcher.connect(self.error_handler, signal=test_signal)dispatcher.connect(self.ok_handler, signal=test_signal)- with LogCapture() as l:+ with LogCapture() as log:result = yield defer.maybeDeferred(self._get_result, test_signal, arg='test',handlers_called=handlers_called@@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase):assert self.error_handler in handlers_calledassert self.ok_handler in handlers_called- self.assertEqual(len(l.records), 1)- record = l.records[0]+ self.assertEqual(len(log.records), 1)+ record = log.records[0]self.assertIn('error_handler', record.getMessage())self.assertEqual(record.levelname, 'ERROR')self.assertEqual(result[0][0], self.error_handler)@@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase):test_signal = object()dispatcher.connect(test_handler, test_signal)- with LogCapture() as l:+ with LogCapture() as log:send_catch_log(test_signal)- self.assertEqual(len(l.records), 1)- self.assertIn("Cannot return deferreds from signal handler", str(l))+ self.assertEqual(len(log.records), 1)+ self.assertIn("Cannot return deferreds from signal handler", str(log))dispatcher.disconnect(test_handler, test_signal)