Scrapy, a fast high-level web crawling & scraping framework for Python.
remove .keys() to avoid creating a tmp list/keyview obj (#4031)
s-sanjay2c14692e6038
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex d4f2234b0..ef8acd29c 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -60,10 +60,12 @@ class Command(ScrapyCommand):@propertydef max_level(self):- levels = list(self.items.keys()) + list(self.requests.keys())- if not levels:- return 0- return max(levels)+ max_items, max_requests = 0, 0+ if self.items:+ max_items = max(self.items)+ if self.requests:+ max_requests = max(self.requests)+ return max(max_items, max_requests)def add_items(self, lvl, new_items):old_items = self.items.get(lvl, [])@@ -84,9 +86,8 @@ class Command(ScrapyCommand):def print_requests(self, lvl=None, colour=True):if lvl is None:- levels = list(self.requests.keys())- if levels:- requests = self.requests[max(levels)]+ if self.requests:+ requests = self.requests[max(self.requests)]else:requests = []else:diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 376d3c84e..57d8471ca 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -60,14 +60,13 @@ class Command(ScrapyCommand):else:self.settings.set('FEED_URI', opts.output, priority='cmdline')feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))- valid_output_formats = feed_exporters.keys()if not opts.output_format:opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")- if opts.output_format not in valid_output_formats:+ if opts.output_format not in feed_exporters:raise UsageError("Unrecognized output format '%s', set one"" using the '-t' switch or as a file extension"" from the supported list %s" % (opts.output_format,- tuple(valid_output_formats)))+ tuple(feed_exporters)))self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')def run(self, args, opts):diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 98e415ad3..62d5d76b4 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -108,6 +108,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}_, _, stderr = yield self.execute(['--spider', self.spider_name,'-a', 'test_arg=1','-c', 'parse',+ '--verbose',self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))@@ -117,12 +118,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}_, _, stderr = yield self.execute(['--spider', self.spider_name,'--meta', raw_json_string,'-c', 'parse_request_with_meta',+ '--verbose',self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))_, _, stderr = yield self.execute(['--spider', self.spider_name,'-m', raw_json_string,'-c', 'parse_request_with_meta',+ '--verbose',self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))@@ -132,6 +135,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}_, _, stderr = yield self.execute(['--spider', self.spider_name,'--cbkwargs', raw_json_string,'-c', 'parse_request_with_cb_kwargs',+ '--verbose',self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))@@ -139,6 +143,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}def test_request_without_meta(self):_, _, stderr = yield self.execute(['--spider', self.spider_name,'-c', 'parse_request_without_meta',+ '--nolinks',self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))@@ -148,6 +153,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}_, _, stderr = yield self.execute(['--spider', self.spider_name,'--pipelines','-c', 'parse',+ '--verbose',self.url('/html')])self.assertIn("INFO: It Works!", _textmode(stderr))