Scrapy, a fast high-level web crawling & scraping framework for Python.
Add cb_kwargs option to the parse command
Eugenio Lacuestae8af6331b5ff
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex 97f8311de..6644d65e4 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -461,6 +461,9 @@ Supported options:* ``--meta`` or ``-m``: additional request meta that will be passed to the callbackrequest. This must be a valid json string. Example: --meta='{"foo" : "bar"}'+* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.+ This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'+* ``--pipelines``: process items through pipelines* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 69418a478..2486f3f23 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -51,12 +51,13 @@ class Command(ScrapyCommand):help="use this callback for parsing, instead looking for a callback")parser.add_option("-m", "--meta", dest="meta",help="inject extra meta into the Request, it must be a valid raw json string")+ parser.add_option("--cb_kwargs", dest="cb_kwargs",+ help="inject extra cb_kwargs into the Request, it must be a valid raw json string")parser.add_option("-d", "--depth", dest="depth", type="int", default=1,help="maximum depth for parsing requests [default: %default]")parser.add_option("-v", "--verbose", dest="verbose", action="store_true",help="print each depth level one by one")-@propertydef max_level(self):levels = list(self.items.keys()) + list(self.requests.keys())@@ -111,10 +112,11 @@ class Command(ScrapyCommand):if not opts.nolinks:self.print_requests(colour=colour)- def run_callback(self, response, cb):+ def run_callback(self, response, callback, cb_kwargs=None):+ cb_kwargs = cb_kwargs or {}items, requests = [], []- for x in iterate_spider_output(cb(response)):+ for x in iterate_spider_output(callback(response, **cb_kwargs)):if isinstance(x, (BaseItem, dict)):items.append(x)elif isinstance(x, Request):@@ -142,8 +144,7 @@ class Command(ScrapyCommand):else:self.spidercls = spidercls_for_request(spider_loader, Request(url))if not self.spidercls:- logger.error('Unable to find spider for: %(url)s',- {'url': url})+ logger.error('Unable to find spider for: %(url)s', {'url': url})# Request requires callback argument as callable or None, not stringrequest = Request(url, None)@@ -160,7 +161,7 @@ class Command(ScrapyCommand):{'url': url})def prepare_request(self, spider, request, opts):- def callback(response):+ def callback(response, **cb_kwargs):# memorize first requestif not self.first_response:self.first_response = response@@ -175,7 +176,7 @@ class Command(ScrapyCommand):if not cb:logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',- {'url': response.url, 'spider': spider.name})+ {'url': response.url, 'spider': spider.name})returnelse:cb = 'parse'@@ -192,7 +193,7 @@ class Command(ScrapyCommand):# parse items and requestsdepth = response.meta['_depth']- items, requests = self.run_callback(response, cb)+ items, requests = self.run_callback(response, cb, cb_kwargs)if opts.pipelines:itemproc = self.pcrawler.engine.scraper.itemprocfor item in items:@@ -207,10 +208,14 @@ class Command(ScrapyCommand):req.callback = callbackreturn requests- #update request meta if any extra meta was passed through the --meta/-m opts.+ # update request meta if any extra meta was passed through the --meta/-m opts.if opts.meta:request.meta.update(opts.meta)+ # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.+ if opts.cb_kwargs:+ request.cb_kwargs.update(opts.cb_kwargs)+request.meta['_depth'] = 1request.meta['_callback'] = request.callbackrequest.callback = callback@@ -221,23 +226,29 @@ class Command(ScrapyCommand):self.process_spider_arguments(opts)self.process_request_meta(opts)+ self.process_request_cb_kwargs(opts)def process_spider_arguments(self, opts):-try:opts.spargs = arglist_to_dict(opts.spargs)except ValueError:raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)def process_request_meta(self, opts):-if opts.meta:try:opts.meta = json.loads(opts.meta)except ValueError:- raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \- "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)+ raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "+ "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)+ def process_request_cb_kwargs(self, opts):+ if opts.cb_kwargs:+ try:+ opts.cb_kwargs = json.loads(opts.cb_kwargs)+ except ValueError:+ raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "+ "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)def run(self, args, opts):# parse argumentsdiff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 02037b866..1404005fb 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -43,6 +43,12 @@ class MySpider(scrapy.Spider):else:self.logger.debug('It Works!')+ def parse_request_with_cb_kwargs(self, response, foo=None, key=None):+ if foo == 'bar' and key == 'value':+ self.logger.debug('It Works!')+ else:+ self.logger.debug('It Does Not Work :(')+def parse_request_without_meta(self, response):foo = response.meta.get('foo', 'bar')@@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}self.url('/html')])self.assertIn("DEBUG: It Works!", _textmode(stderr))+ @defer.inlineCallbacks+ def test_request_with_cb_kwargs(self):+ raw_json_string = '{"foo" : "bar", "key": "value"}'+ _, _, stderr = yield self.execute(['--spider', self.spider_name,+ '--cb_kwargs', raw_json_string,+ '-c', 'parse_request_with_cb_kwargs',+ self.url('/html')])+ self.assertIn("DEBUG: It Works!", _textmode(stderr))@defer.inlineCallbacksdef test_request_without_meta(self):