Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add cb_kwargs option to the parse command

Eugenio Lacuestae8af6331b5ff
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 97f8311de..6644d65e4 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -461,6 +461,9 @@ Supported options:
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
+* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.
+ This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'
+
* ``--pipelines``: process items through pipelines
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 69418a478..2486f3f23 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,12 +51,13 @@ class Command(ScrapyCommand):
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
+ parser.add_option("--cb_kwargs", dest="cb_kwargs",
+ help="inject extra cb_kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
-
@property
def max_level(self):
levels = list(self.items.keys()) + list(self.requests.keys())
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
- def run_callback(self, response, cb):
+ def run_callback(self, response, callback, cb_kwargs=None):
+ cb_kwargs = cb_kwargs or {}
items, requests = [], []
- for x in iterate_spider_output(cb(response)):
+ for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (BaseItem, dict)):
items.append(x)
elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
- logger.error('Unable to find spider for: %(url)s',
- {'url': url})
+ logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
{'url': url})
def prepare_request(self, spider, request, opts):
- def callback(response):
+ def callback(response, **cb_kwargs):
# memorize first request
if not self.first_response:
self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
- {'url': response.url, 'spider': spider.name})
+ {'url': response.url, 'spider': spider.name})
return
else:
cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
# parse items and requests
depth = response.meta['_depth']
- items, requests = self.run_callback(response, cb)
+ items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
req.callback = callback
return requests
- #update request meta if any extra meta was passed through the --meta/-m opts.
+ # update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
request.meta.update(opts.meta)
+ # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.
+ if opts.cb_kwargs:
+ request.cb_kwargs.update(opts.cb_kwargs)
+
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
self.process_spider_arguments(opts)
self.process_request_meta(opts)
+ self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
-
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
-
if opts.meta:
try:
opts.meta = json.loads(opts.meta)
except ValueError:
- raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
- "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+ "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ def process_request_cb_kwargs(self, opts):
+ if opts.cb_kwargs:
+ try:
+ opts.cb_kwargs = json.loads(opts.cb_kwargs)
+ except ValueError:
+ raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "
+ "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)
def run(self, args, opts):
# parse arguments
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 02037b866..1404005fb 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -43,6 +43,12 @@ class MySpider(scrapy.Spider):
else:
self.logger.debug('It Works!')
+ def parse_request_with_cb_kwargs(self, response, foo=None, key=None):
+ if foo == 'bar' and key == 'value':
+ self.logger.debug('It Works!')
+ else:
+ self.logger.debug('It Does Not Work :(')
+
def parse_request_without_meta(self, response):
foo = response.meta.get('foo', 'bar')
@@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
+ @defer.inlineCallbacks
+ def test_request_with_cb_kwargs(self):
+ raw_json_string = '{"foo" : "bar", "key": "value"}'
+ _, _, stderr = yield self.execute(['--spider', self.spider_name,
+ '--cb_kwargs', raw_json_string,
+ '-c', 'parse_request_with_cb_kwargs',
+ self.url('/html')])
+ self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_request_without_meta(self):