Scrapy, a fast high-level web crawling & scraping framework for Python.
Request.cb_kwargs: update in spider middleware
Eugenio Lacuesta312e57382c2c
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 25c0bcf54..c9943faa8 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -10,6 +10,9 @@ from tests.mockserver import MockServerclass InjectArgumentsDownloaderMiddleware(object):+ """+ Make sure downloader middlewares are able to update the keyword arguments+ """def process_request(self, request, spider):if request.callback.__name__ == 'parse_downloader_mw':request.cb_kwargs['from_process_request'] = True@@ -21,12 +24,38 @@ class InjectArgumentsDownloaderMiddleware(object):return response+class InjectArgumentsSpiderMiddleware(object):+ """+ Make sure spider middlewares are able to update the keyword arguments+ """+ def process_start_requests(self, start_requests, spider):+ for request in start_requests:+ if request.callback.__name__ == 'parse_spider_mw':+ request.cb_kwargs['from_process_start_requests'] = True+ yield request++ def process_spider_input(self, response, spider):+ request = response.request+ if request.callback.__name__ == 'parse_spider_mw':+ request.cb_kwargs['from_process_spider_input'] = True+ return None++ def process_spider_output(self, response, result, spider):+ for element in result:+ if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':+ element.cb_kwargs['from_process_spider_output'] = True+ yield element++class KeywordArgumentsSpider(MockServerSpider):name = 'kwargs'custom_settings = {'DOWNLOADER_MIDDLEWARES': {__name__ + '.InjectArgumentsDownloaderMiddleware': 750,- }+ },+ 'SPIDER_MIDDLEWARES': {+ __name__ + '.InjectArgumentsSpiderMiddleware': 750,+ },}checks = list()@@ -41,6 +70,7 @@ class KeywordArgumentsSpider(MockServerSpider):yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)+ yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)def parse_first(self, response, key, number):self.checks.append(key == 'value')@@ -88,13 +118,20 @@ class KeywordArgumentsSpider(MockServerSpider):"""def parse_downloader_mw(self, response, from_process_request, from_process_response):- """- Check if downloader middlewares are able to update the keyword arguments- """self.checks.append(bool(from_process_request))self.checks.append(bool(from_process_response))self.crawler.stats.inc_value('boolean_checks', 2)+ def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):+ self.checks.append(bool(from_process_spider_input))+ self.checks.append(bool(from_process_start_requests))+ self.crawler.stats.inc_value('boolean_checks', 2)+ return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)++ def parse_spider_mw_2(self, response, from_process_spider_output):+ self.checks.append(bool(from_process_spider_output))+ self.crawler.stats.inc_value('boolean_checks', 1)+class CallbackKeywordArgumentsTestCase(TestCase):@@ -129,4 +166,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):if six.PY2:self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")elif six.PY3:- self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")\ No newline at end of file+ self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")