Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Request.cb_kwargs: update in spider middleware

Eugenio Lacuesta312e57382c2c
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 25c0bcf54..c9943faa8 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -10,6 +10,9 @@ from tests.mockserver import MockServer
class InjectArgumentsDownloaderMiddleware(object):
+ """
+ Make sure downloader middlewares are able to update the keyword arguments
+ """
def process_request(self, request, spider):
if request.callback.__name__ == 'parse_downloader_mw':
request.cb_kwargs['from_process_request'] = True
@@ -21,12 +24,38 @@ class InjectArgumentsDownloaderMiddleware(object):
return response
+class InjectArgumentsSpiderMiddleware(object):
+ """
+ Make sure spider middlewares are able to update the keyword arguments
+ """
+ def process_start_requests(self, start_requests, spider):
+ for request in start_requests:
+ if request.callback.__name__ == 'parse_spider_mw':
+ request.cb_kwargs['from_process_start_requests'] = True
+ yield request
+
+ def process_spider_input(self, response, spider):
+ request = response.request
+ if request.callback.__name__ == 'parse_spider_mw':
+ request.cb_kwargs['from_process_spider_input'] = True
+ return None
+
+ def process_spider_output(self, response, result, spider):
+ for element in result:
+ if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
+ element.cb_kwargs['from_process_spider_output'] = True
+ yield element
+
+
class KeywordArgumentsSpider(MockServerSpider):
name = 'kwargs'
custom_settings = {
'DOWNLOADER_MIDDLEWARES': {
__name__ + '.InjectArgumentsDownloaderMiddleware': 750,
- }
+ },
+ 'SPIDER_MIDDLEWARES': {
+ __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+ },
}
checks = list()
@@ -41,6 +70,7 @@ class KeywordArgumentsSpider(MockServerSpider):
yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
+ yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
def parse_first(self, response, key, number):
self.checks.append(key == 'value')
@@ -88,13 +118,20 @@ class KeywordArgumentsSpider(MockServerSpider):
"""
def parse_downloader_mw(self, response, from_process_request, from_process_response):
- """
- Check if downloader middlewares are able to update the keyword arguments
- """
self.checks.append(bool(from_process_request))
self.checks.append(bool(from_process_response))
self.crawler.stats.inc_value('boolean_checks', 2)
+ def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+ self.checks.append(bool(from_process_spider_input))
+ self.checks.append(bool(from_process_start_requests))
+ self.crawler.stats.inc_value('boolean_checks', 2)
+ return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+
+ def parse_spider_mw_2(self, response, from_process_spider_output):
+ self.checks.append(bool(from_process_spider_output))
+ self.crawler.stats.inc_value('boolean_checks', 1)
+
class CallbackKeywordArgumentsTestCase(TestCase):
@@ -129,4 +166,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
if six.PY2:
self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
elif six.PY3:
- self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
\ No newline at end of file
+ self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")