Scrapy, a fast high-level web crawling & scraping framework for Python.
Request.cb_kwargs: update in downloader middleware
Eugenio Lacuestad4d68cf32187
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex ee7117c8a..25c0bcf54 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -9,9 +9,26 @@ from tests.spiders import MockServerSpiderfrom tests.mockserver import MockServer-class KeywordArgumentsSpider(MockServerSpider):+class InjectArgumentsDownloaderMiddleware(object):+ def process_request(self, request, spider):+ if request.callback.__name__ == 'parse_downloader_mw':+ request.cb_kwargs['from_process_request'] = True+ return None++ def process_response(self, request, response, spider):+ if request.callback.__name__ == 'parse_downloader_mw':+ request.cb_kwargs['from_process_response'] = True+ return response++class KeywordArgumentsSpider(MockServerSpider):name = 'kwargs'+ custom_settings = {+ 'DOWNLOADER_MIDDLEWARES': {+ __name__ + '.InjectArgumentsDownloaderMiddleware': 750,+ }+ }+checks = list()def start_requests(self):@@ -23,6 +40,7 @@ class KeywordArgumentsSpider(MockServerSpider):yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)+ yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)def parse_first(self, response, key, number):self.checks.append(key == 'value')@@ -69,6 +87,14 @@ class KeywordArgumentsSpider(MockServerSpider):TypeError: parse_takes_more() missing 1 required positional argument: 'other'"""+ def parse_downloader_mw(self, response, from_process_request, from_process_response):+ """+ Check if downloader middlewares are able to update the keyword arguments+ """+ self.checks.append(bool(from_process_request))+ self.checks.append(bool(from_process_response))+ self.crawler.stats.inc_value('boolean_checks', 2)+class CallbackKeywordArgumentsTestCase(TestCase):