Scrapy, a fast high-level web crawling & scraping framework for Python.
Callback kwargs: more tests
Eugenio Lacuesta70a4d93aa324
diff --git a/tests/spiders.py b/tests/spiders.pyindex a06985837..8c8d50ff5 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -39,6 +39,9 @@ class KeywordArgumentsSpider(MockServerSpider):yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)yield Request(self.mockserver.url('/general_without'), self.parse_general)yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)+ yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)+ yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)+ yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)def parse_first(self, response, key, number):self.checks.append(key == 'value')@@ -66,6 +69,25 @@ class KeywordArgumentsSpider(MockServerSpider):self.checks.append(response.url.endswith('/no_kwargs'))self.crawler.stats.inc_value('boolean_checks')+ def parse_default(self, response, key, number=None, default=99):+ self.checks.append(response.url.endswith('/default'))+ self.checks.append(key == 'value')+ self.checks.append(number == 123)+ self.checks.append(default == 99)+ self.crawler.stats.inc_value('boolean_checks', 4)++ def parse_takes_less(self, response, key):+ """+ Should raise+ TypeError: parse_takes_less() got an unexpected keyword argument 'number'+ """++ def parse_takes_more(self, response, key, number, other):+ """+ Should raise+ TypeError: parse_takes_more() missing 1 required positional argument: 'other'+ """+class FollowAllSpider(MetaSpider):diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 9a39b8cb4..2b3e56ee9 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -4,6 +4,7 @@ import loggingfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial.unittest import TestCase+import sixfrom scrapy.http import Requestfrom scrapy.crawler import CrawlerRunner@@ -15,6 +16,8 @@ from tests.mockserver import MockServerclass CrawlTestCase(TestCase):+ maxDiff = None+def setUp(self):self.mockserver = MockServer()self.mockserver.__enter__()@@ -26,9 +29,25 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_callback_kwargs(self):crawler = self.runner.create_crawler(KeywordArgumentsSpider)- yield crawler.crawl(mockserver=self.mockserver)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)self.assertTrue(all(crawler.spider.checks))self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))+ # check exceptions for argument mismatch+ exceptions = {}+ for line in log.records:+ for key in ('takes_less', 'takes_more'):+ if key in line.getMessage():+ exceptions[key] = line+ self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)+ self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")+ self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)+ # py2 and py3 messages are different+ exc_message = str(exceptions['takes_more'].exc_info[1])+ if six.PY2:+ self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")+ elif six.PY3:+ self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")@defer.inlineCallbacksdef test_follow_all(self):