Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Callback kwargs: more tests

Eugenio Lacuesta70a4d93aa324
diff --git a/tests/spiders.py b/tests/spiders.py
index a06985837..8c8d50ff5 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -39,6 +39,9 @@ class KeywordArgumentsSpider(MockServerSpider):
yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
yield Request(self.mockserver.url('/general_without'), self.parse_general)
yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+ yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+ yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+ yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
def parse_first(self, response, key, number):
self.checks.append(key == 'value')
@@ -66,6 +69,25 @@ class KeywordArgumentsSpider(MockServerSpider):
self.checks.append(response.url.endswith('/no_kwargs'))
self.crawler.stats.inc_value('boolean_checks')
+ def parse_default(self, response, key, number=None, default=99):
+ self.checks.append(response.url.endswith('/default'))
+ self.checks.append(key == 'value')
+ self.checks.append(number == 123)
+ self.checks.append(default == 99)
+ self.crawler.stats.inc_value('boolean_checks', 4)
+
+ def parse_takes_less(self, response, key):
+ """
+ Should raise
+ TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+ """
+
+ def parse_takes_more(self, response, key, number, other):
+ """
+ Should raise
+ TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+ """
+
class FollowAllSpider(MetaSpider):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9a39b8cb4..2b3e56ee9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,6 +4,7 @@ import logging
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
+import six
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
@@ -15,6 +16,8 @@ from tests.mockserver import MockServer
class CrawlTestCase(TestCase):
+ maxDiff = None
+
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
@@ -26,9 +29,25 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_callback_kwargs(self):
crawler = self.runner.create_crawler(KeywordArgumentsSpider)
- yield crawler.crawl(mockserver=self.mockserver)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(all(crawler.spider.checks))
self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+ # check exceptions for argument mismatch
+ exceptions = {}
+ for line in log.records:
+ for key in ('takes_less', 'takes_more'):
+ if key in line.getMessage():
+ exceptions[key] = line
+ self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+ self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+ self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+ # py2 and py3 messages are different
+ exc_message = str(exceptions['takes_more'].exc_info[1])
+ if six.PY2:
+ self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+ elif six.PY3:
+ self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
@defer.inlineCallbacks
def test_follow_all(self):