Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Tests for exceptions on spider callbacks

Eugenio Lacuesta6ed9440ed528
diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
index 4a431d379..092546291 100644
--- a/tests/test_spider_mw.py
+++ b/tests/test_spider_mw.py
@@ -60,11 +60,11 @@ class ProcessSpiderInputSpider(Spider):
class FailProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
- logging.warn('Middleware: will raise ZeroDivisionError')
- raise ZeroDivisionError()
+ logging.warn('Middleware: will raise IndexError')
+ raise IndexError()
-class TestProcessSpiderInputSpider(CommonTestCase):
+class TestProcessSpiderInput(CommonTestCase):
@defer.inlineCallbacks
def test_process_spider_input_errback(self):
"""
@@ -72,6 +72,66 @@ class TestProcessSpiderInputSpider(CommonTestCase):
process_spider_exception chain, it should go directly to the Request errback
"""
log = yield self.crawl_log(ProcessSpiderInputSpider)
- self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log))
- self.assertIn('Middleware: will raise ZeroDivisionError', str(log))
- self.assertIn('Got a Failure on the Request errback', str(log))
+ self.assertNotIn("Middleware: IndexError exception caught", str(log))
+ self.assertIn("Middleware: will raise IndexError", str(log))
+ self.assertIn("Got a Failure on the Request errback", str(log))
+
+
+# ================================================================================
+# (2) exceptions from a spider callback (generator)
+class GeneratorCallbackSpider(Spider):
+ name = 'GeneratorCallbackSpider'
+ start_urls = ['http://localhost:8998']
+ custom_settings = {
+ 'SPIDER_MIDDLEWARES': {
+ # spider
+ __name__ + '.LogExceptionMiddleware': 10,
+ # engine
+ },
+ }
+
+ def parse(self, response):
+ yield {'test': 1}
+ yield {'test': 2}
+ raise ImportError()
+
+
+class TestGeneratorCallback(CommonTestCase):
+ @defer.inlineCallbacks
+ def test_generator_callback(self):
+ """
+ (2) An exception from a spider's callback should
+ be caught by the process_spider_exception chain
+ """
+ log = yield self.crawl_log(GeneratorCallbackSpider)
+ self.assertIn("Middleware: ImportError exception caught", str(log))
+ self.assertIn("'item_scraped_count': 2", str(log))
+
+
+# ================================================================================
+# (3) exceptions from a spider callback (not a generator)
+class NotAGeneratorCallbackSpider(Spider):
+ name = 'NotAGeneratorCallbackSpider'
+ start_urls = ['http://localhost:8998']
+ custom_settings = {
+ 'SPIDER_MIDDLEWARES': {
+ # spider
+ __name__ + '.LogExceptionMiddleware': 10,
+ # engine
+ },
+ }
+
+ def parse(self, response):
+ return [{'test': 1}, {'test': 1/0}]
+
+
+class TestNotAGeneratorCallback(CommonTestCase):
+ @defer.inlineCallbacks
+ def test_not_a_generator_callback(self):
+ """
+ (3) An exception from a spider's callback should
+ be caught by the process_spider_exception chain
+ """
+ log = yield self.crawl_log(NotAGeneratorCallbackSpider)
+ self.assertIn("Middleware: ZeroDivisionError exception caught", str(log))
+ self.assertNotIn("item_scraped_count", str(log))