Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Recover from a callback exception

Eugenio Lacuesta4fca9aba8511
diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
index 092546291..49a60d08b 100644
--- a/tests/test_spider_mw.py
+++ b/tests/test_spider_mw.py
@@ -10,22 +10,8 @@ from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-class CommonTestCase(TestCase):
- @classmethod
- def setUpClass(cls):
- cls.mockserver = MockServer()
- cls.mockserver.__enter__()
-
- @classmethod
- def tearDownClass(cls):
- cls.mockserver.__exit__(None, None, None)
-
- @defer.inlineCallbacks
- def crawl_log(self, spider):
- crawler = get_crawler(spider)
- with LogCapture() as log:
- yield crawler.crawl()
- raise defer.returnValue(log)
+# TEST_URL = 'http://example.org'
+TEST_URL = 'http://localhost:8998'
class LogExceptionMiddleware(object):
@@ -34,6 +20,32 @@ class LogExceptionMiddleware(object):
return None
+# ================================================================================
+# recover from an exception on a spider callback
+class RecoverySpider(Spider):
+ name = 'RecoverySpider'
+ start_urls = [TEST_URL]
+ custom_settings = {
+ 'SPIDER_MIDDLEWARES': {
+ __name__ + '.RecoveryMiddleware': 10,
+ },
+ }
+
+ def parse(self, response):
+ yield {'test': 1}
+ self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
+ if not response.meta.get('dont_fail'):
+ raise ModuleNotFoundError()
+
+class RecoveryMiddleware(object):
+ def process_spider_exception(self, response, exception, spider):
+ logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+ return [
+ {'from': 'process_spider_exception'},
+ Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+ ]
+
+
# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
class ProcessSpiderInputSpider(Spider):
@@ -49,7 +61,7 @@ class ProcessSpiderInputSpider(Spider):
}
def start_requests(self):
- yield Request('http://localhost:8998', callback=self.parse, errback=self.errback)
+ yield Request(TEST_URL, callback=self.parse, errback=self.errback)
def parse(self, response):
return [{'test': 1}, {'test': 2}]
@@ -64,29 +76,14 @@ class FailProcessSpiderInputMiddleware:
raise IndexError()
-class TestProcessSpiderInput(CommonTestCase):
- @defer.inlineCallbacks
- def test_process_spider_input_errback(self):
- """
- (1) An exception from the process_spider_input chain should not be caught by the
- process_spider_exception chain, it should go directly to the Request errback
- """
- log = yield self.crawl_log(ProcessSpiderInputSpider)
- self.assertNotIn("Middleware: IndexError exception caught", str(log))
- self.assertIn("Middleware: will raise IndexError", str(log))
- self.assertIn("Got a Failure on the Request errback", str(log))
-
-
# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(Spider):
name = 'GeneratorCallbackSpider'
- start_urls = ['http://localhost:8998']
+ start_urls = [TEST_URL]
custom_settings = {
'SPIDER_MIDDLEWARES': {
- # spider
__name__ + '.LogExceptionMiddleware': 10,
- # engine
},
}
@@ -96,28 +93,14 @@ class GeneratorCallbackSpider(Spider):
raise ImportError()
-class TestGeneratorCallback(CommonTestCase):
- @defer.inlineCallbacks
- def test_generator_callback(self):
- """
- (2) An exception from a spider's callback should
- be caught by the process_spider_exception chain
- """
- log = yield self.crawl_log(GeneratorCallbackSpider)
- self.assertIn("Middleware: ImportError exception caught", str(log))
- self.assertIn("'item_scraped_count': 2", str(log))
-
-
# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotAGeneratorCallbackSpider(Spider):
name = 'NotAGeneratorCallbackSpider'
- start_urls = ['http://localhost:8998']
+ start_urls = [TEST_URL]
custom_settings = {
'SPIDER_MIDDLEWARES': {
- # spider
__name__ + '.LogExceptionMiddleware': 10,
- # engine
},
}
@@ -125,13 +108,63 @@ class NotAGeneratorCallbackSpider(Spider):
return [{'test': 1}, {'test': 1/0}]
-class TestNotAGeneratorCallback(CommonTestCase):
+# ================================================================================
+class TestSpiderMiddleware(TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.mockserver = MockServer()
+ cls.mockserver.__enter__()
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.mockserver.__exit__(None, None, None)
+
+ @defer.inlineCallbacks
+ def crawl_log(self, spider):
+ crawler = get_crawler(spider)
+ with LogCapture() as log:
+ yield crawler.crawl()
+ raise defer.returnValue(log)
+
+ # @defer.inlineCallbacks
+ # def test_recovery(self):
+ # """
+ # Recover from an exception from a spider's callback. The final item count should be 3
+ # (one from the spider before raising the exception, one from the middleware and one
+ # from the spider when processing the response that was enqueued from the middleware)
+ # """
+ # log = yield self.crawl_log(RecoverySpider)
+ # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
+ # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
+ # self.assertIn("'item_scraped_count': 3", str(log))
+
+ @defer.inlineCallbacks
+ def test_process_spider_input_errback(self):
+ """
+ (1) An exception from the process_spider_input chain should not be caught by the
+ process_spider_exception chain, it should go directly to the Request errback
+ """
+ log1 = yield self.crawl_log(ProcessSpiderInputSpider)
+ self.assertNotIn("Middleware: IndexError exception caught", str(log1))
+ self.assertIn("Middleware: will raise IndexError", str(log1))
+ self.assertIn("Got a Failure on the Request errback", str(log1))
+
+ @defer.inlineCallbacks
+ def test_generator_callback(self):
+ """
+ (2) An exception from a spider's callback should
+ be caught by the process_spider_exception chain
+ """
+ log2 = yield self.crawl_log(GeneratorCallbackSpider)
+ self.assertIn("Middleware: ImportError exception caught", str(log2))
+ self.assertIn("'item_scraped_count': 2", str(log2))
+
@defer.inlineCallbacks
def test_not_a_generator_callback(self):
"""
(3) An exception from a spider's callback should
be caught by the process_spider_exception chain
"""
- log = yield self.crawl_log(NotAGeneratorCallbackSpider)
- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log))
- self.assertNotIn("item_scraped_count", str(log))
+ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
+ self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
+ self.assertNotIn("item_scraped_count", str(log3))