Scrapy, a fast high-level web crawling & scraping framework for Python.
Recover from a callback exception
Eugenio Lacuesta4fca9aba8511
diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.pyindex 092546291..49a60d08b 100644--- a/tests/test_spider_mw.py+++ b/tests/test_spider_mw.py@@ -10,22 +10,8 @@ from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-class CommonTestCase(TestCase):- @classmethod- def setUpClass(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()-- @classmethod- def tearDownClass(cls):- cls.mockserver.__exit__(None, None, None)-- @defer.inlineCallbacks- def crawl_log(self, spider):- crawler = get_crawler(spider)- with LogCapture() as log:- yield crawler.crawl()- raise defer.returnValue(log)+# TEST_URL = 'http://example.org'+TEST_URL = 'http://localhost:8998'class LogExceptionMiddleware(object):@@ -34,6 +20,32 @@ class LogExceptionMiddleware(object):return None+# ================================================================================+# recover from an exception on a spider callback+class RecoverySpider(Spider):+ name = 'RecoverySpider'+ start_urls = [TEST_URL]+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ __name__ + '.RecoveryMiddleware': 10,+ },+ }++ def parse(self, response):+ yield {'test': 1}+ self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))+ if not response.meta.get('dont_fail'):+ raise ModuleNotFoundError()++class RecoveryMiddleware(object):+ def process_spider_exception(self, response, exception, spider):+ logging.warn('Middleware: %s exception caught', exception.__class__.__name__)+ return [+ {'from': 'process_spider_exception'},+ Request(response.url, meta={'dont_fail': True}, dont_filter=True),+ ]++# ================================================================================# (1) exceptions from a spider middleware's process_spider_input methodclass ProcessSpiderInputSpider(Spider):@@ -49,7 +61,7 @@ class ProcessSpiderInputSpider(Spider):}def start_requests(self):- yield Request('http://localhost:8998', callback=self.parse, errback=self.errback)+ yield Request(TEST_URL, callback=self.parse, errback=self.errback)def parse(self, response):return [{'test': 1}, {'test': 2}]@@ -64,29 +76,14 @@ class FailProcessSpiderInputMiddleware:raise IndexError()-class TestProcessSpiderInput(CommonTestCase):- @defer.inlineCallbacks- def test_process_spider_input_errback(self):- """- (1) An exception from the process_spider_input chain should not be caught by the- process_spider_exception chain, it should go directly to the Request errback- """- log = yield self.crawl_log(ProcessSpiderInputSpider)- self.assertNotIn("Middleware: IndexError exception caught", str(log))- self.assertIn("Middleware: will raise IndexError", str(log))- self.assertIn("Got a Failure on the Request errback", str(log))--# ================================================================================# (2) exceptions from a spider callback (generator)class GeneratorCallbackSpider(Spider):name = 'GeneratorCallbackSpider'- start_urls = ['http://localhost:8998']+ start_urls = [TEST_URL]custom_settings = {'SPIDER_MIDDLEWARES': {- # spider__name__ + '.LogExceptionMiddleware': 10,- # engine},}@@ -96,28 +93,14 @@ class GeneratorCallbackSpider(Spider):raise ImportError()-class TestGeneratorCallback(CommonTestCase):- @defer.inlineCallbacks- def test_generator_callback(self):- """- (2) An exception from a spider's callback should- be caught by the process_spider_exception chain- """- log = yield self.crawl_log(GeneratorCallbackSpider)- self.assertIn("Middleware: ImportError exception caught", str(log))- self.assertIn("'item_scraped_count': 2", str(log))--# ================================================================================# (3) exceptions from a spider callback (not a generator)class NotAGeneratorCallbackSpider(Spider):name = 'NotAGeneratorCallbackSpider'- start_urls = ['http://localhost:8998']+ start_urls = [TEST_URL]custom_settings = {'SPIDER_MIDDLEWARES': {- # spider__name__ + '.LogExceptionMiddleware': 10,- # engine},}@@ -125,13 +108,63 @@ class NotAGeneratorCallbackSpider(Spider):return [{'test': 1}, {'test': 1/0}]-class TestNotAGeneratorCallback(CommonTestCase):+# ================================================================================+class TestSpiderMiddleware(TestCase):+ @classmethod+ def setUpClass(cls):+ cls.mockserver = MockServer()+ cls.mockserver.__enter__()++ @classmethod+ def tearDownClass(cls):+ cls.mockserver.__exit__(None, None, None)++ @defer.inlineCallbacks+ def crawl_log(self, spider):+ crawler = get_crawler(spider)+ with LogCapture() as log:+ yield crawler.crawl()+ raise defer.returnValue(log)++ # @defer.inlineCallbacks+ # def test_recovery(self):+ # """+ # Recover from an exception from a spider's callback. The final item count should be 3+ # (one from the spider before raising the exception, one from the middleware and one+ # from the spider when processing the response that was enqueued from the middleware)+ # """+ # log = yield self.crawl_log(RecoverySpider)+ # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))+ # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)+ # self.assertIn("'item_scraped_count': 3", str(log))++ @defer.inlineCallbacks+ def test_process_spider_input_errback(self):+ """+ (1) An exception from the process_spider_input chain should not be caught by the+ process_spider_exception chain, it should go directly to the Request errback+ """+ log1 = yield self.crawl_log(ProcessSpiderInputSpider)+ self.assertNotIn("Middleware: IndexError exception caught", str(log1))+ self.assertIn("Middleware: will raise IndexError", str(log1))+ self.assertIn("Got a Failure on the Request errback", str(log1))++ @defer.inlineCallbacks+ def test_generator_callback(self):+ """+ (2) An exception from a spider's callback should+ be caught by the process_spider_exception chain+ """+ log2 = yield self.crawl_log(GeneratorCallbackSpider)+ self.assertIn("Middleware: ImportError exception caught", str(log2))+ self.assertIn("'item_scraped_count': 2", str(log2))+@defer.inlineCallbacksdef test_not_a_generator_callback(self):"""(3) An exception from a spider's callback shouldbe caught by the process_spider_exception chain"""- log = yield self.crawl_log(NotAGeneratorCallbackSpider)- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log))- self.assertNotIn("item_scraped_count", str(log))+ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)+ self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))+ self.assertNotIn("item_scraped_count", str(log3))