Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix tests
Eugenio Lacuestae7e18db179f2
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 2565ef7af..5622c3179 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -8,10 +8,7 @@ from twisted.internet import deferfrom scrapy import Spider, Requestfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer---# TEST_URL = 'http://example.org'-TEST_URL = 'http://localhost:8998'+from tests.spiders import MockServerSpiderclass LogExceptionMiddleware:@@ -21,16 +18,18 @@ class LogExceptionMiddleware:# ================================================================================-# recover from an exception on a spider callback+# (0) recover from an exception on a spider callbackclass RecoverySpider(Spider):name = 'RecoverySpider'- start_urls = [TEST_URL]custom_settings = {'SPIDER_MIDDLEWARES': {__name__ + '.RecoveryMiddleware': 10,},}+ def start_requests(self):+ yield Request(self.mockserver.url('/status?n=200'))+def parse(self, response):yield {'test': 1}self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))@@ -61,10 +60,11 @@ class ProcessSpiderInputSpider(Spider):}def start_requests(self):- yield Request(TEST_URL, callback=self.parse, errback=self.errback)+ yield Request(url=self.mockserver.url('/status?n=200'),+ callback=self.parse, errback=self.errback)def parse(self, response):- return [{'test': 1}, {'test': 2}]+ return {'from': 'callback'}def errback(self, failure):self.logger.warn('Got a Failure on the Request errback')@@ -80,13 +80,15 @@ class FailProcessSpiderInputMiddleware:# (2) exceptions from a spider callback (generator)class GeneratorCallbackSpider(Spider):name = 'GeneratorCallbackSpider'- start_urls = [TEST_URL]custom_settings = {'SPIDER_MIDDLEWARES': {__name__ + '.LogExceptionMiddleware': 10,},}+ def start_requests(self):+ yield Request(self.mockserver.url('/status?n=200'))+def parse(self, response):yield {'test': 1}yield {'test': 2}@@ -97,13 +99,15 @@ class GeneratorCallbackSpider(Spider):# (3) exceptions from a spider callback (not a generator)class NotAGeneratorCallbackSpider(Spider):name = 'NotAGeneratorCallbackSpider'- start_urls = [TEST_URL]custom_settings = {'SPIDER_MIDDLEWARES': {__name__ + '.LogExceptionMiddleware': 10,},}+ def start_requests(self):+ yield Request(self.mockserver.url('/status?n=200'))+def parse(self, response):return [{'test': 1}, {'test': 1/0}]@@ -123,20 +127,20 @@ class TestSpiderMiddleware(TestCase):def crawl_log(self, spider):crawler = get_crawler(spider)with LogCapture() as log:- yield crawler.crawl()+ yield crawler.crawl(mockserver=self.mockserver)raise defer.returnValue(log)- # @defer.inlineCallbacks- # def test_recovery(self):- # """- # Recover from an exception from a spider's callback. The final item count should be 3- # (one from the spider before raising the exception, one from the middleware and one- # from the spider when processing the response that was enqueued from the middleware)- # """- # log = yield self.crawl_log(RecoverySpider)- # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))- # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)- # self.assertIn("'item_scraped_count': 3", str(log))+ @defer.inlineCallbacks+ def test_recovery(self):+ """+ (0) Recover from an exception in a spider callback. The final item count should be 2+ (one directly from the recovery middleware and one from the spider when processing+ the request that was enqueued from the recovery middleware)+ """+ log = yield self.crawl_log(RecoverySpider)+ self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))+ self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)+ self.assertIn("'item_scraped_count': 2", str(log))@defer.inlineCallbacksdef test_process_spider_input_errback(self):@@ -148,21 +152,24 @@ class TestSpiderMiddleware(TestCase):self.assertNotIn("Middleware: IndexError exception caught", str(log1))self.assertIn("Middleware: will raise IndexError", str(log1))self.assertIn("Got a Failure on the Request errback", str(log1))+ self.assertIn("{'from': 'errback'}", str(log1))+ self.assertNotIn("{'from': 'callback'}", str(log1))+ self.assertIn("'item_scraped_count': 1", str(log1))@defer.inlineCallbacksdef test_generator_callback(self):"""- (2) An exception from a spider's callback should+ (2) An exception from a spider callback (returning a generator) shouldbe caught by the process_spider_exception chain"""log2 = yield self.crawl_log(GeneratorCallbackSpider)self.assertIn("Middleware: ImportError exception caught", str(log2))- self.assertIn("'item_scraped_count': 2", str(log2))+ self.assertNotIn("item_scraped_count", str(log2))@defer.inlineCallbacksdef test_not_a_generator_callback(self):"""- (3) An exception from a spider's callback should+ (3) An exception from a spider callback (returning a list) shouldbe caught by the process_spider_exception chain"""log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)