Scrapy, a fast high-level web crawling & scraping framework for Python.
[test] Override CrawlSpider.parse
Eugenio Lacuesta8d4948f6ca44
diff --git a/tests/spiders.py b/tests/spiders.pyindex 39c8da0b6..dcc475ca7 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider):self.visited += 1-class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):- name = 'crawl_spider_with_errback'+class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):+ """+ A CrawlSpider which overrides the 'parse' method+ """+ name = 'crawl_spider_with_parse_method'custom_settings = {'RETRY_HTTP_CODES': [], # no need to retry}rules = (- Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),+ Rule(LinkExtractor(), callback='parse', follow=True),+ )++ def start_requests(self):+ test_body = b"""+ <html>+ <head><title>Page title<title></head>+ <body>+ <p><a href="/status?n=200">Item 200</a></p> <!-- callback -->+ <p><a href="/status?n=201">Item 201</a></p> <!-- callback -->+ </body>+ </html>+ """+ url = self.mockserver.url("/alpayload")+ yield Request(url, method="POST", body=test_body)++ def parse(self, response):+ self.logger.info('[parse] status %i', response.status)+++class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):+ name = 'crawl_spider_with_errback'+ rules = (+ Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),)def start_requests(self):@@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):url = self.mockserver.url("/alpayload")yield Request(url, method="POST", body=test_body)- def callback(self, response):- self.logger.info('[callback] status %i', response.status)-def errback(self, failure):self.logger.info('[errback] status %i', failure.value.response.status)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex f433fcea6..4299e4bbb 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunnerfrom scrapy.http import Requestfrom scrapy.utils.python import to_unicodefrom tests.mockserver import MockServer-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,- SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)+from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,+ CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,+ DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)class CrawlTestCase(TestCase):@@ -297,6 +298,27 @@ with multiples linesself._assert_retried(log)self.assertIn("Got response 200", str(log))++class CrawlSpiderTestCase(TestCase):++ def setUp(self):+ self.mockserver = MockServer()+ self.mockserver.__enter__()+ self.runner = CrawlerRunner()++ def tearDown(self):+ self.mockserver.__exit__(None, None, None)++ @defer.inlineCallbacks+ def test_crawlspider_with_parse(self):+ self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)++ with LogCapture() as log:+ yield self.runner.join()++ self.assertIn("[parse] status 200", str(log))+ self.assertIn("[parse] status 201", str(log))+@defer.inlineCallbacksdef test_crawlspider_with_errback(self):self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)@@ -304,7 +326,8 @@ with multiples lineswith LogCapture() as log:yield self.runner.join()- self.assertIn("[callback] status 200", str(log))- self.assertIn("[callback] status 201", str(log))+ self.assertIn("[parse] status 200", str(log))+ self.assertIn("[parse] status 201", str(log))self.assertIn("[errback] status 404", str(log))self.assertIn("[errback] status 500", str(log))+ self.assertIn("[errback] status 501", str(log))