Scrapy, a fast high-level web crawling & scraping framework for Python.
Add simple tests for pipelines.
Andrey Rakhmatullin5980d3bbff67
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pynew file mode 100644index 000000000..bc53f5427--- /dev/null+++ b/tests/test_pipelines.py@@ -0,0 +1,71 @@+from twisted.internet import defer+from twisted.internet.defer import Deferred+from twisted.trial import unittest++from scrapy import Spider, signals, Request+from scrapy.utils.test import get_crawler++from tests.mockserver import MockServer+++class SimplePipeline:+ def process_item(self, item, spider):+ item['pipeline_passed'] = True+ return item+++class DeferredPipeline:+ def cb(self, item):+ item['pipeline_passed'] = True+ return item++ def process_item(self, item, spider):+ d = Deferred()+ d.addCallback(self.cb)+ d.callback(item)+ return d+++class ItemSpider(Spider):+ name = 'itemspider'++ def start_requests(self):+ yield Request(self.mockserver.url('/status?n=200'))++ def parse(self, response):+ return {'field': 42}+++class PipelineTestCase(unittest.TestCase):+ def setUp(self):+ self.mockserver = MockServer()+ self.mockserver.__enter__()++ def tearDown(self):+ self.mockserver.__exit__(None, None, None)++ def _on_item_scraped(self, item):+ self.assertIsInstance(item, dict)+ self.assertTrue(item.get('pipeline_passed'))+ self.items.append(item)++ def _create_crawler(self, pipeline_class):+ settings = {+ 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},+ }+ crawler = get_crawler(ItemSpider, settings)+ crawler.signals.connect(self._on_item_scraped, signals.item_scraped)+ self.items = []+ return crawler++ @defer.inlineCallbacks+ def test_simple_pipeline(self):+ crawler = self._create_crawler(SimplePipeline)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertEqual(len(self.items), 1)++ @defer.inlineCallbacks+ def test_deferred_pipeline(self):+ crawler = self._create_crawler(DeferredPipeline)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertEqual(len(self.items), 1)