Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add testcase for catching exception from open_spider() from pipeline

Paul Tremberth61efacdd1f6f
diff --git a/tests/pipelines.py b/tests/pipelines.py
new file mode 100644
index 000000000..d81cfa93d
--- /dev/null
+++ b/tests/pipelines.py
@@ -0,0 +1,11 @@
+"""
+Some pipelines used for testing and benchmarking
+"""
+
+class ZeroDivisionErrorPipeline(object):
+
+ def open_spider(self, spider):
+ a = 1/0
+
+ def process_item(self, item, spider):
+ return item
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 90fd921c8..a0f3c9997 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -250,6 +250,18 @@ with multiples lines
yield self.assertFailure(crawler.crawl(), TestError)
self.assertFalse(crawler.crawling)
+ @defer.inlineCallbacks
+ def test_open_spider_error_on_faulty_pipeline(self):
+ settings = {
+ "ITEM_PIPELINES": {
+ "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+ }
+ }
+ crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+ yield self.assertFailure(
+ self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+ ZeroDivisionError)
+
@defer.inlineCallbacks
def test_crawlerrunner_accepts_crawler(self):
crawler = self.runner.create_crawler(SimpleSpider)