Scrapy, a fast high-level web crawling & scraping framework for Python.
Disable SpiderState extension if no JOBDIR set
Paul Tremberth0349bbf9d369
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex 3799c7c66..2220cbd8f 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -2,6 +2,7 @@ import osfrom six.moves import cPickle as picklefrom scrapy import signals+from scrapy.exceptions import NotConfiguredfrom scrapy.utils.job import job_dirclass SpiderState(object):@@ -12,7 +13,11 @@ class SpiderState(object):@classmethoddef from_crawler(cls, crawler):- obj = cls(job_dir(crawler.settings))+ jobdir = job_dir(crawler.settings)+ if not jobdir:+ raise NotConfigured++ obj = cls(jobdir)crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)return obj