Scrapy, a fast high-level web crawling & scraping framework for Python.
Add SPIDER_LOADER_WARN_ONLY to toggle between spiderloader failure and warning
Paul Tremberth7be773e14ae2
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.pyindex a255b3b94..185a77a40 100644--- a/scrapy/commands/list.py+++ b/scrapy/commands/list.py@@ -4,7 +4,8 @@ from scrapy.commands import ScrapyCommandclass Command(ScrapyCommand):requires_project = True- default_settings = {'LOG_ENABLED': False}+ default_settings = {'LOG_ENABLED': False,+ 'SPIDER_LOADER_WARN_ONLY': True}def short_desc(self):return "List available spiders"diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 1da09e4da..a98033dd1 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -28,6 +28,7 @@ def _import_file(filepath):class Command(ScrapyCommand):requires_project = False+ default_settings = {'SPIDER_LOADER_WARN_ONLY': True}def syntax(self):return "[options] <spider_file>"diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex bce4e6086..bee52f06a 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettingsclass Command(ScrapyCommand):requires_project = False- default_settings = {'LOG_ENABLED': False}+ default_settings = {'LOG_ENABLED': False,+ 'SPIDER_LOADER_WARN_ONLY': True}def syntax(self):return "[options]"diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 594106632..c17aaf442 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn')class Command(ScrapyCommand):requires_project = False- default_settings = {'LOG_ENABLED': False}+ default_settings = {'LOG_ENABLED': False,+ 'SPIDER_LOADER_WARN_ONLY': True}def syntax(self):return "<project_name> [project_dir]"@@ -118,4 +119,4 @@ class Command(ScrapyCommand):_templates_base_dir = self.settings['TEMPLATES_DIR'] or \join(scrapy.__path__[0], 'templates')return join(_templates_base_dir, 'project')-+diff --git a/scrapy/commands/version.py b/scrapy/commands/version.pyindex a9954edb0..e22f98f5a 100644--- a/scrapy/commands/version.py+++ b/scrapy/commands/version.py@@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommandclass Command(ScrapyCommand):- default_settings = {'LOG_ENABLED': False}+ default_settings = {'LOG_ENABLED': False,+ 'SPIDER_LOADER_WARN_ONLY': True}def syntax(self):return "[-v]"diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d73c595d2..854cefc9c 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -250,6 +250,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'+SPIDER_LOADER_WARN_ONLY = FalseSPIDER_MIDDLEWARES = {}diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex 486a4637e..7478faa78 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -19,6 +19,7 @@ class SpiderLoader(object):"""def __init__(self, settings):self.spider_modules = settings.getlist('SPIDER_MODULES')+ self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')self._spiders = {}self._found = defaultdict(list)self._load_all_spiders()@@ -46,10 +47,13 @@ class SpiderLoader(object):for module in walk_modules(name):self._load_spiders(module)except ImportError as e:- msg = ("\n{tb}Could not load spiders from module '{modname}'. "- "Check SPIDER_MODULES setting".format(- modname=name, tb=traceback.format_exc()))- warnings.warn(msg, RuntimeWarning)+ if self.warn_only:+ msg = ("\n{tb}Could not load spiders from module '{modname}'. "+ "See above traceback for details.".format(+ modname=name, tb=traceback.format_exc()))+ warnings.warn(msg, RuntimeWarning)+ else:+ raiseself._check_name_duplicates()@classmethoddiff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 673a2d302..99a61daea 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -91,18 +91,25 @@ class SpiderLoaderTest(unittest.TestCase):self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))self.assertEqual(crawler.spidercls.name, 'spider1')+ def test_bad_spider_modules_exception(self):++ module = 'tests.test_spiderloader.test_spiders.doesnotexist'+ settings = Settings({'SPIDER_MODULES': [module]})+ with self.assertRaises(ImportError):+ SpiderLoader.from_settings(settings)+def test_bad_spider_modules_warning(self):with warnings.catch_warnings(record=True) as w:module = 'tests.test_spiderloader.test_spiders.doesnotexist'- settings = Settings({'SPIDER_MODULES': [module]})+ settings = Settings({'SPIDER_MODULES': [module],+ 'SPIDER_LOADER_WARN_ONLY': True})spider_loader = SpiderLoader.from_settings(settings)self.assertIn("Could not load spiders from module", str(w[0].message))spiders = spider_loader.list()self.assertEqual(spiders, [])-class DuplicateSpiderNameLoaderTest(unittest.TestCase):def setUp(self):