Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add SPIDER_LOADER_WARN_ONLY to toggle between spiderloader failure and warning

Paul Tremberth7be773e14ae2
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..185a77a40 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -4,7 +4,8 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def short_desc(self):
return "List available spiders"
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 1da09e4da..a98033dd1 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -28,6 +28,7 @@ def _import_file(filepath):
class Command(ScrapyCommand):
requires_project = False
+ default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bce4e6086..bee52f06a 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options]"
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 594106632..c17aaf442 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
class Command(ScrapyCommand):
requires_project = False
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "<project_name> [project_dir]"
@@ -118,4 +119,4 @@ class Command(ScrapyCommand):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')
-
+
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index a9954edb0..e22f98f5a 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[-v]"
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d73c595d2..854cefc9c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -250,6 +250,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
+SPIDER_LOADER_WARN_ONLY = False
SPIDER_MIDDLEWARES = {}
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 486a4637e..7478faa78 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -19,6 +19,7 @@ class SpiderLoader(object):
"""
def __init__(self, settings):
self.spider_modules = settings.getlist('SPIDER_MODULES')
+ self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
self._spiders = {}
self._found = defaultdict(list)
self._load_all_spiders()
@@ -46,10 +47,13 @@ class SpiderLoader(object):
for module in walk_modules(name):
self._load_spiders(module)
except ImportError as e:
- msg = ("\n{tb}Could not load spiders from module '{modname}'. "
- "Check SPIDER_MODULES setting".format(
- modname=name, tb=traceback.format_exc()))
- warnings.warn(msg, RuntimeWarning)
+ if self.warn_only:
+ msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+ "See above traceback for details.".format(
+ modname=name, tb=traceback.format_exc()))
+ warnings.warn(msg, RuntimeWarning)
+ else:
+ raise
self._check_name_duplicates()
@classmethod
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 673a2d302..99a61daea 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -91,18 +91,25 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
self.assertEqual(crawler.spidercls.name, 'spider1')
+ def test_bad_spider_modules_exception(self):
+
+ module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+ settings = Settings({'SPIDER_MODULES': [module]})
+ with self.assertRaises(ImportError):
+ SpiderLoader.from_settings(settings)
+
def test_bad_spider_modules_warning(self):
with warnings.catch_warnings(record=True) as w:
module = 'tests.test_spiderloader.test_spiders.doesnotexist'
- settings = Settings({'SPIDER_MODULES': [module]})
+ settings = Settings({'SPIDER_MODULES': [module],
+ 'SPIDER_LOADER_WARN_ONLY': True})
spider_loader = SpiderLoader.from_settings(settings)
self.assertIn("Could not load spiders from module", str(w[0].message))
spiders = spider_loader.list()
self.assertEqual(spiders, [])
-
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):