Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn user instead of failing for wrong SPIDER_MODULES setting
Paul Tremberth948e3cd00328
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex fbf68cec4..265182329 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-from __future__ import absolute_import+import warningsfrom zope.interface import implementer@@ -18,15 +19,21 @@ class SpiderLoader(object):self.spider_modules = settings.getlist('SPIDER_MODULES')self._spiders = {}self._load_all_spiders()-+def _load_spiders(self, module):for spcls in iter_spider_classes(module):self._spiders[spcls.name] = spclsdef _load_all_spiders(self):for name in self.spider_modules:- for module in walk_modules(name):- self._load_spiders(module)+ try:+ for module in walk_modules(name):+ self._load_spiders(module)+ except ImportError as e:+ msg = ("Could not load spiders from module '{}'; "+ "Check SPIDER_MODULES setting "+ "(exception: {})".format(name, str(e)))+ warnings.warn(msg, RuntimeWarning)@classmethoddef from_settings(cls, settings):diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex fbd2c1669..b2ad93b3f 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -1,6 +1,7 @@import sysimport osimport shutil+import warningsfrom zope.interface.verify import verifyObjectfrom twisted.trial import unittest@@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):crawler = runner.create_crawler('spider1')self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))self.assertEqual(crawler.spidercls.name, 'spider1')++ def test_bad_spider_modules_warning(self):++ with warnings.catch_warnings(record=True) as w:+ module = 'tests.test_spiderloader.test_spiders.doesnotexist'+ settings = Settings({'SPIDER_MODULES': [module]})+ spider_loader = SpiderLoader.from_settings(settings)+ self.assertIn("Could not load spiders from module", str(w[0].message))++ spiders = spider_loader.list()+ self.assertEqual(spiders, [])