Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Warn user instead of failing for wrong SPIDER_MODULES setting

Paul Tremberth948e3cd00328
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index fbf68cec4..265182329 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+import warnings
from zope.interface import implementer
@@ -18,15 +19,21 @@ class SpiderLoader(object):
self.spider_modules = settings.getlist('SPIDER_MODULES')
self._spiders = {}
self._load_all_spiders()
-
+
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._spiders[spcls.name] = spcls
def _load_all_spiders(self):
for name in self.spider_modules:
- for module in walk_modules(name):
- self._load_spiders(module)
+ try:
+ for module in walk_modules(name):
+ self._load_spiders(module)
+ except ImportError as e:
+ msg = ("Could not load spiders from module '{}'; "
+ "Check SPIDER_MODULES setting "
+ "(exception: {})".format(name, str(e)))
+ warnings.warn(msg, RuntimeWarning)
@classmethod
def from_settings(cls, settings):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index fbd2c1669..b2ad93b3f 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,6 +1,7 @@
import sys
import os
import shutil
+import warnings
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):
crawler = runner.create_crawler('spider1')
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
self.assertEqual(crawler.spidercls.name, 'spider1')
+
+ def test_bad_spider_modules_warning(self):
+
+ with warnings.catch_warnings(record=True) as w:
+ module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+ settings = Settings({'SPIDER_MODULES': [module]})
+ spider_loader = SpiderLoader.from_settings(settings)
+ self.assertIn("Could not load spiders from module", str(w[0].message))
+
+ spiders = spider_loader.list()
+ self.assertEqual(spiders, [])