Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refactored SpiderLoader class constructor for easier subclassing

Lucas Moauro1ce4c86c2b66
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index a88db27c8..fbf68cec4 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -17,14 +17,17 @@ class SpiderLoader(object):
def __init__(self, settings):
self.spider_modules = settings.getlist('SPIDER_MODULES')
self._spiders = {}
- for name in self.spider_modules:
- for module in walk_modules(name):
- self._load_spiders(module)
-
+ self._load_all_spiders()
+
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._spiders[spcls.name] = spcls
+ def _load_all_spiders(self):
+ for name in self.spider_modules:
+ for module in walk_modules(name):
+ self._load_spiders(module)
+
@classmethod
def from_settings(cls, settings):
return cls(settings)