Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[start_url] Fixes #4133: Raise AttributeError error when empty 'start_urls' and 'start_url' found. Added test.

Mabel Villalba1718e450ef95
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index e9c131e3b..5a35fcdb6 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -68,6 +68,11 @@ class Spider(object_ref):
def start_requests(self):
cls = self.__class__
+ if not self.start_urls and hasattr(self, 'start_url'):
+ raise AttributeError(
+ "Crawling could not start: 'start_urls' not found "
+ "or empty (but found 'start_url' attribute instead, "
+ "did you miss an 's'?)")
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
warnings.warn(
"Spider.make_requests_from_url method is deprecated; it "
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83fb68c2f..0a6640cec 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -391,6 +391,13 @@ class CrawlSpiderTest(SpiderTest):
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
+ def test_start_url(self):
+ spider = self.spider_class("example.com")
+ spider.start_url = 'https://www.example.com'
+
+ with self.assertRaisesRegex(AttributeError,
+ r'^Crawling could not start.*$'):
+ list(spider.start_requests())
class SitemapSpiderTest(SpiderTest):