Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix make_requests_from_url deprcation implementation, add tests

Mikhail Korobova1e8a8525d23
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 138cdbc67..80768b1b0 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -66,11 +66,14 @@ class Spider(object_ref):
crawler.signals.connect(self.close, signals.spider_closed)
def start_requests(self):
- if self.make_requests_from_url is not Spider.make_requests_from_url:
+ cls = self.__class__
+ if cls.make_requests_from_url is not Spider.make_requests_from_url:
warnings.warn(
- "Spider.make_requests_from_url method is deprecated; "
- "it won't be called in future Scrapy releases. "
- "Please override start_requests method instead."
+ "Spider.make_requests_from_url method is deprecated; it "
+ "won't be called in future Scrapy releases. Please "
+ "override Spider.start_requests method instead (see %s.%s)." % (
+ cls.__module__, cls.__name__
+ ),
)
for url in self.start_urls:
yield self.make_requests_from_url(url)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 079734a69..371b8c1ac 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -345,7 +345,7 @@ Sitemap: /sitemap-relative-url.xml
'http://www.example.com/sitemap-relative-url.xml'])
-class BaseSpiderDeprecationTest(unittest.TestCase):
+class DeprecationTest(unittest.TestCase):
def test_basespider_is_deprecated(self):
with warnings.catch_warnings(record=True) as w:
@@ -399,6 +399,29 @@ class BaseSpiderDeprecationTest(unittest.TestCase):
assert isinstance(CrawlSpider(name='foo'), Spider)
assert isinstance(CrawlSpider(name='foo'), BaseSpider)
+ def test_make_requests_from_url_deprecated(self):
+ class MySpider4(Spider):
+ name = 'spider1'
+ start_urls = ['http://example.com']
-if __name__ == '__main__':
- unittest.main()
+ class MySpider5(Spider):
+ name = 'spider2'
+ start_urls = ['http://example.com']
+
+ def make_requests_from_url(self, url):
+ return Request(url + "/foo", dont_filter=True)
+
+ with warnings.catch_warnings(record=True) as w:
+ # spider without overridden make_requests_from_url method
+ # doesn't issue a warning
+ spider1 = MySpider4()
+ self.assertEqual(len(list(spider1.start_requests())), 1)
+ self.assertEqual(len(w), 0)
+
+ # spider with overridden make_requests_from_url issues a warning,
+ # but the method still works
+ spider2 = MySpider5()
+ requests = list(spider2.start_requests())
+ self.assertEqual(len(requests), 1)
+ self.assertEqual(requests[0].url, 'http://example.com/foo')
+ self.assertEqual(len(w), 1)