Scrapy, a fast high-level web crawling & scraping framework for Python.
fix make_requests_from_url deprcation implementation, add tests
Mikhail Korobova1e8a8525d23
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 138cdbc67..80768b1b0 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -66,11 +66,14 @@ class Spider(object_ref):crawler.signals.connect(self.close, signals.spider_closed)def start_requests(self):- if self.make_requests_from_url is not Spider.make_requests_from_url:+ cls = self.__class__+ if cls.make_requests_from_url is not Spider.make_requests_from_url:warnings.warn(- "Spider.make_requests_from_url method is deprecated; "- "it won't be called in future Scrapy releases. "- "Please override start_requests method instead."+ "Spider.make_requests_from_url method is deprecated; it "+ "won't be called in future Scrapy releases. Please "+ "override Spider.start_requests method instead (see %s.%s)." % (+ cls.__module__, cls.__name__+ ),)for url in self.start_urls:yield self.make_requests_from_url(url)diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 079734a69..371b8c1ac 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -345,7 +345,7 @@ Sitemap: /sitemap-relative-url.xml'http://www.example.com/sitemap-relative-url.xml'])-class BaseSpiderDeprecationTest(unittest.TestCase):+class DeprecationTest(unittest.TestCase):def test_basespider_is_deprecated(self):with warnings.catch_warnings(record=True) as w:@@ -399,6 +399,29 @@ class BaseSpiderDeprecationTest(unittest.TestCase):assert isinstance(CrawlSpider(name='foo'), Spider)assert isinstance(CrawlSpider(name='foo'), BaseSpider)+ def test_make_requests_from_url_deprecated(self):+ class MySpider4(Spider):+ name = 'spider1'+ start_urls = ['http://example.com']-if __name__ == '__main__':- unittest.main()+ class MySpider5(Spider):+ name = 'spider2'+ start_urls = ['http://example.com']++ def make_requests_from_url(self, url):+ return Request(url + "/foo", dont_filter=True)++ with warnings.catch_warnings(record=True) as w:+ # spider without overridden make_requests_from_url method+ # doesn't issue a warning+ spider1 = MySpider4()+ self.assertEqual(len(list(spider1.start_requests())), 1)+ self.assertEqual(len(w), 0)++ # spider with overridden make_requests_from_url issues a warning,+ # but the method still works+ spider2 = MySpider5()+ requests = list(spider2.start_requests())+ self.assertEqual(len(requests), 1)+ self.assertEqual(requests[0].url, 'http://example.com/foo')+ self.assertEqual(len(w), 1)