Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove deprecated Spider.make_requests_from_url method
Eugenio Lacuesta91f814455246
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex c13ba4b3c..d8248c606 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -4,14 +4,12 @@ Base class for Scrapy spidersSee documentation in docs/topics/spiders.rst"""import logging-import warningsfrom typing import Optionalfrom scrapy import signalsfrom scrapy.http import Requestfrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spider-from scrapy.utils.deprecate import method_is_overriddenclass Spider(object_ref):@@ -57,34 +55,13 @@ class Spider(object_ref):crawler.signals.connect(self.close, signals.spider_closed)def start_requests(self):- cls = self.__class__if not self.start_urls and hasattr(self, 'start_url'):raise AttributeError("Crawling could not start: 'start_urls' not found ""or empty (but found 'start_url' attribute instead, ""did you miss an 's'?)")- if method_is_overridden(cls, Spider, 'make_requests_from_url'):- warnings.warn(- "Spider.make_requests_from_url method is deprecated; it "- "won't be called in future Scrapy releases. Please "- "override Spider.start_requests method instead "- f"(see {cls.__module__}.{cls.__name__}).",- )- for url in self.start_urls:- yield self.make_requests_from_url(url)- else:- for url in self.start_urls:- yield Request(url, dont_filter=True)-- def make_requests_from_url(self, url):- """ This method is deprecated. """- warnings.warn(- "Spider.make_requests_from_url method is deprecated: "- "it will be removed and not be called by the default "- "Spider.start_requests method in future Scrapy releases. "- "Please override Spider.start_requests method instead."- )- return Request(url, dont_filter=True)+ for url in self.start_urls:+ yield Request(url, dont_filter=True)def _parse(self, response, **kwargs):return self.parse(response, **kwargs)diff --git a/tests/test_spider.py b/tests/test_spider.pyindex d23543f6a..a7c3ee048 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase):assert issubclass(CrawlSpider, Spider)assert isinstance(CrawlSpider(name='foo'), Spider)- def test_make_requests_from_url_deprecated(self):- class MySpider4(Spider):- name = 'spider1'- start_urls = ['http://example.com']-- class MySpider5(Spider):- name = 'spider2'- start_urls = ['http://example.com']-- def make_requests_from_url(self, url):- return Request(url + "/foo", dont_filter=True)-- with warnings.catch_warnings(record=True) as w:- # spider without overridden make_requests_from_url method- # doesn't issue a warning- spider1 = MySpider4()- self.assertEqual(len(list(spider1.start_requests())), 1)- self.assertEqual(len(w), 0)-- # spider without overridden make_requests_from_url method- # should issue a warning when called directly- request = spider1.make_requests_from_url("http://www.example.com")- self.assertTrue(isinstance(request, Request))- self.assertEqual(len(w), 1)-- # spider with overridden make_requests_from_url issues a warning,- # but the method still works- spider2 = MySpider5()- requests = list(spider2.start_requests())- self.assertEqual(len(requests), 1)- self.assertEqual(requests[0].url, 'http://example.com/foo')- self.assertEqual(len(w), 2)-class NoParseMethodSpiderTest(unittest.TestCase):