Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove deprecated Spider.make_requests_from_url method

Eugenio Lacuesta91f814455246
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index c13ba4b3c..d8248c606 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -4,14 +4,12 @@ Base class for Scrapy spiders
See documentation in docs/topics/spiders.rst
"""
import logging
-import warnings
from typing import Optional
from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import method_is_overridden
class Spider(object_ref):
@@ -57,34 +55,13 @@ class Spider(object_ref):
crawler.signals.connect(self.close, signals.spider_closed)
def start_requests(self):
- cls = self.__class__
if not self.start_urls and hasattr(self, 'start_url'):
raise AttributeError(
"Crawling could not start: 'start_urls' not found "
"or empty (but found 'start_url' attribute instead, "
"did you miss an 's'?)")
- if method_is_overridden(cls, Spider, 'make_requests_from_url'):
- warnings.warn(
- "Spider.make_requests_from_url method is deprecated; it "
- "won't be called in future Scrapy releases. Please "
- "override Spider.start_requests method instead "
- f"(see {cls.__module__}.{cls.__name__}).",
- )
- for url in self.start_urls:
- yield self.make_requests_from_url(url)
- else:
- for url in self.start_urls:
- yield Request(url, dont_filter=True)
-
- def make_requests_from_url(self, url):
- """ This method is deprecated. """
- warnings.warn(
- "Spider.make_requests_from_url method is deprecated: "
- "it will be removed and not be called by the default "
- "Spider.start_requests method in future Scrapy releases. "
- "Please override Spider.start_requests method instead."
- )
- return Request(url, dont_filter=True)
+ for url in self.start_urls:
+ yield Request(url, dont_filter=True)
def _parse(self, response, **kwargs):
return self.parse(response, **kwargs)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index d23543f6a..a7c3ee048 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name='foo'), Spider)
- def test_make_requests_from_url_deprecated(self):
- class MySpider4(Spider):
- name = 'spider1'
- start_urls = ['http://example.com']
-
- class MySpider5(Spider):
- name = 'spider2'
- start_urls = ['http://example.com']
-
- def make_requests_from_url(self, url):
- return Request(url + "/foo", dont_filter=True)
-
- with warnings.catch_warnings(record=True) as w:
- # spider without overridden make_requests_from_url method
- # doesn't issue a warning
- spider1 = MySpider4()
- self.assertEqual(len(list(spider1.start_requests())), 1)
- self.assertEqual(len(w), 0)
-
- # spider without overridden make_requests_from_url method
- # should issue a warning when called directly
- request = spider1.make_requests_from_url("http://www.example.com")
- self.assertTrue(isinstance(request, Request))
- self.assertEqual(len(w), 1)
-
- # spider with overridden make_requests_from_url issues a warning,
- # but the method still works
- spider2 = MySpider5()
- requests = list(spider2.start_requests())
- self.assertEqual(len(requests), 1)
- self.assertEqual(requests[0].url, 'http://example.com/foo')
- self.assertEqual(len(w), 2)
-
class NoParseMethodSpiderTest(unittest.TestCase):